diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al new file mode 100644 index 00000000000..42921205af4 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al @@ -0,0 +1,70 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.CRM.BusinessRelation; +using Microsoft.CRM.Contact; +using Microsoft.CRM.Setup; +using Microsoft.CRM.Team; +using Microsoft.Finance.Currency; +using Microsoft.Finance.Dimension; +using Microsoft.Finance.GeneralLedger.Account; +using Microsoft.Finance.GeneralLedger.Setup; +using Microsoft.Finance.SalesTax; +using Microsoft.Finance.VAT.Setup; +using Microsoft.Foundation.Address; +using Microsoft.Foundation.NoSeries; +using Microsoft.Foundation.PaymentTerms; +using Microsoft.Foundation.Shipping; +using Microsoft.Purchases.Setup; +using Microsoft.Purchases.Vendor; +using Microsoft.Sales.Customer; +using Microsoft.Sales.Setup; +using System.Environment; + +/// +/// Assigned to the customer-registered Entra app on the SOURCE (Microsoft Entra Application Card) to grant +/// cross-environment, read-only access to master data. Deliberately NOT part of "Master Data Mgt. - Objects": +/// only this set grants execute on the ODataV4 source API, so local users cannot invoke it. +/// Grants read on the master-data tables synchronized by the default configuration, so cross-environment sync +/// works read-only out of the box. For custom or additional tables, a tenant admin extends this set (or assigns +/// a second, narrowly scoped set alongside it) - which keeps least privilege instead of reaching for SUPER. +/// +permissionset 7242 "MDM Cross-Env Read" +{ + Assignable = true; + Access = Public; + Caption = 'Master Data Mgt. - Cross Environment'; + + Permissions = codeunit "MDM Cross-Env Source API" = X, + tabledata "Salesperson/Purchaser" = R, + tabledata Customer = R, + tabledata Vendor = R, + tabledata Contact = R, + tabledata "Business Relation" = R, + tabledata "Contact Business Relation" = R, + tabledata "Country/Region" = R, + tabledata "Post Code" = R, + tabledata Currency = R, + tabledata "Currency Exchange Rate" = R, + tabledata "Payment Terms" = R, + tabledata "Shipment Method" = R, + tabledata "Shipping Agent" = R, + tabledata "Sales & Receivables Setup" = R, + tabledata "Purchases & Payables Setup" = R, + tabledata "Marketing Setup" = R, + tabledata "No. Series" = R, + tabledata "No. Series Line" = R, + tabledata "G/L Account" = R, + tabledata Dimension = R, + tabledata "Dimension Value" = R, + tabledata "Gen. Business Posting Group" = R, + tabledata "Gen. Product Posting Group" = R, + tabledata "Customer Posting Group" = R, + tabledata "Vendor Posting Group" = R, + tabledata "VAT Business Posting Group" = R, + tabledata "VAT Product Posting Group" = R, + tabledata "VAT Posting Setup" = R, + tabledata "Tax Area" = R, + tabledata "Tax Group" = R, + tabledata "Tax Jurisdiction" = R, + tabledata "Tenant Media" = R; +} diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 1a6a6b01301..71f7d18dfac 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -10,7 +10,25 @@ permissionset 7230 "Master Data Mgt. - Objects" Assignable = false; Access = Public; - Permissions = codeunit * = X, + Permissions = codeunit "Master Data Mgt. Setup Default" = X, + codeunit "Integration Master Data Synch." = X, + codeunit "Master Data Management" = X, + codeunit "Master Data Mgt. Table Couple" = X, + codeunit "Master Data Mgt. Tbl. Uncouple" = X, + codeunit "Master Data Mgt. Subscribers" = X, + codeunit "Master Data Mgt. Upgrade" = X, + codeunit "Master Data Mgt. Install" = X, + codeunit "MDM Local Data Source" = X, + codeunit "MDM Source Response" = X, + codeunit "MDM Http Source Transport" = X, + codeunit "MDM Cross-Env Data Source" = X, + codeunit "MDM Source Connection" = X, + codeunit "MDM Cross-Env Change Detector" = X, + codeunit "MDM Source Capabilities" = X, + codeunit "MDM Inline Media" = X, + codeunit "MDM Source Watermark" = X, + codeunit "MDM Contact Relation Cache" = X, + codeunit "MDM Privacy Notice" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index bdbd419dfa6..795b0dfe47c 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -20,20 +20,21 @@ codeunit 7231 "Integration Master Data Synch." MappingName: Code[20]; begin OnBeforeRun(Rec, IsHandled); - if IsHandled then - exit; - Rec.SetOriginalJobQueueEntryOnHold(OriginalJobQueueEntry, PrevStatus); - if Rec.Direction in [Rec.Direction::ToIntegrationTable, Rec.Direction::Bidirectional] then - LatestModifiedOn[DateType::Local] := PerformScheduledSynchToIntegrationTable(Rec); - if Rec.Direction in [Rec.Direction::FromIntegrationTable, Rec.Direction::Bidirectional] then - LatestModifiedOn[DateType::Integration] := PerformScheduledSynchFromIntegrationTable(Rec); - MappingName := Rec.Name; - if not Rec.Find() then - Session.LogMessage('0000J8M', StrSubstNo(UnableToFindMappingErr, MappingName), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()) - else begin - Rec.UpdateTableMappingModifiedOn(LatestModifiedOn); - Rec.SetOriginalJobQueueEntryStatus(OriginalJobQueueEntry, PrevStatus); + // Run OnAfterRun even when a subscriber handled the run, so run-scoped subscriber state is always released. + if not IsHandled then begin + Rec.SetOriginalJobQueueEntryOnHold(OriginalJobQueueEntry, PrevStatus); + if Rec.Direction in [Rec.Direction::ToIntegrationTable, Rec.Direction::Bidirectional] then + LatestModifiedOn[DateType::Local] := PerformScheduledSynchToIntegrationTable(Rec); + if Rec.Direction in [Rec.Direction::FromIntegrationTable, Rec.Direction::Bidirectional] then + LatestModifiedOn[DateType::Integration] := PerformScheduledSynchFromIntegrationTable(Rec); + MappingName := Rec.Name; + if not Rec.Find() then + Session.LogMessage('0000J8M', StrSubstNo(UnableToFindMappingErr, MappingName), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()) + else begin + Rec.UpdateTableMappingModifiedOn(LatestModifiedOn); + Rec.SetOriginalJobQueueEntryStatus(OriginalJobQueueEntry, PrevStatus); + end; end; OnAfterRun(Rec); @@ -44,10 +45,13 @@ codeunit 7231 "Integration Master Data Synch." SupportedSourceType: Option ,RecordID,GUID; DateType: Option ,Integration,Local; OutOfMapFilter: Boolean; + CrossEnvBatchEndCursor: Text; + CrossEnvBatchHasMore: Boolean; RecordNotFoundErr: Label 'Cannot find %1 record %2.', Comment = '%1 = Source table caption, %2 = The lookup value when searching for the source record'; SourceRecordIsNotInMappingErr: Label 'Cannot find the mapping %2 in table %1.', Comment = '%1 Integration Table Mapping caption, %2 Integration Table Mapping Name'; CannotDetermineSourceOriginErr: Label 'Cannot determine the source origin: %1.', Comment = '%1 the value of the source id'; CopyRecordRefFailedTxt: Label 'Copy record reference failed. Integration Record ID: %1', Locked = true, Comment = '%1 - Business Central record id'; + CrossEnvCopyFailedTelemetryTxt: Label 'Cross-environment copy of a source record failed for table %1.', Locked = true, Comment = '%1 - integration table id'; UnableToFindMappingErr: Label 'Unable to find Integration Table Mapping %1', Locked = true, Comment = '%1 - Mapping name'; FieldKeyTxt: Label '%1-%2', Locked = true; @@ -72,33 +76,53 @@ codeunit 7231 "Integration Master Data Synch." MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; IntegrationRecordRef: RecordRef; + DataSource: Interface "IMDM Data Source"; IntegrationRecordID: Guid; TableFilter: Text; FilterList: List of [Text]; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin OnFindModifiedIntegrationRecords(TempIntegrationRecordRef, IntegrationTableMapping, FailedNotSkippedIdDictionary, IsHandled); if IsHandled then exit; MasterDataManagementSetup.Get(); + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + FindModifiedCrossEnvironmentRecords(TempIntegrationRecordRef, IntegrationTableMapping, FailedNotSkippedIdDictionary); + exit; + end; + DataSource := MasterDataManagementSetup.GetDataSource(); SplitIntegrationTableFilter(IntegrationTableMapping, FilterList); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); foreach TableFilter in FilterList do begin - IntegrationTableMapping.SetIntRecordRefFilter(IntegrationRecordRef, TableFilter); - if IntegrationRecordRef.FindSet() then + if DataSource.GetModifiedSet(IntegrationTableMapping, TableFilter, IntegrationRecordRef) then repeat IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then Session.LogMessage('0000J8Q', StrSubstNo(CopyRecordRefFailedTxt, IntegrationRecordID), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); until IntegrationRecordRef.Next() = 0; + IntegrationRecordRef.Close(); end; + end; + + // Cross-environment reads a BOUNDED batch (MaxPagesPerRun) from the persisted resume cursor, so a large + // initial load drains over several job runs instead of one unbounded, possibly-never-completing job. + local procedure FindModifiedCrossEnvironmentRecords(var TempIntegrationRecordRef: RecordRef; IntegrationTableMapping: Record "Integration Table Mapping"; var FailedNotSkippedIdDictionary: Dictionary of [Guid, Boolean]) + var + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + MasterDataManagement: Codeunit "Master Data Management"; + IntegrationRecordRef: RecordRef; + IntegrationRecordID: Guid; + begin + CrossEnvBatchEndCursor := ''; + CrossEnvBatchHasMore := false; + if CrossEnvDataSource.GetModifiedBatch(IntegrationTableMapping, IntegrationTableMapping.GetIntegrationTableFilter(), IntegrationTableMapping."Source Change Cursor", MaxPagesPerRun(), IntegrationRecordRef, CrossEnvBatchEndCursor, CrossEnvBatchHasMore) then + repeat + IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); + if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then + if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then + Session.LogMessage('0000VAN', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + until IntegrationRecordRef.Next() = 0; IntegrationRecordRef.Close(); end; @@ -207,31 +231,28 @@ codeunit 7231 "Integration Master Data Synch." local procedure CacheFilteredIntegrationRecords(var IntegrationSystemIDFilterList: List of [Text]; IntegrationTableMapping: Record "Integration Table Mapping"; var TempIntegrationRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - MasterDataManagement: Codeunit "Master Data Management"; IntegrationRecordRef: RecordRef; + DataSource: Interface "IMDM Data Source"; IntegrationSystemIDFilter: Text; Cached: Boolean; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin OnCacheFilteredIntegrationRecords(IntegrationSystemIDFilterList, IntegrationTableMapping, TempIntegrationRecordRef, Cached, IsHandled); if (IsHandled) then exit(Cached); MasterDataManagementSetup.Get(); + DataSource := MasterDataManagementSetup.GetDataSource(); foreach IntegrationSystemIDFilter in IntegrationSystemIDFilterList do if IntegrationSystemIDFilter <> '' then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").SetFilter(IntegrationSystemIDFilter); - if IntegrationRecordRef.FindSet() then + // GetByUidFilter returns the ref already positioned on the matching set; a separate FindSet would re-read it. +#pragma warning disable AA0181 + if DataSource.GetByUidFilter(IntegrationTableMapping, IntegrationSystemIDFilter, IntegrationRecordRef) then repeat CopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false); Cached := true; until IntegrationRecordRef.Next() = 0; +#pragma warning restore AA0181 IntegrationRecordRef.Close(); end; exit(Cached); @@ -260,7 +281,6 @@ codeunit 7231 "Integration Master Data Synch." RecordID: RecordID; IntegrationSystemID: Guid; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin case GetSourceType(SourceID) of SupportedSourceType::RecordID: @@ -282,12 +302,7 @@ codeunit 7231 "Integration Master Data Synch." MasterDataManagement.OnGetIntegrationRecordRefBySystemId(IntegrationTableMapping, RecordRef, IntegrationSystemID, IsHandled); if not IsHandled then begin MasterDataManagementSetup.Get(); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecordRef.Open(IntegrationTableMapping."Integration Table ID"); - RecordRef.ChangeCompany(SourceCompanyName); - if not RecordRef.GetBySystemId(IntegrationSystemID) then + if not MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableMapping."Integration Table ID", IntegrationSystemID, RecordRef) then exit(false); end; exit(IntegrationTableMapping.FindFilteredRec(RecordRef, OutOfMapFilter)); @@ -495,6 +510,7 @@ codeunit 7231 "Integration Master Data Synch." SourceRecordRef: RecordRef; JobId: Guid; JobStartDateTime: DateTime; + Drained: Boolean; begin JobStartDateTime := CurrentDateTime(); JobId := @@ -503,11 +519,78 @@ codeunit 7231 "Integration Master Data Synch." if not IsNullGuid(JobId) then begin MasterDataFullSynchRLn.FullSynchStarted(IntegrationTableMapping, JobId, IntegrationTableMapping.Direction::FromIntegrationTable); LatestIntegrationModifiedOn := SynchIntegrationTableToLocalTable(IntegrationTableMapping, IntegrationTableSynch, SourceRecordRef); + Drained := PersistCrossEnvResumeState(IntegrationTableMapping, JobStartDateTime, LatestIntegrationModifiedOn); + IntegrationTableSynch.EndIntegrationSynchJob(); + if Drained then + MasterDataFullSynchRLn.FullSynchFinished(IntegrationTableMapping, IntegrationTableMapping.Direction::FromIntegrationTable); + end; + end; + + // After a cross-environment run, persist the resume cursor and cap the watermark to what was actually + // processed on a partial (page-capped) run, so the next run continues instead of skipping records. Returns + // whether the source was fully drained (same-env and fully-caught-up cross-env both count as drained). + local procedure PersistCrossEnvResumeState(var IntegrationTableMapping: Record "Integration Table Mapping"; JobStartDateTime: DateTime; var LatestIntegrationModifiedOn: DateTime): Boolean + begin + if not IsCrossEnvironmentSynch() then begin if JobStartDateTime > LatestIntegrationModifiedOn then LatestIntegrationModifiedOn := JobStartDateTime; - IntegrationTableSynch.EndIntegrationSynchJob(); - MasterDataFullSynchRLn.FullSynchFinished(IntegrationTableMapping, IntegrationTableMapping.Direction::FromIntegrationTable); + exit(true); + end; + + if CrossEnvBatchHasMore then begin + IntegrationTableMapping."Source Change Cursor" := CopyStr(CrossEnvBatchEndCursor, 1, MaxStrLen(IntegrationTableMapping."Source Change Cursor")); + // Advance the watermark only to the last processed page; the cursor is the authoritative resume point. + LatestIntegrationModifiedOn := CursorModifiedAt(CrossEnvBatchEndCursor); + IntegrationTableMapping.Modify(); + Commit(); + exit(false); end; + + IntegrationTableMapping."Source Change Cursor" := ''; + IntegrationTableMapping.Modify(); + Commit(); + if JobStartDateTime > LatestIntegrationModifiedOn then + LatestIntegrationModifiedOn := JobStartDateTime; + exit(true); + end; + + local procedure IsCrossEnvironmentSynch(): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit(false); + exit(MasterDataManagementSetup."Source Environment Name" <> ''); + end; + + local procedure CursorModifiedAt(CursorText: Text) ModifiedAt: DateTime + var + Cursor: JsonObject; + Token: JsonToken; + begin + if CursorText = '' then + exit(0DT); + if not Cursor.ReadFrom(CursorText) then + exit(0DT); + if Cursor.Get('modifiedAt', Token) then + if Token.IsValue() then + if not Evaluate(ModifiedAt, Token.AsValue().AsText(), 9) then + exit(0DT); + end; + + local procedure MaxPagesPerRun(): Integer + var + MaxPages: Integer; + begin + // Cap pages per run so a large initial load resumes across runs; 0 = unbounded (test override). + MaxPages := 50; + OnGetCrossEnvMaxPagesPerRun(MaxPages); + exit(MaxPages); + end; + + [InternalEvent(false)] + local procedure OnGetCrossEnvMaxPagesPerRun(var MaxPages: Integer) + begin end; internal procedure CreateMasterDataMgtCouplingClone(ForTable: Integer; var TempMasterDataMgtCoupling: Record "Master Data Mgt. Coupling" temporary) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al new file mode 100644 index 00000000000..fcf1542e150 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al @@ -0,0 +1,153 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.CRM.BusinessRelation; +using Microsoft.Integration.SyncEngine; + +/// +/// Resolves source contact business relations cross-environment for the contact auto-create/primary-contact logic. +/// During a synchronization run it prefetches all of a link type's relations in a single call and answers lookups +/// from memory, turning the otherwise O(N) per-contact over-the-wire reads into O(1) per link type. Outside a run +/// (a one-off customer/vendor insert), or when the source reports the set is too large to serve in bulk, it reads +/// the single requested relation over the wire instead. Single instance so the prefetched snapshot survives across +/// the many record inserts of a run; the snapshot is reset at each run boundary. +/// +codeunit 7234 "MDM Contact Relation Cache" +{ + Access = Internal; + SingleInstance = true; + + var + ContactNoByRelation: Dictionary of [Text, Code[20]]; + RelationNoByContact: Dictionary of [Text, Code[20]]; + BulkAttempted: Dictionary of [Integer, Boolean]; + Degraded: Dictionary of [Integer, Boolean]; + InSyncRun: Boolean; + + /// Resolves the source contact number linked to a customer/vendor/bank account relation. + procedure TryGetSourceContactNo(LinkToTable: Enum "Contact Business Relation Link To Table"; RelationNo: Code[20]; var SourceContactNo: Code[20]): Boolean + begin + if UseBulk(LinkToTable) then + exit(ContactNoByRelation.Get(RelationKey(LinkToTable, RelationNo), SourceContactNo) and (SourceContactNo <> '')); + exit(ReadSingleContactNo(LinkToTable, RelationNo, SourceContactNo)); + end; + + /// Resolves the source relation number (customer/vendor No.) linked to a source contact. + procedure TryGetSourceRelationNo(LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]; var RelationNo: Code[20]): Boolean + begin + if UseBulk(LinkToTable) then + exit(RelationNoByContact.Get(ContactKey(LinkToTable, SourceContactNo), RelationNo) and (RelationNo <> '')); + exit(ReadSingleRelationNo(LinkToTable, SourceContactNo, RelationNo)); + end; + + local procedure UseBulk(LinkToTable: Enum "Contact Business Relation Link To Table"): Boolean + begin + if not InSyncRun then + exit(false); + EnsureBulkLoaded(LinkToTable); + exit(not Degraded.ContainsKey(LinkToTable.AsInteger())); + end; + + local procedure EnsureBulkLoaded(LinkToTable: Enum "Contact Business Relation Link To Table") + begin + if BulkAttempted.ContainsKey(LinkToTable.AsInteger()) then + exit; + BulkAttempted.Add(LinkToTable.AsInteger(), true); + LoadLinkType(LinkToTable); + end; + + local procedure LoadLinkType(LinkToTable: Enum "Contact Business Relation Link To Table") + var + FilterContactBusinessRelation: Record "Contact Business Relation"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + SourceRecordRef: RecordRef; + NotIndexed: Boolean; + RelationNo: Code[20]; + SourceContactNo: Code[20]; + begin + FilterContactBusinessRelation.SetRange("Link to Table", LinkToTable); + if not CrossEnvDataSource.TryBulkGetSourceRecordsByFilter(Database::"Contact Business Relation", FilterContactBusinessRelation.GetView(), SourceRecordRef, NotIndexed) then begin + // The source could not serve the whole set (too large to index, or a transient/consent issue): fall back + // to reading the one requested relation per lookup, which preserves per-record error isolation. + Degraded.Add(LinkToTable.AsInteger(), true); + exit; + end; + if not SourceRecordRef.FindSet() then + exit; + repeat + RelationNo := SourceRecordRef.Field(FilterContactBusinessRelation.FieldNo("No.")).Value(); + SourceContactNo := SourceRecordRef.Field(FilterContactBusinessRelation.FieldNo("Contact No.")).Value(); + if (RelationNo <> '') and (SourceContactNo <> '') then begin + AddUnique(ContactNoByRelation, RelationKey(LinkToTable, RelationNo), SourceContactNo); + AddUnique(RelationNoByContact, ContactKey(LinkToTable, SourceContactNo), RelationNo); + end; + until SourceRecordRef.Next() = 0; + end; + + local procedure ReadSingleContactNo(LinkToTable: Enum "Contact Business Relation Link To Table"; RelationNo: Code[20]; var SourceContactNo: Code[20]): Boolean + var + SourceContactBusinessRelation: Record "Contact Business Relation"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + SourceRecordRef: RecordRef; + begin + SourceContactBusinessRelation.SetRange("Link to Table", LinkToTable); + SourceContactBusinessRelation.SetRange("No.", RelationNo); + if not CrossEnvDataSource.GetSourceRecordsByFilter(Database::"Contact Business Relation", SourceContactBusinessRelation.GetView(), SourceRecordRef) then + exit(false); + SourceContactNo := SourceRecordRef.Field(SourceContactBusinessRelation.FieldNo("Contact No.")).Value(); + exit(SourceContactNo <> ''); + end; + + local procedure ReadSingleRelationNo(LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]; var RelationNo: Code[20]): Boolean + var + SourceContactBusinessRelation: Record "Contact Business Relation"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + SourceRecordRef: RecordRef; + begin + SourceContactBusinessRelation.SetRange("Link to Table", LinkToTable); + SourceContactBusinessRelation.SetRange("Contact No.", SourceContactNo); + if not CrossEnvDataSource.GetSourceRecordsByFilter(Database::"Contact Business Relation", SourceContactBusinessRelation.GetView(), SourceRecordRef) then + exit(false); + RelationNo := SourceRecordRef.Field(SourceContactBusinessRelation.FieldNo("No.")).Value(); + exit(RelationNo <> ''); + end; + + local procedure AddUnique(var Cache: Dictionary of [Text, Code[20]]; KeyText: Text; Value: Code[20]) + begin + if not Cache.ContainsKey(KeyText) then + Cache.Add(KeyText, Value); + end; + + local procedure RelationKey(LinkToTable: Enum "Contact Business Relation Link To Table"; RelationNo: Code[20]): Text + begin + exit(Format(LinkToTable.AsInteger()) + '|' + RelationNo); + end; + + local procedure ContactKey(LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]): Text + begin + exit(Format(LinkToTable.AsInteger()) + '|' + SourceContactNo); + end; + + local procedure ClearSnapshot() + begin + Clear(ContactNoByRelation); + Clear(RelationNoByContact); + Clear(BulkAttempted); + Clear(Degraded); + end; + + // The bulk path is used only within a synchronization run; the snapshot is dropped at both boundaries so it is + // fresh per run. MDM sync runs in a background job-queue session, so this state stays isolated to that session. + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Master Data Synch.", 'OnBeforeRun', '', false, false)] + local procedure MarkSyncRunStart(IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) + begin + ClearSnapshot(); + InSyncRun := true; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Master Data Synch.", 'OnAfterRun', '', false, false)] + local procedure MarkSyncRunEnd(IntegrationTableMapping: Record "Integration Table Mapping") + begin + ClearSnapshot(); + InSyncRun := false; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al new file mode 100644 index 00000000000..2413f3b9e62 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -0,0 +1,285 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; +using System.Environment; +using System.Threading; + +/// +/// Cross-environment change detector. One recurring job on the subsidiary polls the source's +/// LastModifiedAtPerTable for the tables it synchronizes and, for those changed since the mapping's watermark, +/// nudges that table's synchronization job to run now. A job already In Process is left alone. +/// +codeunit 7245 "MDM Cross-Env Change Detector" +{ + Access = Internal; + Permissions = tabledata "Master Data Management Setup" = r, + tabledata "Integration Table Mapping" = r, + tabledata "Job Queue Entry" = rm, + tabledata "Scheduled Task" = r; + + var + LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; + DetectorParseFailedTxt: Label 'The cross-environment change detector received an invalid response from the source and skipped this run.', Locked = true; + DetectionContractFailedTxt: Label 'The cross-environment change detector received a response without a valid tables array and skipped this run.', Locked = true; + DetectorTransportFailedTxt: Label 'The cross-environment change detector could not reach the source and skipped this run; the next scheduled run will retry.', Locked = true; + DetectorCapabilitiesFailedTxt: Label 'The cross-environment change detector could not negotiate capabilities with the source (malformed or unsupported response) and skipped this run.', Locked = true; + + trigger OnRun() + begin + DetectChanges(); + end; + + internal procedure DetectChanges() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + SourceConnection: Codeunit "MDM Source Connection"; + SourceResponse: Codeunit "MDM Source Response"; + MasterDataManagement: Codeunit "Master Data Management"; + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + TableIds: JsonArray; + Dimensions: Dictionary of [Text, Text]; + ResponseText: Text; + Supported: Boolean; + begin + if not MasterDataManagementSetup.Get() then + exit; + if not MasterDataManagementSetup."Is Enabled" then + exit; + if MasterDataManagementSetup."Source Environment Name" = '' then + exit; // detector is cross-environment only + + if not CollectSynchronizedTableIds(TableIds) then + exit; + + Transport := SourceConnection.GetTransport(); + // Capability negotiation is a deterministic contract exchange; a failure here (e.g. malformed capabilities) is + // NOT a transport outage, so classify it distinctly instead of masking it as "could not reach the source". + // Still non-fatal to this recurring job: skip the poll, the next scheduled run retries. + if not TryNegotiateDetectionSupport(Transport, Supported) then begin + Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAZ', DetectorCapabilitiesFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, Dimensions); + exit; + end; + // Older source doesn't advertise the detection action: skip rather than error every run. + if not Supported then + exit; + // An operational transport failure (source outage, auth, bad connection state) must NOT error this recurring + // detector job - that would burn its retry budget. Skip this poll; the next scheduled run recovers. + if not TryFetchDetection(Transport, TableIds, ResponseText) then begin + Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAO', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + exit; + end; + if not SourceResponse.TryParse(ResponseText, Response) then begin + Session.LogMessage('0000VAP', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + exit; + end; + + ProcessDetectionResponse(Response); + end; + + // Capability negotiation is a deterministic contract exchange; isolate it so a malformed-capabilities failure is + // classified distinctly from a transport outage. Both are caught by the caller (log + skip), never erroring the job. + [TryFunction] + local procedure TryNegotiateDetectionSupport(Transport: Interface "IMDM Source Transport"; var Supported: Boolean) + var + SourceCapabilities: Codeunit "MDM Source Capabilities"; + begin + Supported := SourceCapabilities.IsSupported(Transport, LastModifiedFeatureTok); + end; + + // Isolates the LastModifiedAtPerTable transport call so an operational transport error skips the poll instead of + // escaping and erroring the recurring detector job. + [TryFunction] + local procedure TryFetchDetection(Transport: Interface "IMDM Source Transport"; TableIds: JsonArray; var ResponseText: Text) + begin + ResponseText := Transport.LastModifiedAtPerTable(WriteArray(TableIds)); + end; + + local procedure CollectSynchronizedTableIds(var TableIds: JsonArray): Boolean + var + IntegrationTableMapping: Record "Integration Table Mapping"; + AddedTables: List of [Integer]; + begin + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); + IntegrationTableMapping.SetLoadFields("Integration Table ID"); + if not IntegrationTableMapping.FindSet() then + exit(false); + repeat + if not AddedTables.Contains(IntegrationTableMapping."Integration Table ID") then begin + AddedTables.Add(IntegrationTableMapping."Integration Table ID"); + TableIds.Add(IntegrationTableMapping."Integration Table ID"); + end; + until IntegrationTableMapping.Next() = 0; + exit(TableIds.Count() > 0); + end; + + local procedure ProcessDetectionResponse(var Response: JsonObject) + var + MasterDataManagement: Codeunit "Master Data Management"; + SourceResponse: Codeunit "MDM Source Response"; + Tables: JsonArray; + TablesToken: JsonToken; + EntryToken: JsonToken; + begin + if SourceResponse.ConsentRequired(Response) then + exit; // source hasn't consented to sharing; the sync job surfaces the actionable error, the detector skips + if (not Response.Get('tables', TablesToken)) or (not TablesToken.IsArray()) then begin + Session.LogMessage('0000VAQ', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + exit; + end; + Tables := TablesToken.AsArray(); + foreach EntryToken in Tables do + if EntryToken.IsObject() then // skip a malformed non-object entry instead of aborting the recurring job + ProcessTableEntry(EntryToken.AsObject()); + end; + + local procedure ProcessTableEntry(Entry: JsonObject) + var + IntegrationTableMapping: Record "Integration Table Mapping"; + LastModifiedAt: DateTime; + TableId: Integer; + HasTimestamp: Boolean; + begin + TableId := GetInteger(Entry, 'tableId'); + if TableId = 0 then + exit; + if not GetBoolean(Entry, 'tableAvailable', true) then + exit; // the sync job itself will report the unavailability + + // No timestamp (indexed:false or empty table): can't compare cheaply, so let the sync job poll (scan). + HasTimestamp := GetDateTime(Entry, 'lastModifiedAt', LastModifiedAt); + + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + IntegrationTableMapping.SetRange("Integration Table ID", TableId); + IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); + IntegrationTableMapping.SetLoadFields("Synch. Modified On Filter"); + if not IntegrationTableMapping.FindSet() then + exit; + repeat + if (not HasTimestamp) or (LastModifiedAt > IntegrationTableMapping."Synch. Modified On Filter") then + NudgeSynchJob(IntegrationTableMapping); + until IntegrationTableMapping.Next() = 0; + end; + + local procedure NudgeSynchJob(IntegrationTableMapping: Record "Integration Table Mapping") + var + JobQueueEntry: Record "Job Queue Entry"; + IsHandled: Boolean; + begin + // In Process / Error / missing jobs are left alone (FindIdleSynchJob returns only idle jobs). + if not FindIdleSynchJob(IntegrationTableMapping, JobQueueEntry) then + exit; + + // Seam so tests can observe the decision and skip the reschedule (jobs can't be scheduled in the test lab). + IsHandled := false; + OnBeforeRescheduleSynchJob(JobQueueEntry, IntegrationTableMapping, IsHandled); + if IsHandled then + exit; + + RescheduleSynchJobNow(JobQueueEntry); + end; + + local procedure FindIdleSynchJob(IntegrationTableMapping: Record "Integration Table Mapping"; var JobQueueEntry: Record "Job Queue Entry"): Boolean + begin + JobQueueEntry.ReadIsolation := IsolationLevel::ReadUncommitted; + JobQueueEntry.SetLoadFields(Status, "System Task ID"); + JobQueueEntry.SetRange("Object Type to Run", JobQueueEntry."Object Type to Run"::Codeunit); + JobQueueEntry.SetRange("Object ID to Run", Codeunit::"Integration Synch. Job Runner"); + JobQueueEntry.SetRange("Record ID to Process", IntegrationTableMapping.RecordId()); + JobQueueEntry.SetRange("Recurring Job", true); + // Only idle jobs. In Process / Error / plain On Hold are excluded, so a running job is left alone. + JobQueueEntry.SetFilter(Status, '%1|%2', JobQueueEntry.Status::Ready, JobQueueEntry.Status::"On Hold with Inactivity Timeout"); + exit(JobQueueEntry.FindFirst()); + end; + + local procedure RescheduleSynchJobNow(JobQueueEntry: Record "Job Queue Entry") + var + JobQueueEntryUpdate: Record "Job Queue Entry"; + ScheduledTask: Record "Scheduled Task"; + NewEarliestStart: DateTime; + begin + NewEarliestStart := CurrentDateTime(); + ScheduledTask.ReadIsolation := IsolationLevel::ReadUncommitted; + if not ScheduledTask.Get(JobQueueEntry."System Task ID") then + exit; + if ScheduledTask."Not Before" <= NewEarliestStart then + exit; // already due to run + + if not TaskScheduler.SetTaskReady(ScheduledTask.ID, NewEarliestStart) then + exit; + + JobQueueEntryUpdate.ReadIsolation := IsolationLevel::UpdLock; + JobQueueEntryUpdate.ID := JobQueueEntry.ID; + if JobQueueEntryUpdate.GetRecLockedExtendedTimeout() then + if JobQueueEntryUpdate.Status in [JobQueueEntryUpdate.Status::Ready, JobQueueEntryUpdate.Status::"On Hold with Inactivity Timeout"] then begin + JobQueueEntryUpdate.Status := JobQueueEntryUpdate.Status::Ready; + JobQueueEntryUpdate."Earliest Start Date/Time" := NewEarliestStart; + JobQueueEntryUpdate.Modify(); + end; + end; + + [InternalEvent(false)] + local procedure OnBeforeRescheduleSynchJob(var JobQueueEntry: Record "Job Queue Entry"; IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) + begin + end; + + local procedure GetInteger(var Container: JsonObject; PropertyName: Text): Integer + var + Token: JsonToken; + Value: Integer; + begin + // A parseable but malformed entry must not throw and kill the recurring job: skip it (returns 0). + if Container.Get(PropertyName, Token) then + if Token.IsValue() and TryReadInteger(Token, Value) then + exit(Value); + exit(0); + end; + + [TryFunction] + local procedure TryReadInteger(Token: JsonToken; var Value: Integer) + begin + Value := Token.AsValue().AsInteger(); + end; + + local procedure GetBoolean(var Container: JsonObject; PropertyName: Text; DefaultValue: Boolean): Boolean + var + Token: JsonToken; + Value: Boolean; + begin + if Container.Get(PropertyName, Token) then + if Token.IsValue() and TryReadBoolean(Token, Value) then + exit(Value); + exit(DefaultValue); + end; + + [TryFunction] + local procedure TryReadBoolean(Token: JsonToken; var Value: Boolean) + begin + Value := Token.AsValue().AsBoolean(); + end; + + local procedure GetDateTime(var Container: JsonObject; PropertyName: Text; var Value: DateTime): Boolean + var + Token: JsonToken; + ValueText: Text; + begin + if not Container.Get(PropertyName, Token) then + exit(false); + if not Token.IsValue() then + exit(false); + ValueText := Token.AsValue().AsText(); + if ValueText = '' then + exit(false); + exit(Evaluate(Value, ValueText, 9)); + end; + + local procedure WriteArray(JsonArrayValue: JsonArray) ResultText: Text + begin + JsonArrayValue.WriteTo(ResultText); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al new file mode 100644 index 00000000000..30cbc1a28ec --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -0,0 +1,591 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; + +/// +/// Reads source master data from another ENVIRONMENT (same tenant) over the ODataV4 source API and materializes +/// results into temporary records, so the existing synchronization engine processes them unchanged. Selected by +/// GetDataSource() when a Source Environment Name is configured. Wire calls go through IMDM Source Transport, +/// which tests swap for an in-process transport (source and subsidiary run in the same environment there). +/// +codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" +{ + Access = Internal; + + var + SourceResponse: Codeunit "MDM Source Response"; + SourceCapabilities: Codeunit "MDM Source Capabilities"; + InlineMedia: Codeunit "MDM Inline Media"; + SourceWatermark: Codeunit "MDM Source Watermark"; + InvalidResponseErr: Label 'The source environment returned an unexpected response for table %1.', Comment = '%1 = table caption'; + TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; + NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; + FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; + SourceConsentRequiredErr: Label 'The source environment has not approved sharing its master data with this environment. Ask an administrator of the source environment to approve the Master Data Management cross-environment privacy notice.'; + SourceProbeFailedErr: Label 'Could not read the change probe from the source environment for table %1.', Comment = '%1 = table caption'; + OpenSynchTablesActionTxt: Label 'Open Synchronization Tables'; + RecordsFeatureTok: Label 'records', Locked = true; + LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; + SourceProbeTelemetryTxt: Label 'The cross-environment source-record probe failed for table %1.', Locked = true, Comment = '%1 = table id'; + SourceConsentTelemetryTxt: Label 'The source environment has not approved cross-environment master data sharing (table %1).', Locked = true, Comment = '%1 = table id'; + ParseFailureTelemetryTxt: Label 'The cross-environment sync received an unusable response for table %1 (reason: %2).', Locked = true, Comment = '%1 = table id, %2 = reason'; + InvalidResponseReasonTok: Label 'InvalidResponse', Locked = true; + TableUnavailableReasonTok: Label 'TableUnavailable', Locked = true; + NotIndexedReasonTok: Label 'NotIndexed', Locked = true; + FieldsUnavailableReasonTok: Label 'FieldsUnavailable', Locked = true; + + procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + EndCursor: Text; + HasMore: Boolean; + begin + // Interface entry point: unbounded (fetch the whole delta). The scheduled cross-env synch uses the + // bounded GetModifiedBatch instead, so a large initial load is drained across several job runs. + exit(GetModifiedBatch(IntegrationTableMapping, TableFilter, CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"), 0, SourceRecordRef, EndCursor, HasMore)); + end; + + procedure GetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + EndCursor: Text; + HasMore: Boolean; + begin + // Full read from the start (selector '{}' = no watermark), then apply the row filter - for coupling/uncoupling. + exit(GetModifiedBatch(IntegrationTableMapping, TableFilter, '{}', 0, SourceRecordRef, EndCursor, HasMore)); + end; + + /// + /// Fetches at most MaxPages pages of changed source records starting from StartCursor (a cursor selector). + /// MaxPages = 0 means unbounded. On return EndCursor holds the resume point and HasMore tells the caller + /// whether more pages remain past the cap, so the next job run can continue from where this one stopped. + /// + internal procedure GetModifiedBatch(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; StartCursor: Text; MaxPages: Integer; var SourceRecordRef: RecordRef; var EndCursor: Text; var HasMore: Boolean): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + FieldIds: Text; + Selector: Text; + PagesFetched: Integer; + begin + SourceRecordRef.Close(); + SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID", true); + InlineMedia.Reset(); // fresh batch: drop the previous page's inline media bytes + SourceWatermark.Reset(); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + FieldIds := BuildFieldIds(IntegrationTableMapping); + if StartCursor <> '' then + Selector := StartCursor + else + Selector := CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"); + EndCursor := ''; + HasMore := false; + repeat + ParseOrError( + IntegrationTableMapping."Integration Table ID", + Transport.GetRecords(IntegrationTableMapping."Integration Table ID", FieldIds, Selector, PageSize(), TableFilter), + Response); + SourceResponse.InsertRecords(Response, SourceRecordRef); + PagesFetched += 1; + HasMore := SourceResponse.HasMore(Response); + if HasMore then begin + EndCursor := SourceResponse.GetNextCursor(Response); + if EndCursor = '' then begin // hasMore without a resume cursor is malformed: don't restart from the watermark and persist a bad resume point + LogParseFailure(IntegrationTableMapping."Integration Table ID", InvalidResponseReasonTok); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableMapping."Integration Table ID")))); + end; + Selector := EndCursor; + end; + until (not HasMore) or ((MaxPages > 0) and (PagesFetched >= MaxPages)); + // The mapping row filter is applied server-side in GetRecords now (parity with same-env), so the materialized + // set is already narrowed; no client-side re-filtering, which would misfire on fields outside the projection. + exit(SourceRecordRef.FindSet()); + end; + + /// + /// Cheap existence probe for the full-synch review: the source reports an empty lastModifiedAt for an empty + /// table, so this avoids counting the whole table over the wire. + /// + internal procedure SourceHasRecords(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + Entry: JsonObject; + Tables: JsonArray; + Token: JsonToken; + TableIds: JsonArray; + TableIdsText: Text; + LastModifiedAtText: Text; + IntegrationTableId: Integer; + BoolValue: Boolean; + begin + IntegrationTableId := IntegrationTableMapping."Integration Table ID"; + // With a row filter the cheap per-table timestamp can't tell whether any MATCHING record exists (parity with + // same-env, which counts filtered rows), so fetch a single filtered record and report on its presence. + if TableFilter <> '' then + exit(SourceHasFilteredRecords(IntegrationTableMapping, TableFilter)); + TableIds.Add(IntegrationTableId); + TableIds.WriteTo(TableIdsText); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, LastModifiedFeatureTok); + // A transport/parse failure is not an empty source; surface it so the full-synch review isn't misled. + if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(TableIdsText), Response) then begin + LogProbeFailure(IntegrationTableId); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + end; + if SourceResponse.ConsentRequired(Response) then begin + LogConsentRequired(IntegrationTableId); + Error(SourceConsentError()); + end; + if (not Response.Get('tables', Token)) or (not Token.IsArray()) then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + Tables := Token.AsArray(); + if Tables.Count() = 0 then begin + LogProbeFailure(IntegrationTableId); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + end; + Tables.Get(0, Token); + if not Token.IsObject() then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + Entry := Token.AsObject(); + if Entry.Get('tableAvailable', Token) then begin + if not (Token.IsValue() and TryGetBoolean(Token, BoolValue)) then begin // malformed contract, not an empty source: keep it in the internal-error path + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + if not BoolValue then + exit(false); + end; + // Unindexed source table: LastModifiedAtPerTable reports indexed:false and no timestamp, so we can't prove + // emptiness cheaply - assume records may exist so the full-synch review isn't wrongly suppressed. + if Entry.Get('indexed', Token) then begin + if not (Token.IsValue() and TryGetBoolean(Token, BoolValue)) then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + if not BoolValue then + exit(true); + end; + if Entry.Get('lastModifiedAt', Token) then + if Token.IsValue() then + LastModifiedAtText := Token.AsValue().AsText(); + exit(LastModifiedAtText <> ''); + end; + + /// + /// Existence probe honoring the mapping row filter: fetches a single matching record from the source so the + /// full-synch review reflects the filtered set, matching how the same-env path counts filtered rows. + /// + local procedure SourceHasFilteredRecords(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + Token: JsonToken; + IntegrationTableId: Integer; + begin + IntegrationTableId := IntegrationTableMapping."Integration Table ID"; + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + // '{}' selector = read from the start (no watermark); PageSize 1 keeps this an existence check, not a count. + // Project only the primary key: the row filter is applied server-side, so no mapped media/blob need materialize. + if not SourceResponse.TryParse(Transport.GetRecords(IntegrationTableId, BuildPrimaryKeyFieldIds(IntegrationTableId), '{}', 1, TableFilter), Response) then begin + LogProbeFailure(IntegrationTableId); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + end; + if SourceResponse.ConsentRequired(Response) then begin + LogConsentRequired(IntegrationTableId); + Error(SourceConsentError()); + end; + // An unavailable table yields no records array; treat as no matching records for the review. + if not SourceResponse.TableAvailable(Response) then + exit(false); + // An unindexed source past the change cap returns indexed:false with an empty page; like the unfiltered probe + // we can't prove emptiness cheaply, so assume records may exist rather than under-reporting to the review. + if not SourceResponse.Indexed(Response) then + exit(true); + // Available and indexed, so a records array is contractually present; its absence is a malformed response, not "no records". + if not (Response.Get('records', Token) and Token.IsArray()) then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + exit(Token.AsArray().Count() > 0); + end; + + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + var + SystemIds: List of [Guid]; + begin + SystemIds.Add(SystemId); + FetchBySystemIds(IntegrationTableId, BuildFieldIdsForTable(IntegrationTableId), SystemIds, SourceRecordRef); + exit(SourceRecordRef.FindFirst()); + end; + + procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean + var + SystemIds: List of [Guid]; + SystemIdValue: Guid; + TextKey: Text; + begin + if ID.IsGuid then + SystemIdValue := ID + else + if ID.IsText then begin + TextKey := ID; + if not Evaluate(SystemIdValue, TextKey) then + exit(false); + end else + // RecordId is environment-specific; the cross-env feed keys on SystemId only. + exit(false); + SystemIds.Add(SystemIdValue); + FetchBySystemIds(IntegrationTableMapping."Integration Table ID", BuildFieldIds(IntegrationTableMapping), SystemIds, SourceRecordRef); + exit(SourceRecordRef.FindFirst()); + end; + + procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + // MDM's UID field is SystemId, so a UID filter is a set of SystemIds. + FetchBySystemIds(IntegrationTableMapping."Integration Table ID", BuildFieldIds(IntegrationTableMapping), ParseSystemIds(UidFilter), SourceRecordRef); + exit(SourceRecordRef.FindSet()); + end; + + local procedure FetchBySystemIds(IntegrationTableId: Integer; FieldIds: Text; SystemIds: List of [Guid]; var SourceRecordRef: RecordRef) + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + begin + SourceRecordRef.Close(); + SourceRecordRef.Open(IntegrationTableId, true); + InlineMedia.Reset(); // fresh fetch: drop any prior inline media bytes + SourceWatermark.Reset(); + if SystemIds.Count() = 0 then + exit; + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + ParseOrError( + IntegrationTableId, + Transport.GetRecords(IntegrationTableId, FieldIds, SystemIdsSelector(SystemIds), PageSize(), ''), + Response); + SourceResponse.InsertRecords(Response, SourceRecordRef); + end; + + // Reads a RELATED source table (not one of the synchronized mappings) narrowed by a row filter, materializing + // matches into a temporary record. Used to resolve the source's contact business relations cross-environment, + // mirroring the same-env ChangeCompany read. Access is gated by the source's cross-environment read permission set. + internal procedure GetSourceRecordsByFilter(TableId: Integer; RowFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + begin + SourceRecordRef.Close(); + SourceRecordRef.Open(TableId, true); + InlineMedia.Reset(); + SourceWatermark.Reset(); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + // Full read from the start (selector '{}' = no watermark), narrowed to the related rows by the row filter. + ParseOrError(TableId, Transport.GetRecords(TableId, BuildFieldIdsForTable(TableId), '{}', PageSize(), RowFilter), Response); + SourceResponse.InsertRecords(Response, SourceRecordRef); + exit(SourceRecordRef.FindSet()); + end; + + // Bulk variant of GetSourceRecordsByFilter that never errors: it returns false (with NotIndexed set when the + // source reports the filtered set is too large to serve without an index) so the caller can fall back to + // per-record reads. Used to prefetch all of a link type's contact business relations in one call, turning the + // per-contact O(N) lookups during a sync run into O(1) calls. + internal procedure TryBulkGetSourceRecordsByFilter(TableId: Integer; RowFilter: Text; var SourceRecordRef: RecordRef; var NotIndexed: Boolean): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + begin + NotIndexed := false; + SourceRecordRef.Close(); + SourceRecordRef.Open(TableId, true); + InlineMedia.Reset(); + SourceWatermark.Reset(); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + if not TryParseCleanResponse(Transport.GetRecords(TableId, BuildFieldIdsForTable(TableId), '{}', PageSize(), RowFilter), Response, NotIndexed) then + exit(false); + // A multi-page filtered result can't be fully materialized in one bulk call; degrade to per-record reads + // rather than caching a partial (silently incomplete) snapshot. + if SourceResponse.HasMore(Response) then + exit(false); + SourceResponse.InsertRecords(Response, SourceRecordRef); + exit(true); + end; + + // Like ParseOrError but non-fatal: returns false instead of erroring, so a failed bulk prefetch degrades to + // per-record reads (which then surface any genuine consent/availability error one record at a time). + local procedure TryParseCleanResponse(ResponseText: Text; var Response: JsonObject; var NotIndexed: Boolean): Boolean + var + UnavailableFields: JsonArray; + begin + Clear(Response); + NotIndexed := false; + if not SourceResponse.TryParse(ResponseText, Response) then + exit(false); + if SourceResponse.ConsentRequired(Response) then + exit(false); + if not SourceResponse.TableAvailable(Response) then + exit(false); + if not SourceResponse.Indexed(Response) then begin + NotIndexed := true; + exit(false); + end; + if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then + exit(false); + exit(true); + end; + + // A malformed wire response is an internal integration defect, not something the user can act on. + local procedure InternalError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + + [TryFunction] + local procedure TryGetBoolean(Token: JsonToken; var Value: Boolean) + begin + Value := Token.AsValue().AsBoolean(); + end; + + // The source declined to share (its cross-environment privacy notice isn't approved); actionable by the SOURCE + // environment's admin, so there is no local navigation action to add. + local procedure SourceConsentError(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := SourceConsentRequiredErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + exit(ErrInfo); + end; + + // The table isn't exposed on the source: a recoverable setup issue, so point the user at Synchronization Tables. + local procedure SynchTablesNavigationError(IntegrationTableId: Integer; MessageText: Text): ErrorInfo + var + IntegrationTableMapping: Record "Integration Table Mapping"; + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.PageNo := Page::"Master Data Synch. Tables"; + // Land on the specific mapping row the user must fix, when it resolves to one. + if GetMappingByIntegrationTableId(IntegrationTableId, IntegrationTableMapping) then + ErrInfo.RecordId := IntegrationTableMapping.RecordId(); + ErrInfo.AddNavigationAction(OpenSynchTablesActionTxt); + exit(ErrInfo); + end; + + local procedure LogParseFailure(IntegrationTableId: Integer; Reason: Text) + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + Session.LogMessage('0000VAR', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + + local procedure LogProbeFailure(IntegrationTableId: Integer) + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + Session.LogMessage('0000VAS', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + + local procedure LogConsentRequired(IntegrationTableId: Integer) + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + Session.LogMessage('0000VDI', StrSubstNo(SourceConsentTelemetryTxt, IntegrationTableId), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + + local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) + var + UnavailableFields: JsonArray; + begin + Clear(Response); + if not SourceResponse.TryParse(ResponseText, Response) then begin + LogParseFailure(IntegrationTableId, InvalidResponseReasonTok); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + if SourceResponse.ConsentRequired(Response) then begin + LogConsentRequired(IntegrationTableId); + Error(SourceConsentError()); + end; + if not SourceResponse.TableAvailable(Response) then begin + LogParseFailure(IntegrationTableId, TableUnavailableReasonTok); + Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(TableUnavailableErr, TableCaption(IntegrationTableId)))); + end; + if not SourceResponse.Indexed(Response) then begin + LogParseFailure(IntegrationTableId, NotIndexedReasonTok); + Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(NotIndexedErr, TableCaption(IntegrationTableId)))); + end; + if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then begin + LogParseFailure(IntegrationTableId, FieldsUnavailableReasonTok); + Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(FieldsUnavailableErr, TableCaption(IntegrationTableId)))); + end; + end; + + // FieldIds = the mapping's integration-side fields + the table's primary-key fields (so temp inserts don't collide). + local procedure BuildFieldIds(IntegrationTableMapping: Record "Integration Table Mapping"): Text + var + IntegrationFieldMapping: Record "Integration Field Mapping"; + FieldIds: JsonArray; + AddedFields: List of [Integer]; + begin + AddPrimaryKeyFields(IntegrationTableMapping."Integration Table ID", FieldIds, AddedFields); + IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); + IntegrationFieldMapping.SetLoadFields("Integration Table Field No."); + if IntegrationFieldMapping.FindSet() then + repeat + if IntegrationFieldMapping."Integration Table Field No." <> 0 then + AddFieldId(FieldIds, AddedFields, IntegrationFieldMapping."Integration Table Field No."); + until IntegrationFieldMapping.Next() = 0; + exit(WriteArray(FieldIds)); + end; + + // Primary-key-only projection for the existence probe: the server-side filter still evaluates against every field, + // so no mapped fields (in particular media/blob) need to be projected just to learn whether a record exists. + local procedure BuildPrimaryKeyFieldIds(IntegrationTableId: Integer): Text + var + FieldIds: JsonArray; + AddedFields: List of [Integer]; + begin + AddPrimaryKeyFields(IntegrationTableId, FieldIds, AddedFields); + exit(WriteArray(FieldIds)); + end; + + local procedure BuildFieldIdsForTable(IntegrationTableId: Integer): Text + var + IntegrationTableMapping: Record "Integration Table Mapping"; + begin + if GetMappingByIntegrationTableId(IntegrationTableId, IntegrationTableMapping) then + exit(BuildFieldIds(IntegrationTableMapping)); + exit(AllNormalFields(IntegrationTableId)); + end; + + local procedure GetMappingByIntegrationTableId(IntegrationTableId: Integer; var IntegrationTableMapping: Record "Integration Table Mapping"): Boolean + begin + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Integration Table ID", IntegrationTableId); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + exit(IntegrationTableMapping.FindFirst()); + end; + + local procedure AddPrimaryKeyFields(IntegrationTableId: Integer; var FieldIds: JsonArray; var AddedFields: List of [Integer]) + var + RecRef: RecordRef; + PrimaryKeyRef: KeyRef; + Index: Integer; + begin + RecRef.Open(IntegrationTableId, true); + PrimaryKeyRef := RecRef.KeyIndex(1); + for Index := 1 to PrimaryKeyRef.FieldCount() do + AddFieldId(FieldIds, AddedFields, PrimaryKeyRef.FieldIndex(Index).Number()); + RecRef.Close(); + end; + + local procedure AllNormalFields(IntegrationTableId: Integer): Text + var + RecRef: RecordRef; + CurrentField: FieldRef; + FieldIds: JsonArray; + AddedFields: List of [Integer]; + Index: Integer; + begin + RecRef.Open(IntegrationTableId, true); + for Index := 1 to RecRef.FieldCount() do begin + CurrentField := RecRef.FieldIndex(Index); + if CurrentField.Class() = FieldClass::Normal then + if not (CurrentField.Type() in [FieldType::MediaSet, FieldType::TableFilter]) then + AddFieldId(FieldIds, AddedFields, CurrentField.Number()); + end; + RecRef.Close(); + exit(WriteArray(FieldIds)); + end; + + local procedure AddFieldId(var FieldIds: JsonArray; var AddedFields: List of [Integer]; FieldNo: Integer) + begin + if AddedFields.Contains(FieldNo) then + exit; + AddedFields.Add(FieldNo); + FieldIds.Add(FieldNo); + end; + + local procedure ParseSystemIds(UidFilter: Text) SystemIds: List of [Guid] + var + Token: Text; + SystemIdValue: Guid; + begin + foreach Token in UidFilter.Split('|') do + if Evaluate(SystemIdValue, Token) then + SystemIds.Add(SystemIdValue); + end; + + local procedure SystemIdsSelector(SystemIds: List of [Guid]): Text + var + Selector: JsonObject; + SystemIdArray: JsonArray; + SystemIdValue: Guid; + SelectorText: Text; + begin + foreach SystemIdValue in SystemIds do + SystemIdArray.Add(Format(SystemIdValue)); + Selector.Add('systemIds', SystemIdArray); + Selector.WriteTo(SelectorText); + exit(SelectorText); + end; + + local procedure CursorSelector(Watermark: DateTime): Text + var + Selector: JsonObject; + SelectorText: Text; + begin + if Watermark = 0DT then + exit('{}'); + Selector.Add('modifiedAt', Format(Watermark, 0, 9)); + Selector.WriteTo(SelectorText); + exit(SelectorText); + end; + + local procedure WriteArray(JsonArrayValue: JsonArray) ResultText: Text + begin + JsonArrayValue.WriteTo(ResultText); + end; + + local procedure TableCaption(IntegrationTableId: Integer) Caption: Text + var + RecRef: RecordRef; + begin + RecRef.Open(IntegrationTableId, true); + Caption := RecRef.Caption(); + RecRef.Close(); + end; + + local procedure PageSize(): Integer + var + Size: Integer; + begin + Size := 1000; + OnGetCrossEnvPageSize(Size); + exit(Size); + end; + + [InternalEvent(false)] + local procedure OnGetCrossEnvPageSize(var PageSize: Integer) + begin + end; + + local procedure GetTransport(): Interface "IMDM Source Transport" + var + SourceConnection: Codeunit "MDM Source Connection"; + begin + exit(SourceConnection.GetTransport()); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al new file mode 100644 index 00000000000..d2c158bd417 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -0,0 +1,681 @@ +namespace Microsoft.Integration.MDM; + +using System.Environment; +using System.Text; +using System.Utilities; + +/// +/// Source-side generic API, published as an ODataV4 web service. The cross-environment implementation of +/// "IMDM Data Source" (running in a subsidiary) calls these unbound actions to read source master data. +/// Runs under the CALLER's permission set with NO permission elevation, so a read-only, table-scoped +/// permission set assigned to the caller's Entra app is the effective access boundary. +/// +codeunit 7241 "MDM Cross-Env Source API" +{ + Access = Public; + + var + SortByChangeFeedKeyTok: Label 'SORTING(Field%1,Field%2)', Locked = true; + SortByModifiedAtTok: Label 'SORTING(Field%1)', Locked = true; + + /// + /// Wire-version negotiation: returns the API contract version and the action/feature names this source + /// supports, so a newer subsidiary only calls actions an older source actually implements. + /// + /// A JSON object with the numeric contract 'version' and a 'features' array of supported action names. + [ServiceEnabled] + procedure GetCapabilities(): Text + var + Capabilities: JsonObject; + Features: JsonArray; + ResultText: Text; + begin + Capabilities.Add('version', ApiVersion()); + Features.Add('records'); + Features.Add('lastModifiedPerTable'); + Capabilities.Add('features', Features); + Capabilities.WriteTo(ResultText); + exit(ResultText); + end; + + /// + /// Returns a page of changed source records for the given table. FieldIds is a JSON array of field + /// numbers. Selector is either a change-feed cursor { modifiedAt, systemId } or a targeted + /// { systemIds } list. Response carries records, hasMore and (cursor mode) nextCursor. The table read + /// runs under the CALLER's permission set, so table-level access is enforced by permissions, not here. + /// + /// The source table ID to read. + /// A JSON array of the field numbers to project. + /// A JSON object: a change-feed cursor { modifiedAt, systemId } or a targeted { systemIds } list. + /// The maximum number of records to return in this page. + /// An optional source row filter (view) restricting which records are returned. + /// A JSON object with 'records', 'hasMore', 'unavailableFields' and, in cursor mode, 'nextCursor'. + [ServiceEnabled] + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text + var + RecRef: RecordRef; + Response: JsonObject; + Records: JsonArray; + UnavailableFields: JsonArray; + SystemIds: JsonArray; + ProjectedFields: List of [Integer]; + CursorModifiedAt: DateTime; + NextModifiedAt: DateTime; + CursorSystemId: Guid; + NextSystemId: Guid; + HasCursor: Boolean; + HasMore: Boolean; + GroupTooLarge: Boolean; + Count: Integer; + ResultText: Text; + begin + if not IsSourceConsented() then + exit(ConsentRequiredResponse()); + Response.Add('tableId', TableId); + + if IsBlockedSourceTable(TableId) then + exit(WriteResponse(Response, false, Records, false)); + if not TryOpenTable(TableId, RecRef) then + exit(WriteResponse(Response, false, Records, false)); + Response.Add('tableAvailable', true); + + // Missing field => HALT that table's sync (no partial records); subsidiary logs a synch error. + // Fields that exist but are media/blob/flow are skipped silently (media sync is deferred). + ResolveProjection(RecRef, FieldIds, ProjectedFields, UnavailableFields); + if UnavailableFields.Count() > 0 then begin + Response.Add('unavailableFields', UnavailableFields); + exit(WriteResponse(Response, true, Records, false)); + end; + + PageSize := ClampPageSize(PageSize); + ApplyProjectionLoadFields(RecRef, ProjectedFields); + + // Targeted mode: caller asked for specific SystemIds (no paging). The mapping row filter is not applied here, + // matching same-env GetById/GetByUidFilter (targeted fetches return the requested records directly). + if SelectorSystemIds(Selector, SystemIds) then begin + FillBySystemIds(RecRef, SystemIds, ProjectedFields, Records); + exit(WriteResponse(Response, true, Records, false)); + end; + + HasCursor := SelectorCursor(Selector, CursorModifiedAt, CursorSystemId); + + if HasCompositeChangeFeedKey(RecRef) then begin + // Bounded paging: the (SystemModifiedAt, SystemId) key lets us split even a big same-timestamp group. + RecRef.SetView(StrSubstNo(SortByChangeFeedKeyTok, SystemModifiedAtFieldNo(), SystemIdFieldNo())); + ApplyRowFilter(RecRef, Filter); + HasMore := FillCursorPage(RecRef, HasCursor, CursorModifiedAt, CursorSystemId, ProjectedFields, PageSize, Records, Count, NextModifiedAt, NextSystemId); + if Count > 0 then + Response.Add('nextCursor', BuildCursor(NextModifiedAt, NextSystemId)); + end else + if HasModifiedAtLeadingKey(RecRef) then begin + // Fallback for tables without the SystemId tiebreak (kept off small/setup tables): drain each + // timestamp group whole so the cursor can advance by SystemModifiedAt alone. Safe while groups + // are small; a group too large to page keylessly asks for the composite key instead. + RecRef.SetView(StrSubstNo(SortByModifiedAtTok, SystemModifiedAtFieldNo())); + ApplyRowFilter(RecRef, Filter); + HasMore := FillDrainPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, PageSize, Records, Count, NextModifiedAt, GroupTooLarge); + if GroupTooLarge then begin + Clear(Records); + Response.Add('indexed', false); + exit(WriteResponse(Response, true, Records, false)); + end; + if Count > 0 then + Response.Add('nextCursor', BuildModifiedAtCursor(NextModifiedAt)); + end else begin + // No SystemModifiedAt index at all: no-code scan fallback. Order by primary key (always indexed), + // filter SystemModifiedAt > watermark, and return the whole changed set in one shot (capped). + // Over the cap, ask for the composite key (only large unindexed tables need it). + ApplyRowFilter(RecRef, Filter); + HasMore := FillScanPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, Records, Count, NextModifiedAt, GroupTooLarge); + if GroupTooLarge then begin + Clear(Records); + Response.Add('indexed', false); + exit(WriteResponse(Response, true, Records, false)); + end; + if Count > 0 then + Response.Add('nextCursor', BuildModifiedAtCursor(NextModifiedAt)); + end; + + Response.Add('records', Records); + Response.Add('hasMore', HasMore); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + local procedure WriteResponse(var Response: JsonObject; TableAvailable: Boolean; var Records: JsonArray; HasMore: Boolean): Text + var + ResultText: Text; + begin + if not Response.Contains('tableAvailable') then + Response.Add('tableAvailable', TableAvailable); + Response.Add('records', Records); + Response.Add('hasMore', HasMore); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + [TryFunction] + local procedure TryOpenTable(TableId: Integer; var RecRef: RecordRef) + begin + RecRef.Open(TableId); + RecRef.ReadIsolation := IsolationLevel::ReadCommitted; + end; + + // Media/infrastructure tables are only reachable inline (BuildMediaValue, tied to a record's media field). Never + // serve them as a top-level read, or a caller holding the Tenant Media read grant could enumerate every blob. + // The environment serving its data must have consented. If not, return a structured signal (not an error) so the + // subsidiary surfaces a clear, actionable message instead of a raw HTTP failure. + local procedure IsSourceConsented(): Boolean + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + exit(MDMPrivacyNotice.IsApproved()); + end; + + local procedure ConsentRequiredResponse(): Text + var + Response: JsonObject; + ResultText: Text; + begin + Response.Add('consentRequired', true); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + local procedure IsBlockedSourceTable(TableId: Integer): Boolean + begin + exit(TableId in [Database::"Tenant Media", Database::"Tenant Media Set", Database::"Tenant Media Thumbnails"]); + end; + + // Applies the mapping's row filter server-side (parity with same-env, which filters the source record directly). + // Re-hydrated as field-level filters so it composes with the change-feed SORTING view already set for paging, + // and works even when the filter references a field outside the projection (which the temp buffer would default). + local procedure ApplyRowFilter(var RecRef: RecordRef; Filter: Text) + var + FilterSource: RecordRef; + SourceField: FieldRef; + FieldFilter: Text; + Index: Integer; + begin + if Filter = '' then + exit; + FilterSource.Open(RecRef.Number()); + FilterSource.SetView(Filter); + for Index := 1 to FilterSource.FieldCount() do begin + SourceField := FilterSource.FieldIndex(Index); + FieldFilter := SourceField.GetFilter(); + if FieldFilter <> '' then + RecRef.Field(SourceField.Number()).SetFilter(FieldFilter); + end; + FilterSource.Close(); + end; + + local procedure ResolveProjection(var RecRef: RecordRef; FieldIds: Text; var ProjectedFields: List of [Integer]; var UnavailableFields: JsonArray) + var + RequestedFields: JsonArray; + Token: JsonToken; + FieldNo: Integer; + begin + if not TryReadJsonArray(FieldIds, RequestedFields) then + exit; + foreach Token in RequestedFields do + if Token.IsValue() and TryReadInteger(Token, FieldNo) then // a malformed field id is skipped, not served as an error + if not RecRef.FieldExist(FieldNo) then + UnavailableFields.Add(FieldNo) + else + if IsProjectableField(RecRef.Field(FieldNo)) then + ProjectedFields.Add(FieldNo); + end; + + local procedure IsProjectableField(FieldReference: FieldRef): Boolean + begin + // Media and Blob are projected inline (base64); MediaSet is deferred and TableFilter carries no data. + exit((FieldReference.Class() = FieldClass::Normal) and + not (FieldReference.Type() in [FieldType::MediaSet, FieldType::TableFilter])); + end; + + // Load only the projected fields (plus the change-feed keys) so wide source tables aren't fully materialized. + local procedure ApplyProjectionLoadFields(var RecRef: RecordRef; ProjectedFields: List of [Integer]) + var + FieldNo: Integer; + begin + RecRef.SetLoadFields(SystemModifiedAtFieldNo(), SystemIdFieldNo()); + foreach FieldNo in ProjectedFields do + RecRef.AddLoadFields(FieldNo); + end; + + local procedure SelectorSystemIds(Selector: Text; var SystemIds: JsonArray): Boolean + var + SelectorObject: JsonObject; + Token: JsonToken; + begin + if not TryReadJsonObject(Selector, SelectorObject) then + exit(false); + if not SelectorObject.Get('systemIds', Token) then + exit(false); + if not Token.IsArray() then + exit(false); + SystemIds := Token.AsArray(); + exit(SystemIds.Count() > 0); + end; + + local procedure SelectorCursor(Selector: Text; var CursorModifiedAt: DateTime; var CursorSystemId: Guid): Boolean + var + SelectorObject: JsonObject; + Token: JsonToken; + begin + if not TryReadJsonObject(Selector, SelectorObject) then + exit(false); + if not SelectorObject.Get('modifiedAt', Token) then + exit(false); + if not Token.IsValue() then + exit(false); + if not Evaluate(CursorModifiedAt, Token.AsValue().AsText(), 9) then + exit(false); + if SelectorObject.Get('systemId', Token) and Token.IsValue() then + Evaluate(CursorSystemId, Token.AsValue().AsText()); + exit(true); + end; + + local procedure HasCompositeChangeFeedKey(var RecRef: RecordRef): Boolean + var + CurrentKey: KeyRef; + Index: Integer; + begin + for Index := 1 to RecRef.KeyCount() do begin + CurrentKey := RecRef.KeyIndex(Index); + if CurrentKey.FieldCount() >= 2 then + if (CurrentKey.FieldIndex(1).Number() = SystemModifiedAtFieldNo()) and + (CurrentKey.FieldIndex(2).Number() = SystemIdFieldNo()) + then + exit(true); + end; + exit(false); + end; + + local procedure HasModifiedAtLeadingKey(var RecRef: RecordRef): Boolean + var + CurrentKey: KeyRef; + Index: Integer; + begin + for Index := 1 to RecRef.KeyCount() do begin + CurrentKey := RecRef.KeyIndex(Index); + if CurrentKey.FieldCount() >= 1 then + if CurrentKey.FieldIndex(1).Number() = SystemModifiedAtFieldNo() then + exit(true); + end; + exit(false); + end; + + // No SystemId tiebreak available, so never split a timestamp group across pages: fill to PageSize, then + // drain the trailing group whole and advance the cursor by SystemModifiedAt with a strict '>'. + local procedure FillDrainPage(var RecRef: RecordRef; HasCursor: Boolean; CursorModifiedAt: DateTime; ProjectedFields: List of [Integer]; PageSize: Integer; var Records: JsonArray; var Count: Integer; var NextModifiedAt: DateTime; var GroupTooLarge: Boolean): Boolean + var + ModifiedAtRef: FieldRef; + CurrentModifiedAt: DateTime; + LastEmittedAt: DateTime; + IgnoredSystemId: Guid; + MaxKeylessGroup: Integer; + PageBytes: Integer; + begin + Count := 0; + PageBytes := 0; + LastEmittedAt := 0DT; + GroupTooLarge := false; + MaxKeylessGroup := 10000; + ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); + if HasCursor then + ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); + if RecRef.FindSet() then + repeat + CurrentModifiedAt := ModifiedAtRef.Value(); + // Stop only at a clean group boundary once the page is full (by count or inline bytes). + if ((Count >= PageSize) or (PageBytes >= MaxPageInlineBytes())) and (CurrentModifiedAt <> LastEmittedAt) then + exit(true); + // A single same-timestamp group that can't be paged cleanly (too many rows OR too many inline bytes) => ask for a key. + if (Count >= MaxKeylessGroup) or (PageBytes >= MaxPageInlineBytes()) then begin + GroupTooLarge := true; + exit(false); + end; + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, IgnoredSystemId, PageBytes); + LastEmittedAt := NextModifiedAt; + Count += 1; + until RecRef.Next() = 0; + exit(false); + end; + + // No SystemModifiedAt index at all: order by primary key (always indexed) and filter SystemModifiedAt > + // watermark. Single-shot up to a cap; a bigger changed set trips TooLarge so the caller asks for a key. + local procedure FillScanPage(var RecRef: RecordRef; HasCursor: Boolean; CursorModifiedAt: DateTime; ProjectedFields: List of [Integer]; var Records: JsonArray; var Count: Integer; var MaxModifiedAt: DateTime; var TooLarge: Boolean): Boolean + var + ModifiedAtRef: FieldRef; + CurrentModifiedAt: DateTime; + IgnoredModifiedAt: DateTime; + IgnoredSystemId: Guid; + MaxUnindexedRecords: Integer; + PageBytes: Integer; + begin + Count := 0; + PageBytes := 0; + TooLarge := false; + MaxUnindexedRecords := 10000; + ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); + if HasCursor then + ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); + if RecRef.FindSet() then + repeat + // Unindexed scan can't resume mid-set, so too many records OR too many inline bytes => ask for a key. + if (Count >= MaxUnindexedRecords) or (PageBytes >= MaxPageInlineBytes()) then begin + TooLarge := true; + exit(false); + end; + CurrentModifiedAt := ModifiedAtRef.Value(); + if CurrentModifiedAt > MaxModifiedAt then + MaxModifiedAt := CurrentModifiedAt; + AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId, PageBytes); + Count += 1; + until RecRef.Next() = 0; + exit(false); + end; + + local procedure FillCursorPage(var RecRef: RecordRef; HasCursor: Boolean; CursorModifiedAt: DateTime; CursorSystemId: Guid; ProjectedFields: List of [Integer]; PageSize: Integer; var Records: JsonArray; var Count: Integer; var NextModifiedAt: DateTime; var NextSystemId: Guid): Boolean + var + ModifiedAtRef: FieldRef; + SystemIdRef: FieldRef; + PageBytes: Integer; + begin + Count := 0; + ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); + SystemIdRef := RecRef.Field(SystemIdFieldNo()); + + // Pass 1: records at exactly the cursor timestamp but a later SystemId (DB uniqueidentifier order). + if HasCursor then begin + ModifiedAtRef.SetRange(CursorModifiedAt); + SystemIdRef.SetFilter('>%1', CursorSystemId); + if RecRef.FindSet() then + repeat + if Count = PageSize then + exit(true); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId, PageBytes); + Count += 1; + if PageBytes >= MaxPageInlineBytes() then + exit(true); // inline-byte budget: >=1 record emitted; resume from NextModifiedAt/NextSystemId + until RecRef.Next() = 0; + ModifiedAtRef.SetRange(); + SystemIdRef.SetRange(); + end; + + // Pass 2: records strictly after the cursor timestamp (or all records on the first call). + if HasCursor then + ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); + if RecRef.FindSet() then + repeat + if Count = PageSize then + exit(true); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId, PageBytes); + Count += 1; + if PageBytes >= MaxPageInlineBytes() then + exit(true); + until RecRef.Next() = 0; + + exit(false); + end; + + local procedure FillBySystemIds(var RecRef: RecordRef; SystemIds: JsonArray; ProjectedFields: List of [Integer]; var Records: JsonArray) + var + SystemIdRef: FieldRef; + FilterBuilder: TextBuilder; + Token: JsonToken; + SystemIdValue: Guid; + IgnoredModifiedAt: DateTime; + IgnoredSystemId: Guid; + IgnoredPageBytes: Integer; + FilterText: Text; + begin + foreach Token in SystemIds do + if Token.IsValue() and Evaluate(SystemIdValue, Token.AsValue().AsText()) then begin + if FilterBuilder.Length() > 0 then + FilterBuilder.Append('|'); + FilterBuilder.Append(Format(SystemIdValue)); + end; + FilterText := FilterBuilder.ToText(); + if FilterText = '' then + exit; + + SystemIdRef := RecRef.Field(SystemIdFieldNo()); + SystemIdRef.SetFilter(FilterText); + if RecRef.FindSet() then + repeat + AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId, IgnoredPageBytes); + until RecRef.Next() = 0; + end; + + local procedure AppendRecord(var RecRef: RecordRef; ProjectedFields: List of [Integer]; var Records: JsonArray; var LastModifiedAt: DateTime; var LastSystemId: Guid; var PageBytes: Integer) + var + CurrentField: FieldRef; + RecordObject: JsonObject; + FieldsObject: JsonObject; + FieldNo: Integer; + begin + LastModifiedAt := RecRef.Field(SystemModifiedAtFieldNo()).Value(); + LastSystemId := RecRef.Field(SystemIdFieldNo()).Value(); + RecordObject.Add('systemId', Format(LastSystemId)); + RecordObject.Add('systemModifiedAt', FormatFieldValue(RecRef.Field(SystemModifiedAtFieldNo()))); + foreach FieldNo in ProjectedFields do begin + CurrentField := RecRef.Field(FieldNo); + case CurrentField.Type() of + FieldType::Media: + FieldsObject.Add(Format(FieldNo), BuildMediaValue(CurrentField, PageBytes)); + FieldType::Blob: + FieldsObject.Add(Format(FieldNo), BuildBlobValue(CurrentField, PageBytes)); + else + FieldsObject.Add(Format(FieldNo), FormatFieldValue(CurrentField)); + end; + end; + RecordObject.Add('fields', FieldsObject); + Records.Add(RecordObject); + end; + + local procedure BuildCursor(ModifiedAt: DateTime; SystemId: Guid): JsonObject + var + Cursor: JsonObject; + begin + Cursor.Add('modifiedAt', Format(ModifiedAt, 0, 9)); + Cursor.Add('systemId', Format(SystemId)); + exit(Cursor); + end; + + local procedure BuildModifiedAtCursor(ModifiedAt: DateTime): JsonObject + var + Cursor: JsonObject; + begin + Cursor.Add('modifiedAt', Format(ModifiedAt, 0, 9)); + exit(Cursor); + end; + + // Invariant (XML) format so field values round-trip via Evaluate(..., 9) on the subsidiary. + local procedure FormatFieldValue(FieldReference: FieldRef): Text + begin + exit(Format(FieldReference.Value(), 0, 9)); + end; + + // Single Media field: emit { media, name, mimeType, length, content(base64) }, or { media, empty } when the + // source has no picture, or { media, skipped, length } when it exceeds the inline cap. + local procedure BuildMediaValue(FieldReference: FieldRef; var PageBytes: Integer): JsonObject + var + TenantMedia: Record "Tenant Media"; + Base64Convert: Codeunit "Base64 Convert"; + MediaValue: JsonObject; + MediaId: Guid; + ContentInStream: InStream; + begin + MediaValue.Add('media', true); + MediaId := FieldReference.Value(); + if IsNullGuid(MediaId) then begin + MediaValue.Add('empty', true); + exit(MediaValue); + end; + TenantMedia.SetAutoCalcFields(Content); + if not TenantMedia.Get(MediaId) then begin + MediaValue.Add('empty', true); + exit(MediaValue); + end; + if TenantMedia.Content.Length() > MaxInlineContentSize() then begin + MediaValue.Add('skipped', true); + MediaValue.Add('length', TenantMedia.Content.Length()); + exit(MediaValue); + end; + MediaValue.Add('name', TenantMedia."File Name"); + MediaValue.Add('mimeType', TenantMedia."Mime Type"); + MediaValue.Add('length', TenantMedia.Content.Length()); + TenantMedia.Content.CreateInStream(ContentInStream); + MediaValue.Add('content', Base64Convert.ToBase64(ContentInStream)); + PageBytes += TenantMedia.Content.Length(); + exit(MediaValue); + end; + + // Blob field: emit { blob, length, content(base64) }, or { blob, empty }, or { blob, skipped, length }. + local procedure BuildBlobValue(FieldReference: FieldRef; var PageBytes: Integer): JsonObject + var + Base64Convert: Codeunit "Base64 Convert"; + TempBlob: Codeunit "Temp Blob"; + BlobValue: JsonObject; + ContentInStream: InStream; + begin + BlobValue.Add('blob', true); + TempBlob.FromFieldRef(FieldReference); + if not TempBlob.HasValue() then begin + BlobValue.Add('empty', true); + exit(BlobValue); + end; + if TempBlob.Length() > MaxInlineContentSize() then begin + BlobValue.Add('skipped', true); + BlobValue.Add('length', TempBlob.Length()); + exit(BlobValue); + end; + BlobValue.Add('length', TempBlob.Length()); + TempBlob.CreateInStream(ContentInStream); + BlobValue.Add('content', Base64Convert.ToBase64(ContentInStream)); + PageBytes += TempBlob.Length(); + exit(BlobValue); + end; + + // 512 KB raw. Its base64 form (~700 KB) stays under BC's 1,000,000-byte single-stream-read limit; do not + // raise toward 1 MB, where the encoded value would exceed that limit on a single read. + local procedure MaxInlineContentSize(): Integer + begin + exit(512 * 1024); + end; + + // Cap inline media/blob bytes per page (~3 MB raw, ~4 MB base64) so a media-heavy page stays well under the + // 8-minute operation timeout and memory; the composite cursor resumes the remaining records on the next page. + local procedure MaxPageInlineBytes(): Integer + var + MaxBytes: Integer; + begin + MaxBytes := 3 * 1024 * 1024; + OnGetMaxPageInlineBytes(MaxBytes); + exit(MaxBytes); + end; + + [InternalEvent(false)] + local procedure OnGetMaxPageInlineBytes(var MaxBytes: Integer) + begin + end; + + local procedure TryReadJsonArray(Value: Text; var JsonArrayValue: JsonArray): Boolean + begin + exit(JsonArrayValue.ReadFrom(Value)); + end; + + local procedure TryReadJsonObject(Value: Text; var JsonObjectValue: JsonObject): Boolean + begin + exit(JsonObjectValue.ReadFrom(Value)); + end; + + [TryFunction] + local procedure TryReadInteger(Token: JsonToken; var Value: Integer) + begin + Value := Token.AsValue().AsInteger(); + end; + + local procedure ClampPageSize(PageSize: Integer): Integer + begin + if PageSize <= 0 then + exit(100); + if PageSize > 1000 then + exit(1000); + exit(PageSize); + end; + + local procedure SystemIdFieldNo(): Integer + begin + exit(2000000000); + end; + + local procedure SystemModifiedAtFieldNo(): Integer + begin + exit(2000000003); + end; + + /// + /// Change detection: for each requested table id, returns its latest source modification timestamp so + /// the subsidiary detector can decide which per-table sync jobs to reschedule. Read from the change-feed + /// index tip (FindLast), so no scan and no summary table to maintain. + /// + /// A JSON array of the source table IDs to probe. + /// A JSON object with a 'tables' array of { tableId, tableAvailable, indexed, lastModifiedAt } entries. + [ServiceEnabled] + procedure LastModifiedAtPerTable(TableIds: Text): Text + var + RequestedTables: JsonArray; + Tables: JsonArray; + Response: JsonObject; + Token: JsonToken; + ResultText: Text; + TableId: Integer; + begin + if not IsSourceConsented() then + exit(ConsentRequiredResponse()); + if TryReadJsonArray(TableIds, RequestedTables) then + foreach Token in RequestedTables do + if Token.IsValue() and TryReadInteger(Token, TableId) then // a malformed table id is skipped, not served as an error + Tables.Add(BuildTableModifiedAt(TableId)); + Response.Add('tables', Tables); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + local procedure BuildTableModifiedAt(TableId: Integer): JsonObject + var + RecRef: RecordRef; + Entry: JsonObject; + begin + Entry.Add('tableId', TableId); + if IsBlockedSourceTable(TableId) or (not TryOpenTable(TableId, RecRef)) then begin + Entry.Add('tableAvailable', false); + exit(Entry); + end; + Entry.Add('tableAvailable', true); + if not HasModifiedAtLeadingKey(RecRef) then begin + Entry.Add('indexed', false); + exit(Entry); + end; + RecRef.SetView(StrSubstNo(SortByModifiedAtTok, SystemModifiedAtFieldNo())); + // Detection only needs an approximate max, so read uncommitted: never takes or waits on a lock, even + // where snapshot isolation is off (OnPrem). Data reads (GetRecords) stay at committed isolation. + RecRef.ReadIsolation := IsolationLevel::ReadUncommitted; + RecRef.SetLoadFields(SystemModifiedAtFieldNo()); + if RecRef.FindLast() then + Entry.Add('lastModifiedAt', Format(RecRef.Field(SystemModifiedAtFieldNo()).Value(), 0, 9)) + else + Entry.Add('lastModifiedAt', ''); // empty table: no changes to detect + exit(Entry); + end; + + local procedure ApiVersion(): Integer + begin + exit(1); + end; +} + diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al new file mode 100644 index 00000000000..1bba9a4b3af --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -0,0 +1,381 @@ +namespace Microsoft.Integration.MDM; + +using System.Azure.Identity; +using System.Environment; +using System.Reflection; +using System.Security.Authentication; +using System.Telemetry; +using System.Utilities; + +/// +/// Production transport: calls the source environment's ODataV4 web service with an app-only (client +/// credentials) token. Same-tenant by construction — the OAuth authority (tenant + ring) is derived from THIS +/// environment, so there is no tenant-id setting to point the connection at another tenant; production and the +/// TIE/PPE ring use different Entra authorities and Business Central resource audiences. Tests never hit +/// this: they inject an in-process transport that calls the source API directly. +/// +codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" +{ + Access = Internal; + + var + CachedToken: SecretText; + TokenExpiresAt: DateTime; + MaxRetriesValue: Integer; + NotConfiguredErr: Label 'The cross-environment connection to the source is not configured yet.'; + OpenSetupActionTxt: Label 'Open Master Data Management Setup'; + NonSaaSErr: Label 'Cross-environment synchronization is only available in online environments.'; + NoTokenErr: Label 'Could not acquire an access token for the source environment. Check the client ID and secret.'; + SendFailedErr: Label 'The request to the source environment could not be sent. Check the source environment URL.'; + InvalidSourceUrlErr: Label 'The source environment URL is not a valid Business Central endpoint.'; + InvalidSourceUrlAuditTxt: Label 'Blocked a cross-environment request: the configured source environment URL host ''%1'' is not a valid Business Central endpoint.', Comment = '%1 = the rejected host'; + HttpErr: Label 'The source environment returned HTTP %1.', Comment = '%1 = HTTP status code'; + ServiceNameTok: Label 'MDMCrossEnvSource', Locked = true; + ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true; + ScopePPETok: Label 'https://api.businesscentral.dynamics-tie.com/.default', Locked = true; + TokenEndpointTok: Label 'https://login.microsoftonline.com/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; + TokenEndpointPPETok: Label 'https://login.windows-ppe.net/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; + SourceHostProdTok: Label 'api.businesscentral.dynamics.com', Locked = true; + SourceHostPPETok: Label 'api.businesscentral.dynamics-tie.com', Locked = true; + ActionUrlTok: Label '%1/ODataV4/%2_%3?company=%4', Locked = true, Comment = '%1 = base url, %2 = service, %3 = action, %4 = company'; + BaseUrlTok: Label 'https://%1/v2.0/%2/%3', Locked = true, Comment = '%1 = api host, %2 = Entra tenant id, %3 = source environment name'; + TelemetryCategoryTok: Label 'MDM Cross-Environment', Locked = true; + TokenAcquiredAuditTxt: Label 'Acquired an application access token to read master data from source environment %1.', Comment = '%1 = source environment name'; + TokenFailedAuditTxt: Label 'Failed to acquire an application access token for source environment %1.', Comment = '%1 = source environment name'; + AccessDeniedAuditTxt: Label 'Source environment %1 denied the master data request (HTTP %2).', Comment = '%1 = source environment name, %2 = HTTP status code'; + RequestFailedTelemetryTxt: Label 'Cross-environment %1 request failed with HTTP %2.', Locked = true, Comment = '%1 = action name, %2 = HTTP status code'; + TransportFailedTelemetryTxt: Label 'Cross-environment %1 request could not be sent to the source environment.', Locked = true, Comment = '%1 = action name'; + + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text + var + Body: JsonObject; + BodyText: Text; + begin + Body.Add('tableId', TableId); + Body.Add('fieldIds', FieldIds); + Body.Add('selector', Selector); + Body.Add('pageSize', PageSize); + Body.Add('filter', Filter); + Body.WriteTo(BodyText); + exit(InvokeAction('GetRecords', BodyText)); + end; + + procedure LastModifiedAtPerTable(TableIds: Text): Text + var + Body: JsonObject; + BodyText: Text; + begin + Body.Add('tableIds', TableIds); + Body.WriteTo(BodyText); + exit(InvokeAction('LastModifiedAtPerTable', BodyText)); + end; + + procedure GetCapabilities(): Text + begin + exit(InvokeAction('GetCapabilities', '{}')); + end; + + local procedure InvokeAction(ActionName: Text; RequestBody: Text): Text + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + EnvironmentInformation: Codeunit "Environment Information"; + PrivacyNotice: Codeunit "MDM Privacy Notice"; + ResponseMessage: HttpResponseMessage; + ResponseBodyText: Text; + RetryAfter: Duration; + Attempt: Integer; + begin + GetConfiguredSetup(MasterDataManagementSetup); + if not EnvironmentInformation.IsSaaSInfrastructure() then + Error(NonSaaSErr); + PrivacyNotice.CheckApproved(); + + for Attempt := 0 to MaxRetries() do + if TrySend(MasterDataManagementSetup, ActionName, RequestBody, ResponseMessage) then begin + ResponseMessage.Content().ReadAs(ResponseBodyText); + if ResponseMessage.IsSuccessStatusCode() then + exit(UnwrapODataValue(ResponseBodyText)); + if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin + LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); + Error(SetupNavigationError(StrSubstNo(HttpErr, ResponseMessage.HttpStatusCode()))); + end; + Sleep(RetryAfter); + end else begin + // Transport-level failure (dropped connection, timeout): these reads are idempotent, so retry while + // attempts remain instead of aborting the sync run on a transient network blip. + if Attempt >= MaxRetries() then begin + LogTransportFailure(ActionName); + Error(SetupNavigationError(SendFailedErr)); + end; + Sleep(TransportRetryBackoff()); + end; + end; + + local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage) + var + AuditLog: Codeunit "Audit Log"; + Dimensions: Dictionary of [Text, Text]; + begin + // The response body can echo source-environment record content, so it is never emitted to telemetry; only + // the action and HTTP status (non-content diagnostics) are logged. + Dimensions.Add('Category', TelemetryCategoryTok); + Dimensions.Add('Action', ActionName); + Dimensions.Add('HttpStatusCode', Format(ResponseMessage.HttpStatusCode())); + Session.LogMessage('0000VAT', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + // Security audit: an authorization failure crossing the environment boundary. + if ResponseMessage.HttpStatusCode() in [401, 403] then + AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); + end; + + local procedure LogTransportFailure(ActionName: Text) + var + Dimensions: Dictionary of [Text, Text]; + begin + // GetLastErrorText() can contain record keys or file names, so the raw error is never emitted to telemetry; + // only the action is logged. + Dimensions.Add('Category', TelemetryCategoryTok); + Dimensions.Add('Action', ActionName); + Session.LogMessage('0000VAU', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + end; + + local procedure TrySend(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage): Boolean + var + HttpClient: HttpClient; + RequestMessage: HttpRequestMessage; + RequestHeaders: HttpHeaders; + HttpContent: HttpContent; + ContentHeaders: HttpHeaders; + begin + HttpClient.Timeout := 100000; // explicit 100s cap so a stalled connection can't hang a background sync run + RequestMessage.Method('POST'); + RequestMessage.SetRequestUri(BuildActionUrl(MasterDataManagementSetup, ActionName)); + RequestMessage.GetHeaders(RequestHeaders); + RequestHeaders.Add('Accept', 'application/json'); + RequestHeaders.Add('Authorization', SecretStrSubstNo('Bearer %1', GetBearerToken(MasterDataManagementSetup))); + + HttpContent.WriteFrom(RequestBody); + HttpContent.GetHeaders(ContentHeaders); + ContentHeaders.Remove('Content-Type'); + ContentHeaders.Add('Content-Type', 'application/json'); + RequestMessage.Content(HttpContent); + + // A transport-level failure returns false so the caller can retry the read-only call. + exit(HttpClient.Send(RequestMessage, ResponseMessage)); + end; + + local procedure BuildActionUrl(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text): Text + var + BaseUrl: Text; + begin + BaseUrl := DelChr(MasterDataManagementSetup."Source Environment URL", '>', '/'); + ValidateSourceHost(BaseUrl); + exit(StrSubstNo(ActionUrlTok, BaseUrl, ServiceNameTok, ActionName, UriEncodeCompany(MasterDataManagementSetup."Source Company Name"))); + end; + + // The source must be a Business Central SaaS endpoint over HTTPS. Validating the configured URL before the + // bearer token is attached stops the setup field from redirecting the authenticated call to an arbitrary host (SSRF). + local procedure ValidateSourceHost(BaseUrl: Text) + var + AuditLog: Codeunit "Audit Log"; + Uri: Codeunit Uri; + Host: Text; + begin + // Only the standard Business Central API hosts are allowed - production and the TIE ring. Embed/ISV + // deployments must be on a normal cluster that serves api.businesscentral.dynamics.com. Matching the host + // EXACTLY (not a dynamics.com suffix) blocks SSRF via look-alike hosts; a malformed URL surfaces the + // actionable setup error instead of the URI parser's raw exception. + if TryInitUri(Uri, BaseUrl) then begin + Host := LowerCase(Uri.GetHost()); + if (Uri.GetScheme() = 'https') and ((Host = SourceHostProdTok) or (Host = SourceHostPPETok)) then + exit; + end; + AuditLog.LogAuditMessage(StrSubstNo(InvalidSourceUrlAuditTxt, Host), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); + Error(SetupNavigationError(InvalidSourceUrlErr)); + end; + + [TryFunction] + local procedure TryInitUri(var Uri: Codeunit Uri; BaseUrl: Text) + begin + Uri.Init(BaseUrl); + end; + + // Test seam: exercise the source-host allow-list without a live environment or the SaaS gate. + internal procedure ValidateSourceHostUrl(BaseUrl: Text) + begin + ValidateSourceHost(BaseUrl); + end; + + // The ODataV4 envelope for an action returning Text is { "@odata.context": "...", "value": "" }; + // the source API's own JSON is that inner value. Fall back to the raw body if the shape is unexpected. + local procedure UnwrapODataValue(ResponseBody: Text): Text + var + Envelope: JsonObject; + ValueToken: JsonToken; + begin + if Envelope.ReadFrom(ResponseBody) then + if Envelope.Get('value', ValueToken) then + if ValueToken.IsValue() then + exit(ValueToken.AsValue().AsText()); + exit(ResponseBody); + end; + + // Test seam: exercise OData envelope unwrapping without a live transport. + internal procedure UnwrapODataValueForTest(ResponseBody: Text): Text + begin + exit(UnwrapODataValue(ResponseBody)); + end; + + local procedure GetBearerToken(var MasterDataManagementSetup: Record "Master Data Management Setup"): SecretText + begin + // Cached in-instance for the lifetime of a single sync run (the paging loop reuses this transport). + if (TokenExpiresAt <> 0DT) and (TokenExpiresAt > CurrentDateTime()) then + exit(CachedToken); + CachedToken := AcquireToken(MasterDataManagementSetup); + TokenExpiresAt := CurrentDateTime() + (3500 * 1000); // refresh a little before the ~1h token lifetime + exit(CachedToken); + end; + + [NonDebuggable] + local procedure AcquireToken(var MasterDataManagementSetup: Record "Master Data Management Setup") Token: SecretText + var + OAuth2: Codeunit OAuth2; + AzureADTenant: Codeunit "Azure AD Tenant"; + AuditLog: Codeunit "Audit Log"; + Scopes: List of [Text]; + TokenEndpoint: Text; + begin + Scopes.Add(GetBcResourceScope()); + TokenEndpoint := StrSubstNo(GetTokenEndpointTemplate(), AzureADTenant.GetAadTenantId()); + // Prefer a cached/refreshed token (no round-trip when a valid one exists); fall back to a fresh + // client-credentials grant if the cache misses, errors, or returns an empty token. + if not TryAcquireTokenFromCache(MasterDataManagementSetup, TokenEndpoint, Scopes, Token) then + Clear(Token); + if Token.IsEmpty() then + if not OAuth2.AcquireTokenWithClientCredentials( + MasterDataManagementSetup."Source OAuth Client Id", + MasterDataManagementSetup.GetSourceClientSecret(), + TokenEndpoint, '', Scopes, Token) or Token.IsEmpty() + then begin + AuditLog.LogAuditMessage(StrSubstNo(TokenFailedAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Failure, AuditCategory::Authentication, 4, 0); + Error(SetupNavigationError(NoTokenErr)); + end; + AuditLog.LogAuditMessage(StrSubstNo(TokenAcquiredAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Success, AuditCategory::Authentication, 4, 0); + end; + + // Reuses a token from the platform (MSAL) cache when one is valid; a cache miss or error is treated as "no token" + // so the caller falls back to a fresh client-credentials grant. + [TryFunction] + [NonDebuggable] + local procedure TryAcquireTokenFromCache(MasterDataManagementSetup: Record "Master Data Management Setup"; TokenEndpoint: Text; Scopes: List of [Text]; var Token: SecretText) + var + OAuth2: Codeunit OAuth2; + begin + if not OAuth2.AcquireAuthorizationCodeTokenFromCache(MasterDataManagementSetup."Source OAuth Client Id", MasterDataManagementSetup.GetSourceClientSecret(), '', TokenEndpoint, Scopes, Token) then + Clear(Token); + end; + + // Production and the TIE/PPE ring use different Entra authorities and BC resource audiences. Cross-env is same + // tenant and same ring, so detect the ring from this environment's web URL and mint the token for the right one. + local procedure IsPPE(): Boolean + begin + exit(StrPos(LowerCase(GetUrl(ClientType::Web)), 'businesscentral.dynamics-tie.com') <> 0); + end; + + local procedure GetBcResourceScope(): Text + begin + if IsPPE() then + exit(ScopePPETok); + exit(ScopeTok); + end; + + local procedure GetTokenEndpointTemplate(): Text + begin + if IsPPE() then + exit(TokenEndpointPPETok); + exit(TokenEndpointTok); + end; + + // Same-tenant, same-ring: the source web-service base URL is fully derivable from the source environment name, + // this environment's tenant, and the ring host, so the wizard constructs it instead of asking for a free-text URL. + internal procedure BuildSourceApiBaseUrl(EnvironmentName: Text): Text + var + AzureADTenant: Codeunit "Azure AD Tenant"; + begin + exit(StrSubstNo(BaseUrlTok, GetSourceApiHost(), AzureADTenant.GetAadTenantId(), EnvironmentName)); + end; + + local procedure GetSourceApiHost(): Text + begin + if IsPPE() then + exit(SourceHostPPETok); + exit(SourceHostProdTok); + end; + + local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") + begin + if not MasterDataManagementSetup.Get() then + Error(SetupNavigationError(NotConfiguredErr)); + if not MasterDataManagementSetup.IsCrossEnvConnectionConfigured() then + Error(SetupNavigationError(NotConfiguredErr)); + end; + + local procedure SetupNavigationError(MessageText: Text): ErrorInfo + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + // The remedy is always the setup page, even before the record exists; only RecordId needs an existing record. + ErrInfo.PageNo := Page::"Master Data Management Setup"; + ErrInfo.AddNavigationAction(OpenSetupActionTxt); + if MasterDataManagementSetup.Get() then + ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); + exit(ErrInfo); + end; + + local procedure ShouldRetry(var ResponseMessage: HttpResponseMessage; Attempt: Integer; var RetryAfter: Duration): Boolean + begin + if Attempt >= MaxRetries() then + exit(false); + if not (ResponseMessage.HttpStatusCode() in [408, 429, 502, 503, 504]) then + exit(false); + RetryAfter := RetryAfterDuration(ResponseMessage); + exit(true); + end; + + local procedure RetryAfterDuration(var ResponseMessage: HttpResponseMessage) RetryAfter: Duration + var + ResponseHeaders: HttpHeaders; + Values: array[10] of Text; + Seconds: Integer; + begin + RetryAfter := 5000; // default backoff when the source gives no Retry-After + ResponseHeaders := ResponseMessage.Headers(); + if ResponseHeaders.GetValues('Retry-After', Values) then + if Evaluate(Seconds, Values[1]) then + if Seconds > 0 then + RetryAfter := Seconds * 1000; + if RetryAfter > 60000 then + RetryAfter := 60000; // never wait more than a minute inside a job + end; + + local procedure UriEncodeCompany(CompanyNameText: Text): Text + var + TypeHelper: Codeunit "Type Helper"; + begin + exit(TypeHelper.UrlEncode(CompanyNameText)); + end; + + local procedure MaxRetries(): Integer + begin + if MaxRetriesValue = 0 then + MaxRetriesValue := 2; + exit(MaxRetriesValue); + end; + + local procedure TransportRetryBackoff(): Duration + begin + exit(5000); // fixed backoff for transport failures, where no Retry-After header is available + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al new file mode 100644 index 00000000000..b5a3c083408 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al @@ -0,0 +1,100 @@ +namespace Microsoft.Integration.MDM; + +using System.Text; +using System.Utilities; + +/// +/// Per-batch cache of inline Media bytes carried in a cross-environment GetRecords response. The materializer +/// stashes the decoded content keyed by the source record's SystemId and the Media field number; the field +/// transfer subscriber (UpdateMedia, cross-environment branch) reads it to build the destination Tenant Media. +/// A Media field only holds a Tenant Media GUID, which is meaningless in the subsidiary, so the bytes travel +/// here instead of on the temporary source record. Single instance so the value survives from materialization +/// through the synchronization write; Reset() clears it between batches. +/// +codeunit 7232 "MDM Inline Media" +{ + Access = Internal; + SingleInstance = true; + + var + ContentByKey: Dictionary of [Text, Text]; + NameByKey: Dictionary of [Text, Text]; + MimeByKey: Dictionary of [Text, Text]; + ClearedByKey: Dictionary of [Text, Boolean]; + MalformedMediaErr: Label 'The source returned media content that could not be decoded.', Locked = true; + + procedure Reset() + begin + Clear(ContentByKey); + Clear(NameByKey); + Clear(MimeByKey); + Clear(ClearedByKey); + end; + + procedure Put(SystemId: Guid; FieldNo: Integer; FileName: Text; MimeType: Text; ContentBase64: Text) + var + MediaKey: Text; + begin + MediaKey := MakeKey(SystemId, FieldNo); + ContentByKey.Set(MediaKey, ContentBase64); + NameByKey.Set(MediaKey, FileName); + MimeByKey.Set(MediaKey, MimeType); + end; + + // The source reported the media field empty (cleared): record it so the transfer clears the destination media + // instead of leaving stale bytes. Distinct from an absent entry, which means "not projected / leave untouched". + procedure PutCleared(SystemId: Guid; FieldNo: Integer) + begin + ClearedByKey.Set(MakeKey(SystemId, FieldNo), true); + end; + + procedure IsCleared(SystemId: Guid; FieldNo: Integer): Boolean + begin + exit(ClearedByKey.ContainsKey(MakeKey(SystemId, FieldNo))); + end; + + procedure Contains(SystemId: Guid; FieldNo: Integer): Boolean + begin + exit(ContentByKey.ContainsKey(MakeKey(SystemId, FieldNo))); + end; + + procedure TryGet(SystemId: Guid; FieldNo: Integer; var FileName: Text; var MimeType: Text; var TempBlob: Codeunit "Temp Blob"): Boolean + var + MediaKey: Text; + ContentBase64: Text; + begin + MediaKey := MakeKey(SystemId, FieldNo); + if not ContentByKey.Get(MediaKey, ContentBase64) then + exit(false); + NameByKey.Get(MediaKey, FileName); + MimeByKey.Get(MediaKey, MimeType); + if not TryDecodeContent(ContentBase64, TempBlob) then // undecodable media content is a broken record entry from the source + Error(MalformedMediaContent()); + exit(true); + end; + + [TryFunction] + local procedure TryDecodeContent(ContentBase64: Text; var TempBlob: Codeunit "Temp Blob") + var + Base64Convert: Codeunit "Base64 Convert"; + ContentOutStream: OutStream; + begin + TempBlob.CreateOutStream(ContentOutStream); + Base64Convert.FromBase64(ContentBase64, ContentOutStream); + end; + + local procedure MalformedMediaContent(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MalformedMediaErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + + local procedure MakeKey(SystemId: Guid; FieldNo: Integer): Text + begin + exit(Format(SystemId, 0, 4) + '|' + Format(FieldNo)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al new file mode 100644 index 00000000000..3133c5cb65b --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -0,0 +1,86 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; + +/// +/// Reads source master data from another company in the same environment via ChangeCompany. +/// This preserves today's behavior; it is the implementation used whenever no source environment is set. +/// +codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" +{ + Access = Internal; + + procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + IntegrationTableMapping.SetIntRecordRefFilter(SourceRecordRef, TableFilter); + exit(SourceRecordRef.FindSet()); + end; + + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableId, SourceRecordRef); + exit(SourceRecordRef.GetBySystemId(SystemId)); + end; + + procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean + var + RecId: RecordID; + SystemId: Guid; + TextKey: Text; + begin + SourceRecordRef.Close(); + // MDM always maps the integration UID to the SystemId field, so exact-key lookups seek by SystemId instead of a filtered find. + if ID.IsGuid then begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + SystemId := ID; + exit(SourceRecordRef.GetBySystemId(SystemId)); + end; + + if ID.IsRecordId then begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + RecId := ID; + if RecId.TableNo = IntegrationTableMapping."Table ID" then + exit(SourceRecordRef.Get(ID)); + end; + + if ID.IsText then begin + TextKey := ID; + if not Evaluate(SystemId, TextKey) then + exit(false); + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + exit(SourceRecordRef.GetBySystemId(SystemId)); + end; + end; + + procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + SourceRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").SetFilter(UidFilter); + exit(SourceRecordRef.FindSet()); + end; + + procedure GetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + if TableFilter <> '' then + SourceRecordRef.SetView(TableFilter); + exit(SourceRecordRef.FindSet()); + end; + + local procedure OpenSourceRecordRef(IntegrationTableId: Integer; var SourceRecordRef: RecordRef) + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataManagement: Codeunit "Master Data Management"; + SourceCompanyName: Text[30]; + begin + MasterDataManagementSetup.Get(); + if SourceRecordRef.Number() <> 0 then + SourceRecordRef.Close(); // a re-fetch may pass an already-open ref; start from a clean handle + SourceRecordRef.Open(IntegrationTableId); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableId); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + SourceRecordRef.ChangeCompany(SourceCompanyName); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al new file mode 100644 index 00000000000..2774c842c25 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al @@ -0,0 +1,70 @@ +namespace Microsoft.Integration.MDM; + +using System.Privacy; + +/// +/// Registers the privacy notice for cross-environment master data synchronization and gates data transfer on +/// its approval. The setup wizard records the durable platform approval; the HTTP transport verifies it before +/// every outbound call, so no master data leaves the environment without recorded per-integration consent. +/// +codeunit 7242 "MDM Privacy Notice" +{ + Access = Internal; + InherentEntitlements = X; + InherentPermissions = X; + + var + PrivacyNoticeIdTok: Label 'MDMCrossEnvSync', Locked = true; + IntegrationServiceNameTxt: Label 'Master Data Management - cross-environment synchronization'; + NotApprovedErr: Label 'Cross-environment master data synchronization requires the privacy notice to be approved. Open Master Data Management Setup and approve sharing data between Business Central environments.'; + OpenSetupActionTxt: Label 'Open Master Data Management Setup'; + PrivacyLinkTok: Label 'https://go.microsoft.com/fwlink/?linkid=521839', Locked = true; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Privacy Notice", OnRegisterPrivacyNotices, '', false, false)] + local procedure RegisterPrivacyNotice(var TempPrivacyNotice: Record "Privacy Notice" temporary) + begin + TempPrivacyNotice.Init(); + TempPrivacyNotice.ID := PrivacyNoticeIdTok; + TempPrivacyNotice."Integration Service Name" := IntegrationServiceNameTxt; + TempPrivacyNotice.Link := PrivacyLinkTok; + if not TempPrivacyNotice.Insert() then; + end; + + procedure GetPrivacyNoticeId(): Code[50] + begin + exit(PrivacyNoticeIdTok); + end; + + procedure IsApproved(): Boolean + var + PrivacyNotice: Codeunit "Privacy Notice"; + begin + exit(PrivacyNotice.GetPrivacyNoticeApprovalState(PrivacyNoticeIdTok, false) = "Privacy Notice Approval State"::Agreed); + end; + + // Interactive: shows the platform notice and records the admin's decision (used from the setup wizard). + procedure ConfirmApproval(): Boolean + var + PrivacyNotice: Codeunit "Privacy Notice"; + begin + exit(PrivacyNotice.ConfirmPrivacyNoticeApproval(PrivacyNoticeIdTok, false)); + end; + + // Non-interactive gate for background/transport paths: fail closed if consent isn't recorded. + procedure CheckApproved() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + ErrInfo: ErrorInfo; + begin + if IsApproved() then + exit; + ErrInfo.Message := NotApprovedErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + // The remedy is always the setup page, even before the record exists; only RecordId needs an existing record. + ErrInfo.PageNo := Page::"Master Data Management Setup"; + ErrInfo.AddNavigationAction(OpenSetupActionTxt); + if MasterDataManagementSetup.Get() then + ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); + Error(ErrInfo); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al new file mode 100644 index 00000000000..e0f0f724f56 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -0,0 +1,112 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Wire-contract negotiation: caches the source's advertised version and feature names (GetCapabilities) so the +/// subsidiary only calls actions the source actually supports. A newer subsidiary talking to an older source thus +/// fails with a clear message instead of a 404. SingleInstance: one source connection per session. +/// +codeunit 7246 "MDM Source Capabilities" +{ + Access = Internal; + SingleInstance = true; + + var + Negotiated: Boolean; + NegotiatedForUrl: Text; + ContractVersion: Integer; + SupportedFeatures: List of [Text]; + UnsupportedFeatureErr: Label 'The source environment does not support the required ''%1'' capability. Update the Master Data Management app on the source environment.', Comment = '%1 = capability name'; + CapabilitiesParseErr: Label 'The source environment returned an invalid capabilities response.'; + CapabilitiesParseTelemetryTxt: Label 'The source environment returned a malformed capabilities response during cross-environment negotiation.', Locked = true; + + procedure EnsureSupported(Transport: Interface "IMDM Source Transport"; Feature: Text) + begin + if not IsSupported(Transport, Feature) then + Error(UnsupportedFeatureErr, Feature); + end; + + procedure IsSupported(Transport: Interface "IMDM Source Transport"; Feature: Text): Boolean + begin + Negotiate(Transport); + exit(SupportedFeatures.Contains(Feature)); + end; + + procedure ContractVersionNo(Transport: Interface "IMDM Source Transport"): Integer + begin + Negotiate(Transport); + exit(ContractVersion); + end; + + // Clears the cached negotiation (used by tests that swap the injected transport). + procedure Reset() + begin + Negotiated := false; + NegotiatedForUrl := ''; + ContractVersion := 0; + Clear(SupportedFeatures); + end; + + // A malformed capabilities response is an internal contract failure; keep the detail in telemetry and show a generic error. + local procedure InternalError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + + local procedure Negotiate(Transport: Interface "IMDM Source Transport") + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataManagement: Codeunit "Master Data Management"; + Capabilities: JsonObject; + FeaturesToken: JsonToken; + VersionToken: JsonToken; + FeatureToken: JsonToken; + CurrentSource: Text; + begin + if MasterDataManagementSetup.Get() then + CurrentSource := MasterDataManagementSetup."Source Environment URL"; + // Re-negotiate when the configured source changes so a switched environment can't reuse the previous + // source's cached feature/version data. + if Negotiated and (NegotiatedForUrl = CurrentSource) then + exit; + Negotiated := false; + ContractVersion := 0; + Clear(SupportedFeatures); + // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading + // "capability unsupported / update the source app" error instead of the real bad-response problem. + if not Capabilities.ReadFrom(Transport.GetCapabilities()) then + RaiseCapabilitiesParseError(MasterDataManagement); + // A present-but-malformed version/features (wrong token kind or non-integer version) is a contract failure, + // not a valid negotiation: keep it on the internal-diagnostic path instead of throwing a raw runtime error. + if Capabilities.Get('version', VersionToken) then + if not (VersionToken.IsValue() and TryReadInteger(VersionToken, ContractVersion)) then + RaiseCapabilitiesParseError(MasterDataManagement); + if Capabilities.Get('features', FeaturesToken) then begin + if not FeaturesToken.IsArray() then + RaiseCapabilitiesParseError(MasterDataManagement); + foreach FeatureToken in FeaturesToken.AsArray() do begin + if not FeatureToken.IsValue() then + RaiseCapabilitiesParseError(MasterDataManagement); + SupportedFeatures.Add(FeatureToken.AsValue().AsText()); + end; + end; + Negotiated := true; + NegotiatedForUrl := CurrentSource; + end; + + local procedure RaiseCapabilitiesParseError(MasterDataManagement: Codeunit "Master Data Management") + begin + Session.LogMessage('0000VAV', CapabilitiesParseTelemetryTxt, Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Error(InternalError(CapabilitiesParseErr)); + end; + + [TryFunction] + local procedure TryReadInteger(Token: JsonToken; var Value: Integer) + begin + Value := Token.AsValue().AsInteger(); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al new file mode 100644 index 00000000000..03e18d9aa0e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al @@ -0,0 +1,23 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Single resolution point for the source transport, shared by the cross-environment data source and the change +/// detector. Defaults to the HTTP transport; tests inject an in-process transport via OnResolveSourceTransport. +/// +codeunit 7244 "MDM Source Connection" +{ + Access = Internal; + + procedure GetTransport() Transport: Interface "IMDM Source Transport" + var + HttpTransport: Codeunit "MDM Http Source Transport"; + begin + Transport := HttpTransport; + OnResolveSourceTransport(Transport); + end; + + [InternalEvent(false)] + local procedure OnResolveSourceTransport(var Transport: Interface "IMDM Source Transport") + begin + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al new file mode 100644 index 00000000000..2d9b5a9feb6 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -0,0 +1,411 @@ +namespace Microsoft.Integration.MDM; + +using System.Text; +using System.Utilities; + +/// +/// Parses a GetRecords/LastModifiedAtPerTable JSON response from the source and materializes records into a +/// temporary RecordRef, so the existing synchronization engine can read them as if they were local. Also +/// exposes the response's control fields (tableAvailable / indexed / unavailableFields / hasMore / nextCursor) +/// so the caller can raise the right synchronization error. +/// +codeunit 7248 "MDM Source Response" +{ + Access = Internal; + + var + SourceWatermark: Codeunit "MDM Source Watermark"; + SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; + BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; + MalformedControlFieldErr: Label 'The source returned a malformed value for the response control field ''%1''.', Comment = '%1 = response control field name'; + MalformedRecordErr: Label 'The source returned a malformed record entry.', Locked = true; + + procedure TryParse(ResponseText: Text; var Response: JsonObject): Boolean + begin + exit(Response.ReadFrom(ResponseText)); + end; + + procedure TableAvailable(var Response: JsonObject): Boolean + begin + exit(ReadControlBoolean(Response, 'tableAvailable', true)); + end; + + procedure Indexed(var Response: JsonObject): Boolean + begin + // 'indexed' is only emitted when false (a too-large unindexed/keyless table). + exit(ReadControlBoolean(Response, 'indexed', true)); + end; + + procedure GetUnavailableFields(var Response: JsonObject; var UnavailableFields: JsonArray): Boolean + var + Token: JsonToken; + begin + if not Response.Get('unavailableFields', Token) then + exit(false); + if not Token.IsArray() then + exit(false); + UnavailableFields := Token.AsArray(); + exit(UnavailableFields.Count() > 0); + end; + + procedure HasMore(var Response: JsonObject): Boolean + begin + exit(ReadControlBoolean(Response, 'hasMore', false)); + end; + + // True when the source declined to share because its cross-environment privacy notice isn't approved. + procedure ConsentRequired(var Response: JsonObject): Boolean + begin + exit(ReadControlBoolean(Response, 'consentRequired', false)); + end; + + // Control fields (tableAvailable/indexed/hasMore) are always booleans in the contract; a present-but-malformed + // token is an internal contract failure, so surface it as an internal diagnostic instead of a raw runtime throw. + local procedure ReadControlBoolean(var Response: JsonObject; PropertyName: Text; DefaultValue: Boolean): Boolean + var + Token: JsonToken; + Value: Boolean; + begin + if not Response.Get(PropertyName, Token) then + exit(DefaultValue); + if not (Token.IsValue() and TryReadBoolean(Token, Value)) then + Error(MalformedControlField(PropertyName)); + exit(Value); + end; + + [TryFunction] + local procedure TryReadBoolean(Token: JsonToken; var Value: Boolean) + begin + Value := Token.AsValue().AsBoolean(); + end; + + [TryFunction] + local procedure TryFromBase64(ContentBase64: Text; ContentOutStream: OutStream) + var + Base64Convert: Codeunit "Base64 Convert"; + begin + Base64Convert.FromBase64(ContentBase64, ContentOutStream); + end; + + local procedure MalformedControlField(PropertyName: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := StrSubstNo(MalformedControlFieldErr, PropertyName); + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + + local procedure MalformedRecordEntry(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MalformedRecordErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + + // The nextCursor object, re-serialized so the caller can pass it straight back as the next Selector. + procedure GetNextCursor(var Response: JsonObject): Text + var + Token: JsonToken; + CursorText: Text; + begin + if not Response.Get('nextCursor', Token) then + exit(''); + if not Token.IsObject() then // a non-object cursor is malformed; report "no cursor" so the caller fails the response instead of replaying from the start + exit(''); + Token.WriteTo(CursorText); + exit(CursorText); + end; + + // Inserts the response's records into TempSourceRecordRef (already opened temporary on the table). + // Returns the number inserted. Requires the caller to have requested the primary-key fields so the + // temporary inserts don't collide. + procedure InsertRecords(var Response: JsonObject; var TempSourceRecordRef: RecordRef): Integer + var + RecordsToken: JsonToken; + RecordToken: JsonToken; + RecordsArray: JsonArray; + Count: Integer; + begin + if not Response.Get('records', RecordsToken) then // available+indexed responses always carry a records array; absence is a broken contract + Error(MalformedRecordEntry()); + if not RecordsToken.IsArray() then + Error(MalformedRecordEntry()); + RecordsArray := RecordsToken.AsArray(); + foreach RecordToken in RecordsArray do begin + if not RecordToken.IsObject() then + Error(MalformedRecordEntry()); + InsertRecord(RecordToken.AsObject(), TempSourceRecordRef); + Count += 1; + end; + exit(Count); + end; + + local procedure InsertRecord(RecordObject: JsonObject; var TempSourceRecordRef: RecordRef) + var + DestField: FieldRef; + FieldsToken: JsonToken; + ValueToken: JsonToken; + FieldsObject: JsonObject; + FieldName: Text; + SystemIdValue: Guid; + SystemModifiedAtValue: DateTime; + FieldNo: Integer; + begin + TempSourceRecordRef.Init(); + if not GetGuid(RecordObject, 'systemId', SystemIdValue) then // systemId is the record identity and dedup key; a missing/unparsable one is a broken record + Error(MalformedRecordEntry()); + if RecordObject.Get('fields', FieldsToken) then begin + if not FieldsToken.IsObject() then // a non-object 'fields' is a broken contract, routed as an internal error + Error(MalformedRecordEntry()); + FieldsObject := FieldsToken.AsObject(); + foreach FieldName in FieldsObject.Keys() do + if Evaluate(FieldNo, FieldName) then + if TempSourceRecordRef.FieldExist(FieldNo) then begin + FieldsObject.Get(FieldName, ValueToken); + DestField := TempSourceRecordRef.Field(FieldNo); + case DestField.Type() of + FieldType::Media: + ApplyInlineMedia(SystemIdValue, FieldNo, TempSourceRecordRef.Number(), ValueToken); + FieldType::Blob: + ApplyInlineBlob(DestField, FieldNo, TempSourceRecordRef.Number(), ValueToken); + else begin + if not ValueToken.IsValue() then // an object/array for a scalar field is a broken contract + Error(MalformedRecordEntry()); + SetFieldFromText(DestField, ValueToken.AsValue().AsText()); + end; + end; + end; + end; + if not IsNullGuid(SystemIdValue) then + TempSourceRecordRef.Field(TempSourceRecordRef.SystemIdNo()).Value := SystemIdValue; + // A record re-modified between page fetches can arrive on two pages under the advancing cursor; keep the + // newest copy instead of aborting the whole batch on the duplicate primary key. + if not TryInsertTempRecord(TempSourceRecordRef) then + TempSourceRecordRef.Modify(false); + // The temp row can't hold SystemModifiedAt (the platform ignores the write), so stash the source watermark + // in a side cache the sync loop reads back via GetRowLastModifiedOn. + if GetDateTime(RecordObject, 'systemModifiedAt', SystemModifiedAtValue) then + SourceWatermark.Put(SystemIdValue, SystemModifiedAtValue); + end; + + [TryFunction] + local procedure TryInsertTempRecord(var TempSourceRecordRef: RecordRef) + begin + TempSourceRecordRef.Insert(false); + end; + + // Media bytes travel in a per-batch cache keyed by (SystemId, fieldNo); the temp record's Media field only + // holds a GUID that is meaningless in the subsidiary. UpdateMedia (cross-env) reads the cache during transfer. + local procedure ApplyInlineMedia(SystemId: Guid; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) + var + InlineMedia: Codeunit "MDM Inline Media"; + MediaObject: JsonObject; + ContentToken: JsonToken; + NameToken: JsonToken; + MimeToken: JsonToken; + FileName: Text; + MimeType: Text; + begin + if not ValueToken.IsObject() then + exit; + MediaObject := ValueToken.AsObject(); + if IsSkipped(MediaObject) then begin + LogSkippedField(TableId, FieldNo, MediaObject); + exit; + end; + if IsEmptyField(MediaObject) then begin + InlineMedia.PutCleared(SystemId, FieldNo); // source cleared the picture: mirror it on the destination + exit; + end; + if not MediaObject.Get('content', ContentToken) then + exit; // no content and not flagged empty: leave the destination picture untouched + if not ContentToken.IsValue() then // a non-scalar content payload is a broken record entry + Error(MalformedRecordEntry()); + if MediaObject.Get('name', NameToken) and NameToken.IsValue() then + FileName := NameToken.AsValue().AsText(); + if MediaObject.Get('mimeType', MimeToken) and MimeToken.IsValue() then + MimeType := MimeToken.AsValue().AsText(); + InlineMedia.Put(SystemId, FieldNo, FileName, MimeType, ContentToken.AsValue().AsText()); + end; + + // Blob bytes are placed directly on the temp source record; the framework's record transfer carries them to + // the destination (the same path same-env uses for mapped blobs), so no destination-side apply is needed. + local procedure ApplyInlineBlob(var DestField: FieldRef; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) + var + TempBlob: Codeunit "Temp Blob"; + BlobObject: JsonObject; + ContentToken: JsonToken; + ContentOutStream: OutStream; + begin + if not ValueToken.IsObject() then + exit; + BlobObject := ValueToken.AsObject(); + if IsSkipped(BlobObject) then begin + LogSkippedField(TableId, FieldNo, BlobObject); + exit; + end; + if IsEmptyField(BlobObject) then begin + Clear(TempBlob); + TempBlob.ToFieldRef(DestField); // source cleared the blob: write empty so the transfer clears the destination + exit; + end; + if not BlobObject.Get('content', ContentToken) then + exit; // no content and not flagged empty: leave the destination untouched + if not ContentToken.IsValue() then // a non-scalar content payload is a broken record entry + Error(MalformedRecordEntry()); + TempBlob.CreateOutStream(ContentOutStream); + if not TryFromBase64(ContentToken.AsValue().AsText(), ContentOutStream) then // undecodable content is a broken record entry + Error(MalformedRecordEntry()); + TempBlob.ToFieldRef(DestField); + end; + + local procedure IsSkipped(FieldObject: JsonObject): Boolean + begin + exit(ReadRecordBoolean(FieldObject, 'skipped')); + end; + + local procedure IsEmptyField(FieldObject: JsonObject): Boolean + begin + exit(ReadRecordBoolean(FieldObject, 'empty')); + end; + + // Per-record media/blob flags are booleans in the contract; a present-but-malformed token is a broken record + // entry, so route it through the same internal malformed-record path as the rest of record materialization. + local procedure ReadRecordBoolean(FieldObject: JsonObject; PropertyName: Text): Boolean + var + Token: JsonToken; + Value: Boolean; + begin + if not FieldObject.Get(PropertyName, Token) then + exit(false); + if not (Token.IsValue() and TryReadBoolean(Token, Value)) then + Error(MalformedRecordEntry()); + exit(Value); + end; + + // Over-cap media/blob is not synchronized (v1). We can't error (it would retry the record every run) and MDM + // surfaces no synch warnings, so the skip is emitted as telemetry only; the record's other fields still sync. + local procedure LogSkippedField(TableId: Integer; FieldNo: Integer; FieldObject: JsonObject) + var + MasterDataManagement: Codeunit "Master Data Management"; + Dimensions: Dictionary of [Text, Text]; + LengthToken: JsonToken; + begin + // The source record identifier (systemId) is customer data, so it is not emitted; only the table, field, + // and length (non-identifying diagnostics) are logged. + Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); + Dimensions.Add('TableId', Format(TableId)); + Dimensions.Add('FieldNo', Format(FieldNo)); + if FieldObject.Get('length', LengthToken) and LengthToken.IsValue() then + Dimensions.Add('Length', Format(LengthToken.AsValue().AsBigInteger())); + Session.LogMessage('0000VAW', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + end; + + // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. A + // failed conversion means a drifted/malformed source value; error rather than silently syncing a defaulted one. + local procedure SetFieldFromText(var DestField: FieldRef; ValueText: Text) + var + DateFormulaValue: DateFormula; + IntegerValue: Integer; + BigIntegerValue: BigInteger; + DecimalValue: Decimal; + BooleanValue: Boolean; + DateValue: Date; + TimeValue: Time; + DateTimeValue: DateTime; + DurationValue: Duration; + GuidValue: Guid; + Converted: Boolean; + ErrInfo: ErrorInfo; + begin + Converted := true; + case DestField.Type() of + FieldType::Text, FieldType::Code: + DestField.Value := CopyStr(ValueText, 1, DestField.Length()); + FieldType::Integer, FieldType::Option: + if Evaluate(IntegerValue, ValueText, 9) then + DestField.Value := IntegerValue + else + Converted := false; + FieldType::BigInteger: + if Evaluate(BigIntegerValue, ValueText, 9) then + DestField.Value := BigIntegerValue + else + Converted := false; + FieldType::Decimal: + if Evaluate(DecimalValue, ValueText, 9) then + DestField.Value := DecimalValue + else + Converted := false; + FieldType::Boolean: + if Evaluate(BooleanValue, ValueText, 9) then + DestField.Value := BooleanValue + else + Converted := false; + FieldType::Date: + if Evaluate(DateValue, ValueText, 9) then + DestField.Value := DateValue + else + Converted := false; + FieldType::Time: + if Evaluate(TimeValue, ValueText, 9) then + DestField.Value := TimeValue + else + Converted := false; + FieldType::DateTime: + if Evaluate(DateTimeValue, ValueText, 9) then + DestField.Value := DateTimeValue + else + Converted := false; + FieldType::Duration: + if Evaluate(DurationValue, ValueText, 9) then + DestField.Value := DurationValue + else + Converted := false; + FieldType::DateFormula: + if Evaluate(DateFormulaValue, ValueText, 9) then + DestField.Value := DateFormulaValue + else + Converted := false; + FieldType::Guid: + if Evaluate(GuidValue, ValueText) then + DestField.Value := GuidValue + else + Converted := false; + end; + if not Converted then begin + // Internal contract failure the user can't fix: detail goes to telemetry, user sees a generic dialog. + ErrInfo.Message := StrSubstNo(BadFieldValueErr, DestField.Caption(), Format(DestField.Type())); + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + Error(ErrInfo); + end; + end; + + local procedure GetGuid(var Container: JsonObject; PropertyName: Text; var Value: Guid): Boolean + var + Token: JsonToken; + begin + if not Container.Get(PropertyName, Token) then + exit(false); + if not Token.IsValue() then + exit(false); + exit(Evaluate(Value, Token.AsValue().AsText())); + end; + + local procedure GetDateTime(var Container: JsonObject; PropertyName: Text; var Value: DateTime): Boolean + var + Token: JsonToken; + begin + if not Container.Get(PropertyName, Token) then + exit(false); + if not Token.IsValue() then + exit(false); + exit(Evaluate(Value, Token.AsValue().AsText(), 9)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al new file mode 100644 index 00000000000..584e1a3867f --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al @@ -0,0 +1,38 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Per-batch cache of the source SystemModifiedAt watermark carried in a cross-environment GetRecords response. +/// The materializer stashes it keyed by the source record's SystemId; the synchronization engine reads it back +/// (GetRowLastModifiedOn) to obtain the real source change time. The platform does not let a temporary record +/// hold SystemModifiedAt (writes are ignored), so the watermark travels here instead of on the temp source row. +/// Single instance so the value survives from materialization through the synchronization write; Reset() clears +/// it between batches. +/// +codeunit 7239 "MDM Source Watermark" +{ + Access = Internal; + SingleInstance = true; + + var + ModifiedAtByKey: Dictionary of [Text, DateTime]; + + procedure Reset() + begin + Clear(ModifiedAtByKey); + end; + + procedure Put(SystemId: Guid; ModifiedAt: DateTime) + begin + ModifiedAtByKey.Set(MakeKey(SystemId), ModifiedAt); + end; + + procedure TryGet(SystemId: Guid; var ModifiedAt: DateTime): Boolean + begin + exit(ModifiedAtByKey.Get(MakeKey(SystemId), ModifiedAt)); + end; + + local procedure MakeKey(SystemId: Guid): Text + begin + exit(Format(SystemId, 0, 4)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index 2a04f766028..895c0671732 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -70,6 +70,8 @@ codeunit 7233 "Master Data Management" BothRecordsModifiedToNAVQst: Label 'Both %1 and the %4 %2 record have been changed since the last synchronization, or synchronization has never been performed. If you continue with synchronization, data in %3 will be overwritten with data from %4. Are you sure you want to synchronize?', Comment = '%1 is a formatted RecordID, such as ''Customer 1234''. %2 is the caption of a Business Central table. %3 - product name, %4 = Business Central product name'; NoOf: Option ,Scheduled,Failed,Skipped,Total; CategoryTok: Label 'AL Master Data Management', Locked = true; + InvalidIntegrationRecordSystemIdErr: Label 'Invalid integration record system id.', Locked = true; + EmptyIntegrationRecordSystemIdErr: Label 'Empty integration record system id.', Locked = true; DeletionConflictHandledRemoveCouplingTxt: Label 'Deletion conflict handled by removing the coupling to the deleted record.', Locked = true; DeletionConflictHandledRestoreRecordTxt: Label 'Deletion conflict handled by restoring the deleted record.', Locked = true; ResetAllCustomIntegrationTableMappingsLbl: Label 'One or more of the selected integration table mappings is custom. \\To restore a custom table mapping, you must subscribe to the event OnBeforeResetTableMapping in codeunit "Master Data Mgt. Setup Default" and implement the defaults for each custom table mapping. \\Do you want to continue?'; @@ -349,46 +351,15 @@ codeunit 7233 "Master Data Management" internal procedure GetIntegrationRecordRef(var IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var IntegrationRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - IDFieldRef: FieldRef; - RecordID: RecordID; - TextKey: Text; Found: Boolean; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin OnGetIntegrationRecordRefByIntegrationSystemId(IntegrationTableMapping, ID, IntegrationRecordRef, Found, IsHandled); if IsHandled then exit(Found); - IntegrationRecordRef.Close(); MasterDataManagementSetup.Get(); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - if ID.IsGuid then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IDFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); - IDFieldRef.SetFilter(ID); - exit(IntegrationRecordRef.FindFirst()); - end; - - if ID.IsRecordId then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - RecordID := ID; - if RecordID.TableNo = IntegrationTableMapping."Table ID" then - exit(IntegrationRecordRef.Get(ID)); - end; - - if ID.IsText then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IDFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); - TextKey := ID; - IDFieldRef.SetFilter('%1', TextKey); - exit(IntegrationRecordRef.FindFirst()); - end; + exit(MasterDataManagementSetup.GetDataSource().GetById(IntegrationTableMapping, ID, IntegrationRecordRef)); end; local procedure GetRecordRef(RecVariant: Variant; var RecordRef: RecordRef): Integer @@ -686,7 +657,6 @@ codeunit 7233 "Master Data Management" LocalRecordRef: RecordRef; IntegrationRecordRef: RecordRef; CountFailed: Integer; - SourceCompanyName: Text[30]; begin AddIntegrationTableMapping(IntegrationTableMapping); IntegrationTableMapping.SetTableFilter(LocalTableFilter); @@ -701,17 +671,15 @@ codeunit 7233 "Master Data Management" until LocalRecordRef.Next() = 0 end else begin MasterDataManagementSetup.Get(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationRecordRef.SetView(IntegrationTableFilter); - if IntegrationRecordRef.FindSet() then + // Route the source read so uncoupling works against the local company or another environment. + // GetByFilter returns the ref already positioned on the matching set; a separate FindSet would re-read it. +#pragma warning disable AA0181 + if MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, IntegrationTableFilter, IntegrationRecordRef) then repeat if not PerformUncoupling(IntegrationTableMapping, LocalRecordRef, IntegrationRecordRef) then CountFailed += 1; until IntegrationRecordRef.Next() = 0; +#pragma warning restore AA0181 end; IntegrationTableMapping.Delete(true); exit(CountFailed = 0); @@ -2031,6 +1999,16 @@ codeunit 7233 "Master Data Management" end; end; + local procedure InternalError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Record Management", 'OnUpdateIntegrationTableCouplingForRecordRef', '', false, false)] local procedure HandleOnUpdateIntegrationTableCouplingForRecordRef(IntegrationTableConnectionType: TableConnectionType; IntegrationTableUid: Variant; RecordRef: RecordRef; var IsHandled: Boolean) var @@ -2047,11 +2025,11 @@ codeunit 7233 "Master Data Management" exit; if not IntegrationTableUid.IsGuid() then - Error('Invalid integration record system id.'); + Error(InternalError(InvalidIntegrationRecordSystemIdErr)); IntegrationSystemId := IntegrationTableUid; if IntegrationSystemId = SysId then - Error('Empty integration record system id.'); + Error(InternalError(EmptyIntegrationRecordSystemIdErr)); if not MasterDataMgtCoupling.FindSystemIdByRecordRef(SysId, RecordRef) then Error(IntegrationRecordNotFoundErr, Format(RecordRef.RecordId(), 0, 1)); @@ -2156,7 +2134,6 @@ codeunit 7233 "Master Data Management" MasterDataManagementSetup: Record "Master Data Management Setup"; IsHandled: Boolean; Found: Boolean; - SourceCompanyName: Text[30]; begin OnGetIntegrationRecordRefFromCoupling(IntegrationTableID, MasterDataMgtCoupling, RecRef, Found, IsHandled); if IsHandled then @@ -2166,12 +2143,7 @@ codeunit 7233 "Master Data Management" exit(false); MasterDataManagementSetup.Get(); - RecRef.Open(IntegrationTableID); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableID); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecRef.ChangeCompany(SourceCompanyName); - exit(RecRef.GetBySystemId(MasterDataMgtCoupling."Integration System ID")); + exit(MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableID, MasterDataMgtCoupling."Integration System ID", RecRef)); end; internal procedure RemoveSubsidiarySubscriptionFromMasterCompany(MasterCompanyName: Text[30]; SubsidiaryCompanyName: Text[30]) @@ -2192,23 +2164,15 @@ codeunit 7233 "Master Data Management" local procedure FindCouplingByIntegrationSystemID(var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; IntegrationSystemID: Guid) Found: Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - RecRef: RecordRef; - RecId: RecordId; - SourceCompanyName: Text[30]; + IntegrationRecordRef: RecordRef; begin Clear(MasterDataMgtCoupling."Integration System ID"); MasterDataManagementSetup.Get(); MasterDataMgtCoupling.Reset(); MasterDataMgtCoupling.SetRange("Integration System ID", IntegrationSystemID); if MasterDataMgtCoupling.FindFirst() then - if MasterDataMgtCoupling.FindRecordId(RecId) then begin - RecRef.Open(RecId.TableNo()); - OnSetSourceCompanyName(SourceCompanyName, RecId.TableNo()); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecRef.ChangeCompany(SourceCompanyName); - Found := RecRef.Get(RecId); - end; + // Route through the data source: the source record is in the local company or another environment. + Found := GetIntegrationRecordRef(MasterDataMgtCoupling."Table ID", MasterDataMgtCoupling, IntegrationRecordRef); end; internal procedure GetIntegrationRecRefCount(var IntegrationTableMapping: Record "Integration Table Mapping"): Integer @@ -2217,11 +2181,18 @@ codeunit 7233 "Master Data Management" INtegrationVendor: Record Vendor; IntegrationCustomer: Record Customer; MasterDataManagementSetup: Record "Master Data Management Setup"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; IntegrationRecRef: RecordRef; IntegrationRecRefCount: Integer; SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + // Cross-environment: the review only needs existence; probe the source instead of counting over the wire. + if CrossEnvDataSource.SourceHasRecords(IntegrationTableMapping, IntegrationTableMapping.GetIntegrationTableFilter()) then + exit(1); + exit(0); + end; OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); if SourceCompanyName = '' then SourceCompanyName := MasterDataManagementSetup."Company Name"; @@ -2330,9 +2301,6 @@ codeunit 7233 "Master Data Management" var MasterDataManagementSetup: Record "Master Data Management Setup"; IntegrationRecordRef: RecordRef; - IntegrationSystemIdFieldRef: FieldRef; - IntegrationTableView: Text; - SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); @@ -2341,22 +2309,29 @@ codeunit 7233 "Master Data Management" IntegrationTableMapping.SetFilter("Integration Table ID", '<>0'); if IntegrationTableMapping.FindSet() then repeat - IntegrationRecordRef.Close(); - IntegrationTableView := IntegrationTableMapping.GetIntegrationTableFilter(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationSystemIdFieldRef := IntegrationRecordRef.Field(IntegrationRecordRef.SystemIdNo); - IntegrationRecordRef.SetView(IntegrationTableView); - IntegrationSystemIdFieldRef.SetRange(MasterDataMgtCoupling."Integration System ID"); - if not IntegrationRecordRef.IsEmpty() then - exit(true); + // Route through the data source: find which enabled mapping's source table holds this record, then + // confirm it is within that mapping's integration table filter (GetBySystemId is a key lookup and ignores it). + if MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableMapping."Integration Table ID", MasterDataMgtCoupling."Integration System ID", IntegrationRecordRef) then + if MasterDataManagementSetup.IsCrossEnvironment() or IntegrationRecordMatchesMappingFilter(IntegrationTableMapping, IntegrationRecordRef, MasterDataMgtCoupling."Integration System ID") then + exit(true); until IntegrationTableMapping.Next() = 0; exit(false); end; + // GetBySystemId retrieves by key and ignores filters; same-environment must re-check the mapping's integration + // table filter here (the previous SetView-based lookup enforced it). Cross-environment applies the filter source-side. + local procedure IntegrationRecordMatchesMappingFilter(IntegrationTableMapping: Record "Integration Table Mapping"; var IntegrationRecordRef: RecordRef; IntegrationSystemId: Guid): Boolean + var + IntegrationTableFilter: Text; + begin + IntegrationTableFilter := IntegrationTableMapping.GetIntegrationTableFilter(); + if IntegrationTableFilter = '' then + exit(true); + IntegrationRecordRef.SetView(IntegrationTableFilter); + IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").SetRange(IntegrationSystemId); + exit(not IntegrationRecordRef.IsEmpty()); + end; + internal procedure CheckSetupPermissions() var MasterDataManagementSetup: Record "Master Data Management Setup"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al new file mode 100644 index 00000000000..9face0a2edf --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al @@ -0,0 +1,32 @@ +namespace Microsoft.Integration.MDM; + +using System.Upgrade; + +/// +/// Codeunit Master Data Mgt. Install (ID 7243). +/// +codeunit 7243 "Master Data Mgt. Install" +{ + Access = Internal; + Subtype = Install; + + trigger OnInstallAppPerDatabase() + var + MasterDataMgtUpgrade: Codeunit "Master Data Mgt. Upgrade"; + begin + // Fresh install (incl. package/base-image build) never fires the upgrade trigger, so publish the source endpoint here too. + MasterDataMgtUpgrade.RegisterCrossEnvSourceWebService(); + end; + + trigger OnInstallAppPerCompany() + var + UpgradeTag: Codeunit "Upgrade Tag"; + AppInfo: ModuleInfo; + begin + NavApp.GetCurrentModuleInfo(AppInfo); + // Only a genuine first install (no preserved data) may skip the historical per-company migrations. A reinstall + // over preserved data keeps DataVersion non-zero and must let those migrations run against the existing data. + if AppInfo.DataVersion() = Version.Create(0, 0, 0, 0) then + UpgradeTag.SetAllUpgradeTags(); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al index 16ce36cbb3e..32105963dd5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al @@ -49,6 +49,7 @@ codeunit 7230 "Master Data Mgt. Setup Default" JobQueueEntryNameTok: Label ' %1 - %2 synchronization job.', Comment = '%1 = The Integration Table Name to synchronized (ex. CUSTOMER), %2 = Business Central product name'; UncoupleJobQueueEntryNameTok: Label ' %1 uncouple job.', Comment = '%1 = Integration mapping description, for example, CUSTOMER <-> CUSTOMER'; CoupleJobQueueEntryNameTok: Label ' %1 coupling job.', Comment = '%1 = Integration mapping description, for example, CUSTOMER <-> CUSTOMER'; + ChangeDetectorJobDescriptionTxt: Label 'Master Data Management cross-environment change detection.'; IntegrationTablePrefixTok: Label 'Business Central', Comment = 'Product name', Locked = true; CustomerConfigTemplateCodeTok: Label 'MDMCUST', Comment = 'Customer template code for new customers created from source company data. Max length 10.', Locked = true; VendorConfigTemplateCodeTok: Label 'MDMVEND', Comment = 'Vendor template code for new vendors created from source company data. Max length 10.', Locked = true; @@ -97,6 +98,8 @@ codeunit 7230 "Master Data Mgt. Setup Default" ResetDimensionValueMapping('MDM_DIMENSIONVALUE', (not MasterDataManagementSetup."Delay Job Scheduling")); SetCustomIntegrationsTableMappings(MasterDataManagementSetup); + + UpdateChangeDetectorJob(MasterDataManagementSetup); end; internal procedure ResetSalesPeopleSystemUserMapping(IntegrationTableMappingName: Code[20]; ShouldRecreateJobQueueEntry: Boolean) @@ -1435,6 +1438,50 @@ codeunit 7230 "Master Data Mgt. Setup Default" exit(Codeunit.Run(Codeunit::"Job Queue - Enqueue", JobQueueEntry)) end; + // One recurring detector job per subsidiary; the cross-environment analog of same-env's event-driven reschedule. + internal procedure UpdateChangeDetectorJob(MasterDataManagementSetup: Record "Master Data Management Setup") + begin + if MasterDataManagementSetup."Is Enabled" and (MasterDataManagementSetup."Source Environment Name" <> '') then + EnsureChangeDetectorJob() + else + RemoveChangeDetectorJob(); + end; + + local procedure EnsureChangeDetectorJob() + var + JobQueueEntry: Record "Job Queue Entry"; + begin + JobQueueEntry.SetRange("Object Type to Run", JobQueueEntry."Object Type to Run"::Codeunit); + JobQueueEntry.SetRange("Object ID to Run", Codeunit::"MDM Cross-Env Change Detector"); + if not JobQueueEntry.IsEmpty() then + exit; + + JobQueueEntry.InitRecurringJob(ChangeDetectorIntervalInMinutes()); + JobQueueEntry."Object Type to Run" := JobQueueEntry."Object Type to Run"::Codeunit; + JobQueueEntry."Object ID to Run" := Codeunit::"MDM Cross-Env Change Detector"; + JobQueueEntry."Run in User Session" := false; + JobQueueEntry.Description := CopyStr(ChangeDetectorJobDescriptionTxt, 1, MaxStrLen(JobQueueEntry.Description)); + JobQueueEntry."Maximum No. of Attempts to Run" := 10; + JobQueueEntry.Status := JobQueueEntry.Status::Ready; + JobQueueEntry."Rerun Delay (sec.)" := 30; + JobQueueEntry."Job Queue Category Code" := JobQueueCategoryLbl; + Codeunit.Run(Codeunit::"Job Queue - Enqueue", JobQueueEntry); + end; + + local procedure RemoveChangeDetectorJob() + var + JobQueueEntry: Record "Job Queue Entry"; + begin + JobQueueEntry.SetRange("Object Type to Run", JobQueueEntry."Object Type to Run"::Codeunit); + JobQueueEntry.SetRange("Object ID to Run", Codeunit::"MDM Cross-Env Change Detector"); + JobQueueEntry.DeleteTasks(); + end; + + local procedure ChangeDetectorIntervalInMinutes(): Integer + begin + exit(1); + end; + internal procedure RecreateJobQueueEntryFromIntTableMapping(IntegrationTableMapping: Record "Integration Table Mapping"; IntervalInMinutes: Integer; ShouldRecreateJobQueueEntry: Boolean; InactivityTimeoutPeriod: Integer) begin RecreateJobQueueEntryFromIntTableMapping(IntegrationTableMapping, IntervalInMinutes, ShouldRecreateJobQueueEntry, InactivityTimeoutPeriod, ProductName.Short(), false); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 2b1e32cb7c8..30fddaed90b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -17,6 +17,7 @@ using System.IO; using System.Reflection; using System.Telemetry; using System.Threading; +using System.Utilities; codeunit 7237 "Master Data Mgt. Subscribers" { @@ -36,7 +37,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" UnsupportedKeyLengthErr: label 'Table %1 has a primary key that consists of %2 fields. Off-the page, synchronization engine doesn''t support renaming with primary key length of more than 10 fields.\\Subscribe to event OnRenameDestination in codeunit "Master Data Management" to implement the rename.', Comment = '%1 - a table caption, %2 - an integer'; MappingDoesNotAllowDirectionErr: label 'The only supported direction for the data synchronization is %1.', Comment = '%1 - a text: From Integration Table'; RunningFullSynchTelemetryTxt: Label 'Running full synch job for table mapping %1', Locked = true; - SetContactNoFromSourceCompanyTxt: Label 'For %1 %2, initialized company contact No. to be equal the No. of the company contact from the source company %3.', Locked = true; + SetContactNoFromSourceCompanyTxt: Label 'Initialized the %1 company contact number to match the source company contact.', Locked = true; [EventSubscriber(ObjectType::Table, Database::"Integration Table Mapping", 'OnAfterDeleteEvent', '', false, false)] local procedure HandleOnAfterDeleteIntegrationTableMapping(var Rec: Record "Integration Table Mapping"; RunTrigger: Boolean) @@ -134,20 +135,24 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IsJobQueueEntryDataSynchJob(Sender, IntegrationTableMapping) then begin MasterDataManagementSetup.Get(); - if MasterDataManagementSetup."Is Enabled" then begin - MasterDataManagement.OnSetIntegrationTableFilter(IntegrationTableMapping, RecRef, IsHandled); - if not IsHandled then begin - RecRef.Open(IntegrationTableMapping."Integration Table ID", false); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecRef.ChangeCompany(SourceCompanyName); - IntegrationTableMapping.SetIntRecordRefFilter(RecRef); + if MasterDataManagementSetup."Is Enabled" then + if MasterDataManagementSetup."Source Environment Name" <> '' then + // Cross-environment: the change detector governs when this job is nudged; let it run and fetch the delta. + Result := true + else begin + MasterDataManagement.OnSetIntegrationTableFilter(IntegrationTableMapping, RecRef, IsHandled); + if not IsHandled then begin + RecRef.Open(IntegrationTableMapping."Integration Table ID", false); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + RecRef.ChangeCompany(SourceCompanyName); + IntegrationTableMapping.SetIntRecordRefFilter(RecRef); + end; + if not RecRef.IsEmpty() then + Result := true; + RecRef.Close(); end; - if not RecRef.IsEmpty() then - Result := true; - RecRef.Close(); - end; end; end; @@ -316,6 +321,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" MediaUpdated: Boolean; SourceMediaName, DestinationMediaName : Text; begin + if IsCrossEnvironmentSync() then + exit(UpdateMediaCrossEnvironment(SourceFieldRef, DestinationFieldRef, NewValue)); + SourceTenantMedia.SetAutoCalcFields(Content); DestinationTenantMedia.SetAutoCalcFields(Content); @@ -354,6 +362,70 @@ codeunit 7237 "Master Data Mgt. Subscribers" exit(MediaUpdated); end; + local procedure IsCrossEnvironmentSync(): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit(false); + exit(MasterDataManagementSetup.IsCrossEnvironment()); + end; + + // Cross-env: the source Tenant Media lives in another environment, so the bytes arrive inline (per-batch + // cache) rather than via the source field's GUID. Build the destination Tenant Media from them, keeping the + // same length+name change check. Returns the new media id via NewValue during transfer -> single write. + local procedure UpdateMediaCrossEnvironment(var SourceFieldRef: FieldRef; var DestinationFieldRef: FieldRef; var NewValue: Variant): Boolean + var + DestinationTenantMedia: Record "Tenant Media"; + InlineMedia: Codeunit "MDM Inline Media"; + TempBlob: Codeunit "Temp Blob"; + SourceRecordRef: RecordRef; + SourceSystemId, DestinationMediaId, EmptyGuid : Guid; + MediaInStream: InStream; + MediaOutStream: OutStream; + FileName, MimeType, DestinationName : Text; + SourceLength, DestinationLength : Integer; + begin + SourceRecordRef := SourceFieldRef.Record(); + SourceSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + // Source cleared the picture: mirror it by deleting the destination media and emptying the field. + if InlineMedia.IsCleared(SourceSystemId, SourceFieldRef.Number()) then begin + DestinationMediaId := DestinationFieldRef.Value(); + if (DestinationMediaId <> EmptyGuid) and DestinationTenantMedia.Get(DestinationMediaId) then + DestinationTenantMedia.Delete(); + NewValue := EmptyGuid; + exit(true); + end; + if not InlineMedia.TryGet(SourceSystemId, SourceFieldRef.Number(), FileName, MimeType, TempBlob) then + exit(false); // no inline bytes (over-cap skip or field not projected): leave the destination untouched + SourceLength := TempBlob.Length(); + + DestinationMediaId := DestinationFieldRef.Value(); + DestinationTenantMedia.SetAutoCalcFields(Content); + if DestinationTenantMedia.Get(DestinationMediaId) then begin + DestinationLength := DestinationTenantMedia.Content.Length(); + DestinationName := DestinationTenantMedia."File Name"; + end; + if (SourceLength = DestinationLength) and (FileName = DestinationName) then + exit(false); // unchanged + + if DestinationMediaId <> EmptyGuid then + if DestinationTenantMedia.Get(DestinationMediaId) then + DestinationTenantMedia.Delete(); + + Clear(DestinationTenantMedia); + DestinationTenantMedia.ID := CreateGuid(); + DestinationTenantMedia."Company Name" := CopyStr(CompanyName(), 1, MaxStrLen(DestinationTenantMedia."Company Name")); + DestinationTenantMedia."File Name" := CopyStr(FileName, 1, MaxStrLen(DestinationTenantMedia."File Name")); + DestinationTenantMedia."Mime Type" := CopyStr(MimeType, 1, MaxStrLen(DestinationTenantMedia."Mime Type")); + TempBlob.CreateInStream(MediaInStream); + DestinationTenantMedia.Content.CreateOutStream(MediaOutStream); + CopyStream(MediaOutStream, MediaInStream); + DestinationTenantMedia.Insert(); + NewValue := DestinationTenantMedia.ID; + exit(true); + end; + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Table Synch.", 'OnDetermineSynchDirection', '', false, false)] local procedure OnDetermineSynchDirection(var CurrentIntegrationTableMapping: Record "Integration Table Mapping"; var TableID: Integer; var ErrorMessage: Text; var IsHandled: Boolean) begin @@ -501,7 +573,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagement: Codeunit "Master Data Management"; BeforeRenameDestinationRecordRef: RecordRef; IsHandled: Boolean; - SourceCompanyName: Text[30]; + SourceSystemId: Guid; begin if not MasterDataManagement.IsEnabled() then exit; @@ -512,11 +584,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagementSetup.Get(); MasterDataManagement.OnGetIntegrationRecordRef(IntegrationTableMapping, SourceRecordRef, IsHandled); if not IsHandled then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - SourceRecordRef.ChangeCompany(SourceCompanyName); - SourceRecordRef.GetBySystemId(SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value()); + // Route the source re-fetch: the record lives in the local company or another environment. + SourceSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + if MasterDataManagementSetup.GetDataSource().GetBySystemId(SourceRecordRef.Number(), SourceSystemId, SourceRecordRef) then; end; BeforeRenameDestinationRecordRef.Open(DestinationRecordRef.Number()); BeforeRenameDestinationRecordRef.GetBySystemId(DestinationRecordRef.Field(DestinationRecordRef.SystemIdNo()).Value()); @@ -683,29 +753,41 @@ codeunit 7237 "Master Data Mgt. Subscribers" var MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; + SourceWatermark: Codeunit "MDM Source Watermark"; IntegrationRecordRef: RecordRef; - ModifiedFieldRef: FieldRef; IsHandled: Boolean; IntRecSystemId: Guid; - SourceCompanyName: Text[30]; + SourceSystemId: Guid; + SourceModifiedAt: DateTime; begin MasterDataManagementSetup.Get(); + // Cross-environment: FromRecordRef is the source row materialized from the fetched batch. Its SystemModifiedAt + // can't be carried on a temp row (the platform ignores the write), so the real watermark rides a side cache; + // fall back to the row's own value on a cache miss. + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + SourceSystemId := FromRecordRef.Field(FromRecordRef.SystemIdNo()).Value(); + if SourceWatermark.TryGet(SourceSystemId, SourceModifiedAt) then + exit(SourceModifiedAt); + exit(ModifiedOnFromRecordRef(IntegrationTableMapping, FromRecordRef)); + end; + IntegrationRecordRef.Open(FromRecordRef.Number, false); IntRecSystemId := FromRecordRef.Field(FromRecordRef.SystemIdNo).Value(); MasterDataManagement.OnGetIntegrationRecordRefBySystemId(IntegrationTableMapping, IntegrationRecordRef, IntRecSystemId, IsHandled); - if not IsHandled then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationRecordRef.GetBySystemId(IntRecSystemId); - end; - if FromRecordRef.Number() = IntegrationTableMapping."Integration Table ID" then begin - ModifiedFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Int. Tbl. Modified On Fld. No."); - exit(ModifiedFieldRef.Value()); - end; + if not IsHandled then + // Route the source re-fetch: the record lives in the local company or another environment. + if MasterDataManagementSetup.GetDataSource().GetBySystemId(FromRecordRef.Number, IntRecSystemId, IntegrationRecordRef) then; + exit(ModifiedOnFromRecordRef(IntegrationTableMapping, IntegrationRecordRef)); + end; - ModifiedFieldRef := IntegrationRecordRef.Field(IntegrationRecordRef.SystemModifiedAtNo()); + local procedure ModifiedOnFromRecordRef(IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef): DateTime + var + ModifiedFieldRef: FieldRef; + begin + if SourceRecordRef.Number() = IntegrationTableMapping."Integration Table ID" then + ModifiedFieldRef := SourceRecordRef.Field(IntegrationTableMapping."Int. Tbl. Modified On Fld. No.") + else + ModifiedFieldRef := SourceRecordRef.Field(SourceRecordRef.SystemModifiedAtNo()); exit(ModifiedFieldRef.Value()); end; @@ -718,7 +800,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" LocalContact: Record Contact; IntegrationTableMapping: Record "Integration Table Mapping"; MasterDataManagement: Codeunit "Master Data Management"; + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; SourceCompanyName: Text[30]; + SourceContactNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit; @@ -726,13 +810,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup.Get() then exit; - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - - if not Company.Get(SourceCompanyName) then - exit; - IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Table ID", Database::Customer); IntegrationTableMapping.SetRange("Integration Table ID", Database::Customer); @@ -741,6 +818,24 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; + if MasterDataManagementSetup.IsCrossEnvironment() then begin + // Contact business relations are not replicated locally; resolve the source contact number cross-env + // (bulk-prefetched for the whole run, or read per-record outside a run). + if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::Customer, Customer."No.", SourceContactNo) then + if not LocalContact.Get(SourceContactNo) then begin + Contact."No." := SourceContactNo; + Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + IsHandled := true; + end; + exit; + end; + + // Same-environment reads the source company's relations directly via ChangeCompany. + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then exit; @@ -749,7 +844,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactBusinessRelation.FindFirst() then if not LocalContact.Get(ContactBusinessRelation."Contact No.") then begin Contact."No." := ContactBusinessRelation."Contact No."; - Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption(), Customer.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; end; @@ -763,7 +858,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" LocalContact: Record Contact; IntegrationTableMapping: Record "Integration Table Mapping"; MasterDataManagement: Codeunit "Master Data Management"; + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; SourceCompanyName: Text[30]; + SourceContactNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit; @@ -771,13 +868,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup.Get() then exit; - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - - if not Company.Get(SourceCompanyName) then - exit; - IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Table ID", Database::Vendor); IntegrationTableMapping.SetRange("Integration Table ID", Database::Vendor); @@ -786,6 +876,24 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; + if MasterDataManagementSetup.IsCrossEnvironment() then begin + // Contact business relations are not replicated locally; resolve the source contact number cross-env + // (bulk-prefetched for the whole run, or read per-record outside a run). + if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::Vendor, Vendor."No.", SourceContactNo) then + if not LocalContact.Get(SourceContactNo) then begin + Contact."No." := SourceContactNo; + Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + IsHandled := true; + end; + exit; + end; + + // Same-environment reads the source company's relations directly via ChangeCompany. + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then exit; @@ -794,7 +902,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactBusinessRelation.FindFirst() then if not LocalContact.Get(ContactBusinessRelation."Contact No.") then begin Contact."No." := ContactBusinessRelation."Contact No."; - Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption(), Vendor.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; end; @@ -808,7 +916,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" LocalContact: Record Contact; IntegrationTableMapping: Record "Integration Table Mapping"; MasterDataManagement: Codeunit "Master Data Management"; + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; SourceCompanyName: Text[30]; + SourceContactNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit; @@ -816,13 +926,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup.Get() then exit; - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - - if not Company.Get(SourceCompanyName) then - exit; - IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Table ID", Database::"Bank Account"); IntegrationTableMapping.SetRange("Integration Table ID", Database::"Bank Account"); @@ -831,6 +934,24 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; + if MasterDataManagementSetup.IsCrossEnvironment() then begin + // Contact business relations are not replicated locally; resolve the source contact number cross-env + // (bulk-prefetched for the whole run, or read per-record outside a run). + if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::"Bank Account", BankAccount."No.", SourceContactNo) then + if not LocalContact.Get(SourceContactNo) then begin + Contact."No." := SourceContactNo; + Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + IsHandled := true; + end; + exit; + end; + + // Same-environment reads the source company's relations directly via ChangeCompany. + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then exit; @@ -839,7 +960,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactBusinessRelation.FindFirst() then if not LocalContact.Get(ContactBusinessRelation."Contact No.") then begin Contact."No." := ContactBusinessRelation."Contact No."; - Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption(), BankAccount.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; end; @@ -913,6 +1034,10 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup."Is Enabled" then exit; + // Media synchronization is not supported cross-environment (deferred); skip so pictures are not cleared. + if MasterDataManagementSetup."Source Environment Name" <> '' then + exit; + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); IntegrationTableMapping.SetRange("Delete After Synchronization", false); @@ -1085,69 +1210,111 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagement: Codeunit "Master Data Management"; RecRef: RecordRef; RecordModifiedAfterLastSync: Boolean; + CrossEnvironment: Boolean; LinkType: Enum "Contact Business Relation Link To Table"; + RelationNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit(false); MasterDataManagementSetup.Get(); + // Cross-environment: the source's contact business relations are NOT replicated locally, so read the matching + // source relation over the wire to resolve the related customer/vendor (numbers are aligned by synchronization). + CrossEnvironment := MasterDataManagementSetup.IsCrossEnvironment(); DestinationRecordRef.SetTable(Contact); - IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); + if not CrossEnvironment then + IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); SourceRecordRef.SetTable(IntegrationContact); - IntegrationContactBusinessRelation.ChangeCompany(MasterDataManagementSetup."Company Name"); + if not CrossEnvironment then + IntegrationContactBusinessRelation.ChangeCompany(MasterDataManagementSetup."Company Name"); case IntegrationContact."Contact Business Relation" of IntegrationContact."Contact Business Relation"::Customer: begin - IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); - if IntegrationContactBusinessRelation.FindByContact(LinkType::Customer, IntegrationContact."No.") then - if IntegrationCustomer.Get(IntegrationContactBusinessRelation."No.") then - if FindCustomerByIntegrationSystemId(IntegrationCustomer.SystemId, Customer) then - if Customer."Primary Contact No." = '' then - if IntegrationTableMapping.FindMapping(Database::Customer, Database::Customer) then - if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin - RecRef.GetTable(Customer); - RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); - Customer.Validate("Primary Contact No.", Contact."No."); - Customer.Modify(); - if not RecordModifiedAfterLastSync then begin - MasterDataMgtCoupling.SetRange("Local System ID", Customer.SystemId); - if MasterDataMgtCoupling.FindFirst() then begin - MasterDataMgtCoupling."Last Synch. Modified On" := Customer.SystemModifiedAt; - MasterDataMgtCoupling.Modify(); - end; + if not CrossEnvironment then + IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); + if ResolveSourceRelationNo(CrossEnvironment, IntegrationContactBusinessRelation, LinkType::Customer, IntegrationContact."No.", RelationNo) then + if ResolvePrimaryContactCustomer(CrossEnvironment, RelationNo, IntegrationCustomer, Customer) then + if Customer."Primary Contact No." = '' then + if IntegrationTableMapping.FindMapping(Database::Customer, Database::Customer) then + if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin + RecRef.GetTable(Customer); + RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); + Customer.Validate("Primary Contact No.", Contact."No."); + Customer.Modify(); + if not RecordModifiedAfterLastSync then begin + MasterDataMgtCoupling.SetRange("Local System ID", Customer.SystemId); + if MasterDataMgtCoupling.FindFirst() then begin + MasterDataMgtCoupling."Last Synch. Modified On" := Customer.SystemModifiedAt; + MasterDataMgtCoupling.Modify(); end; - exit(true); end; + exit(true); + end; end; IntegrationContact."Contact Business Relation"::Vendor: begin - IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); - if IntegrationContactBusinessRelation.FindByContact(LinkType::Vendor, IntegrationContact."No.") then - if IntegrationVendor.Get(IntegrationContactBusinessRelation."No.") then - if FindVendorByIntegrationSystemId(IntegrationVendor.SystemId, Vendor) then - if Vendor."Primary Contact No." = '' then - if IntegrationTableMapping.FindMapping(Database::Vendor, Database::Vendor) then - if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin - RecRef.GetTable(Vendor); - RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); - Vendor.Validate("Primary Contact No.", Contact."No."); - Vendor.Modify(); - if not RecordModifiedAfterLastSync then begin - MasterDataMgtCoupling.SetRange("Local System ID", Vendor.SystemId); - if MasterDataMgtCoupling.FindFirst() then begin - MasterDataMgtCoupling."Last Synch. Modified On" := Vendor.SystemModifiedAt; - MasterDataMgtCoupling.Modify(); - end; + if not CrossEnvironment then + IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); + if ResolveSourceRelationNo(CrossEnvironment, IntegrationContactBusinessRelation, LinkType::Vendor, IntegrationContact."No.", RelationNo) then + if ResolvePrimaryContactVendor(CrossEnvironment, RelationNo, IntegrationVendor, Vendor) then + if Vendor."Primary Contact No." = '' then + if IntegrationTableMapping.FindMapping(Database::Vendor, Database::Vendor) then + if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin + RecRef.GetTable(Vendor); + RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); + Vendor.Validate("Primary Contact No.", Contact."No."); + Vendor.Modify(); + if not RecordModifiedAfterLastSync then begin + MasterDataMgtCoupling.SetRange("Local System ID", Vendor.SystemId); + if MasterDataMgtCoupling.FindFirst() then begin + MasterDataMgtCoupling."Last Synch. Modified On" := Vendor.SystemModifiedAt; + MasterDataMgtCoupling.Modify(); end; - exit(true); end; + exit(true); + end; end; else exit(false) end; end; + // Same-environment maps the source customer to the destination via its source SystemId coupling; cross-environment + // resolves the destination customer directly by No. (numbers are aligned by synchronization). + local procedure ResolvePrimaryContactCustomer(CrossEnvironment: Boolean; CustomerNo: Code[20]; var IntegrationCustomer: Record Customer; var Customer: Record Customer): Boolean + begin + if CrossEnvironment then + exit(Customer.Get(CustomerNo)); + if not IntegrationCustomer.Get(CustomerNo) then + exit(false); + exit(FindCustomerByIntegrationSystemId(IntegrationCustomer.SystemId, Customer)); + end; + + local procedure ResolvePrimaryContactVendor(CrossEnvironment: Boolean; VendorNo: Code[20]; var IntegrationVendor: Record Vendor; var Vendor: Record Vendor): Boolean + begin + if CrossEnvironment then + exit(Vendor.Get(VendorNo)); + if not IntegrationVendor.Get(VendorNo) then + exit(false); + exit(FindVendorByIntegrationSystemId(IntegrationVendor.SystemId, Vendor)); + end; + + // Resolves the source contact business relation's "No." (the related customer/vendor number) for a given source + // contact. Same-environment reads the source company's relation via ChangeCompany; cross-environment reads the + // matching relation over the wire, since contact business relations are not replicated locally. + local procedure ResolveSourceRelationNo(CrossEnvironment: Boolean; var IntegrationContactBusinessRelation: Record "Contact Business Relation"; LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]; var RelationNo: Code[20]): Boolean + var + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; + begin + if CrossEnvironment then + exit(ContactRelationCache.TryGetSourceRelationNo(LinkToTable, SourceContactNo, RelationNo)); + if not IntegrationContactBusinessRelation.FindByContact(LinkToTable, SourceContactNo) then + exit(false); + RelationNo := IntegrationContactBusinessRelation."No."; + exit(true); + end; + local procedure FindCustomerByIntegrationSystemId(IntegrationSystemId: Guid; var Customer: Record Customer): Boolean var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; @@ -1198,6 +1365,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" exit; // all contacts have parent company set MasterDataManagementSetup.Get(); + // Cross-environment: related contact resolution reads the source company directly; deferred for now. + if MasterDataManagementSetup."Source Environment Name" <> '' then + exit; IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al index a01934bdc0f..e30179bbba5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al @@ -26,9 +26,9 @@ codeunit 7235 "Master Data Mgt. Table Couple" SynchActionType: Option "None",Insert,Modify,ForceModify,IgnoreUnchanged,Fail,Skip,Delete,Uncouple,Couple; NoMatchingCriteriaDefinedErr: Label 'You must specify which fields on the table %1 should be used for match-based coupling.', Comment = '%1 - integration table mapping name'; NoMatchFoundErr: Label 'Failed to couple %2 record(s), because no unique uncoupled matching entity was found in %1 with the specified matching criteria.', Comment = '%1 - comma-separated list of field names, %2 - A URL, %3 - an integer, number of records'; - NoMatchFoundTelemetryErr: Label 'No matching entity was found for %1 in %3 by matching on following fields: %2.', Locked = true; - SingleMatchAlreadyCoupledTelemetryErr: Label 'Single matching entity was found for %1 in %3 by matching on following fields: %2, but it is already coupled.', Locked = true; - MultipleMatchesFoundTelemetryErr: Label 'Multiple matching entities found for %1 in %3 by matching on following fields: %2.', Locked = true; + NoMatchFoundTelemetryErr: Label 'No matching entity was found by matching on the following fields: %1.', Locked = true; + SingleMatchAlreadyCoupledTelemetryErr: Label 'A single matching entity was found by matching on the following fields: %1, but it is already coupled.', Locked = true; + MultipleMatchesFoundTelemetryErr: Label 'Multiple matching entities were found by matching on the following fields: %1.', Locked = true; NoMatchingCriteriaDefinedTelemetryErr: Label 'User is trying to schedule match based coupling for integration table mapping %1 without having specified the matchin criteria.', Locked = true; NoConflictResolutionStrategyDefinedTelemetryErr: Label 'User is trying to schedule match based coupling for integration table mapping %1 without having specified the conflict resolution strategy.', Locked = true; SkippingPostCouplingSynchTelemetryUserChoiceMsg: Label 'Skipping post-coupling synchronization for integration table mapping %1, because the user chose not to run it.', Locked = true; @@ -89,7 +89,6 @@ codeunit 7235 "Master Data Mgt. Table Couple" FilterList: List of [Text]; MatchPriorityList: List of [Integer]; MatchPriority: Integer; - SourceCompanyName: Text[30]; begin // collect the matching criteria fields in a temporary record IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); @@ -117,16 +116,10 @@ codeunit 7235 "Master Data Mgt. Table Couple" // iterate through integration records and for each of them try to find a match in local system MasterDataManagementSetup.Get(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); IntegrationMasterDataSynch.SplitIntegrationTableFilter(IntegrationTableMapping, FilterList); foreach TableFilter in FilterList do begin - if TableFilter <> '' then - IntegrationRecordRef.SetView(TableFilter); - if IntegrationRecordRef.FindSet() then + // Route the source read so match-based coupling works against the local company or another environment. + if MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, TableFilter, IntegrationRecordRef) then repeat if GuiAllowed() then begin RecordNumber += 1; @@ -290,28 +283,29 @@ codeunit 7235 "Master Data Mgt. Table Couple" exit(StrSubstNo(NoMatchFoundErr, GetIntegrationOrgCompanyName(), ErrorCount)); end; - local procedure GetNoMatchFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text + local procedure GetNoMatchFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text var MatchingFieldNameList: Text; begin - MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, MatchIntegrationFieldMapping); - exit(StrSubstNo(NoMatchFoundTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); + MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); + // Record SystemId and source company name are kept out of the telemetry message; only the matching field names. + exit(StrSubstNo(NoMatchFoundTelemetryErr, MatchingFieldNameList)); end; - local procedure GetMultipleMatchesFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text + local procedure GetMultipleMatchesFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text var MatchingFieldNameList: Text; begin - MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, MatchIntegrationFieldMapping); - exit(StrSubstNo(MultipleMatchesFoundTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); + MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); + exit(StrSubstNo(MultipleMatchesFoundTelemetryErr, MatchingFieldNameList)); end; - local procedure GetSingleMatchAlreadyCoupledTelemetryErrorMessage(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text + local procedure GetSingleMatchAlreadyCoupledTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text var MatchingFieldNameList: Text; begin - MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, MatchIntegrationFieldMapping); - exit(StrSubstNo(SingleMatchAlreadyCoupledTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); + MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); + exit(StrSubstNo(SingleMatchAlreadyCoupledTelemetryErr, MatchingFieldNameList)); end; local procedure GetMappingNameWithParent(var IntegrationTableMapping: Record "Integration Table Mapping"): Text @@ -321,15 +315,15 @@ codeunit 7235 "Master Data Mgt. Table Couple" exit(IntegrationTableMapping.Name); end; - local procedure GetMatchingFieldNameList(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary) MatchingFieldNameList: Text + local procedure GetMatchingFieldNameList(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary) MatchingFieldNameList: Text begin - MatchIntegrationFieldMapping.FindSet(); + TempMatchIntegrationFieldMapping.FindSet(); repeat if MatchingFieldNameList = '' then - MatchingFieldNameList := LocalRecordRef.Field(MatchIntegrationFieldMapping."Field No.").Name() + MatchingFieldNameList := LocalRecordRef.Field(TempMatchIntegrationFieldMapping."Field No.").Name() else - MatchingFieldNameList += ', ' + LocalRecordRef.Field(MatchIntegrationFieldMapping."Field No.").Name() - until MatchIntegrationFieldMapping.Next() = 0; + MatchingFieldNameList += ', ' + LocalRecordRef.Field(TempMatchIntegrationFieldMapping."Field No.").Name() + until TempMatchIntegrationFieldMapping.Next() = 0; end; local procedure GetIntegrationOrgCompanyName(): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al index f77882b2867..724d047db0a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al @@ -88,20 +88,28 @@ codeunit 7236 "Master Data Mgt. Tbl. Uncouple" SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationTableMapping.SetIntRecordRefFilter(IntegrationRecordRef); - if IntegrationRecordRef.FindSet() then - repeat - if TempMasterDataMgtCoupling.IsIntegrationRecordRefCoupled(IntegrationRecordRef) then begin - TempMasterDataMgtCoupling.Delete(); - Clear(LocalRecordRef); - IntegrationTableSynch.Uncouple(LocalRecordRef, IntegrationRecordRef); - end; - until IntegrationRecordRef.Next() = 0; + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + // Cross-environment: read the whole filtered source set over the wire (no modified-on watermark). + if not MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, IntegrationTableMapping.GetIntegrationTableFilter(), IntegrationRecordRef) then + exit; + end else begin + IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + IntegrationRecordRef.ChangeCompany(SourceCompanyName); + IntegrationTableMapping.SetIntRecordRefFilter(IntegrationRecordRef); + if not IntegrationRecordRef.FindSet() then + exit; + end; + + repeat + if TempMasterDataMgtCoupling.IsIntegrationRecordRefCoupled(IntegrationRecordRef) then begin + TempMasterDataMgtCoupling.Delete(); + Clear(LocalRecordRef); + IntegrationTableSynch.Uncouple(LocalRecordRef, IntegrationRecordRef); + end; + until IntegrationRecordRef.Next() = 0; end; local procedure UncoupleAllCoupledRecords(var IntegrationTableMapping: Record "Integration Table Mapping"; var IntegrationTableSynch: Codeunit "Integration Table Synch."; var TempMasterDataMgtCoupling: Record "Master Data Mgt. Coupling" temporary) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al index 023b432fa52..6ad9d221ae1 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al @@ -2,6 +2,7 @@ namespace Microsoft.Integration.MDM; using Microsoft.Integration.SyncEngine; using System.Automation; +using System.Integration; using System.Threading; using System.Upgrade; @@ -13,13 +14,48 @@ codeunit 7238 "Master Data Mgt. Upgrade" Access = Internal; Subtype = Upgrade; Permissions = tabledata "Integration Field Mapping" = rimd, - tabledata "Integration Table Mapping" = rimd; + tabledata "Integration Table Mapping" = rimd, + tabledata "Tenant Web Service" = rimd; trigger OnUpgradePerCompany() begin UpgradeJobQueueEntryFrequencies(); end; + trigger OnUpgradePerDatabase() + begin + RegisterCrossEnvSourceWebService(); + end; + + // Guaranteed provisioning path: install codeunits are skipped when BC is pre-baked into a package and mounted per tenant. + // NOT gated by an upgrade tag: SetAllUpgradeTags on a fresh install can set the tag without the service ever being + // created, which would permanently skip registration and 404 the source API. Ensure the record exists AND points at the + // right object and is published instead - idempotent and self-healing on every install and upgrade. Access stays gated by + // the "Cross Env" permission set, not by publishing. + internal procedure RegisterCrossEnvSourceWebService() + var + TenantWebService: Record "Tenant Web Service"; + WebServiceManagement: Codeunit "Web Service Management"; + begin + if not TenantWebService.Get(TenantWebService."Object Type"::Codeunit, CrossEnvSourceWebServiceName()) then begin + WebServiceManagement.CreateTenantWebService(TenantWebService."Object Type"::Codeunit, Codeunit::"MDM Cross-Env Source API", CrossEnvSourceWebServiceName(), true); + exit; + end; + + if (TenantWebService."Object ID" = Codeunit::"MDM Cross-Env Source API") and TenantWebService.Published then + exit; + + // Repair a pre-existing row that points at the wrong object or is unpublished, rather than trusting the name alone. + TenantWebService.Validate("Object ID", Codeunit::"MDM Cross-Env Source API"); + TenantWebService.Validate(Published, true); + TenantWebService.Modify(true); + end; + + internal procedure CrossEnvSourceWebServiceName(): Text[240] + begin + exit('MDMCrossEnvSource'); + end; + internal procedure UpgradeJobQueueEntryFrequencies() var IntegrationTableMapping: Record "Integration Table Mapping"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al b/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al new file mode 100644 index 00000000000..ee50e8e584d --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al @@ -0,0 +1,23 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Selects the master data source implementation. Non-extensible and internal: partners must not add +/// source types. The value is derived from setup (Source Environment Name), never stored directly. +/// +enum 7239 "MDM Data Source Type" implements "IMDM Data Source" +{ + Access = Internal; + Extensible = false; + + value(0; LocalCompany) + { + Caption = 'Local Company'; + Implementation = "IMDM Data Source" = "MDM Local Data Source"; + } + + value(1; CrossEnvironment) + { + Caption = 'Cross Environment'; + Implementation = "IMDM Data Source" = "MDM Cross-Env Data Source"; + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al new file mode 100644 index 00000000000..2753cc5a4db --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al @@ -0,0 +1,47 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; + +/// +/// Abstracts reading source-company master data so the same synchronization logic can run against a +/// local company (ChangeCompany) or, in a future release, a remote environment over OData. +/// Sealed to Microsoft: partners must not implement it or ride the synchronization credentials. +/// +interface "IMDM Data Source" +{ + Access = Internal; + + /// + /// Opens SourceRecordRef on the source integration table for the given mapping and applies the + /// supplied table filter. Returns true if at least one record matches. + /// + procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Fetches a single source record from the given integration table by its SystemId into + /// SourceRecordRef. Returns true if the record was found. + /// + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Fetches a single source integration-table record by its identifier into SourceRecordRef. + /// The identifier is the integration UID field value - for Master Data Management the SystemId - + /// passed as a Guid or its text form. A RecordId is environment-specific and only resolvable by the + /// local same-environment implementation; the cross-environment feed keys on SystemId, so it returns + /// false for a RecordId. Returns true if found. + /// + procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Opens the source integration table and returns the set of records whose integration UID field + /// matches UidFilter (a filter expression, e.g. a list of SystemIds). Returns true if any matched. + /// + procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Opens SourceRecordRef on the source integration table and returns ALL records matching TableFilter + /// (the whole set, not just those modified since the watermark) - used by coupling and uncoupling. + /// Returns true if at least one record matches. + /// + procedure GetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al new file mode 100644 index 00000000000..a8bea45fc13 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al @@ -0,0 +1,17 @@ +namespace Microsoft.Integration.MDM; + +/// +/// The wire contract the cross-environment data source calls on the source's ODataV4 web service. One method +/// per unbound action; JSON in, JSON out. Kept as a seam so the HTTP/OAuth transport can be swapped for an +/// in-process transport in tests (the source and subsidiary run in the same environment there). +/// +interface "IMDM Source Transport" +{ + Access = Internal; + + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text; + + procedure LastModifiedAtPerTable(TableIds: Text): Text; + + procedure GetCapabilities(): Text; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al new file mode 100644 index 00000000000..af669575d96 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -0,0 +1,315 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Guided setup for reading master data from another Business Central ENVIRONMENT (same tenant). Mirrors the +/// Intercompany cross-environment partner wizard: consent, then the source environment/company and the +/// Microsoft Entra client-credentials application, then an optional connection test. The client secret is +/// write-only and stored in module-scoped Isolated Storage; it is never read back to the page. +/// +page 7232 "MDM Connection Details" +{ + Caption = 'Cross-Environment Connection Setup'; + PageType = NavigatePage; + ApplicationArea = Suite; + UsageCategory = None; + Permissions = tabledata "Master Data Management Setup" = imd; + + layout + { + area(Content) + { + group(WelcomeTab) + { + ShowCaption = false; + Visible = Step = Step::Welcome; + group(Introduction) + { + Caption = 'Welcome'; + InstructionalText = 'This guide helps you connect to a company in a different Business Central environment so you can synchronize master data from it. Before you continue, make sure the source environment has this extension installed and exposes its data, and that you have a Microsoft Entra application with a read-only permission set on the source. When you choose Next, you are asked to review and accept the privacy terms for sharing data between environments.'; + } + } + group(ConnectionTab) + { + ShowCaption = false; + Visible = Step = Step::Connection; + group(SourceConnectionDetails) + { + Caption = 'Source environment connection details'; + InstructionalText = 'Provide the environment and company you want to read master data from.'; + + field(SourceEnvironmentName; SourceEnvironmentName) + { + Caption = 'Source Environment'; + ApplicationArea = Suite; + ShowMandatory = true; + ToolTip = 'Specifies the name of the source Business Central environment.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + field(SourceCompanyName; SourceCompanyName) + { + Caption = 'Source Company Name'; + ApplicationArea = Suite; + ShowMandatory = true; + ToolTip = 'Specifies the name of the company in the source environment that data is read from.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + } + group(OAuth2ConnectionDetails) + { + Caption = 'Authentication details'; + InstructionalText = 'Provide the Microsoft Entra application that this environment uses to authenticate to the source environment. Register it as a single-tenant application, because synchronization only connects to environments in the same Microsoft Entra tenant.'; + + field(OAuth2ClientId; OAuth2ClientId) + { + Caption = 'Client ID'; + ApplicationArea = Suite; + ExtendedDatatype = Masked; + ShowMandatory = true; + ToolTip = 'Specifies the application (client) ID of the Microsoft Entra authentication application.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + field(OAuth2ClientSecret; OAuth2ClientSecret) + { + Caption = 'Client Secret'; + ApplicationArea = Suite; + ExtendedDatatype = Masked; + ShowMandatory = not SecretAlreadyStored; + ToolTip = 'Specifies the client secret of the Microsoft Entra authentication application. The secret is stored securely and is not shown again after you enter it.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + } + } + group(TestConnectionTab) + { + ShowCaption = false; + Visible = Step = Step::TestConnection; + group(VerifyConnection) + { + Caption = 'Verify connection'; + InstructionalText = 'Optionally test that the source environment can be reached with the details you entered. Choose Test Connection, or choose Next to continue.'; + } + } + group(FinishTab) + { + ShowCaption = false; + Visible = Step = Step::Finish; + group(AllDone) + { + Caption = 'All done'; + InstructionalText = 'You''re all set. Choose Finish to save your connection settings. You can enable data synchronization right away.'; + } + } + } + } + + actions + { + area(Processing) + { + action(TestConnection) + { + ApplicationArea = Suite; + Caption = 'Test connection'; + ToolTip = 'Test that the source environment can be reached with the current URL and credentials.'; + Visible = TestConnectionEnabled; + Image = InteractionTemplateSetup; + InFooterBar = true; + + trigger OnAction() + begin + TestConnectionToSource(); + end; + } + action(ActionBack) + { + ApplicationArea = Suite; + Caption = 'Back'; + ToolTip = 'Go to the previous step.'; + Enabled = BackEnabled; + Image = PreviousRecord; + InFooterBar = true; + + trigger OnAction() + begin + PreviousStep(); + end; + } + action(ActionNext) + { + ApplicationArea = Suite; + Caption = 'Next'; + ToolTip = 'Go to the next step.'; + Enabled = NextEnabled; + Image = NextRecord; + InFooterBar = true; + + trigger OnAction() + begin + NextStep(); + end; + } + action(ActionFinish) + { + ApplicationArea = Suite; + Caption = 'Finish'; + ToolTip = 'Save the connection settings.'; + Enabled = FinishEnabled; + Image = Approve; + InFooterBar = true; + + trigger OnAction() + begin + SaveConfiguration(); + EnableSynchronizationOnFinish(); + CurrPage.Close(); + end; + } + } + } + + trigger OnOpenPage() + begin + LoadConfiguration(); + Step := Step::Welcome; + SetControls(); + end; + + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + Step: Option Welcome,Connection,TestConnection,Finish; + NextEnabled, BackEnabled, FinishEnabled, TestConnectionEnabled : Boolean; + SecretAlreadyStored: Boolean; + SourceEnvironmentName: Text[100]; + SourceCompanyName: Text[100]; + OAuth2ClientId: Text[100]; + [NonDebuggable] + OAuth2ClientSecret: Text; + ConnectionOkMsg: Label 'Successfully connected to the source environment (contract version %1).', Comment = '%1 = wire contract version'; + ConnectionFailedErr: Label 'Could not connect to the source environment. Check the source environment, company, and credentials, then try again.'; + EnableNowQst: Label 'Your cross-environment connection is saved. Do you want to enable data synchronization now?'; + + local procedure LoadConfiguration() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit; + SourceEnvironmentName := MasterDataManagementSetup."Source Environment Name"; + SourceCompanyName := MasterDataManagementSetup."Source Company Name"; + OAuth2ClientId := MasterDataManagementSetup."Source OAuth Client Id"; + SecretAlreadyStored := not IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"); + end; + + [NonDebuggable] + local procedure SaveConfiguration() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + HttpSourceTransport: Codeunit "MDM Http Source Transport"; + begin + if not MasterDataManagementSetup.Get() then begin + MasterDataManagementSetup.Init(); + MasterDataManagementSetup.Insert(); + end; + MasterDataManagementSetup.Validate("Source Environment Name", SourceEnvironmentName); + // Same-tenant, same-ring: derive the source web-service URL from the environment name instead of asking for it. + MasterDataManagementSetup."Source Environment URL" := CopyStr(HttpSourceTransport.BuildSourceApiBaseUrl(SourceEnvironmentName), 1, MaxStrLen(MasterDataManagementSetup."Source Environment URL")); + MasterDataManagementSetup."Source Company Name" := SourceCompanyName; + MasterDataManagementSetup."Source OAuth Client Id" := OAuth2ClientId; + if OAuth2ClientSecret <> '' then begin + MasterDataManagementSetup.SetSourceClientSecret(OAuth2ClientSecret); + // Minimize the plain-text window: the secret is now encrypted at rest, so drop the wizard copy. + Clear(OAuth2ClientSecret); + SecretAlreadyStored := true; + end; + MasterDataManagementSetup.Modify(true); + end; + + // Enabling here starts synchronization straight from Finish so the admin doesn't have to flip the setup toggle afterward. + local procedure EnableSynchronizationOnFinish() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit; + if MasterDataManagementSetup."Is Enabled" then + exit; + if not Confirm(EnableNowQst) then + exit; + MasterDataManagementSetup.Validate("Is Enabled", true); + MasterDataManagementSetup.Modify(true); + end; + + [NonDebuggable] + local procedure TestConnectionToSource() + var + SourceConnection: Codeunit "MDM Source Connection"; + Transport: Interface "IMDM Source Transport"; + Capabilities: JsonObject; + VersionToken: JsonToken; + VersionValue: Integer; + VersionText: Text; + begin + // Persist first so the transport reads the details entered in the wizard. + SaveConfiguration(); + Commit(); + Transport := SourceConnection.GetTransport(); + if not Capabilities.ReadFrom(Transport.GetCapabilities()) then + Error(ConnectionFailedErr); + if Capabilities.Get('version', VersionToken) then begin + if not (VersionToken.IsValue() and Evaluate(VersionValue, VersionToken.AsValue().AsText())) then // malformed version is a broken capabilities contract, not user-actionable + Error(ConnectionFailedErr); + VersionText := Format(VersionValue); + end; + Message(ConnectionOkMsg, VersionText); + end; + + local procedure NextStep() + begin + // Leaving the Welcome step requires the durable platform privacy-notice approval; show it once if needed. + if (Step = Step::Welcome) and not (MDMPrivacyNotice.IsApproved() or MDMPrivacyNotice.ConfirmApproval()) then + exit; + Step += 1; + SetControls(); + end; + + local procedure PreviousStep() + begin + Step -= 1; + SetControls(); + end; + + local procedure SetControls() + begin + BackEnabled := Step > Step::Welcome; + TestConnectionEnabled := Step = Step::TestConnection; + FinishEnabled := Step = Step::Finish; + NextEnabled := (Step < Step::Finish) and StepIsComplete(); + end; + + local procedure StepIsComplete(): Boolean + begin + case Step of + Step::Connection: + exit((SourceEnvironmentName <> '') and (SourceCompanyName <> '') and + (OAuth2ClientId <> '') and ((OAuth2ClientSecret <> '') or SecretAlreadyStored)); + else + exit(true); + end; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index e73c7cb6008..613d9ffdd47 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -29,8 +29,23 @@ page 7230 "Master Data Management Setup" { ApplicationArea = Suite; Editable = IsEditable; + Visible = not CrossEnvConfigured; // same-environment source company; hidden once a source environment is configured ToolTip = 'Specifies the name of the source company that you synchronize data from.'; } + field("Source Environment Name"; Rec."Source Environment Name") + { + ApplicationArea = Suite; + Editable = false; + Visible = CrossEnvConfigured; + ToolTip = 'Specifies the source Business Central environment that master data is read from. Use the Cross-Environment Setup action to change it.'; + } + field("Source Company Name"; Rec."Source Company Name") + { + ApplicationArea = Suite; + Editable = false; + Visible = CrossEnvConfigured; + ToolTip = 'Specifies the company in the source environment that master data is read from. Use the Cross-Environment Setup action to change it.'; + } field("Is Enabled"; Rec."Is Enabled") { ApplicationArea = Suite; @@ -51,10 +66,46 @@ page 7230 "Master Data Management Setup" { area(Processing) { + action(ConnectionDetails) + { + ApplicationArea = Suite; + Caption = 'Cross-environment setup'; + Image = LinkAccount; + ToolTip = 'Set up the connection to a company in a different Business Central environment for cross-environment synchronization.'; + + trigger OnAction() + begin + Page.RunModal(Page::"MDM Connection Details"); + if Rec.Get() then; // the wizard may have configured or cleared the cross-environment connection + RefreshData(); + CurrPage.Update(false); + end; + } + action(ClearCrossEnvSetup) + { + ApplicationArea = Suite; + Caption = 'Clear cross-environment setup'; + Image = RemoveLine; + Enabled = IsEditable; + Visible = CrossEnvConfigured; + ToolTip = 'Remove the cross-environment connection - the source environment, company, and stored credentials - and return to same-environment synchronization.'; + + trigger OnAction() + begin + if not Confirm(ClearCrossEnvConfirmQst, false) then + exit; + Rec.Validate("Source Environment Name", ''); // clear the source env; runs the enabled guard and detector cleanup + Rec.ClearCrossEnvConnection(); + Rec.Modify(true); + if Rec.Get() then; + RefreshData(); + CurrPage.Update(false); + end; + } action(ResetConfiguration) { ApplicationArea = Suite; - Caption = 'Use Default Synchronization Setup'; + Caption = 'Use default synchronization setup'; Image = ResetStatus; ToolTip = 'Resets the synchronization tables, fields, and job queue entries to the default values for the connection with the source company. All current synchronization tables are deleted and recreated.'; @@ -73,7 +124,7 @@ page 7230 "Master Data Management Setup" action(ExportSetup) { ApplicationArea = Suite; - Caption = 'Export Setup'; + Caption = 'Export setup'; Image = ExportFile; ToolTip = 'Export the setup tables.'; @@ -90,7 +141,7 @@ page 7230 "Master Data Management Setup" action(ImportSetup) { ApplicationArea = Suite; - Caption = 'Import Setup'; + Caption = 'Import setup'; Image = Import; ToolTip = 'Import the setup tables.'; @@ -114,7 +165,7 @@ page 7230 "Master Data Management Setup" action(StartInitialSynchAction) { ApplicationArea = Suite; - Caption = 'Start Initial Synchronization'; + Caption = 'Start initial synchronization'; Enabled = Rec."Is Enabled"; Image = RefreshLines; ToolTip = 'Start all the default synchronization jobs for synchronizing data from the source company. Data is synchronized according to the mappings defined on the Synchronization Tables page.'; @@ -123,7 +174,7 @@ page 7230 "Master Data Management Setup" action(SynchronizeNow) { ApplicationArea = Suite; - Caption = 'Synchronize Modified Records'; + Caption = 'Synchronize modified records'; Enabled = Rec."Is Enabled"; Image = Refresh; ToolTip = 'Synchronize records that have been modified since the last time they were synchronized.'; @@ -151,7 +202,7 @@ page 7230 "Master Data Management Setup" action("Synch. Job Queue Entries") { ApplicationArea = Suite; - Caption = 'Synch. Job Queue Entries'; + Caption = 'Synch. job queue entries'; Image = JobListSetup; ToolTip = 'View the job queue entries that manage the scheduled data synchronization.'; @@ -170,7 +221,7 @@ page 7230 "Master Data Management Setup" action(IntegrationTableMappings) { ApplicationArea = Suite; - Caption = 'Synchronization Tables'; + Caption = 'Synchronization tables'; Image = MapAccounts; ToolTip = 'View the list of tables to synchronize.'; @@ -192,6 +243,9 @@ page 7230 "Master Data Management Setup" actionref(IntegrationTableMappings_Promoted; IntegrationTableMappings) { } + actionref(ConnectionDetails_Promoted; ConnectionDetails) + { + } actionref("Synch. Job Queue Entries_Promoted"; "Synch. Job Queue Entries") { } @@ -226,7 +280,7 @@ page 7230 "Master Data Management Setup" trigger OnQueryClosePage(CloseAction: Action): Boolean begin if not Rec."Is Enabled" then - if not Confirm(StrSubstNo(EnableServiceQst, CurrPage.Caption()), true) then + if not Confirm(EnableServiceQst, true, CurrPage.Caption()) then exit(false); end; @@ -238,7 +292,7 @@ page 7230 "Master Data Management Setup" begin IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Delete After Synchronization", false); - Session.LogMessage('0000JIW', CompanyName(), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JIW', SetupExportedTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); Xmlport.Run(Xmlport::ExportMDMSetup, false, false, IntegrationTableMapping); end; @@ -247,18 +301,22 @@ page 7230 "Master Data Management Setup" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('0000JIX', CompanyName(), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JIX', SetupImportedTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); Xmlport.Run(XmlPort::ImportMDMSetup, false, true); end; var + SetupExportedTelemetryTxt: Label 'Master data synchronization setup was exported.', Locked = true; + SetupImportedTelemetryTxt: Label 'Master data synchronization setup was imported.', Locked = true; ResetIntegrationTableMappingConfirmQst: Label 'This will restore the default synchronization table setup and synchronization jobs. \\All existing customizations to synchronization table setup and jobs will be overwritten.\\Do you want to continue?'; ImportIntegrationTableMappingConfirmQst: Label 'This will import the synchronization table setup from a chosen file. \\Existing synchronization tables and fields will be overwritten with the version from the file.\\Existing synchronization job queue entries will not be overwritten. Do you want to continue?'; EnableServiceQst: Label 'The %1 is not enabled. Are you sure you want to exit?', Comment = '%1 = This Page Caption (Business Central Connection Setup)'; SynchronizeModifiedQst: Label 'This will synchronize all modified records in all integration table mappings. \\The synchronization will run in the background so you can continue with other tasks. \\Do you want to continue?'; SyncNowScheduledMsg: Label 'Synchronization of modified records is scheduled. \\You can view details on the %1 page.', Comment = '%1 = The localized caption of page Integration Synch. Job List'; SetupSuccessfulMsg: Label 'The default setup for Business Central synchronization has completed successfully.'; + ClearCrossEnvConfirmQst: Label 'This removes the cross-environment connection and its stored credentials, and returns to same-environment synchronization. Do you want to continue?'; IsEditable: Boolean; + CrossEnvConfigured: Boolean; SynchronizationImportedMsg: label 'The synchronization setup is imported. \\To view or edit the synchronization table setup, choose action Synchronization Tables.\\To view or edit the synchronization field setup, select a synchronization table and choose action Synchronization Fields.'; NoCoupledRecordsMsg: label 'No records are currently coupled to records from the source company. \\Choose the action Start Initial Synchronization.'; @@ -270,6 +328,7 @@ page 7230 "Master Data Management Setup" local procedure UpdateEnableFlags() begin IsEditable := (not Rec."Is Enabled"); + CrossEnvConfigured := Rec.IsCrossEnvironment(); end; local procedure GetJobQueueEntriesObjectIDToRunFilter(): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al index ac51870b1bb..ca63af7e94f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al @@ -94,7 +94,7 @@ page 7236 "Master Data Synch. Fields" action(FieldMapping) { ApplicationArea = Suite; - Caption = 'Update Fields'; + Caption = 'Update fields'; Image = Relationship; ToolTip = 'Updates field mappings to match table schema. Use this action if you added fields to the table with an extension.'; @@ -147,15 +147,15 @@ page 7236 "Master Data Synch. Fields" until IntegrationFieldMapping.Next() = 0; if FieldsAdded * FieldsRemoved > 0 then begin - Message(StrSubstNo(FieldsAddedAndRemovedTxt, FieldsAdded, FieldsRemoved)); + Message(FieldsAddedAndRemovedTxt, FieldsAdded, FieldsRemoved); exit; end; if FieldsAdded > 0 then - Message(StrSubstNo(FieldsAddedTxt, FieldsAdded)); + Message(FieldsAddedTxt, FieldsAdded); if FieldsRemoved > 0 then - Message(StrSubstNo(FieldsRemovedTxt, FieldsRemoved)); + Message(FieldsRemovedTxt, FieldsRemoved); end; } action(Enable) diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al index d437aac1c68..bf09f7885e3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al @@ -80,14 +80,14 @@ page 7233 "Master Data Synch. Tables" Error(TableMetadataNotFoundErr, AllObjWithCaption."Object ID"); if not TableMetadata.DataPerCompany then - Error(TableNotPerCompanyErr, AllObjWithCaption."Object Name"); + Error(TableNotPerCompanyErr, AllObjWithCaption."Object Caption"); if TableMetadata.TableType <> TableMetadata.TableType::Normal then - Error(TableNotOfTypeNormalErr, AllObjWithCaption."Object Name"); + Error(TableNotOfTypeNormalErr, AllObjWithCaption."Object Caption"); RecRef.Open(AllObjWithCaption."Object ID"); if not RecRef.WritePermission() then - Error(TablePermissionMissingErr, AllObjWithCaption."Object Name"); + Error(TablePermissionMissingErr, AllObjWithCaption."Object Caption"); RecRef.Close(); FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, AllObjWithCaption."Object ID"); @@ -95,7 +95,7 @@ page 7233 "Master Data Synch. Tables" IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); if RelatedTablesToAdd.Count() > 0 then - if Confirm(StrSubstno(RelatedTablesQst, RelatedTablesToAddText)) then begin + if Confirm(RelatedTablesQst, false, RelatedTablesToAddText) then begin IntegrationTableMapping.Validate(Status, IntegrationTableMapping.Status::Disabled); IntegrationTableMapping.Modify(); foreach RelatedTableNo in RelatedTablesToAdd do @@ -105,7 +105,7 @@ page 7233 "Master Data Synch. Tables" IntegrationTableMapping.Validate(Status, IntegrationTableMapping.Status::Disabled); IntegrationTableMapping.Modify(); end; - Message(StrSubstNo(RelatedTablesAddedMsg, AllObjWithCaption."Object Name", RelatedTablesToAddText)); + Message(RelatedTablesAddedMsg, AllObjWithCaption."Object Caption", RelatedTablesToAddText); exit; end; @@ -231,7 +231,7 @@ page 7233 "Master Data Synch. Tables" action(ResetConfiguration) { ApplicationArea = Suite; - Caption = 'Use Default Synchronization Setup'; + Caption = 'Use default synchronization setup'; Image = ResetStatus; ToolTip = 'Resets the tables, fields and synchronization jobs to the default values for the connection with the source company. All default synchronization table definitions are deleted and recreated.'; @@ -323,7 +323,7 @@ page 7233 "Master Data Synch. Tables" action(SynchronizeNow) { ApplicationArea = Suite; - Caption = 'Synchronize Modified Records'; + Caption = 'Synchronize modified records'; Enabled = HasRecords and (Rec."Parent Name" = '') and DataSynchEnabled; Image = Refresh; ToolTip = 'Synchronize records that have been modified since the last time they were synchronized.'; @@ -352,7 +352,7 @@ page 7233 "Master Data Synch. Tables" action(SynchronizeAll) { ApplicationArea = Suite; - Caption = 'Run Full Synchronization'; + Caption = 'Run full synchronization'; Enabled = HasRecords and (Rec."Parent Name" = '') and DataSynchEnabled; Image = RefreshLines; ToolTip = 'Start a job for full synchronization from records in the chosen source company for each of the selected tables.'; @@ -409,7 +409,7 @@ page 7233 "Master Data Synch. Tables" action(RemoveCoupling) { ApplicationArea = Suite; - Caption = 'Delete Couplings'; + Caption = 'Delete couplings'; Enabled = HasRecords and (Rec."Parent Name" = '') and DataSynchEnabled; Image = UnLinkAccount; ToolTip = 'Delete couplings for the selected tables.'; @@ -543,9 +543,9 @@ page 7233 "Master Data Synch. Tables" if RecRef.WritePermission() then begin RelatedTablesToAdd.Add(Field.RelationTableNo); if RelatedTablesToAddText = '' then - RelatedTablesToAddText := TableMetadata.Name + RelatedTablesToAddText := RecRef.Caption() else - RelatedTablesToAddText += ', ' + TableMetadata.Name; + RelatedTablesToAddText += ', ' + RecRef.Caption(); FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, Field.RelationTableNo, TopLevelTableId); end; RecRef.Close(); @@ -655,8 +655,8 @@ page 7233 "Master Data Synch. Tables" UserEditedIntegrationTableFilterTxt: Label 'The user edited the Integration Table Filter on %1 mapping.', Locked = true; EditIntegrationTableFilterTxt: Label ''; NoCoupledRecordsMsg: label 'No records of this table are currently coupled to records from the source company. \\Choose the action Run Full Synchronization.'; - RelatedTablesQst: label 'The chosen table has a relation to the following tables that are currently not included in the synchronization: %1. \\Do you want to synchronize these tables too?', Comment = '%1 - comma-separated list of table names'; - RelatedTablesAddedMsg: label 'Table %1 and related tables: %2 are added to the synchronization with state set to Disabled. \\Open Synchronization Tables page, choose synchronization fields for each of the added tables and then set their status to Enabled.', Comment = '%1 - a table name, %2 - comma-separated list of table names'; + RelatedTablesQst: label 'The chosen table has a relation to the following tables that are currently not included in the synchronization: %1. \\Do you want to synchronize these tables too?', Comment = '%1 - comma-separated list of table captions'; + RelatedTablesAddedMsg: label 'Table %1 and related tables: %2 are added to the synchronization with state set to Disabled. \\Open Synchronization Tables page, choose synchronization fields for each of the added tables and then set their status to Enabled.', Comment = '%1 - a table caption, %2 - comma-separated list of table captions'; TableMetadataNotFoundErr: label 'Metadata for table %1 cannot be loaded. Choose another table.', Comment = '%1 - a table name'; TableNotPerCompanyErr: label 'Table %1 is shared across all companies of this environment. Choose another table.', Comment = '%1 - a table name'; TableNotOfTypeNormalErr: label 'Table %1 is either declared as temporary, a query or as an interface for accessing an external entity. Choose another table.', Comment = '%1 - a table name'; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al new file mode 100644 index 00000000000..00e1a1f7451 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.CRM.Contact; + +tableextension 7246 "MDM Contact" extends Contact +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al new file mode 100644 index 00000000000..c748d9ecef9 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Finance.Currency; + +tableextension 7251 "MDM Currency Exch. Rate" extends "Currency Exchange Rate" +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al new file mode 100644 index 00000000000..2910298f215 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Sales.Customer; + +tableextension 7244 "MDM Customer" extends Customer +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al new file mode 100644 index 00000000000..74459e3dc5e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Foundation.Address; + +tableextension 7250 "MDM Post Code" extends "Post Code" +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al new file mode 100644 index 00000000000..db87851e6e7 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Purchases.Vendor; + +tableextension 7245 "MDM Vendor" extends Vendor +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index c8b1a20280b..6690eb05170 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -27,10 +27,17 @@ table 7230 "Master Data Management Setup" DataClassification = SystemMetadata; trigger OnValidate() + var + MasterDataMgtSetupDefault: Codeunit "Master Data Mgt. Setup Default"; begin if "Is Enabled" then - if "Company Name" = '' then - Error(MustPickSourceCompanyErr); + if IsCrossEnvironment() then begin + if not IsCrossEnvConnectionConfigured() then + Error(BuildConfigureConnectionError()); + end else + if "Company Name" = '' then + Error(MustPickSourceCompanyErr); + MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); end; } field(151; "Company Name"; Text[30]) @@ -65,7 +72,7 @@ table 7230 "Master Data Management Setup" if (xRec."Company Name" <> '') and (xRec."Company Name" <> Rec."Company Name") then if not MasterDataMgtCoupling.IsEmpty() then - if not Confirm(StrSubstNo(CouplingsWillBeDeletedQst, xRec."Company Name")) then + if not Confirm(CouplingsWillBeDeletedQst, false, xRec."Company Name") then Error(''); CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); @@ -79,6 +86,43 @@ table 7230 "Master Data Management Setup" Caption = 'Delay Synchronization Job Scheduling'; DataClassification = SystemMetadata; } + field(155; "Source Environment Name"; Text[100]) + { + Caption = 'Source Environment'; + DataClassification = OrganizationIdentifiableInformation; + + trigger OnValidate() + var + MasterDataMgtSetupDefault: Codeunit "Master Data Mgt. Setup Default"; + begin + // Re-pointing an enabled setup would leave stale couplings/cursors against the old source; force a disable/rebuild. + if "Is Enabled" and ("Source Environment Name" <> xRec."Source Environment Name") then + Error(CannotChangeSourceWhileEnabledErr); + MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); + end; + } + field(156; "Source Environment URL"; Text[250]) + { + Caption = 'Source Environment URL'; + DataClassification = OrganizationIdentifiableInformation; + } + field(157; "Source Company Name"; Text[100]) + { + Caption = 'Source Company'; + DataClassification = OrganizationIdentifiableInformation; + } + field(158; "Source OAuth Client Id"; Text[100]) + { + Caption = 'Source Client ID'; + ExtendedDatatype = Masked; + DataClassification = SystemMetadata; + } + field(159; "Source Client Secret Key"; Guid) + { + Caption = 'Source Client Secret Key'; + ExtendedDatatype = Masked; + DataClassification = SystemMetadata; + } } keys @@ -141,6 +185,69 @@ table 7230 "Master Data Management Setup" exit(Result); end; + internal procedure IsCrossEnvConnectionConfigured(): Boolean + begin + exit(("Source Environment URL" <> '') and ("Source Company Name" <> '') and ("Source OAuth Client Id" <> '') and (not IsNullGuid("Source Client Secret Key"))); + end; + + internal procedure IsCrossEnvironment(): Boolean + begin + exit("Source Environment Name" <> ''); + end; + + // Reverting to same-environment: drop the source connection details and the stored secret. + internal procedure ClearCrossEnvConnection() + begin + "Source Environment URL" := ''; + "Source Company Name" := ''; + "Source OAuth Client Id" := ''; + if not IsNullGuid("Source Client Secret Key") then + if not IsolatedStorage.Delete("Source Client Secret Key", DataScope::Company) then; + Clear("Source Client Secret Key"); + end; + + [NonDebuggable] + internal procedure SetSourceClientSecret(ClientSecret: SecretText) + begin + "Source Client Secret Key" := SetSecret("Source Client Secret Key", ClientSecret); + end; + + internal procedure GetSourceClientSecret(): SecretText + begin + exit(GetSecret("Source Client Secret Key")); + end; + + // Mirrors the Intercompany connection pattern: secrets live in module-scoped Isolated Storage, keyed by a Guid. + [NonDebuggable] + local procedure SetSecret(SecretKey: Guid; SecretValue: SecretText): Guid + var + EnvironmentInformation: Codeunit "Environment Information"; + NewSecretKey: Guid; + begin + if not IsNullGuid(SecretKey) then + if not IsolatedStorage.Delete(SecretKey, DataScope::Company) then; + + NewSecretKey := CreateGuid(); + if EncryptionEnabled() then + IsolatedStorage.SetEncrypted(NewSecretKey, SecretValue, DataScope::Company) + else begin + // On SaaS (the only runtime for cross-env) encryption is always available, so refuse to store the + // secret unencrypted there; off-SaaS dev/test/on-prem may lack encryption, so fall back as Intercompany does. + if EnvironmentInformation.IsSaaSInfrastructure() then + Error(EncryptionRequiredErr); + IsolatedStorage.Set(NewSecretKey, SecretValue, DataScope::Company); + end; + + exit(NewSecretKey); + end; + + local procedure GetSecret(SecretKey: Guid) SecretValue: SecretText + begin + if IsNullGuid(SecretKey) then + exit; + if not IsolatedStorage.Get(SecretKey, DataScope::Company, SecretValue) then; + end; + local procedure EnableConnection() var MasterDataMgtSubscriber: Record "Master Data Mgt. Subscriber"; @@ -159,11 +266,35 @@ table 7230 "Master Data Management Setup" ResetConfig := Confirm(ResetConfigQst); if ResetConfig then MasterDataMgtSetupDefault.ResetConfiguration(Rec); + + if IsCrossEnvironment() then begin + // Cross-environment: the source is a different environment; never write to its subscriber table. + // Drop any stale local-company subscription left from a prior local-source setup. + if "Company Name" <> '' then begin + CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); + MasterDataManagement.RemoveSubsidiarySubscriptionFromMasterCompany("Company Name", CurrentCompanyName); + end; + Message(SynchronizationEnabledMsg, "Source Company Name"); + LogCrossEnvironmentEnabled(MasterDataManagement.GetTelemetryCategory()); + exit; + end; + CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); MasterDataManagement.AddSubsidiarySubscriptionToMasterCompany(Rec."Company Name", CurrentCompanyName); - Message(StrSubstNo(SynchronizationEnabledMsg, Rec."Company Name")); - Session.LogMessage('0000JIM', Rec."Company Name", Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); - Session.LogMessage('0000JIN', CurrentCompanyName, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Message(SynchronizationEnabledMsg, Rec."Company Name"); + // Company names are tenant data: keep them out of the free-text telemetry message. + Session.LogMessage('0000JIM', SynchronizationEnabledSourceTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JIN', SynchronizationEnabledSubscriberTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + + // Env name is organization-identifiable: keep it out of the free-text message and in a structured dimension. + local procedure LogCrossEnvironmentEnabled(TelemetryCategory: Text) + var + Dimensions: Dictionary of [Text, Text]; + begin + Dimensions.Add('Category', TelemetryCategory); + Dimensions.Add('sourceEnvironment', "Source Environment Name"); + Session.LogMessage('0000VAX', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); end; local procedure GetConfigurationUpdates(var IsEnabledChanged: Boolean) @@ -186,11 +317,13 @@ table 7230 "Master Data Management Setup" begin CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(Rec."Company Name")); - MasterDataManagement.RemoveSubsidiarySubscriptionFromMasterCompany(Rec."Company Name", CurrentCompanyName); + // Cross-environment: the source is a different environment; never touch its subscriber table. + if not IsCrossEnvironment() then + MasterDataManagement.RemoveSubsidiarySubscriptionFromMasterCompany(Rec."Company Name", CurrentCompanyName); UpdateDataSynchJobQueueEntriesStatus(); if not MasterDataMgtCoupling.IsEmpty() then - if Confirm(StrSubstNo(KeepTheCouplingsQst, Rec."Company Name")) then + if Confirm(KeepTheCouplingsQst, false, Rec."Company Name") then exit else begin IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); @@ -245,11 +378,37 @@ table 7230 "Master Data Management Setup" until IntegrationTableMapping.Next() = 0; end; + internal procedure GetDataSource(): Interface "IMDM Data Source" + begin + if "Source Environment Name" <> '' then + exit(Enum::"MDM Data Source Type"::CrossEnvironment); + exit(Enum::"MDM Data Source Type"::LocalCompany); + end; + + local procedure BuildConfigureConnectionError(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MustConfigureConnectionErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.RecordId := Rec.RecordId(); + ErrInfo.PageNo := Page::"Master Data Management Setup"; + ErrInfo.AddNavigationAction(OpenSetupNavigationTxt); + exit(ErrInfo); + end; + var SynchronizationEnabledMsg: label 'The synchronization of data from company %1 is enabled. \\To review the tables and fields that will be synchronized, choose action Synchronization Tables. \\To perform the initial synchronization of data from %1, choose Start Initial Synchronization. \\After the initial synchronization is done, job queue entries will continue to synchronize modifications.', Comment = '%1 - a company name'; CouplingsWillBeDeletedQst: label 'All the couplings with records from previous source company %1 will be deleted. Do you want to continue?', Comment = '%1 - a company name'; KeepTheCouplingsQst: label 'Data synchronization with company %1 is disabled. \\We recommend to keep the table setup and coupling information, especially if you intend to reenable the synchronization with the same company. \\Do you want to keep the table setup and coupling information?', Comment = '%1 - a company name'; MustNotPickCurrentCompanyErr: label 'You are currently signed into this company. \\Choose a different company to synchronize data with.'; MustPickSourceCompanyErr: label 'You must choose a source company to synchronize data from.'; + MustConfigureConnectionErr: label 'Enter the cross-environment connection details before you enable synchronization.'; + OpenSetupNavigationTxt: label 'Open Master Data Management Setup'; + CannotChangeSourceWhileEnabledErr: label 'You cannot change the source environment while synchronization is enabled. Disable synchronization first, then change the source.'; + EncryptionRequiredErr: label 'Enable data encryption before saving the source connection secret. Cross-environment credentials are never stored unencrypted.'; + CrossEnvEnabledTelemetryTxt: label 'Cross-environment master data synchronization was enabled.', Locked = true; + SynchronizationEnabledSourceTelemetryTxt: label 'Master data synchronization was enabled for the source company.', Locked = true; + SynchronizationEnabledSubscriberTelemetryTxt: label 'Master data synchronization was enabled for the subscriber company.', Locked = true; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al index ced28c054e8..b504722605e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al @@ -42,6 +42,14 @@ tableextension 7235 MasterDataMgtTableMapping extends "Integration Table Mapping Commit(); end; } + field(7236; "Source Change Cursor"; Text[150]) + { + // Composite (SystemModifiedAt, SystemId) resume point ({"modifiedAt":...,"systemId":...}, ~93 chars). + // Empty means "caught up": the run drained the source and the watermark is authoritative. + // Internal sync resume state (the systemId is a pagination pointer, not business data), so SystemMetadata. + Caption = 'Source Change Cursor'; + DataClassification = SystemMetadata; + } } var diff --git a/src/Apps/W1/MasterDataManagement/test library/app.json b/src/Apps/W1/MasterDataManagement/test library/app.json index 226805516eb..2ead3d8eaac 100644 --- a/src/Apps/W1/MasterDataManagement/test library/app.json +++ b/src/Apps/W1/MasterDataManagement/test library/app.json @@ -22,6 +22,20 @@ "screenshots": [], "platform": "29.0.0.0", "target": "Cloud", + "idRanges": [ + { + "from": 139757, + "to": 139758 + }, + { + "from": 139929, + "to": 139930 + }, + { + "from": 139934, + "to": 139934 + } + ], "resourceExposurePolicy": { "allowDebugging": false, "allowDownloadingSource": true, diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 4716511930b..506b307abb7 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -3,36 +3,67 @@ codeunit 139757 "Library - Master Data Mgt." { Access = Public; + /// Invokes the field-transfer subscriber logic that resolves the value to synchronize for a field. + /// The source field being transferred. + /// The destination field receiving the value. + /// Returns the value to apply to the destination field. + /// Returns true when the subscriber resolved the value. + /// Returns whether the resolved value still needs type conversion. procedure HandleOnTransferFieldData(SourceFieldRef: FieldRef; DestinationFieldRef: FieldRef; var NewValue: Variant; var IsValueFound: Boolean; var NeedsConversion: Boolean) begin MasterDataMgtSubscribers.HandleOnTransferFieldData(SourceFieldRef, DestinationFieldRef, NewValue, IsValueFound, NeedsConversion); end; + /// Renames the destination record before modification when the source primary key has changed. + /// The integration table mapping being synchronized. + /// The source record providing the new primary key. + /// The destination record to rename in place. procedure RenameIfNeededOnBeforeModifyRecord(IntegrationTableMapping: Record "Integration Table Mapping"; SourceRecordRef: RecordRef; var DestinationRecordRef: RecordRef) begin MasterDataMgtSubscribers.RenameIfNeededOnBeforeModifyRecord(IntegrationTableMapping, SourceRecordRef, DestinationRecordRef); end; + /// Determines whether the source record was modified after the last synchronization. + /// The connection type of the integration table. + /// The integration table mapping being synchronized. + /// The source record to evaluate. + /// Returns true when the source record changed since the last synch. + /// Returns true when the subscriber handled the evaluation. procedure HandleOnWasModifiedAfterLastSynch(IntegrationTableConnectionType: TableConnectionType; IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef; var SourceWasChanged: Boolean; var IsHandled: Boolean) begin MasterDataMgtSubscribers.HandleOnWasModifiedAfterLastSynch(IntegrationTableConnectionType, IntegrationTableMapping, SourceRecordRef, SourceWasChanged, IsHandled); end; + /// Resolves the local record ID coupled to an integration SystemId, synchronizing it first if needed. + /// The SystemId of the integration record. + /// The table ID to resolve against. + /// Returns the coupled local record ID. + /// Returns true when the subscriber handled the resolution. procedure HandleOnFindAndSynchRecordIDFromIntegrationSystemId(IntegrationSystemId: Guid; TableId: Integer; var LocalRecordID: RecordID; var IsHandled: Boolean) begin MasterDataMgtSubscribers.HandleOnFindAndSynchRecordIDFromIntegrationSystemId(IntegrationSystemId, TableId, LocalRecordID, IsHandled); end; + /// Evaluates whether a data-synchronization job queue entry needs to run. + /// The job queue entry being evaluated. + /// Returns true when the job needs to run. procedure HandleOnFindingIfJobNeedsToBeRun(var Sender: Record "Job Queue Entry"; var Result: Boolean) begin MasterDataMgtSubscribers.HandleOnFindingIfJobNeedsToBeRun(Sender, Result); end; + /// Runs the post-run handling for a data-synchronization job queue entry. + /// The job queue entry that finished running. procedure HandleOnAfterJobQueueEntryRun(var JobQueueEntry: Record "Job Queue Entry") begin MasterDataMgtSubscribers.HandleOnAfterJobQueueEntryRun(JobQueueEntry); end; + /// Finds the tables related to a synchronization table, as offered when adding it to the setup. + /// The tables already in the synchronization setup. + /// Returns the related table IDs proposed for adding. + /// Returns the display text for the proposed related tables. + /// The table whose related tables are resolved. procedure FindRelatedTables(var ExistingSynchTableNos: List of [Integer]; var RelatedTablesToAdd: List of [Integer]; var RelatedTablesToAddText: Text; TableId: Integer) var MasterDataSynchTables: Page "Master Data Synch. Tables"; @@ -40,6 +71,264 @@ codeunit 139757 "Library - Master Data Mgt." MasterDataSynchTables.FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, TableId); end; + /// Sets the source company on Master Data Management Setup to the current company. + procedure SetSourceCompanyToCurrent() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Company Name" := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Company Name")); + MasterDataManagementSetup.Modify(false); + end; + + /// Finds the enabled Master Data Management mapping whose source table holds the coupling's integration record, respecting the mapping's integration table filter. + /// Returns the matched mapping. + /// The coupling whose integration record is resolved. + /// True if an enabled mapping holds the record within its integration table filter; otherwise false. + procedure FindMappingByIntegrationRecordId(var IntegrationTableMapping: Record "Integration Table Mapping"; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"): Boolean + begin + exit(MasterDataManagement.FindMappingByIntegrationRecordId(IntegrationTableMapping, MasterDataMgtCoupling)); + end; + + /// Gets the integration record reference for a coupling. + /// The integration table ID to resolve. + /// The coupling whose integration record is requested. + /// Returns the resolved integration record reference. + /// True if the integration record was found; otherwise false. + procedure GetIntegrationRecordRefByCoupling(IntegrationTableID: Integer; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; var RecRef: RecordRef): Boolean + begin + exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableID, MasterDataMgtCoupling, RecRef)); + end; + + /// Gets the integration record reference identified by a coupling ID. + /// The integration table mapping to resolve against. + /// The coupling ID to resolve. + /// Returns the resolved integration record reference. + /// True if the integration record was found; otherwise false. + procedure GetIntegrationRecordRefById(var IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var RecRef: RecordRef): Boolean + begin + exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableMapping, ID, RecRef)); + end; + + /// Gets the set of modified source records for a table mapping from the configured data source. + /// The integration table mapping to read. + /// The source table filter to apply. + /// Returns the record reference positioned on the modified set. + /// True if any modified records were found; otherwise false. + procedure DataSourceGetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetModifiedSet(IntegrationTableMapping, TableFilter, SourceRecordRef)); + end; + + /// Gets source records matching a UID filter from the configured data source. + /// The integration table mapping to read. + /// The UID filter to apply. + /// Returns the record reference positioned on the matching set. + /// True if any matching records were found; otherwise false. + procedure DataSourceGetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef)); + end; + + /// Gets a single source record by its coupling ID (SystemId as GUID or text) from the configured data source. + /// The integration table mapping to read. + /// The record ID (SystemId as a GUID or its text form). + /// Returns the record reference positioned on the found record. + /// True if the record was found; otherwise false. + procedure DataSourceGetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetById(IntegrationTableMapping, ID, SourceRecordRef)); + end; + + /// Gets source records matching a table filter from the configured data source. + /// The integration table mapping to read. + /// The source table filter to apply. + /// Returns the record reference positioned on the matching set. + /// True if any matching records were found; otherwise false. + procedure DataSourceGetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, TableFilter, SourceRecordRef)); + end; + + /// Gets the count of integration records for a table mapping. + /// The integration table mapping to count. + /// The number of integration records. + procedure GetIntegrationRecRefCount(IntegrationTableMapping: Record "Integration Table Mapping"): Integer + begin + exit(MasterDataManagement.GetIntegrationRecRefCount(IntegrationTableMapping)); + end; + + /// Gets a single source record by its SystemId from the configured data source. + /// The integration table ID to read. + /// The SystemId of the source record. + /// Returns the record reference positioned on the found record. + /// True if the record was found; otherwise false. + procedure DataSourceGetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableId, SystemId, SourceRecordRef)); + end; + + /// Gets a cursor-paged batch of modified source records from the cross-environment data source. + /// The integration table mapping to read. + /// The source table filter to apply. + /// The cursor to resume from; empty starts a new scan. + /// The maximum number of pages to fetch in this call. + /// Returns the record reference positioned on the fetched batch. + /// Returns the cursor to resume from on the next call. + /// Returns true if more records remain beyond this batch. + /// True if any records were fetched; otherwise false. + procedure DataSourceGetModifiedBatch(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; StartCursor: Text; MaxPages: Integer; var SourceRecordRef: RecordRef; var EndCursor: Text; var HasMore: Boolean): Boolean + var + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + begin + exit(CrossEnvDataSource.GetModifiedBatch(IntegrationTableMapping, TableFilter, StartCursor, MaxPages, SourceRecordRef, EndCursor, HasMore)); + end; + + /// Reads a related source table narrowed by a row filter from the cross-environment data source. + /// The source table ID to read. + /// The source row filter (view) restricting which records are returned. + /// Returns the record reference positioned on the materialized set. + /// True if any matching records were found; otherwise false. + procedure DataSourceGetRecordsByFilter(TableId: Integer; RowFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + begin + exit(CrossEnvDataSource.GetSourceRecordsByFilter(TableId, RowFilter, SourceRecordRef)); + end; + + // Setting a Source Environment Name routes GetDataSource() to the cross-environment implementation. + /// Sets the source environment name, routing the data source to the cross-environment implementation. + /// The source environment name to set. + procedure SetSourceEnvironmentName(EnvironmentName: Text) + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Source Environment Name" := CopyStr(EnvironmentName, 1, MaxStrLen(MasterDataManagementSetup."Source Environment Name")); + MasterDataManagementSetup.Modify(false); + end; + + /// Runs the cross-environment change detector once. + procedure RunChangeDetector() + var + MDMCrossEnvChangeDetector: Codeunit "MDM Cross-Env Change Detector"; + begin + MDMCrossEnvChangeDetector.DetectChanges(); + end; + + /// Validates a source environment URL against the HTTP transport's host allow-list; errors if it is not a valid Business Central endpoint. + /// The source environment base URL to validate. + procedure ValidateHttpTransportSourceHost(BaseUrl: Text) + var + MDMHttpSourceTransport: Codeunit "MDM Http Source Transport"; + begin + MDMHttpSourceTransport.ValidateSourceHostUrl(BaseUrl); + end; + + /// Unwraps the ODataV4 action envelope the HTTP transport receives, returning the inner value (or the raw body). + /// The raw response body to unwrap. + /// The inner OData value, or the body unchanged if it is not a value-envelope. + procedure UnwrapHttpTransportODataValue(ResponseBody: Text): Text + var + MDMHttpSourceTransport: Codeunit "MDM Http Source Transport"; + begin + exit(MDMHttpSourceTransport.UnwrapODataValueForTest(ResponseBody)); + end; + + /// Checks whether the inline media cache holds an entry for a record field. + /// The SystemId of the source record. + /// The field number of the media/blob field. + /// True if the cache contains the entry; otherwise false. + procedure InlineMediaCacheContains(SystemId: Guid; FieldNo: Integer): Boolean + var + InlineMedia: Codeunit "MDM Inline Media"; + begin + exit(InlineMedia.Contains(SystemId, FieldNo)); + end; + + /// Checks whether an inline media field was marked cleared (empty on the source) for destination removal. + /// The SystemId of the source record. + /// The field number of the media field. + /// True if the field was marked cleared; otherwise false. + procedure InlineMediaIsCleared(SystemId: Guid; FieldNo: Integer): Boolean + var + InlineMedia: Codeunit "MDM Inline Media"; + begin + exit(InlineMedia.IsCleared(SystemId, FieldNo)); + end; + + /// Reads the source SystemModifiedAt watermark cached during cross-environment materialization. + /// The SystemId of the source record. + /// Returns the cached source SystemModifiedAt. + /// True if the watermark was cached; otherwise false. + procedure TryGetSourceWatermark(SystemId: Guid; var ModifiedAt: DateTime): Boolean + var + SourceWatermark: Codeunit "MDM Source Watermark"; + begin + exit(SourceWatermark.TryGet(SystemId, ModifiedAt)); + end; + + /// Returns the registered privacy-notice ID that gates cross-environment synchronization. + /// The privacy-notice ID code. + procedure PrivacyNoticeId(): Code[50] + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + exit(MDMPrivacyNotice.GetPrivacyNoticeId()); + end; + + /// Returns whether the cross-environment privacy notice is currently approved. + /// True if the notice is approved; otherwise false. + procedure PrivacyNoticeIsApproved(): Boolean + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + exit(MDMPrivacyNotice.IsApproved()); + end; + + /// Runs the fail-closed transport gate; errors when the privacy notice is not approved. + procedure PrivacyNoticeCheckApproved() + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + MDMPrivacyNotice.CheckApproved(); + end; + + /// Removes any recorded approval for the cross-env privacy notice, resetting it to Not set. + procedure PrivacyNoticeResetApproval() + var + PrivacyNoticeApproval: Record "Privacy Notice Approval"; + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + PrivacyNoticeApproval.SetRange(ID, MDMPrivacyNotice.GetPrivacyNoticeId()); + PrivacyNoticeApproval.DeleteAll(); + end; + + /// Records approval for the cross-env privacy notice (for tests exercising the consent-gated source API). + procedure ApproveCrossEnvPrivacyNotice() + var + PrivacyNotice: Codeunit "Privacy Notice"; + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + PrivacyNotice.SetApprovalState(MDMPrivacyNotice.GetPrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; + MasterDataManagement: Codeunit "Master Data Management"; } diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al new file mode 100644 index 00000000000..7588116c26e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al @@ -0,0 +1,101 @@ +#pragma warning disable AA0247 +codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" +{ + // Test transport: injected into the cross-env data source via OnResolveSourceTransport so the subsidiary and + // source run in ONE environment. Pass-through calls the real source API (codeunit 7241) for round-trip tests; + // a canned response lets tests exercise the subsidiary's error handling. SingleInstance so the injected + // instance shares the state the test sets. + SingleInstance = true; + Access = Public; + + var + CannedResponse: Text; + CannedCapabilities: Text; + Active: Boolean; + UseCanned: Boolean; + UseCannedCapabilities: Boolean; + + /// Activates the in-process transport so it is injected in place of the HTTP transport. + procedure Activate() + begin + Active := true; + end; + + /// Deactivates the transport and clears all canned state and negotiated capabilities. + procedure Deactivate() + var + SourceCapabilities: Codeunit "MDM Source Capabilities"; + begin + Active := false; + UseCanned := false; + UseCannedCapabilities := false; + Clear(CannedResponse); + Clear(CannedCapabilities); + SourceCapabilities.Reset(); // clear negotiated capabilities between tests + end; + + /// Sets a canned records/last-modified response returned instead of calling the real source API. + /// The raw JSON response to return. + procedure SetCannedResponse(Response: Text) + begin + CannedResponse := Response; + UseCanned := true; + end; + + /// Sets a canned capabilities response returned instead of calling the real source API. + /// The raw JSON capabilities response to return. + procedure SetCannedCapabilities(Response: Text) + begin + CannedCapabilities := Response; + UseCannedCapabilities := true; + end; + + /// Returns records for a table, using the canned response if one is set, otherwise the real source API. + /// The source table ID to read. + /// The projected field IDs. + /// The cursor/systemId selector. + /// The page size. + /// The optional source row filter (view). + /// The raw JSON records response. + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + begin + if UseCanned then + exit(CannedResponse); + exit(SourceApi.GetRecords(TableId, FieldIds, Selector, PageSize, Filter)); + end; + + /// Returns the last-modified-per-table probe, using the canned response if set, otherwise the real source API. + /// The JSON array of table IDs to probe. + /// The raw JSON last-modified response. + procedure LastModifiedAtPerTable(TableIds: Text): Text + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + begin + if UseCanned then + exit(CannedResponse); + exit(SourceApi.LastModifiedAtPerTable(TableIds)); + end; + + /// Returns the source capabilities, using the canned capabilities if set, otherwise the real source API. + /// The raw JSON capabilities response. + procedure GetCapabilities(): Text + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + begin + if UseCannedCapabilities then + exit(CannedCapabilities); + exit(SourceApi.GetCapabilities()); + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Source Connection", 'OnResolveSourceTransport', '', false, false)] + local procedure InjectTransport(var Transport: Interface "IMDM Source Transport") + var + InProcessTransport: Codeunit "MDM In-Process Transport"; + begin + if not Active then + exit; + Transport := InProcessTransport; // SingleInstance: same stateful instance the test configured + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al new file mode 100644 index 00000000000..e40b1f8868d --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al @@ -0,0 +1,51 @@ +#pragma warning disable AA0247 +codeunit 139930 "MDM Test Detector Probe" +{ + // Captures which tables the change detector decided to nudge and short-circuits the real reschedule, so the + // detector's decision logic can be asserted without scheduling background jobs (not possible in the test lab). + SingleInstance = true; + Access = Public; + + var + NudgedTableIds: List of [Integer]; + Active: Boolean; + + /// Activates the probe so it captures detector nudges and clears any previously captured table IDs. + procedure Activate() + begin + Active := true; + Clear(NudgedTableIds); + end; + + /// Deactivates the probe and clears the captured table IDs. + procedure Deactivate() + begin + Active := false; + Clear(NudgedTableIds); + end; + + /// Checks whether the change detector nudged the sync job for a given table. + /// The integration table ID to check. + /// True if the table was nudged since activation; otherwise false. + procedure WasNudged(TableId: Integer): Boolean + begin + exit(NudgedTableIds.Contains(TableId)); + end; + + /// Returns the number of distinct tables the detector nudged since activation. + /// The nudge count. + procedure NudgeCount(): Integer + begin + exit(NudgedTableIds.Count()); + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Change Detector", 'OnBeforeRescheduleSynchJob', '', false, false)] + local procedure CaptureNudge(var JobQueueEntry: Record "Job Queue Entry"; IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) + begin + if not Active then + exit; + if not NudgedTableIds.Contains(IntegrationTableMapping."Integration Table ID") then + NudgedTableIds.Add(IntegrationTableMapping."Integration Table ID"); + IsHandled := true; // the test lab cannot reschedule background jobs + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al new file mode 100644 index 00000000000..f092e41491f --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al @@ -0,0 +1,53 @@ +#pragma warning disable AA0247 +codeunit 139934 "MDM Test Paging Config" +{ + // Test hook: forces a small cross-environment page size so paging/resume can be exercised with a few records. + // SingleInstance so the flag the test sets is the one the static subscriber reads. + SingleInstance = true; + Access = Public; + + var + Active: Boolean; + PageSizeValue: Integer; + InlineBytesActive: Boolean; + InlineBytesValue: Integer; + + /// Activates a forced cross-environment page size for paging/resume tests. + /// The page size to force. + procedure Activate(NewPageSize: Integer) + begin + Active := true; + PageSizeValue := NewPageSize; + end; + + /// Activates a forced maximum inline-bytes cap so the over-cap media skip path can be exercised. + /// The maximum inline bytes to force. + procedure ActivateInlineBytes(NewMaxBytes: Integer) + begin + InlineBytesActive := true; + InlineBytesValue := NewMaxBytes; + end; + + /// Deactivates all forced paging and inline-bytes overrides. + procedure Deactivate() + begin + Active := false; + PageSizeValue := 0; + InlineBytesActive := false; + InlineBytesValue := 0; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Data Source", 'OnGetCrossEnvPageSize', '', false, false)] + local procedure HandleGetCrossEnvPageSize(var PageSize: Integer) + begin + if Active then + PageSize := PageSizeValue; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Source API", 'OnGetMaxPageInlineBytes', '', false, false)] + local procedure HandleGetMaxPageInlineBytes(var MaxBytes: Integer) + begin + if InlineBytesActive then + MaxBytes := InlineBytesValue; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al index 445bfa8f36c..fc223f05f88 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al @@ -19,6 +19,14 @@ table 139757 "MDM Test Table A" Caption = 'TableB Reference'; TableRelation = "MDM Test Table B"."Primary Key"; } + field(4; "Test Blob"; Blob) + { + Caption = 'Test Blob'; + } + field(5; "Test Image"; Media) + { + Caption = 'Test Image'; + } } keys @@ -27,5 +35,8 @@ table 139757 "MDM Test Table A" { Clustered = true; } + key(ChangeFeed; SystemModifiedAt, SystemId) + { + } } } diff --git a/src/Apps/W1/MasterDataManagement/test/app.json b/src/Apps/W1/MasterDataManagement/test/app.json index f5d33c4bf0e..fb5dbba085a 100644 --- a/src/Apps/W1/MasterDataManagement/test/app.json +++ b/src/Apps/W1/MasterDataManagement/test/app.json @@ -40,6 +40,20 @@ "screenshots": [], "platform": "29.0.0.0", "target": "Cloud", + "idRanges": [ + { + "from": 139758, + "to": 139758 + }, + { + "from": 139770, + "to": 139770 + }, + { + "from": 139931, + "to": 139933 + } + ], "resourceExposurePolicy": { "allowDebugging": false, "allowDownloadingSource": true, diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al new file mode 100644 index 00000000000..514a59bb359 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -0,0 +1,987 @@ +#pragma warning disable AA0247 +codeunit 139932 "MDM Cross-Env Consumer Tests" +{ + Subtype = Test; + TestType = IntegrationTest; + TestPermissions = Disabled; + EventSubscriberInstance = Manual; + + var + Assert: Codeunit Assert; + LibrarySalesLib: Codeunit "Library - Sales"; + WizardPrivacyNoticeOpenCount: Integer; + InvalidSourceHostErr: Label 'not a valid Business Central endpoint', Locked = true; + + [Test] + procedure CrossEnvGetBySystemIdRoundTripsSourceRecord() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + MaterializedModifiedAt: DateTime; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] The cross-env data source fetches a record over the transport and materializes it (fields included). + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + Customer.Name := CopyStr(LibraryRandomText(), 1, MaxStrLen(Customer.Name)); + Customer.Modify(); + + // [GIVEN] a subsidiary configured for cross-env with the in-process (pass-through) transport + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [WHEN] the record is fetched by SystemId + Found := LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + + // [THEN] the materialized record carries the same SystemId and field values, and the source SystemModifiedAt + // watermark survives materialization (the sync loop reads it via the side cache, not the temp row) + Assert.IsTrue(Found, 'Cross-env GetBySystemId should find the source record'); + Assert.AreEqual(Customer.SystemId, SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(), 'Wrong SystemId materialized'); + Assert.AreEqual(Customer.Name, Format(SourceRecordRef.Field(Customer.FieldNo(Name)).Value()), 'Name should round-trip through the wire'); + Assert.IsTrue(LibraryMasterDataMgt.TryGetSourceWatermark(Customer.SystemId, MaterializedModifiedAt), 'Source watermark should be cached during materialization'); + Assert.AreEqual(Customer.SystemModifiedAt, MaterializedModifiedAt, 'Source SystemModifiedAt should survive materialization'); + + CleanUp(); + end; + + [Test] + procedure HttpTransportRejectsNonBusinessCentralHosts() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Security] + // [SCENARIO] The source-host allow-list accepts only the exact HTTPS Business Central API hosts (SSRF guard). + Initialize(); + + // [GIVEN] the exact Business Central API hosts (production and TIE) over HTTPS [THEN] validation passes + LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://api.businesscentral.dynamics.com/v2.0/CRONUS/Production'); + LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://api.businesscentral.dynamics-tie.com/v2.0/CRONUS/Sandbox'); + + // [GIVEN] a non-HTTPS scheme [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('http://api.businesscentral.dynamics.com/v2.0/CRONUS/Production'); + Assert.ExpectedError(InvalidSourceHostErr); + + // [GIVEN] any non-standard dynamics.com host (e.g. an Embed/ISV per-cluster hostname) [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.api.bc.dynamics.com'); + Assert.ExpectedError(InvalidSourceHostErr); + + // [GIVEN] a subdomain of the allowed host [THEN] validation is rejected (exact host only) + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://sub.api.businesscentral.dynamics.com'); + Assert.ExpectedError(InvalidSourceHostErr); + + // [GIVEN] a host outside the allow-list [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://evil.example.com'); + Assert.ExpectedError(InvalidSourceHostErr); + + // [GIVEN] a look-alike host that only embeds the allowed host as a non-final label [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://api.businesscentral.dynamics.com.evil.example.com'); + Assert.ExpectedError(InvalidSourceHostErr); + + CleanUp(); + end; + + [Test] + procedure CrossEnvTransferBlockedUntilPrivacyNoticeApproved() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + PrivacyNotice: Codeunit "Privacy Notice"; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] Cross-env data transfer is gated on the privacy notice: blocked until approved, allowed after. + Initialize(); + + // [GIVEN] the cross-environment privacy notice is not approved + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Disagreed); + + // [THEN] the gate reports not approved and the transport check fails closed + Assert.IsFalse(LibraryMasterDataMgt.PrivacyNoticeIsApproved(), 'Gate should report not approved before consent'); + asserterror LibraryMasterDataMgt.PrivacyNoticeCheckApproved(); + Assert.ExpectedError('privacy notice to be approved'); + + // [WHEN] the admin approves the notice + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); + + // [THEN] the gate reports approved and the transport check passes + Assert.IsTrue(LibraryMasterDataMgt.PrivacyNoticeIsApproved(), 'Gate should report approved after consent'); + LibraryMasterDataMgt.PrivacyNoticeCheckApproved(); + + // reset approval so it does not leak into later tests + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Disagreed); + CleanUp(); + end; + + [Test] + procedure CrossEnvSyncSurfacesSourceConsentRequired() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] When the source hasn't approved sharing, the subsidiary sync surfaces a clear, actionable error + // (pointing at the source admin) rather than a raw transport failure. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); // the (in-process) source has not consented + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('has not approved sharing its master data'); + + CleanUp(); + end; + + [Test] + [HandlerFunctions('PrivacyNoticeModalHandler')] + procedure WizardConsentOpensPrivacyNotice() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + ConnectionWizard: TestPage "MDM Connection Details"; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] Choosing Next on the Welcome step opens the platform privacy notice - a regression guard that the + // wizard actually calls ConfirmApproval (the handler below fires only if the notice dialog is shown). + Initialize(); + + // [GIVEN] the privacy notice has no recorded decision, so continuing past Welcome must prompt it + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + WizardPrivacyNoticeOpenCount := 0; + + // [WHEN] the admin chooses Next on the Welcome step + ConnectionWizard.OpenEdit(); + ConnectionWizard.ActionNext.Invoke(); + ConnectionWizard.Close(); + + // [THEN] the privacy-notice dialog was shown exactly once - proving the wizard invoked ConfirmApproval + Assert.AreEqual(1, WizardPrivacyNoticeOpenCount, 'Choosing Next on the Welcome step should open the privacy notice exactly once (call ConfirmApproval)'); + + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + CleanUp(); + end; + + [Test] + procedure CrossEnvGetByUidFilterAndGetByIdMaterializeSourceRecords() + var + Customer1: Record Customer; + Customer2: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + UidFilter: Text; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] Cross-env GetByUidFilter (pipe-split selector) and GetById (variant->SystemId) fetch source records. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer1); + LibrarySalesLib.CreateCustomer(Customer2); + CreateMinimalCustomerMapping(IntegrationTableMapping); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [GIVEN] a pipe-delimited UID filter of two source SystemIds + UidFilter := Format(Customer1.SystemId) + '|' + Format(Customer2.SystemId); + + // [WHEN] fetched via the cross-env UID filter [THEN] both records materialize (exercises ParseSystemIds pipe-split) + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef), 'GetByUidFilter should return records'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer1.SystemId), 'UID filter should include the first customer'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer2.SystemId), 'UID filter should include the second customer'); + + // [WHEN] fetched via GetById with a text SystemId [THEN] the record materializes (exercises the variant->SystemId conversion) + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetById(IntegrationTableMapping, Format(Customer1.SystemId), SourceRecordRef), 'GetById should return the record'); + Assert.AreEqual(1, SourceRecordRef.Count(), 'GetById should return exactly one record'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer1.SystemId), 'GetById should include the requested customer'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvGetRecordsByFilterMaterializesContactBusinessRelation() + var + ContactBusinessRelation: Record "Contact Business Relation"; + FilterContactBusinessRelation: Record "Contact Business Relation"; + SourceContact: Record Contact; + OtherContact: Record Contact; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + RelationNo: Code[20]; + OtherRelationNo: Code[20]; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] The over-the-wire filtered read resolves a source Contact Business Relation (used to align + // auto-created contact numbers), which is not replicated locally cross-environment, and the row + // filter narrows the result to the requested link server-side. + Initialize(); + + // [GIVEN] two source contact business relations under distinct customer numbers, each linked to its own source + // contact; synthetic numbers keep a customer's own auto-created relation from matching the filter + RelationNo := UniqueCode(); + OtherRelationNo := UniqueCode(); + SourceContact := SeedContactRelation(RelationNo); + OtherContact := SeedContactRelation(OtherRelationNo); + + // [GIVEN] a subsidiary configured for cross-env with the in-process (pass-through) transport + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [WHEN] the relation is read over the wire by a row filter on (Link to Table, No.) of the first relation + FilterContactBusinessRelation.SetRange("Link to Table", FilterContactBusinessRelation."Link to Table"::Customer); + FilterContactBusinessRelation.SetRange("No.", RelationNo); + + // [THEN] exactly the matching relation materializes with the source contact number preserved (the other is filtered out) + Assert.IsTrue( + LibraryMasterDataMgt.DataSourceGetRecordsByFilter(Database::"Contact Business Relation", FilterContactBusinessRelation.GetView(), SourceRecordRef), + 'The filtered read should return the source contact business relation'); + Assert.AreEqual(1, SourceRecordRef.Count(), 'The row filter should narrow the result to the one requested relation'); + Assert.IsTrue(SourceRecordRef.FindFirst(), 'The materialized relation should be positioned'); + Assert.AreEqual( + SourceContact."No.", + Format(SourceRecordRef.Field(ContactBusinessRelation.FieldNo("Contact No.")).Value()), + 'The source Contact No. should round-trip through the wire'); + + // seeded contacts/relations live outside the MDMXENV artifact set, so remove them explicitly + ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::Customer); + ContactBusinessRelation.SetFilter("No.", '%1|%2', RelationNo, OtherRelationNo); + ContactBusinessRelation.DeleteAll(); + SourceContact.Delete(); + OtherContact.Delete(); + CleanUp(); + end; + + [Test] + procedure HttpTransportUnwrapsODataValueEnvelope() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] The HTTP transport unwraps the ODataV4 { value: } envelope and passes other bodies through. + Initialize(); + + // [GIVEN] an OData action envelope [THEN] the inner value is returned + Assert.AreEqual('{"records":[]}', LibraryMasterDataMgt.UnwrapHttpTransportODataValue('{"@odata.context":"x","value":"{\"records\":[]}"}'), 'Envelope value should be unwrapped'); + + // [GIVEN] a body that is not a value-envelope [THEN] it is returned unchanged + Assert.AreEqual('{"records":[]}', LibraryMasterDataMgt.UnwrapHttpTransportODataValue('{"records":[]}'), 'A non-envelope body should pass through unchanged'); + + CleanUp(); + end; + + [ModalPageHandler] + procedure PrivacyNoticeModalHandler(var PrivacyNoticePage: TestPage "Privacy Notice") + begin + // Reached only if the wizard actually opened the notice. + WizardPrivacyNoticeOpenCount += 1; + end; + + [Test] + procedure CrossEnvGetModifiedSetMaterializesSourceChange() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Cross-env GetModifiedSet pages the source feed over the transport and materializes the changes. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + CreateMinimalCustomerMapping(IntegrationTableMapping); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Found := LibraryMasterDataMgt.DataSourceGetModifiedSet(IntegrationTableMapping, '', SourceRecordRef); + + Assert.IsTrue(Found, 'Cross-env GetModifiedSet should return source records'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer.SystemId), 'Materialized set should contain the seeded customer'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenSourceTableUnavailable() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A tableAvailable:false response surfaces as a clear synchronization error. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse('{"tableId":18,"tableAvailable":false,"records":[],"hasMore":false}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('not available on the source environment'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenSourceTableNotIndexed() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] An indexed:false response (too-large unindexed table) asks for the composite key. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse('{"tableId":18,"tableAvailable":true,"indexed":false,"records":[],"hasMore":false}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('Add a key on SystemModifiedAt and SystemId'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenFieldsUnavailable() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] An unavailableFields response halts the table with a clear synchronization error. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse('{"tableId":18,"tableAvailable":true,"unavailableFields":[5],"records":[],"hasMore":false}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('do not exist on table'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvIntegrationRecRefCountReportsExistence() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The full-synch review existence probe reports records for a non-empty cross-env source table. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + CreateMinimalCustomerMapping(IntegrationTableMapping); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.AreEqual(1, LibraryMasterDataMgt.GetIntegrationRecRefCount(IntegrationTableMapping), 'A non-empty source table should report records to the full-synch review'); + + CleanUp(); + end; + + [Test] + [HandlerFunctions('ConfirmHandlerNo')] + procedure ConnectionDetailsWizardSavesConfiguration() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + PrivacyNotice: Codeunit "Privacy Notice"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + AzureADTenant: Codeunit "Azure AD Tenant"; + ConnectionDetails: TestPage "MDM Connection Details"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The Connection Details wizard collects the source connection details and saves them (secret to Isolated Storage). + Initialize(); + // Pre-approve so ticking consent doesn't open the notice dialog in this configuration-focused test. + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); + + ConnectionDetails.OpenEdit(); + // Welcome step: consent is pre-approved above, so Next advances without prompting. + ConnectionDetails.ActionNext.Invoke(); + // Connection step: provide the source environment and credentials (the URL is derived from the environment name). + ConnectionDetails.SourceEnvironmentName.SetValue('CONTOSO-PROD'); + ConnectionDetails.SourceCompanyName.SetValue('CRONUS'); + ConnectionDetails.OAuth2ClientId.SetValue('11111111-2222-3333-4444-555555555555'); + ConnectionDetails.OAuth2ClientSecret.SetValue('super-secret'); + ConnectionDetails.ActionNext.Invoke(); // -> Test Connection step + ConnectionDetails.ActionNext.Invoke(); // -> Finish step (optional test skipped) + ConnectionDetails.ActionFinish.Invoke(); + + // [THEN] the setup holds the connection and a stored client secret + MasterDataManagementSetup.Get(); + Assert.AreEqual('CONTOSO-PROD', MasterDataManagementSetup."Source Environment Name", 'Source environment not saved'); + // The URL is constructed from the current tenant, ring host, and source environment name. + Assert.IsTrue( + StrPos(MasterDataManagementSetup."Source Environment URL", '/v2.0/' + AzureADTenant.GetAadTenantId() + '/CONTOSO-PROD') > 0, + 'Source URL should embed the tenant id and source environment name'); + Assert.IsTrue( + StrPos(LowerCase(MasterDataManagementSetup."Source Environment URL"), 'https://api.businesscentral.dynamics') = 1, + 'Source URL should target the Business Central API host'); + Assert.AreEqual('CRONUS', MasterDataManagementSetup."Source Company Name", 'Source company not saved'); + Assert.AreEqual('11111111-2222-3333-4444-555555555555', MasterDataManagementSetup."Source OAuth Client Id", 'Source client id not saved'); + Assert.IsFalse(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Client secret should be stored'); + + // Restore privacy state so this configuration test doesn't leak consent into later privacy-notice scenarios. + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + CleanUp(); + end; + + [Test] + [HandlerFunctions('ConfirmHandlerYes')] + procedure ClearCrossEnvSetupRevertsToSameEnvironment() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + SetupPage: TestPage "Master Data Management Setup"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Clear Cross-Environment Setup removes the source environment, company, client id, and the stored + // secret, reverting the setup to same-environment synchronization. + Initialize(); + // [GIVEN] a configured cross-environment connection with a stored secret + MasterDataManagementSetup.Get(); + MasterDataManagementSetup.Validate("Source Environment Name", 'CONTOSO-PROD'); + MasterDataManagementSetup."Source Environment URL" := 'https://api.businesscentral.dynamics.com/v2.0/contoso-prod'; + MasterDataManagementSetup."Source Company Name" := 'CRONUS'; + MasterDataManagementSetup."Source OAuth Client Id" := '11111111-2222-3333-4444-555555555555'; + MasterDataManagementSetup."Source Client Secret Key" := CreateGuid(); // simulate a stored secret key + MasterDataManagementSetup.Modify(true); + Assert.AreNotEqual('', MasterDataManagementSetup."Source Environment Name", 'Precondition: setup should be cross-environment'); + + // [WHEN] the user runs Clear Cross-Environment Setup and confirms + SetupPage.OpenEdit(); + SetupPage.ClearCrossEnvSetup.Invoke(); + SetupPage.Close(); + + // [THEN] every cross-environment field and the stored secret are cleared + MasterDataManagementSetup.Get(); + Assert.AreEqual('', MasterDataManagementSetup."Source Environment Name", 'Source environment should be cleared'); + Assert.AreEqual('', MasterDataManagementSetup."Source Environment URL", 'Source URL should be cleared'); + Assert.AreEqual('', MasterDataManagementSetup."Source Company Name", 'Source company should be cleared'); + Assert.AreEqual('', MasterDataManagementSetup."Source OAuth Client Id", 'Source client id should be cleared'); + Assert.IsTrue(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Secret key should be cleared'); // empty Source Environment Name (asserted above) means same-environment again + + CleanUp(); + end; + + [Test] + procedure ClearCrossEnvSetupDisabledWhileSynchronizationEnabled() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + SetupPage: TestPage "Master Data Management Setup"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Clear Cross-Environment Setup is disabled while synchronization is enabled, so the user must + // disable synchronization before the connection can be cleared. + Initialize(); + // [GIVEN] a cross-environment setup with synchronization enabled + MasterDataManagementSetup.Get(); + MasterDataManagementSetup.Validate("Source Environment Name", 'CONTOSO-PROD'); + MasterDataManagementSetup."Is Enabled" := true; // set directly to skip the enable side effects; the action's Enabled binding is what we assert + MasterDataManagementSetup.Modify(false); + + // [THEN] the Clear Cross-Environment Setup action is disabled on the setup page + SetupPage.OpenEdit(); + Assert.IsFalse(SetupPage.ClearCrossEnvSetup.Enabled(), 'Clear action should be disabled while synchronization is enabled'); + SetupPage.Close(); + + // reset the enabled flag so it does not leak into later tests + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Is Enabled" := false; + MasterDataManagementSetup.Modify(false); + CleanUp(); + end; + + [Test] + procedure CrossEnvGetByFilterReturnsRecordsPastTheWatermark() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + FilterSourceRef: RecordRef; + ModifiedSetSourceRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] GetByFilter returns the whole filtered set (used by coupling/uncoupling), ignoring the mapping's + // watermark, unlike GetModifiedSet which only returns changes after the watermark. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + CreateMinimalCustomerMapping(IntegrationTableMapping); + // [GIVEN] a watermark far in the future, so the seeded customer is not "modified since" + IntegrationTableMapping."Synch. Modified On Filter" := CreateDateTime(DMY2Date(1, 1, 2099), 0T); + IntegrationTableMapping.Modify(); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [THEN] the watermark-based read excludes it, but the full filtered read includes it + Assert.IsFalse( + LibraryMasterDataMgt.DataSourceGetModifiedSet(IntegrationTableMapping, '', ModifiedSetSourceRef) and ContainsSystemId(ModifiedSetSourceRef, Customer.SystemId), + 'GetModifiedSet should not return records at or before the watermark'); + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetByFilter(IntegrationTableMapping, '', FilterSourceRef), 'GetByFilter should return records'); + Assert.IsTrue(ContainsSystemId(FilterSourceRef, Customer.SystemId), 'GetByFilter should return the seeded customer regardless of the watermark'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenSourceLacksRecordsCapability() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The subsidiary negotiates capabilities first and fails clearly if the source doesn't advertise 'records'. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedCapabilities('{"version":1,"features":["lastModifiedPerTable"]}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('does not support the required'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvGetModifiedBatchResumesAcrossRuns() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + PagingConfig: Codeunit "MDM Test Paging Config"; + SourceRecordRef: RecordRef; + SeededSystemId: Guid; + SeededSystemIds: List of [Guid]; + CollectedSystemIds: List of [Guid]; + PerRunCounts: List of [Integer]; + Watermark: DateTime; + Cursor: Text; + EndCursor: Text; + PreviousCursor: Text; + Index: Integer; + Runs: Integer; + HasMore: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A page-capped batch (MaxPages=1) drains a multi-page change set across several runs, resuming + // from the returned cursor each time, and covers every source record exactly once. + Initialize(); + + // [GIVEN] five source customers created strictly after a captured watermark (so only they are in the feed) + Watermark := CurrentDateTime(); + Sleep(100); + for Index := 1 to 5 do begin + LibrarySalesLib.CreateCustomer(Customer); + SeededSystemIds.Add(Customer.SystemId); + end; + CreateMinimalCustomerMapping(IntegrationTableMapping); + IntegrationTableMapping."Synch. Modified On Filter" := Watermark; + IntegrationTableMapping.Modify(); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + PagingConfig.Activate(2); // two records per page + + // [WHEN] the batch is drained one page per run, resuming from the returned cursor + Cursor := ''; + repeat + PreviousCursor := Cursor; + LibraryMasterDataMgt.DataSourceGetModifiedBatch(IntegrationTableMapping, '', Cursor, 1, SourceRecordRef, EndCursor, HasMore); + // Each resumed run must advance from the cursor it was given - never restart from the start or stall. + Assert.AreNotEqual(PreviousCursor, EndCursor, 'Each resumed run must advance the cursor'); + PerRunCounts.Add(SourceRecordRef.Count()); + CollectSystemIds(SourceRecordRef, CollectedSystemIds); + Cursor := EndCursor; + Runs += 1; + until not HasMore; + + // [THEN] the set drained in a stable 2 + 2 + 1 sequence across exactly three resumed runs + Assert.AreEqual(3, Runs, 'A 5-record set at 2/page and 1 page/run must take exactly three runs'); + Assert.AreEqual(2, PerRunCounts.Get(1), 'First run should return a full page of two records'); + Assert.AreEqual(2, PerRunCounts.Get(2), 'Second run should return a full page of two records'); + Assert.AreEqual(1, PerRunCounts.Get(3), 'Final run should return the remaining single record'); + // [THEN] every seeded record was returned exactly once (no overlap, no gaps) + Assert.AreEqual(5, CollectedSystemIds.Count(), 'Every seeded record should be returned exactly once across runs'); + foreach SeededSystemId in SeededSystemIds do + Assert.IsTrue(CollectedSystemIds.Contains(SeededSystemId), 'Every seeded record should be covered by the resumed batches'); + + PagingConfig.Deactivate(); + CleanUp(); + end; + + [Test] + procedure CrossEnvBlobFieldRoundTrips() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + TempBlob: Codeunit "Temp Blob"; + SourceRecordRef: RecordRef; + BlobField: FieldRef; + BlobInStream: InStream; + BlobText: Text; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] A Blob field is projected inline (base64) and materialized back onto the temp source record. + Initialize(); + CreateTestTableAWithBlob(TestTableA, 'the quick brown fox'); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should be materialized'); + BlobField := SourceRecordRef.Field(TestTableA.FieldNo("Test Blob")); + TempBlob.FromFieldRef(BlobField); + Assert.IsTrue(TempBlob.HasValue(), 'Blob should round-trip onto the materialized record'); + TempBlob.CreateInStream(BlobInStream, TextEncoding::UTF8); + BlobInStream.ReadText(BlobText); + Assert.AreEqual('the quick brown fox', BlobText, 'Blob content should round-trip through the wire'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvOversizeBlobIsSkipped() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + TempBlob: Codeunit "Temp Blob"; + SourceRecordRef: RecordRef; + BlobField: FieldRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A Blob over the 512 KB inline cap is skipped: the record materializes without the blob. + Initialize(); + CreateTestTableAWithBlob(TestTableA, PadStr('', 600000, 'A')); // > 512 KB raw + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should still materialize'); + BlobField := SourceRecordRef.Field(TestTableA.FieldNo("Test Blob")); + TempBlob.FromFieldRef(BlobField); + Assert.IsFalse(TempBlob.HasValue(), 'Over-cap blob must be skipped, not synchronized'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvMediaFieldIsCachedForApply() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A Media field's bytes are cached (keyed by SystemId+fieldNo) for the transfer-time apply. + Initialize(); + CreateTestTableAWithImage(TestTableA, 'small picture bytes'); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should be materialized'); + Assert.IsTrue( + LibraryMasterDataMgt.InlineMediaCacheContains(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'Inline media bytes should be cached for the transfer-time apply'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvEmptyMediaFieldMarksDestinationCleared() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] A source record whose Media field is empty is marked cleared so the destination picture is + // removed during transfer (mirroring a source deletion), instead of leaving a stale image behind. + Initialize(); + Clear(TestTableA); + TestTableA."Primary Key" := CopyStr('M' + Format(LibraryRandomInt()), 1, MaxStrLen(TestTableA."Primary Key")); + TestTableA.Insert(); // no Test Image: the source serializes the media field as empty + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should be materialized'); + Assert.IsTrue( + LibraryMasterDataMgt.InlineMediaIsCleared(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'An empty source media field must be marked cleared so the destination picture is removed'); + Assert.IsFalse( + LibraryMasterDataMgt.InlineMediaCacheContains(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'An empty source media field must not cache any bytes'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvOversizeMediaIsSkipped() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A Media over the 512 KB cap is skipped: no bytes are cached (telemetry-only warning). + Initialize(); + CreateTestTableAWithImage(TestTableA, PadStr('', 600000, 'A')); // > 512 KB raw + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should still materialize'); + Assert.IsFalse( + LibraryMasterDataMgt.InlineMediaCacheContains(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'Over-cap media must not be cached (skipped, telemetry only)'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvPageStopsAtInlineByteBudget() + var + TestTableA: Record "MDM Test Table A"; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + PagingConfig: Codeunit "MDM Test Paging Config"; + SourceRecordRef: RecordRef; + Watermark: DateTime; + EndCursor: Text; + HasMore: Boolean; + Index: Integer; + Count: Integer; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A page stops at the inline-byte budget: media-heavy records page out even under the record count cap. + Initialize(); + + // [GIVEN] three media records created after a captured watermark, and a byte budget so small any one media fills it + Watermark := CurrentDateTime(); + Sleep(100); + for Index := 1 to 3 do + CreateTestTableAWithImage(TestTableA, 'picture bytes'); + CreateTestTableAMapping(IntegrationTableMapping); + IntegrationTableMapping."Synch. Modified On Filter" := Watermark; + IntegrationTableMapping.Modify(); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + PagingConfig.ActivateInlineBytes(1); // 1-byte budget: the first inlined media ends the page + + // [WHEN] a single page is fetched + LibraryMasterDataMgt.DataSourceGetModifiedBatch(IntegrationTableMapping, '', '', 1, SourceRecordRef, EndCursor, HasMore); + if SourceRecordRef.FindSet() then + repeat + Count += 1; + until SourceRecordRef.Next() = 0; + + // [THEN] the byte budget capped the page to one record, with more remaining + Assert.AreEqual(1, Count, 'The inline-byte budget should stop the page after the first media record'); + Assert.IsTrue(HasMore, 'More records should remain past the byte-budget cap'); + + PagingConfig.Deactivate(); + CleanUp(); + end; + + local procedure CreateTestTableAMapping(var IntegrationTableMapping: Record "Integration Table Mapping") + var + IntegrationFieldMapping: Record "Integration Field Mapping"; + begin + IntegrationTableMapping.Init(); + IntegrationTableMapping.Name := CopyStr('MDMXENV' + Format(LibraryRandomInt()), 1, MaxStrLen(IntegrationTableMapping.Name)); + IntegrationTableMapping.Type := IntegrationTableMapping.Type::"Master Data Management"; + IntegrationTableMapping."Table ID" := Database::"MDM Test Table A"; + IntegrationTableMapping."Integration Table ID" := Database::"MDM Test Table A"; + IntegrationTableMapping."Integration Table UID Fld. No." := 2000000000; // SystemId + IntegrationTableMapping."Int. Tbl. Modified On Fld. No." := 2000000003; // SystemModifiedAt + IntegrationTableMapping."Delete After Synchronization" := false; + IntegrationTableMapping.Insert(); + // map the Media field (5) so the batch requests it and the page carries inline bytes + IntegrationFieldMapping.Init(); + IntegrationFieldMapping."Integration Table Mapping Name" := IntegrationTableMapping.Name; + IntegrationFieldMapping."Field No." := 5; + IntegrationFieldMapping."Integration Table Field No." := 5; + IntegrationFieldMapping.Insert(true); + end; + + local procedure CreateTestTableAWithBlob(var TestTableA: Record "MDM Test Table A"; Content: Text) + var + BlobOutStream: OutStream; + begin + Clear(TestTableA); + TestTableA."Primary Key" := CopyStr('B' + Format(LibraryRandomInt()), 1, MaxStrLen(TestTableA."Primary Key")); + TestTableA."Test Blob".CreateOutStream(BlobOutStream, TextEncoding::UTF8); + BlobOutStream.WriteText(Content); + TestTableA.Insert(); + end; + + local procedure CreateTestTableAWithImage(var TestTableA: Record "MDM Test Table A"; Content: Text) + var + TempBlob: Codeunit "Temp Blob"; + MediaInStream: InStream; + MediaOutStream: OutStream; + begin + Clear(TestTableA); + TestTableA."Primary Key" := CopyStr('M' + Format(LibraryRandomInt()), 1, MaxStrLen(TestTableA."Primary Key")); + TestTableA.Insert(); + TempBlob.CreateOutStream(MediaOutStream, TextEncoding::UTF8); + MediaOutStream.WriteText(Content); + TempBlob.CreateInStream(MediaInStream, TextEncoding::UTF8); + TestTableA."Test Image".ImportStream(MediaInStream, 'pic.bin', 'application/octet-stream'); + TestTableA.Modify(); + end; + + local procedure SeedContactRelation(RelationNo: Code[20]) SourceContact: Record Contact + var + ContactBusinessRelation: Record "Contact Business Relation"; + begin + SourceContact.Init(); + SourceContact."No." := UniqueCode(); + SourceContact.Insert(); + ContactBusinessRelation.Init(); + ContactBusinessRelation."Contact No." := SourceContact."No."; + ContactBusinessRelation."Business Relation Code" := 'MDMTEST'; + ContactBusinessRelation."Link to Table" := ContactBusinessRelation."Link to Table"::Customer; + ContactBusinessRelation."No." := RelationNo; + ContactBusinessRelation.Insert(); + end; + + local procedure UniqueCode(): Code[20] + begin + exit(CopyStr(DelChr(Format(CreateGuid()), '=', '{}-'), 1, 20)); + end; + + local procedure Initialize() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + InProcessTransport: Codeunit "MDM In-Process Transport"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + PagingConfig: Codeunit "MDM Test Paging Config"; + begin + InProcessTransport.Deactivate(); + PagingConfig.Deactivate(); // a paging test that failed before its CleanUp must not leak its forced page cap into later tests + LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); // the source API is consent-gated; approve for the gated paths + // SetSourceEnvironmentName validates the setup, which schedules the detector job and commits, so a mapping + // created earlier in a test survives AutoRollback; clear leftovers to keep tests independent. + DeleteTestArtifacts(); + if not MasterDataManagementSetup.Get() then begin + MasterDataManagementSetup.Init(); + MasterDataManagementSetup.Insert(); + end; + MasterDataManagementSetup."Source Environment Name" := ''; + MasterDataManagementSetup.Modify(false); + end; + + [ConfirmHandler] + procedure ConfirmHandlerYes(Question: Text; var Reply: Boolean) + begin + Reply := true; + end; + + [ConfirmHandler] + procedure ConfirmHandlerNo(Question: Text; var Reply: Boolean) + begin + Reply := false; + end; + + local procedure CleanUp() + var + InProcessTransport: Codeunit "MDM In-Process Transport"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + PagingConfig: Codeunit "MDM Test Paging Config"; + begin + InProcessTransport.Deactivate(); + PagingConfig.Deactivate(); + LibraryMasterDataMgt.SetSourceEnvironmentName(''); + DeleteTestArtifacts(); + end; + + local procedure DeleteTestArtifacts() + var + IntegrationTableMapping: Record "Integration Table Mapping"; + IntegrationFieldMapping: Record "Integration Field Mapping"; + TestTableA: Record "MDM Test Table A"; + begin + IntegrationFieldMapping.SetFilter("Integration Table Mapping Name", 'MDMXENV*'); + IntegrationFieldMapping.DeleteAll(); + IntegrationTableMapping.SetFilter(Name, 'MDMXENV*'); + IntegrationTableMapping.DeleteAll(); + TestTableA.DeleteAll(); // deterministic LibraryRandom keys collide across tests; the setup commit survives rollback + end; + + local procedure CreateMinimalCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") + begin + IntegrationTableMapping.Init(); + IntegrationTableMapping.Name := CopyStr('MDMXENV' + Format(LibraryRandomInt()), 1, MaxStrLen(IntegrationTableMapping.Name)); + IntegrationTableMapping.Type := IntegrationTableMapping.Type::"Master Data Management"; + IntegrationTableMapping."Table ID" := Database::Customer; + IntegrationTableMapping."Integration Table ID" := Database::Customer; + IntegrationTableMapping."Integration Table UID Fld. No." := 2000000000; // SystemId + IntegrationTableMapping."Int. Tbl. Modified On Fld. No." := 2000000003; // SystemModifiedAt + IntegrationTableMapping."Delete After Synchronization" := false; + IntegrationTableMapping.Insert(); + end; + + local procedure ContainsSystemId(var SourceRecordRef: RecordRef; SystemIdValue: Guid): Boolean + begin + if SourceRecordRef.FindSet() then + repeat + if Format(SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value()) = Format(SystemIdValue) then + exit(true); + until SourceRecordRef.Next() = 0; + exit(false); + end; + + local procedure CollectSystemIds(var SourceRecordRef: RecordRef; var CollectedSystemIds: List of [Guid]) + var + SystemIdValue: Guid; + begin + // Record every occurrence (no de-dup) so a record returned on two pages makes the count exceed the seeded set. + if SourceRecordRef.FindSet() then + repeat + SystemIdValue := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + CollectedSystemIds.Add(SystemIdValue); + until SourceRecordRef.Next() = 0; + end; + + local procedure LibraryRandomText(): Text + var + LibraryRandomCu: Codeunit "Library - Random"; + begin + exit(LibraryRandomCu.RandText(20)); + end; + + local procedure LibraryRandomInt(): Integer + var + LibraryRandomCu: Codeunit "Library - Random"; + begin + exit(LibraryRandomCu.RandIntInRange(1, 999999)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al new file mode 100644 index 00000000000..96003fb47bc --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al @@ -0,0 +1,269 @@ +#pragma warning disable AA0247 +codeunit 139933 "MDM Cross-Env Detector Tests" +{ + Subtype = Test; + TestType = IntegrationTest; + TestPermissions = Disabled; + EventSubscriberInstance = Manual; + + var + Assert: Codeunit Assert; + + [Test] + procedure DetectorNudgesChangedEnabledTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] A table changed since its watermark gets its synchronization job nudged. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); // an idle (Ready) job exists + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, ChangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsTrue(DetectorProbe.WasNudged(Database::Customer), 'A changed enabled table should be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorSkipsUnchangedTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A table whose source timestamp is not past the watermark is not nudged. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, UnchangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'An unchanged table should not be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorSkipsDisabledTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A disabled table is never polled or nudged, even if the source reports a change. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), false); // disabled + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, ChangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'A disabled table should not be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorLeavesInProcessJobAlone() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A changed table whose job is already In Process is left alone (no idle job to nudge). + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, true); // job In Process + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, ChangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'A table whose job is In Process should be left alone'); + CleanUp(); + end; + + [Test] + procedure DetectorSkipsUnavailableTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A table the source reports as unavailable is not nudged (the sync job reports the error). + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, false, ChangedDateTime(), false)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'An unavailable table should not be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorNudgesAvailableTableWithoutTimestamp() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] A table the source reports available but WITHOUT a lastModifiedAt (keyless/unindexed) is nudged + // so the sync job scans it, rather than being silently skipped. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, 0DT, false)); // available, no timestamp + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsTrue(DetectorProbe.WasNudged(Database::Customer), 'An available table without a timestamp should be nudged to scan'); + CleanUp(); + end; + + local procedure Initialize() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + InProcessTransport.Deactivate(); + DetectorProbe.Deactivate(); + // RunChangeDetector calls DetectChanges() directly (no Codeunit.Run) and the detector never commits, so it runs + // inside the test transaction; DeleteTestArtifacts() defensively clears any residue from an aborted prior run. + DeleteTestArtifacts(); + if not MasterDataManagementSetup.Get() then begin + MasterDataManagementSetup.Init(); + MasterDataManagementSetup.Insert(); + end; + // Direct assignment avoids the OnValidate that (de)provisions the detector job. + MasterDataManagementSetup."Is Enabled" := true; + MasterDataManagementSetup."Source Environment Name" := 'PROD'; + MasterDataManagementSetup.Modify(false); + end; + + local procedure CleanUp() + var + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + InProcessTransport.Deactivate(); + DetectorProbe.Deactivate(); + DeleteTestArtifacts(); + end; + + local procedure DeleteTestArtifacts() + var + Mapping: Record "Integration Table Mapping"; + JobQueueEntry: Record "Job Queue Entry"; + begin + Mapping.SetFilter(Name, 'MDMXD*'); + if Mapping.FindSet() then + repeat + JobQueueEntry.SetRange("Record ID to Process", Mapping.RecordId()); + JobQueueEntry.DeleteAll(); + until Mapping.Next() = 0; + Mapping.DeleteAll(); + end; + + local procedure CreateMapping(var Mapping: Record "Integration Table Mapping"; TableId: Integer; Watermark: DateTime; Enabled: Boolean) + var + LibraryRandom: Codeunit "Library - Random"; + begin + Mapping.Init(); + Mapping.Name := CopyStr('MDMXD' + Format(LibraryRandom.RandIntInRange(1, 999999)), 1, MaxStrLen(Mapping.Name)); + Mapping.Type := Mapping.Type::"Master Data Management"; + Mapping."Table ID" := TableId; + Mapping."Integration Table ID" := TableId; + Mapping."Integration Table UID Fld. No." := 2000000000; // SystemId + Mapping."Int. Tbl. Modified On Fld. No." := 2000000003; // SystemModifiedAt + Mapping."Synch. Modified On Filter" := Watermark; + Mapping."Delete After Synchronization" := false; + if Enabled then + Mapping.Status := Mapping.Status::Enabled + else + Mapping.Status := Mapping.Status::Disabled; + Mapping.Insert(); + end; + + // Insert-only (no scheduling): the test lab can't schedule background jobs, and the detector only reads status. + local procedure InsertSynchJob(Mapping: Record "Integration Table Mapping"; InProcess: Boolean) + var + JobQueueEntry: Record "Job Queue Entry"; + begin + JobQueueEntry.Init(); + JobQueueEntry.ID := CreateGuid(); + JobQueueEntry."Object Type to Run" := JobQueueEntry."Object Type to Run"::Codeunit; + JobQueueEntry."Object ID to Run" := Codeunit::"Integration Synch. Job Runner"; + JobQueueEntry."Record ID to Process" := Mapping.RecordId(); + JobQueueEntry."Recurring Job" := true; + if InProcess then + JobQueueEntry.Status := JobQueueEntry.Status::"In Process" + else + JobQueueEntry.Status := JobQueueEntry.Status::Ready; + JobQueueEntry.Insert(false); + end; + + local procedure CannedLastModified(TableId: Integer; TableAvailable: Boolean; LastModifiedAt: DateTime; IncludeTimestamp: Boolean) ResultText: Text + var + Response: JsonObject; + Entry: JsonObject; + Tables: JsonArray; + begin + Entry.Add('tableId', TableId); + Entry.Add('tableAvailable', TableAvailable); + if IncludeTimestamp then + Entry.Add('lastModifiedAt', Format(LastModifiedAt, 0, 9)); + Tables.Add(Entry); + Response.Add('tables', Tables); + Response.WriteTo(ResultText); + end; + + local procedure WatermarkDateTime(): DateTime + begin + exit(CreateDateTime(DMY2Date(1, 1, 2020), 0T)); + end; + + local procedure ChangedDateTime(): DateTime + begin + exit(CreateDateTime(DMY2Date(1, 1, 2030), 0T)); + end; + + local procedure UnchangedDateTime(): DateTime + begin + exit(CreateDateTime(DMY2Date(1, 1, 2019), 0T)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al new file mode 100644 index 00000000000..78d95d5218e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -0,0 +1,310 @@ +#pragma warning disable AA0247 +codeunit 139931 "MDM Cross-Env Source Tests" +{ + Subtype = Test; + TestType = IntegrationTest; + TestPermissions = Disabled; + + var + Assert: Codeunit Assert; + LibrarySales: Codeunit "Library - Sales"; + LibraryRandom: Codeunit "Library - Random"; + + [Test] + procedure GetCapabilitiesReturnsVersionAndFeatures() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Token: JsonToken; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] GetCapabilities advertises the contract version and supported features. + Response.ReadFrom(SourceApi.GetCapabilities()); + + Response.Get('version', Token); + Assert.AreEqual(1, Token.AsValue().AsInteger(), 'Unexpected capability version'); + Assert.IsTrue(FeaturesContain(Response, 'records'), 'records feature should be advertised'); + Assert.IsTrue(FeaturesContain(Response, 'lastModifiedPerTable'), 'lastModifiedPerTable feature should be advertised'); + end; + + [Test] + procedure GetRecordsSignalsConsentRequiredWhenNotApproved() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Token: JsonToken; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] Without source consent the API returns a structured consentRequired signal (not data, not a raw + // error), so the subsidiary can surface a clear message; after approval it serves data. + LibrarySales.CreateCustomer(Customer); + + // [GIVEN] this environment has not approved sharing [THEN] the source signals consentRequired and no data + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100, '')); + Assert.IsTrue(Response.Get('consentRequired', Token) and Token.AsValue().AsBoolean(), 'Source should signal consentRequired when not approved'); + Assert.IsFalse(Response.Contains('records'), 'No records should be served without consent'); + + // [WHEN] the environment approves sharing [THEN] the source serves data with no consent signal + Clear(Response); + LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100, '')); + Assert.IsFalse(Response.Contains('consentRequired'), 'The consent signal should be absent once approved'); + end; + + local procedure ApproveSourceConsent() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); + end; + + [Test] + procedure GetRecordsBySystemIdReturnsRequestedFields() + var + Customer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + RecordObject: JsonObject; + FieldsObject: JsonObject; + RecordsArray: JsonArray; + Token: JsonToken; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] GetRecords with a systemIds selector returns just that record with the requested fields. + ApproveSourceConsent(); + LibrarySales.CreateCustomer(Customer); + Customer.Name := CopyStr(LibraryRandom.RandText(20), 1, MaxStrLen(Customer.Name)); + Customer.Modify(); + + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100, '')); + + Response.Get('records', Token); + RecordsArray := Token.AsArray(); + Assert.AreEqual(1, RecordsArray.Count(), 'Expected exactly the requested record'); + RecordsArray.Get(0, Token); + RecordObject := Token.AsObject(); + RecordObject.Get('systemId', Token); + Assert.AreEqual(Format(Customer.SystemId), Token.AsValue().AsText(), 'Wrong systemId returned'); + RecordObject.Get('fields', Token); + FieldsObject := Token.AsObject(); + FieldsObject.Get(Format(Customer.FieldNo(Name)), Token); + Assert.AreEqual(Customer.Name, Token.AsValue().AsText(), 'Wrong Name value returned'); + end; + + [Test] + procedure GetRecordsCursorModePagesWithHasMore() + var + Customer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Watermark: DateTime; + Response: JsonObject; + NextCursor: Text; + Index: Integer; + SeededSystemIds: List of [Guid]; + Page1SystemIds: List of [Guid]; + Page2SystemIds: List of [Guid]; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Cursor mode pages ascending by (SystemModifiedAt, SystemId) and reports hasMore / nextCursor. + ApproveSourceConsent(); + Watermark := CurrentDateTime(); + Sleep(50); // ensure the seeded records sort strictly after the watermark + for Index := 1 to 3 do begin + Sleep(20); // distinct, strictly increasing SystemModifiedAt so creation order == cursor order + LibrarySales.CreateCustomer(Customer); + SeededSystemIds.Add(Customer.SystemId); + end; + + // [WHEN] the first page of size 2 is requested from the watermark + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), CursorSelector(Watermark), 2, '')); + + // [THEN] two records come back and hasMore is true + Assert.AreEqual(2, RecordCount(Response), 'First page should hold the page size'); + Assert.IsTrue(GetBoolean(Response, 'hasMore'), 'hasMore should be true while records remain'); + CollectResponseSystemIds(Response, Page1SystemIds); + NextCursor := NextCursorText(Response); + + // [WHEN] the next page is requested with the returned cursor + Clear(Response); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), NextCursor, 2, '')); + + // [THEN] the remaining record comes back and hasMore is false + Assert.AreEqual(1, RecordCount(Response), 'Second page should hold the remaining record'); + Assert.IsFalse(GetBoolean(Response, 'hasMore'), 'hasMore should be false on the last page'); + CollectResponseSystemIds(Response, Page2SystemIds); + + // [THEN] the pages follow the ascending (SystemModifiedAt, SystemId) order: first two seeded on page one, last on page two + Assert.AreEqual(2, Page1SystemIds.Count(), 'First page should contain exactly two records'); + Assert.AreEqual(SeededSystemIds.Get(1), Page1SystemIds.Get(1), 'First page, first record should be the earliest-modified customer'); + Assert.AreEqual(SeededSystemIds.Get(2), Page1SystemIds.Get(2), 'First page, second record should be the second-earliest customer'); + Assert.AreEqual(1, Page2SystemIds.Count(), 'Second page should contain exactly one record'); + Assert.AreEqual(SeededSystemIds.Get(3), Page2SystemIds.Get(1), 'Second page should contain the latest-modified customer'); + end; + + [Test] + procedure LastModifiedAtPerTableReturnsLatestTimestamp() + var + Customer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Entry: JsonObject; + Tables: JsonArray; + Token: JsonToken; + LastModified: DateTime; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] LastModifiedAtPerTable returns each table's latest modification timestamp. + ApproveSourceConsent(); + LibrarySales.CreateCustomer(Customer); + + Response.ReadFrom(SourceApi.LastModifiedAtPerTable(TableIdsArray(Database::Customer))); + + Response.Get('tables', Token); + Tables := Token.AsArray(); + Assert.AreEqual(1, Tables.Count(), 'Expected one table entry'); + Tables.Get(0, Token); + Entry := Token.AsObject(); + Entry.Get('lastModifiedAt', Token); + Assert.IsTrue(Evaluate(LastModified, Token.AsValue().AsText(), 9), 'lastModifiedAt should be a round-trippable timestamp'); + Assert.IsTrue(LastModified >= Customer.SystemModifiedAt, 'lastModifiedAt should be at least the just-created customer'); + end; + + [Test] + procedure GetRecordsRejectsTenantMediaInfrastructureTable() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Security] + // [SCENARIO] The source API refuses to serve Tenant Media as a top-level table, so a caller holding the media + // read grant cannot enumerate blobs directly; media stays reachable only inline via a record's media field. + ApproveSourceConsent(); + Response.ReadFrom(SourceApi.GetRecords(Database::"Tenant Media", FieldIdsArray(1), CursorSelector(CurrentDateTime()), 10, '')); + + // [THEN] the table is reported unavailable and no records are returned + Assert.IsFalse(GetBoolean(Response, 'tableAvailable'), 'Tenant Media must not be served as a top-level table'); + Assert.AreEqual(0, RecordCount(Response), 'A blocked table must return no records'); + end; + + [Test] + procedure GetRecordsAppliesRowFilterOnUnprojectedField() + var + MatchCustomer: Record Customer; + OtherCustomer: Record Customer; + FilterCustomer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Watermark: DateTime; + SystemIds: List of [Guid]; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] The source applies the mapping row filter server-side even when it references a field outside the + // projection (parity with same-env), so only matching records are returned rather than filtered post-hoc. + ApproveSourceConsent(); + Watermark := CurrentDateTime(); + Sleep(50); // ensure the seeded records sort strictly after the watermark + LibrarySales.CreateCustomer(MatchCustomer); + MatchCustomer.Blocked := MatchCustomer.Blocked::All; + MatchCustomer.Modify(); + LibrarySales.CreateCustomer(OtherCustomer); // Blocked = " ": excluded by the filter + + // [WHEN] records are fetched projecting only Name, with a filter on the (unprojected) Blocked field + FilterCustomer.SetRange(Blocked, MatchCustomer.Blocked::All); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(MatchCustomer.FieldNo(Name)), CursorSelector(Watermark), 100, FilterCustomer.GetView(false))); + + // [THEN] only the customer matching the row filter comes back + CollectResponseSystemIds(Response, SystemIds); + Assert.IsTrue(SystemIds.Contains(MatchCustomer.SystemId), 'The customer matching the row filter should be returned'); + Assert.IsFalse(SystemIds.Contains(OtherCustomer.SystemId), 'A customer outside the row filter must be excluded server-side'); + end; + + local procedure FeaturesContain(var Response: JsonObject; Feature: Text): Boolean + var + Features: JsonArray; + FeatureToken: JsonToken; + Token: JsonToken; + begin + Response.Get('features', Token); + Features := Token.AsArray(); + foreach FeatureToken in Features do + if FeatureToken.AsValue().AsText() = Feature then + exit(true); + exit(false); + end; + + local procedure RecordCount(var Response: JsonObject): Integer + var + Token: JsonToken; + begin + Response.Get('records', Token); + exit(Token.AsArray().Count()); + end; + + local procedure CollectResponseSystemIds(var Response: JsonObject; var SystemIds: List of [Guid]) + var + RecordsToken: JsonToken; + RecordToken: JsonToken; + SystemIdToken: JsonToken; + SystemIdValue: Guid; + begin + // Record every returned systemId (no de-dup) so a repeat across pages is caught by the count assertion. + Response.Get('records', RecordsToken); + foreach RecordToken in RecordsToken.AsArray() do + if RecordToken.AsObject().Get('systemId', SystemIdToken) then + if Evaluate(SystemIdValue, SystemIdToken.AsValue().AsText()) then + SystemIds.Add(SystemIdValue); + end; + + local procedure GetBoolean(var Response: JsonObject; PropertyName: Text): Boolean + var + Token: JsonToken; + begin + Response.Get(PropertyName, Token); + exit(Token.AsValue().AsBoolean()); + end; + + local procedure NextCursorText(var Response: JsonObject) CursorText: Text + var + Token: JsonToken; + begin + Response.Get('nextCursor', Token); + Token.WriteTo(CursorText); + end; + + local procedure FieldIdsArray(FieldNo: Integer) ResultText: Text + var + FieldIds: JsonArray; + begin + FieldIds.Add(FieldNo); + FieldIds.WriteTo(ResultText); + end; + + local procedure TableIdsArray(TableId: Integer) ResultText: Text + var + TableIds: JsonArray; + begin + TableIds.Add(TableId); + TableIds.WriteTo(ResultText); + end; + + local procedure SystemIdsSelector(SystemId: Guid) ResultText: Text + var + Selector: JsonObject; + SystemIds: JsonArray; + begin + SystemIds.Add(Format(SystemId)); + Selector.Add('systemIds', SystemIds); + Selector.WriteTo(ResultText); + end; + + local procedure CursorSelector(Watermark: DateTime) ResultText: Text + var + Selector: JsonObject; + begin + Selector.Add('modifiedAt', Format(Watermark, 0, 9)); + Selector.WriteTo(ResultText); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al index c9f9f776695..da471ad5052 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al @@ -17,6 +17,8 @@ codeunit 139770 "Master Data Mgt. Setup Tests" LibraryVariableStorage: Codeunit "Library - Variable Storage"; LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; InitializeHandled: Boolean; + IncorrectTablesListErr: Label 'Synchronization tables list is incorrect.'; + UnexpectedConfirmErr: Label 'Unexpected confirmation dialog: %1', Locked = true; [Test] [HandlerFunctions('SynchronizationEnabledMessageHandler')] @@ -96,6 +98,54 @@ codeunit 139770 "Master Data Mgt. Setup Tests" Initialize(); MasterDataManagementSetup.Init(); asserterror MasterDataManagementSetup.Validate("Company Name", CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Company Name"))); + Assert.ExpectedError('You are currently signed into this company'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure EnableCrossEnvironmentAllowsSameSourceCompanyName() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataMgtSubscriber: Record "Master Data Mgt. Subscriber"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Cross-env enables with a source company whose name equals the current company (different environment), + // and never writes to the source subscriber table. + Initialize(); + MasterDataManagementSetup.Init(); + MasterDataManagementSetup."Source Environment Name" := 'CONTOSO-PROD'; + MasterDataManagementSetup."Source Environment URL" := 'https://example/v2.0/contoso-prod'; + MasterDataManagementSetup."Source Company Name" := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Source Company Name")); + MasterDataManagementSetup."Source OAuth Client Id" := '11111111-2222-3333-4444-555555555555'; + MasterDataManagementSetup."Source Client Secret Key" := CreateGuid(); // simulate a stored secret + MasterDataManagementSetup.Insert(); + + MasterDataManagementSetup.Validate("Is Enabled", true); + MasterDataManagementSetup.Modify(true); + + // [THEN] no subscriber row was written for the current company + MasterDataMgtSubscriber.SetRange("Company Name", CompanyName()); + Assert.AreEqual(0, MasterDataMgtSubscriber.Count(), 'Cross-env enable must not write to the source subscriber table'); + + // cleanup: disable to remove the detector job + MasterDataManagementSetup.Validate("Is Enabled", false); + MasterDataManagementSetup.Modify(true); + end; + + [Test] + procedure EnableCrossEnvironmentRequiresConnectionDetails() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Enabling cross-env without a configured connection is blocked with a clear error. + Initialize(); + MasterDataManagementSetup.Init(); + MasterDataManagementSetup."Source Environment Name" := 'CONTOSO-PROD'; // env set, but URL/company/client id/secret missing + MasterDataManagementSetup.Insert(); + + asserterror MasterDataManagementSetup.Validate("Is Enabled", true); + Assert.ExpectedError('connection details'); end; [Test] @@ -126,6 +176,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" MasterDataMgtCoupling."Local System ID" := EmptyGuid; MasterDataMgtCoupling.Insert(); + LibraryVariableStorage.Enqueue('keep the table setup and coupling'); MasterDataManagementSetup.Validate("Is Enabled", false); BindSubscription(MasterDataMgtSetupTests); MasterDataManagementSetup.Modify(true); @@ -140,6 +191,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" Assert.IsTrue(IntegrationTableMapping.Count() > 0, ''); Assert.IsTrue(IntegrationFieldMapping.Count() > 0, ''); Assert.AreEqual(1, MasterDataMgtCoupling.Count(), ''); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -171,8 +223,10 @@ codeunit 139770 "Master Data Mgt. Setup Tests" // reset configuration MasterDataManagementSetupPage.OpenEdit(); + LibraryVariableStorage.Enqueue('restore the default synchronization table setup'); MasterDataManagementSetupPage.ResetConfiguration.Invoke(); VerifyDefaultSetup(); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -277,6 +331,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" IntegrationFieldMapping.SetRange(Status); IntegrationFieldMapping.SetRange("Field Caption", ''); Assert.IsTrue(IntegrationFieldMapping.Count() = 0, 'All synchronization fields for the added table should have a caption.'); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -309,6 +364,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" MasterDataMgtCoupling."Local System ID" := EmptyGuid; MasterDataMgtCoupling.Insert(); + LibraryVariableStorage.Enqueue('keep the table setup and coupling'); MasterDataManagementSetup.Validate("Is Enabled", false); BindSubscription(MasterDataMgtSetupTests); MasterDataManagementSetup.Modify(true); @@ -323,6 +379,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" Assert.AreEqual(0, IntegrationTableMapping.Count(), ''); Assert.AreEqual(0, IntegrationFieldMapping.Count(), ''); Assert.AreEqual(0, MasterDataMgtCoupling.Count(), ''); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -331,7 +388,6 @@ codeunit 139770 "Master Data Mgt. Setup Tests" SynchTables: List of [Integer]; RelatedTablesToAdd: List of [Integer]; TablesToAddText: Text; - IncorrectTablesListErr: Label 'Synchronization tables list is incorrect.'; begin // [SCENARIO] When selecting a table that has a self-reference or other reference that create a cycle, duplicate records are not added to the setup list @@ -356,6 +412,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" exit; LibrarySetupStorage.Restore(); + LibraryVariableStorage.Clear(); BindSubscription(MasterDataMgtSynchTests); IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); @@ -409,12 +466,14 @@ codeunit 139770 "Master Data Mgt. Setup Tests" [ConfirmHandler] internal procedure ConfirmHandlerYes(Question: Text; var Reply: Boolean) begin + Assert.IsTrue(StrPos(Question, LibraryVariableStorage.DequeueText()) > 0, StrSubstNo(UnexpectedConfirmErr, Question)); Reply := true; end; [ConfirmHandler] internal procedure ConfirmHandlerNo(Question: Text; var Reply: Boolean) begin + Assert.IsTrue(StrPos(Question, LibraryVariableStorage.DequeueText()) > 0, StrSubstNo(UnexpectedConfirmErr, Question)); Reply := false; end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al index e415b8ce696..330c50664f9 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al @@ -722,6 +722,199 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" Assert.IsTrue(IntegrationSynchJobErrorsSecond.Get(IntegrationSynchJobErrorsSecond."No."), 'The newly inserted error row should exist'); end; + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceFetchesSourceRecordByCouplingSystemId() + var + SourceCustomer: Record Customer; + MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; + IntegrationRecordRef: RecordRef; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The local data source (IMDM Data Source.GetBySystemId) fetches the source record for a coupling. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer coupled by its SystemId + LibrarySales.CreateCustomer(SourceCustomer); + MasterDataMgtCoupling.Init(); + MasterDataMgtCoupling."Integration System ID" := SourceCustomer.SystemId; + MasterDataMgtCoupling."Local System ID" := CreateGuid(); + MasterDataMgtCoupling."Table ID" := Database::Customer; + MasterDataMgtCoupling.Insert(); + + // [WHEN] the coupling-based GetIntegrationRecordRef is invoked + Found := LibraryMasterDataMgt.GetIntegrationRecordRefByCoupling(Database::Customer, MasterDataMgtCoupling, IntegrationRecordRef); + + // [THEN] the local data source returns the source customer + Assert.IsTrue(Found, 'GetBySystemId should find the source record'); + Assert.AreEqual(SourceCustomer.SystemId, IntegrationRecordRef.Field(IntegrationRecordRef.SystemIdNo()).Value(), 'Wrong record fetched by GetBySystemId'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceFetchesSourceRecordById() + var + SourceCustomer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + IntegrationRecordRef: RecordRef; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The local data source (IMDM Data Source.GetById) fetches the source record by its SystemId. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer + LibrarySales.CreateCustomer(SourceCustomer); + GetCustomerMapping(IntegrationTableMapping); + + // [WHEN] GetIntegrationRecordRef by id (GUID) is invoked + Found := LibraryMasterDataMgt.GetIntegrationRecordRefById(IntegrationTableMapping, SourceCustomer.SystemId, IntegrationRecordRef); + + // [THEN] the local data source returns the source customer + Assert.IsTrue(Found, 'GetById should find the source record'); + Assert.AreEqual(SourceCustomer.SystemId, IntegrationRecordRef.Field(IntegrationRecordRef.SystemIdNo()).Value(), 'Wrong record fetched by GetById'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceFetchesSourceRecordsByUidFilter() + var + SourceCustomer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + SourceRecordRef: RecordRef; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The local data source (IMDM Data Source.GetByUidFilter) returns records matching a UID (SystemId) filter. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer + LibrarySales.CreateCustomer(SourceCustomer); + GetCustomerMapping(IntegrationTableMapping); + + // [WHEN] GetByUidFilter is invoked with the customer's SystemId as the filter + Found := LibraryMasterDataMgt.DataSourceGetByUidFilter(IntegrationTableMapping, Format(SourceCustomer.SystemId), SourceRecordRef); + + // [THEN] the local data source returns the source customer + Assert.IsTrue(Found, 'GetByUidFilter should find the source record'); + Assert.AreEqual(SourceCustomer.SystemId, SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(), 'Wrong record fetched by GetByUidFilter'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceReturnsModifiedSet() + var + SourceCustomer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + SourceRecordRef: RecordRef; + Found: Boolean; + FoundOurs: Boolean; + RecSystemId: Guid; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The local data source (IMDM Data Source.GetModifiedSet) returns the source records for the mapping. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer + LibrarySales.CreateCustomer(SourceCustomer); + GetCustomerMapping(IntegrationTableMapping); + + // [WHEN] GetModifiedSet is invoked + Found := LibraryMasterDataMgt.DataSourceGetModifiedSet(IntegrationTableMapping, '', SourceRecordRef); + + // [THEN] the created source customer is in the returned set + Assert.IsTrue(Found, 'GetModifiedSet should return source records'); + if SourceRecordRef.FindSet() then + repeat + RecSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + if RecSystemId = SourceCustomer.SystemId then + FoundOurs := true; + until (SourceRecordRef.Next() = 0) or FoundOurs; + Assert.IsTrue(FoundOurs, 'The created source customer should be in the modified set'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure ReadFailsWhenSourceEnvironmentNameIsSet() + var + SourceCustomer: Record Customer; + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; + IntegrationRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Setting Source Environment Name routes reads to the cross-environment source, which fails until a connection is configured. + Initialize(); + + // [GIVEN] a coupled source customer and a source environment name set on the setup + LibrarySales.CreateCustomer(SourceCustomer); + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Source Environment Name" := 'CONTOSOENV'; + MasterDataManagementSetup.Modify(false); + MasterDataMgtCoupling.Init(); + MasterDataMgtCoupling."Integration System ID" := SourceCustomer.SystemId; + MasterDataMgtCoupling."Local System ID" := CreateGuid(); + MasterDataMgtCoupling."Table ID" := Database::Customer; + MasterDataMgtCoupling.Insert(); + + // [WHEN] a read that resolves the data source is invoked + asserterror LibraryMasterDataMgt.GetIntegrationRecordRefByCoupling(Database::Customer, MasterDataMgtCoupling, IntegrationRecordRef); + + // [THEN] it fails because the cross-environment connection to the source is not configured yet + Assert.ExpectedError('The cross-environment connection to the source is not configured yet'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure FindMappingByIntegrationRecordIdRespectsIntegrationTableFilter() + var + IntegrationTableMapping: Record "Integration Table Mapping"; + CustomerInFilter: Record Customer; + CustomerOutsideFilter: Record Customer; + MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; + CustomerRecRef: RecordRef; + begin + // [SCENARIO] FindMappingByIntegrationRecordId matches a mapping only when the source record is within the + // mapping's integration table filter - GetBySystemId is a key lookup that ignores the filter. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] two source customers, and the Customer mapping's integration table filter includes only the first + LibrarySales.CreateCustomer(CustomerInFilter); + LibrarySales.CreateCustomer(CustomerOutsideFilter); + GetCustomerMapping(IntegrationTableMapping); + CustomerRecRef.Open(Database::Customer); + CustomerRecRef.Field(CustomerInFilter.FieldNo("No.")).SetRange(CustomerInFilter."No."); + IntegrationTableMapping.SetIntegrationTableFilter(CustomerRecRef.GetView()); + CustomerRecRef.Close(); + IntegrationTableMapping.Modify(); + + // [WHEN] resolving the mapping for the in-filter record [THEN] it matches + Clear(MasterDataMgtCoupling); + MasterDataMgtCoupling."Integration System ID" := CustomerInFilter.SystemId; + Assert.IsTrue(LibraryMasterDataMgt.FindMappingByIntegrationRecordId(IntegrationTableMapping, MasterDataMgtCoupling), 'A source record within the integration table filter should match its mapping.'); + + // [WHEN] resolving the mapping for the out-of-filter record [THEN] it does not match (the filter is enforced) + Clear(IntegrationTableMapping); + Clear(MasterDataMgtCoupling); + MasterDataMgtCoupling."Integration System ID" := CustomerOutsideFilter.SystemId; + Assert.IsFalse(LibraryMasterDataMgt.FindMappingByIntegrationRecordId(IntegrationTableMapping, MasterDataMgtCoupling), 'A source record outside the integration table filter must not match the mapping.'); + end; + + local procedure GetCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") + begin + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Table ID", Database::Customer); + IntegrationTableMapping.SetRange("Integration Table ID", Database::Customer); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + IntegrationTableMapping.FindFirst(); + end; + local procedure CreateCoupledCustomers(var SourceCustomer: Record Customer; var DestinationCustomer: Record Customer; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling") begin LibrarySales.CreateCustomer(SourceCustomer);