From 000c18a62834c4eac6d615157d2753fd06fac532 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 24 Aug 2026 19:37:20 +0200 Subject: [PATCH 01/64] [Master Data Management] Cross-environment synchonization --- .../codeunits/MDMLocalDataSource.Codeunit.al | 39 +++++++++++++++++++ .../app/src/enums/MDMDataSourceType.Enum.al | 17 ++++++++ .../interfaces/IMDMDataSource.Interface.al | 25 ++++++++++++ .../tables/MasterDataManagementSetup.Table.al | 13 +++++++ 4 files changed, 94 insertions(+) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al new file mode 100644 index 00000000000..8af3535b39b --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -0,0 +1,39 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; + +/// +/// Reads source master data from another company in the same environment via ChangeCompany. +/// This preserves today's behavior; it is the implementation used whenever no source environment is set. +/// +codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" +{ + Access = Internal; + + procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping, SourceRecordRef); + IntegrationTableMapping.SetIntRecordRefFilter(SourceRecordRef, TableFilter); + exit(SourceRecordRef.FindSet()); + end; + + procedure GetBySystemId(IntegrationTableMapping: Record "Integration Table Mapping"; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping, SourceRecordRef); + exit(SourceRecordRef.GetBySystemId(SystemId)); + end; + + local procedure OpenSourceRecordRef(IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef) + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataManagement: Codeunit "Master Data Management"; + SourceCompanyName: Text[30]; + begin + MasterDataManagementSetup.Get(); + SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID"); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + SourceRecordRef.ChangeCompany(SourceCompanyName); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al b/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al new file mode 100644 index 00000000000..2128b5affbb --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al @@ -0,0 +1,17 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Selects the master data source implementation. Non-extensible and internal: partners must not add +/// source types. The value is derived from setup (Source Environment Name), never stored directly. +/// +enum 7239 "MDM Data Source Type" implements "IMDM Data Source" +{ + Access = Internal; + Extensible = false; + + value(0; LocalCompany) + { + Caption = 'Local Company'; + Implementation = "IMDM Data Source" = "MDM Local Data Source"; + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al new file mode 100644 index 00000000000..2e855106efa --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al @@ -0,0 +1,25 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; + +/// +/// Abstracts reading source-company master data so the same synchronization logic can run against a +/// local company (ChangeCompany) or, in a future release, a remote environment over OData. +/// Sealed to Microsoft: partners must not implement it or ride the synchronization credentials. +/// +interface "IMDM Data Source" +{ + Access = Internal; + + /// + /// Opens SourceRecordRef on the source integration table for the given mapping and applies the + /// supplied table filter. Returns true if at least one record matches. + /// + procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Fetches a single source integration-table record by its SystemId into SourceRecordRef. + /// Returns true if the record was found. + /// + procedure GetBySystemId(IntegrationTableMapping: Record "Integration Table Mapping"; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index c8b1a20280b..12459e82cee 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -79,6 +79,11 @@ table 7230 "Master Data Management Setup" Caption = 'Delay Synchronization Job Scheduling'; DataClassification = SystemMetadata; } + field(155; "Source Environment Name"; Text[100]) + { + Caption = 'Source Environment'; + DataClassification = OrganizationIdentifiableInformation; + } } keys @@ -245,6 +250,13 @@ table 7230 "Master Data Management Setup" until IntegrationTableMapping.Next() = 0; end; + internal procedure GetDataSource(): Interface "IMDM Data Source" + begin + if "Source Environment Name" <> '' then + Error(CrossEnvNotYetSupportedErr); + exit(Enum::"MDM Data Source Type"::LocalCompany); + end; + var SynchronizationEnabledMsg: label 'The synchronization of data from company %1 is enabled. \\To review the tables and fields that will be synchronized, choose action Synchronization Tables. \\To perform the initial synchronization of data from %1, choose Start Initial Synchronization. \\After the initial synchronization is done, job queue entries will continue to synchronize modifications.', Comment = '%1 - a company name'; CouplingsWillBeDeletedQst: label 'All the couplings with records from previous source company %1 will be deleted. Do you want to continue?', Comment = '%1 - a company name'; @@ -252,4 +264,5 @@ table 7230 "Master Data Management Setup" MustNotPickCurrentCompanyErr: label 'You are currently signed into this company. \\Choose a different company to synchronize data with.'; MustPickSourceCompanyErr: label 'You must choose a source company to synchronize data from.'; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; + CrossEnvNotYetSupportedErr: label 'Cross-environment synchronization is not yet available.'; } From 23be6cb11dd390073f093e3f24e9a5927522cf4d Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 25 Aug 2026 15:34:22 +0200 Subject: [PATCH 02/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 9 +++- .../IntegrationMasterDataSynch.Codeunit.al | 21 +++------- .../codeunits/MDMLocalDataSource.Codeunit.al | 42 ++++++++++++++++--- .../MasterDataManagement.Codeunit.al | 41 +----------------- .../interfaces/IMDMDataSource.Interface.al | 12 ++++-- 5 files changed, 60 insertions(+), 65 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 1a6a6b01301..878e4106fb4 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -10,7 +10,14 @@ permissionset 7230 "Master Data Mgt. - Objects" Assignable = false; Access = Public; - Permissions = codeunit * = X, + Permissions = codeunit "Master Data Mgt. Setup Default" = X, + codeunit "Integration Master Data Synch." = X, + codeunit "Master Data Management" = X, + codeunit "Master Data Mgt. Table Couple" = X, + codeunit "Master Data Mgt. Tbl. Uncouple" = X, + codeunit "Master Data Mgt. Subscribers" = X, + codeunit "Master Data Mgt. Upgrade" = X, + codeunit "MDM Local Data Source" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index bdbd419dfa6..a3a12fe98c0 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -71,35 +71,30 @@ codeunit 7231 "Integration Master Data Synch." var MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; + DataSource: Interface "IMDM Data Source"; IntegrationRecordRef: RecordRef; IntegrationRecordID: Guid; TableFilter: Text; FilterList: List of [Text]; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin OnFindModifiedIntegrationRecords(TempIntegrationRecordRef, IntegrationTableMapping, FailedNotSkippedIdDictionary, IsHandled); if IsHandled then exit; MasterDataManagementSetup.Get(); + DataSource := MasterDataManagementSetup.GetDataSource(); SplitIntegrationTableFilter(IntegrationTableMapping, FilterList); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); foreach TableFilter in FilterList do begin - IntegrationTableMapping.SetIntRecordRefFilter(IntegrationRecordRef, TableFilter); - if IntegrationRecordRef.FindSet() then + if DataSource.GetModifiedSet(IntegrationTableMapping, TableFilter, IntegrationRecordRef) then repeat IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then Session.LogMessage('0000J8Q', StrSubstNo(CopyRecordRefFailedTxt, IntegrationRecordID), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); until IntegrationRecordRef.Next() = 0; + IntegrationRecordRef.Close(); end; - IntegrationRecordRef.Close(); end; [TryFunction] @@ -260,7 +255,6 @@ codeunit 7231 "Integration Master Data Synch." RecordID: RecordID; IntegrationSystemID: Guid; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin case GetSourceType(SourceID) of SupportedSourceType::RecordID: @@ -282,12 +276,7 @@ codeunit 7231 "Integration Master Data Synch." MasterDataManagement.OnGetIntegrationRecordRefBySystemId(IntegrationTableMapping, RecordRef, IntegrationSystemID, IsHandled); if not IsHandled then begin MasterDataManagementSetup.Get(); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecordRef.Open(IntegrationTableMapping."Integration Table ID"); - RecordRef.ChangeCompany(SourceCompanyName); - if not RecordRef.GetBySystemId(IntegrationSystemID) then + if not MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableMapping."Integration Table ID", IntegrationSystemID, RecordRef) then exit(false); end; exit(IntegrationTableMapping.FindFilteredRec(RecordRef, OutOfMapFilter)); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al index 8af3535b39b..0bf601fe1d5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -12,26 +12,56 @@ codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean begin - OpenSourceRecordRef(IntegrationTableMapping, SourceRecordRef); + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); IntegrationTableMapping.SetIntRecordRefFilter(SourceRecordRef, TableFilter); exit(SourceRecordRef.FindSet()); end; - procedure GetBySystemId(IntegrationTableMapping: Record "Integration Table Mapping"; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean begin - OpenSourceRecordRef(IntegrationTableMapping, SourceRecordRef); + OpenSourceRecordRef(IntegrationTableId, SourceRecordRef); exit(SourceRecordRef.GetBySystemId(SystemId)); end; - local procedure OpenSourceRecordRef(IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef) + procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean + var + IDFieldRef: FieldRef; + RecId: RecordID; + TextKey: Text; + begin + SourceRecordRef.Close(); + if ID.IsGuid then begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + IDFieldRef := SourceRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); + IDFieldRef.SetFilter(ID); + exit(SourceRecordRef.FindFirst()); + end; + + if ID.IsRecordId then begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + RecId := ID; + if RecId.TableNo = IntegrationTableMapping."Table ID" then + exit(SourceRecordRef.Get(ID)); + end; + + if ID.IsText then begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + IDFieldRef := SourceRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); + TextKey := ID; + IDFieldRef.SetFilter('%1', TextKey); + exit(SourceRecordRef.FindFirst()); + end; + end; + + local procedure OpenSourceRecordRef(IntegrationTableId: Integer; var SourceRecordRef: RecordRef) var MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); - SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); + SourceRecordRef.Open(IntegrationTableId); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableId); if SourceCompanyName = '' then SourceCompanyName := MasterDataManagementSetup."Company Name"; SourceRecordRef.ChangeCompany(SourceCompanyName); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index 2a04f766028..febf099ba23 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -349,46 +349,15 @@ codeunit 7233 "Master Data Management" internal procedure GetIntegrationRecordRef(var IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var IntegrationRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - IDFieldRef: FieldRef; - RecordID: RecordID; - TextKey: Text; Found: Boolean; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin OnGetIntegrationRecordRefByIntegrationSystemId(IntegrationTableMapping, ID, IntegrationRecordRef, Found, IsHandled); if IsHandled then exit(Found); - IntegrationRecordRef.Close(); MasterDataManagementSetup.Get(); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - if ID.IsGuid then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IDFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); - IDFieldRef.SetFilter(ID); - exit(IntegrationRecordRef.FindFirst()); - end; - - if ID.IsRecordId then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - RecordID := ID; - if RecordID.TableNo = IntegrationTableMapping."Table ID" then - exit(IntegrationRecordRef.Get(ID)); - end; - - if ID.IsText then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IDFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); - TextKey := ID; - IDFieldRef.SetFilter('%1', TextKey); - exit(IntegrationRecordRef.FindFirst()); - end; + exit(MasterDataManagementSetup.GetDataSource().GetById(IntegrationTableMapping, ID, IntegrationRecordRef)); end; local procedure GetRecordRef(RecVariant: Variant; var RecordRef: RecordRef): Integer @@ -2156,7 +2125,6 @@ codeunit 7233 "Master Data Management" MasterDataManagementSetup: Record "Master Data Management Setup"; IsHandled: Boolean; Found: Boolean; - SourceCompanyName: Text[30]; begin OnGetIntegrationRecordRefFromCoupling(IntegrationTableID, MasterDataMgtCoupling, RecRef, Found, IsHandled); if IsHandled then @@ -2166,12 +2134,7 @@ codeunit 7233 "Master Data Management" exit(false); MasterDataManagementSetup.Get(); - RecRef.Open(IntegrationTableID); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableID); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecRef.ChangeCompany(SourceCompanyName); - exit(RecRef.GetBySystemId(MasterDataMgtCoupling."Integration System ID")); + exit(MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableID, MasterDataMgtCoupling."Integration System ID", RecRef)); end; internal procedure RemoveSubsidiarySubscriptionFromMasterCompany(MasterCompanyName: Text[30]; SubsidiaryCompanyName: Text[30]) diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al index 2e855106efa..8d8caf5cd0d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al @@ -18,8 +18,14 @@ interface "IMDM Data Source" procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean; /// - /// Fetches a single source integration-table record by its SystemId into SourceRecordRef. - /// Returns true if the record was found. + /// Fetches a single source record from the given integration table by its SystemId into + /// SourceRecordRef. Returns true if the record was found. /// - procedure GetBySystemId(IntegrationTableMapping: Record "Integration Table Mapping"; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean; + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Fetches a single source integration-table record by its id (the integration UID field value, + /// a RecordId, or a business-key text) into SourceRecordRef. Returns true if found. + /// + procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean; } From 700203cb8404f39dbe81f8bf95530082084ecbd3 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 25 Aug 2026 17:38:06 +0200 Subject: [PATCH 03/64] [Master Data Management] Cross-environment synchonization --- .../codeunits/IntegrationMasterDataSynch.Codeunit.al | 12 +++--------- .../app/src/codeunits/MDMLocalDataSource.Codeunit.al | 7 +++++++ .../app/src/interfaces/IMDMDataSource.Interface.al | 6 ++++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index a3a12fe98c0..a8f9851ac08 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -202,27 +202,21 @@ codeunit 7231 "Integration Master Data Synch." local procedure CacheFilteredIntegrationRecords(var IntegrationSystemIDFilterList: List of [Text]; IntegrationTableMapping: Record "Integration Table Mapping"; var TempIntegrationRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - MasterDataManagement: Codeunit "Master Data Management"; + DataSource: Interface "IMDM Data Source"; IntegrationRecordRef: RecordRef; IntegrationSystemIDFilter: Text; Cached: Boolean; IsHandled: Boolean; - SourceCompanyName: Text[30]; begin OnCacheFilteredIntegrationRecords(IntegrationSystemIDFilterList, IntegrationTableMapping, TempIntegrationRecordRef, Cached, IsHandled); if (IsHandled) then exit(Cached); MasterDataManagementSetup.Get(); + DataSource := MasterDataManagementSetup.GetDataSource(); foreach IntegrationSystemIDFilter in IntegrationSystemIDFilterList do if IntegrationSystemIDFilter <> '' then begin - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").SetFilter(IntegrationSystemIDFilter); - if IntegrationRecordRef.FindSet() then + if DataSource.GetByUidFilter(IntegrationTableMapping, IntegrationSystemIDFilter, IntegrationRecordRef) then repeat CopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false); Cached := true; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al index 0bf601fe1d5..38b6cc8a598 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -53,6 +53,13 @@ codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" end; end; + procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + SourceRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").SetFilter(UidFilter); + exit(SourceRecordRef.FindSet()); + end; + local procedure OpenSourceRecordRef(IntegrationTableId: Integer; var SourceRecordRef: RecordRef) var MasterDataManagementSetup: Record "Master Data Management Setup"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al index 8d8caf5cd0d..cbfb8269f2d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al @@ -28,4 +28,10 @@ interface "IMDM Data Source" /// a RecordId, or a business-key text) into SourceRecordRef. Returns true if found. /// procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Opens the source integration table and returns the set of records whose integration UID field + /// matches UidFilter (a filter expression, e.g. a list of SystemIds). Returns true if any matched. + /// + procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean; } From 2dd3eef657ee6809395d35608dd5810be417be2f Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 26 Aug 2026 11:21:47 +0200 Subject: [PATCH 04/64] [Master Data Management] Cross-environment synchonization --- .../src/LibraryMasterDataMgt.Codeunit.al | 36 +++++ .../src/MasterDataMgtSynchTests.Codeunit.al | 151 ++++++++++++++++++ 2 files changed, 187 insertions(+) diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 4716511930b..1522f9fd53a 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -40,6 +40,42 @@ codeunit 139757 "Library - Master Data Mgt." MasterDataSynchTables.FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, TableId); end; + procedure SetSourceCompanyToCurrent() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Company Name" := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Company Name")); + MasterDataManagementSetup.Modify(false); + end; + + procedure GetIntegrationRecordRefByCoupling(IntegrationTableID: Integer; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; var RecRef: RecordRef): Boolean + begin + exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableID, MasterDataMgtCoupling, RecRef)); + end; + + procedure GetIntegrationRecordRefById(var IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var RecRef: RecordRef): Boolean + begin + exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableMapping, ID, RecRef)); + end; + + procedure DataSourceGetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetModifiedSet(IntegrationTableMapping, TableFilter, SourceRecordRef)); + end; + + procedure DataSourceGetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef)); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; + MasterDataManagement: Codeunit "Master Data Management"; } diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al index e415b8ce696..f792804eb98 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al @@ -722,6 +722,157 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" Assert.IsTrue(IntegrationSynchJobErrorsSecond.Get(IntegrationSynchJobErrorsSecond."No."), 'The newly inserted error row should exist'); end; + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceFetchesSourceRecordByCouplingSystemId() + var + SourceCustomer: Record Customer; + MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; + IntegrationRecordRef: RecordRef; + Found: Boolean; + begin + // [SCENARIO] The local data source (IMDM Data Source.GetBySystemId) fetches the source record for a coupling. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer coupled by its SystemId + LibrarySales.CreateCustomer(SourceCustomer); + MasterDataMgtCoupling.Init(); + MasterDataMgtCoupling."Integration System ID" := SourceCustomer.SystemId; + MasterDataMgtCoupling."Local System ID" := CreateGuid(); + MasterDataMgtCoupling."Table ID" := Database::Customer; + MasterDataMgtCoupling.Insert(); + + // [WHEN] the coupling-based GetIntegrationRecordRef is invoked + Found := LibraryMasterDataMgt.GetIntegrationRecordRefByCoupling(Database::Customer, MasterDataMgtCoupling, IntegrationRecordRef); + + // [THEN] the local data source returns the source customer + Assert.IsTrue(Found, 'GetBySystemId should find the source record'); + Assert.AreEqual(SourceCustomer.SystemId, IntegrationRecordRef.Field(IntegrationRecordRef.SystemIdNo()).Value(), 'Wrong record fetched by GetBySystemId'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceFetchesSourceRecordById() + var + SourceCustomer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + IntegrationRecordRef: RecordRef; + Found: Boolean; + begin + // [SCENARIO] The local data source (IMDM Data Source.GetById) fetches the source record by its SystemId. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer + LibrarySales.CreateCustomer(SourceCustomer); + GetCustomerMapping(IntegrationTableMapping); + + // [WHEN] GetIntegrationRecordRef by id (GUID) is invoked + Found := LibraryMasterDataMgt.GetIntegrationRecordRefById(IntegrationTableMapping, SourceCustomer.SystemId, IntegrationRecordRef); + + // [THEN] the local data source returns the source customer + Assert.IsTrue(Found, 'GetById should find the source record'); + Assert.AreEqual(SourceCustomer.SystemId, IntegrationRecordRef.Field(IntegrationRecordRef.SystemIdNo()).Value(), 'Wrong record fetched by GetById'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceFetchesSourceRecordsByUidFilter() + var + SourceCustomer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + SourceRecordRef: RecordRef; + Found: Boolean; + begin + // [SCENARIO] The local data source (IMDM Data Source.GetByUidFilter) returns records matching a UID (SystemId) filter. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer + LibrarySales.CreateCustomer(SourceCustomer); + GetCustomerMapping(IntegrationTableMapping); + + // [WHEN] GetByUidFilter is invoked with the customer's SystemId as the filter + Found := LibraryMasterDataMgt.DataSourceGetByUidFilter(IntegrationTableMapping, Format(SourceCustomer.SystemId), SourceRecordRef); + + // [THEN] the local data source returns the source customer + Assert.IsTrue(Found, 'GetByUidFilter should find the source record'); + Assert.AreEqual(SourceCustomer.SystemId, SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(), 'Wrong record fetched by GetByUidFilter'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure LocalDataSourceReturnsModifiedSet() + var + SourceCustomer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + SourceRecordRef: RecordRef; + Found: Boolean; + FoundOurs: Boolean; + RecSystemId: Guid; + begin + // [SCENARIO] The local data source (IMDM Data Source.GetModifiedSet) returns the source records for the mapping. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] a source customer + LibrarySales.CreateCustomer(SourceCustomer); + GetCustomerMapping(IntegrationTableMapping); + + // [WHEN] GetModifiedSet is invoked + Found := LibraryMasterDataMgt.DataSourceGetModifiedSet(IntegrationTableMapping, '', SourceRecordRef); + + // [THEN] the created source customer is in the returned set + Assert.IsTrue(Found, 'GetModifiedSet should return source records'); + if SourceRecordRef.FindSet() then + repeat + RecSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + if RecSystemId = SourceCustomer.SystemId then + FoundOurs := true; + until (SourceRecordRef.Next() = 0) or FoundOurs; + Assert.IsTrue(FoundOurs, 'The created source customer should be in the modified set'); + end; + + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure ReadFailsWhenSourceEnvironmentNameIsSet() + var + SourceCustomer: Record Customer; + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; + IntegrationRecordRef: RecordRef; + begin + // [SCENARIO] Setting Source Environment Name makes the data-source resolver fail (cross-environment not yet supported). + Initialize(); + + // [GIVEN] a coupled source customer and a source environment name set on the setup + LibrarySales.CreateCustomer(SourceCustomer); + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Source Environment Name" := 'CONTOSOENV'; + MasterDataManagementSetup.Modify(false); + MasterDataMgtCoupling.Init(); + MasterDataMgtCoupling."Integration System ID" := SourceCustomer.SystemId; + MasterDataMgtCoupling."Local System ID" := CreateGuid(); + MasterDataMgtCoupling."Table ID" := Database::Customer; + MasterDataMgtCoupling.Insert(); + + // [WHEN] a read that resolves the data source is invoked + asserterror LibraryMasterDataMgt.GetIntegrationRecordRefByCoupling(Database::Customer, MasterDataMgtCoupling, IntegrationRecordRef); + + // [THEN] it fails because cross-environment synchronization is not yet available + Assert.ExpectedError('Cross-environment synchronization is not yet available'); + end; + + local procedure GetCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") + begin + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Table ID", Database::Customer); + IntegrationTableMapping.SetRange("Integration Table ID", Database::Customer); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + IntegrationTableMapping.FindFirst(); + end; + local procedure CreateCoupledCustomers(var SourceCustomer: Record Customer; var DestinationCustomer: Record Customer; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling") begin LibrarySales.CreateCustomer(SourceCustomer); From 354a74ef8b90ac5fb7444ea38a32b4f229ee2f6b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 26 Aug 2026 11:22:01 +0200 Subject: [PATCH 05/64] [Master Data Management] Cross-environment synchonization --- .../test/src/MasterDataMgtSynchTests.Codeunit.al | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al index f792804eb98..3769252fd6c 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al @@ -731,6 +731,7 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" IntegrationRecordRef: RecordRef; Found: Boolean; begin + // [FEATURE] [AI test 0.4] // [SCENARIO] The local data source (IMDM Data Source.GetBySystemId) fetches the source record for a coupling. Initialize(); LibraryMasterDataMgt.SetSourceCompanyToCurrent(); @@ -760,6 +761,7 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" IntegrationRecordRef: RecordRef; Found: Boolean; begin + // [FEATURE] [AI test 0.4] // [SCENARIO] The local data source (IMDM Data Source.GetById) fetches the source record by its SystemId. Initialize(); LibraryMasterDataMgt.SetSourceCompanyToCurrent(); @@ -785,6 +787,7 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" SourceRecordRef: RecordRef; Found: Boolean; begin + // [FEATURE] [AI test 0.4] // [SCENARIO] The local data source (IMDM Data Source.GetByUidFilter) returns records matching a UID (SystemId) filter. Initialize(); LibraryMasterDataMgt.SetSourceCompanyToCurrent(); @@ -812,6 +815,7 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" FoundOurs: Boolean; RecSystemId: Guid; begin + // [FEATURE] [AI test 0.4] // [SCENARIO] The local data source (IMDM Data Source.GetModifiedSet) returns the source records for the mapping. Initialize(); LibraryMasterDataMgt.SetSourceCompanyToCurrent(); @@ -843,6 +847,7 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; IntegrationRecordRef: RecordRef; begin + // [FEATURE] [AI test 0.4] // [SCENARIO] Setting Source Environment Name makes the data-source resolver fail (cross-environment not yet supported). Initialize(); From ffd1d43714bf09ac40f112ee94969f6ac040e9f6 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 26 Aug 2026 13:20:10 +0200 Subject: [PATCH 06/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtCrossEnv.PermissionSet.al | 17 +++++ .../MasterDataMgtObjects.PermissionSet.al | 1 + .../MDMCrossEnvSourceAPI.Codeunit.al | 65 +++++++++++++++++++ .../MasterDataMgtInstall.Codeunit.al | 18 +++++ .../MasterDataMgtUpgrade.Codeunit.al | 41 +++++++++++- 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al new file mode 100644 index 00000000000..f325430e9a7 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al @@ -0,0 +1,17 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Assigned to the customer-registered Entra app on the SOURCE (Microsoft Entra Application Card) to grant +/// cross-environment, read-only access to master data. Deliberately NOT part of "Master Data Mgt. - Objects": +/// only this set grants execute on the ODataV4 source API, so local users cannot invoke it. +/// The per-synchronized-table read permissions are added dynamically as a tenant permission set as the user +/// edits Synchronization Tables (see design doc); this static set covers only the API surface. +/// +permissionset 7242 "MDM Cross-Env Read" +{ + Assignable = true; + Access = Public; + Caption = 'Master Data Mgt. - Cross Environment'; + + Permissions = codeunit "MDM Cross-Env Source API" = X; +} diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 878e4106fb4..95fac7d45fb 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -17,6 +17,7 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "Master Data Mgt. Tbl. Uncouple" = X, codeunit "Master Data Mgt. Subscribers" = X, codeunit "Master Data Mgt. Upgrade" = X, + codeunit "Master Data Mgt. Install" = X, codeunit "MDM Local Data Source" = X, page * = X, table * = X, diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al new file mode 100644 index 00000000000..aa62f86ba60 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -0,0 +1,65 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Source-side generic API, published as an ODataV4 web service. The cross-environment implementation of +/// "IMDM Data Source" (running in a subsidiary) calls these unbound actions to read source master data. +/// Runs under the CALLER's permission set with NO permission elevation, so a read-only, table-scoped +/// permission set assigned to the caller's Entra app is the effective access boundary. +/// +codeunit 7241 "MDM Cross-Env Source API" +{ + Access = Public; + + var + NotYetImplementedErr: Label 'This master data management source action is not yet available.'; + + /// + /// Wire-version negotiation: returns the API contract version and the action/feature names this source + /// supports, so a newer subsidiary only calls actions an older source actually implements. + /// + [ServiceEnabled] + procedure GetCapabilities(): Text + var + Capabilities: JsonObject; + Features: JsonArray; + ResultText: Text; + begin + Capabilities.Add('version', ApiVersion()); + Features.Add('records'); + Features.Add('lastModifiedPerTable'); + Capabilities.Add('features', Features); + Capabilities.WriteTo(ResultText); + exit(ResultText); + end; + + /// + /// Returns a page of changed source records for the given table. Selector is either a change-feed + /// cursor { modifiedAt, systemId, pageSize } or a targeted { systemIds } list. Response carries the + /// records, nextCursor and hasMore. Only tables/fields in a configured mapping are served. + /// + [ServiceEnabled] + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + begin + // TODO(cross-env): authorize (calling app + mapping scope), project FieldIds, page by the composite + // cursor (SystemModifiedAt, SystemId), serialize field types, and report unavailable table/fields. + Error(NotYetImplementedErr); + end; + + /// + /// Change detection: for each requested table id, returns its latest source modification timestamp so + /// the subsidiary detector can decide which per-table sync jobs to reschedule. + /// + [ServiceEnabled] + procedure LastModifiedAtPerTable(TableIds: Text): Text + begin + // TODO(cross-env): read the trigger-maintained (TableId, LastModifiedAt) summary table (not yet built) + // rather than scanning; return [{ tableId, lastModifiedAt }]. + Error(NotYetImplementedErr); + end; + + local procedure ApiVersion(): Integer + begin + exit(1); + end; +} + diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al new file mode 100644 index 00000000000..799809e2c97 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al @@ -0,0 +1,18 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Codeunit Master Data Mgt. Install (ID 7243). +/// +codeunit 7243 "Master Data Mgt. Install" +{ + Access = Internal; + Subtype = Install; + + trigger OnInstallAppPerDatabase() + var + MasterDataMgtUpgrade: Codeunit "Master Data Mgt. Upgrade"; + begin + // Fresh install (incl. package/base-image build) never fires the upgrade trigger, so publish the source endpoint here too. + MasterDataMgtUpgrade.RegisterCrossEnvSourceWebService(); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al index 023b432fa52..10336b6aecc 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al @@ -2,6 +2,7 @@ namespace Microsoft.Integration.MDM; using Microsoft.Integration.SyncEngine; using System.Automation; +using System.Integration; using System.Threading; using System.Upgrade; @@ -13,13 +14,40 @@ codeunit 7238 "Master Data Mgt. Upgrade" Access = Internal; Subtype = Upgrade; Permissions = tabledata "Integration Field Mapping" = rimd, - tabledata "Integration Table Mapping" = rimd; + tabledata "Integration Table Mapping" = rimd, + tabledata "Tenant Web Service" = rimd; trigger OnUpgradePerCompany() begin UpgradeJobQueueEntryFrequencies(); end; + trigger OnUpgradePerDatabase() + begin + RegisterCrossEnvSourceWebService(); + end; + + // Guaranteed provisioning path: install codeunits are skipped when BC is pre-baked into a package and mounted per tenant. + internal procedure RegisterCrossEnvSourceWebService() + var + WebServiceManagement: Codeunit "Web Service Management"; + TenantWebService: Record "Tenant Web Service"; + UpgradeTag: Codeunit "Upgrade Tag"; + begin + if UpgradeTag.HasUpgradeTag(GetCrossEnvWebServiceUpgradeTag()) then + exit; + + // Idempotent: creates or updates the service. Access stays gated by the dedicated "Cross Env" permission set, not by publishing. + WebServiceManagement.CreateTenantWebService(TenantWebService."Object Type"::Codeunit, Codeunit::"MDM Cross-Env Source API", CrossEnvSourceWebServiceName(), true); + + UpgradeTag.SetUpgradeTag(GetCrossEnvWebServiceUpgradeTag()); + end; + + internal procedure CrossEnvSourceWebServiceName(): Text[240] + begin + exit('MDMCrossEnvSource'); + end; + internal procedure UpgradeJobQueueEntryFrequencies() var IntegrationTableMapping: Record "Integration Table Mapping"; @@ -101,10 +129,21 @@ codeunit 7238 "Master Data Mgt. Upgrade" exit('MS-543635-MDMJobQueueFrequency-20240830'); end; + local procedure GetCrossEnvWebServiceUpgradeTag(): Code[250] + begin + exit('MS-647660-MDMCrossEnvWebService-20260826'); + end; + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Upgrade Tag", 'OnGetPerCompanyUpgradeTags', '', false, false)] local procedure RegisterPerCompanyTags(var PerCompanyUpgradeTags: List of [Code[250]]) begin PerCompanyUpgradeTags.Add(GetSynchTableCaptionUpgradeTag()); PerCompanyUpgradeTags.Add(GetJobQueueFrequencyUpgradeTag()); end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Upgrade Tag", 'OnGetPerDatabaseUpgradeTags', '', false, false)] + local procedure RegisterPerDatabaseTags(var PerDatabaseUpgradeTags: List of [Code[250]]) + begin + PerDatabaseUpgradeTags.Add(GetCrossEnvWebServiceUpgradeTag()); + end; } \ No newline at end of file From a85a414c370e412bc4b53138ebf8921116ebf7aa Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 26 Aug 2026 19:15:57 +0200 Subject: [PATCH 07/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 3 + .../MDMCrossEnvDataSource.Codeunit.al | 260 +++++++++++ .../MDMCrossEnvSourceAPI.Codeunit.al | 433 +++++++++++++++++- .../MDMHttpSourceTransport.Codeunit.al | 31 ++ .../codeunits/MDMSourceResponse.Codeunit.al | 177 +++++++ .../app/src/enums/MDMDataSourceType.Enum.al | 6 + .../IMDMSourceTransport.Interface.al | 17 + .../app/src/tables/MDMContact.TableExt.al | 14 + .../tables/MDMCurrencyExchRate.TableExt.al | 14 + .../app/src/tables/MDMCustomer.TableExt.al | 14 + .../app/src/tables/MDMPostCode.TableExt.al | 14 + .../app/src/tables/MDMVendor.TableExt.al | 14 + .../tables/MasterDataManagementSetup.Table.al | 3 +- 13 files changed, 985 insertions(+), 15 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 95fac7d45fb..1fd896ed926 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -19,6 +19,9 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "Master Data Mgt. Upgrade" = X, codeunit "Master Data Mgt. Install" = X, codeunit "MDM Local Data Source" = X, + codeunit "MDM Source Response" = X, + codeunit "MDM Http Source Transport" = X, + codeunit "MDM Cross-Env Data Source" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al new file mode 100644 index 00000000000..f1bf69d0996 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -0,0 +1,260 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; + +/// +/// Reads source master data from another ENVIRONMENT (same tenant) over the ODataV4 source API and materializes +/// results into temporary records, so the existing synchronization engine processes them unchanged. Selected by +/// GetDataSource() when a Source Environment Name is configured. Wire calls go through IMDM Source Transport, +/// which tests swap for an in-process transport (source and subsidiary run in the same environment there). +/// +codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" +{ + Access = Internal; + + var + SourceResponse: Codeunit "MDM Source Response"; + InvalidResponseErr: Label 'The source environment returned an unexpected response for table %1.', Comment = '%1 = table caption'; + TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; + NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; + FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; + + procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + FieldIds: Text; + Selector: Text; + MoreToFetch: Boolean; + begin + SourceRecordRef.Close(); + SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID", true); + Transport := GetTransport(); + FieldIds := BuildFieldIds(IntegrationTableMapping); + // Watermark = 'Synchronize Changes Since'; the wire pages the delta, all accumulated into the temp ref. + Selector := CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"); + repeat + ParseOrError( + IntegrationTableMapping."Integration Table ID", + Transport.GetRecords(IntegrationTableMapping."Integration Table ID", FieldIds, Selector, PageSize()), + Response); + SourceResponse.InsertRecords(Response, SourceRecordRef); + MoreToFetch := SourceResponse.HasMore(Response); + if MoreToFetch then + Selector := SourceResponse.GetNextCursor(Response); + until not MoreToFetch; + IntegrationTableMapping.SetIntRecordRefFilter(SourceRecordRef, TableFilter); + exit(SourceRecordRef.FindSet()); + end; + + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + var + SystemIds: List of [Guid]; + begin + SystemIds.Add(SystemId); + FetchBySystemIds(IntegrationTableId, BuildFieldIdsForTable(IntegrationTableId), SystemIds, SourceRecordRef); + exit(SourceRecordRef.FindFirst()); + end; + + procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean + var + SystemIds: List of [Guid]; + SystemIdValue: Guid; + TextKey: Text; + begin + if ID.IsGuid then + SystemIdValue := ID + else + if ID.IsText then begin + TextKey := ID; + if not Evaluate(SystemIdValue, TextKey) then + exit(false); + end else + // RecordId is environment-specific; the cross-env feed keys on SystemId only. + exit(false); + SystemIds.Add(SystemIdValue); + FetchBySystemIds(IntegrationTableMapping."Integration Table ID", BuildFieldIds(IntegrationTableMapping), SystemIds, SourceRecordRef); + exit(SourceRecordRef.FindFirst()); + end; + + procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + // MDM's UID field is SystemId, so a UID filter is a set of SystemIds. + FetchBySystemIds(IntegrationTableMapping."Integration Table ID", BuildFieldIds(IntegrationTableMapping), ParseSystemIds(UidFilter), SourceRecordRef); + exit(SourceRecordRef.FindSet()); + end; + + local procedure FetchBySystemIds(IntegrationTableId: Integer; FieldIds: Text; SystemIds: List of [Guid]; var SourceRecordRef: RecordRef) + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + begin + SourceRecordRef.Close(); + SourceRecordRef.Open(IntegrationTableId, true); + if SystemIds.Count() = 0 then + exit; + Transport := GetTransport(); + ParseOrError( + IntegrationTableId, + Transport.GetRecords(IntegrationTableId, FieldIds, SystemIdsSelector(SystemIds), PageSize()), + Response); + SourceResponse.InsertRecords(Response, SourceRecordRef); + end; + + local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) + var + UnavailableFields: JsonArray; + begin + Clear(Response); + if not SourceResponse.TryParse(ResponseText, Response) then + Error(InvalidResponseErr, TableCaption(IntegrationTableId)); + if not SourceResponse.TableAvailable(Response) then + Error(TableUnavailableErr, TableCaption(IntegrationTableId)); + if not SourceResponse.Indexed(Response) then + Error(NotIndexedErr, TableCaption(IntegrationTableId)); + if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then + Error(FieldsUnavailableErr, TableCaption(IntegrationTableId)); + end; + + // FieldIds = the mapping's integration-side fields + the table's primary-key fields (so temp inserts don't collide). + local procedure BuildFieldIds(IntegrationTableMapping: Record "Integration Table Mapping"): Text + var + IntegrationFieldMapping: Record "Integration Field Mapping"; + FieldIds: JsonArray; + AddedFields: List of [Integer]; + begin + AddPrimaryKeyFields(IntegrationTableMapping."Integration Table ID", FieldIds, AddedFields); + IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); + if IntegrationFieldMapping.FindSet() then + repeat + if IntegrationFieldMapping."Integration Table Field No." <> 0 then + AddFieldId(FieldIds, AddedFields, IntegrationFieldMapping."Integration Table Field No."); + until IntegrationFieldMapping.Next() = 0; + exit(WriteArray(FieldIds)); + end; + + local procedure BuildFieldIdsForTable(IntegrationTableId: Integer): Text + var + IntegrationTableMapping: Record "Integration Table Mapping"; + begin + if GetMappingByIntegrationTableId(IntegrationTableId, IntegrationTableMapping) then + exit(BuildFieldIds(IntegrationTableMapping)); + exit(AllNormalFields(IntegrationTableId)); + end; + + local procedure GetMappingByIntegrationTableId(IntegrationTableId: Integer; var IntegrationTableMapping: Record "Integration Table Mapping"): Boolean + begin + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Integration Table ID", IntegrationTableId); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + exit(IntegrationTableMapping.FindFirst()); + end; + + local procedure AddPrimaryKeyFields(IntegrationTableId: Integer; var FieldIds: JsonArray; var AddedFields: List of [Integer]) + var + RecRef: RecordRef; + PrimaryKeyRef: KeyRef; + Index: Integer; + begin + RecRef.Open(IntegrationTableId, true); + PrimaryKeyRef := RecRef.KeyIndex(1); + for Index := 1 to PrimaryKeyRef.FieldCount() do + AddFieldId(FieldIds, AddedFields, PrimaryKeyRef.FieldIndex(Index).Number()); + RecRef.Close(); + end; + + local procedure AllNormalFields(IntegrationTableId: Integer): Text + var + RecRef: RecordRef; + FieldIds: JsonArray; + AddedFields: List of [Integer]; + CurrentField: FieldRef; + Index: Integer; + begin + RecRef.Open(IntegrationTableId, true); + for Index := 1 to RecRef.FieldCount() do begin + CurrentField := RecRef.FieldIndex(Index); + if CurrentField.Class() = FieldClass::Normal then + if not (CurrentField.Type() in [FieldType::Blob, FieldType::Media, FieldType::MediaSet]) then + AddFieldId(FieldIds, AddedFields, CurrentField.Number()); + end; + RecRef.Close(); + exit(WriteArray(FieldIds)); + end; + + local procedure AddFieldId(var FieldIds: JsonArray; var AddedFields: List of [Integer]; FieldNo: Integer) + begin + if AddedFields.Contains(FieldNo) then + exit; + AddedFields.Add(FieldNo); + FieldIds.Add(FieldNo); + end; + + local procedure ParseSystemIds(UidFilter: Text) SystemIds: List of [Guid] + var + Token: Text; + SystemIdValue: Guid; + begin + foreach Token in UidFilter.Split('|') do + if Evaluate(SystemIdValue, Token) then + SystemIds.Add(SystemIdValue); + end; + + local procedure SystemIdsSelector(SystemIds: List of [Guid]): Text + var + Selector: JsonObject; + SystemIdArray: JsonArray; + SystemIdValue: Guid; + SelectorText: Text; + begin + foreach SystemIdValue in SystemIds do + SystemIdArray.Add(Format(SystemIdValue)); + Selector.Add('systemIds', SystemIdArray); + Selector.WriteTo(SelectorText); + exit(SelectorText); + end; + + local procedure CursorSelector(Watermark: DateTime): Text + var + Selector: JsonObject; + SelectorText: Text; + begin + if Watermark = 0DT then + exit('{}'); + Selector.Add('modifiedAt', Format(Watermark, 0, 9)); + Selector.WriteTo(SelectorText); + exit(SelectorText); + end; + + local procedure WriteArray(JsonArrayValue: JsonArray) ResultText: Text + begin + JsonArrayValue.WriteTo(ResultText); + end; + + local procedure TableCaption(IntegrationTableId: Integer) Caption: Text + var + RecRef: RecordRef; + begin + RecRef.Open(IntegrationTableId, true); + Caption := RecRef.Caption(); + RecRef.Close(); + end; + + local procedure PageSize(): Integer + begin + exit(1000); + end; + + local procedure GetTransport() Transport: Interface "IMDM Source Transport" + var + HttpTransport: Codeunit "MDM Http Source Transport"; + begin + Transport := HttpTransport; + OnResolveSourceTransport(Transport); + end; + + [InternalEvent(false)] + local procedure OnResolveSourceTransport(var Transport: Interface "IMDM Source Transport") + begin + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index aa62f86ba60..edfaa2c5ad9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -10,9 +10,6 @@ codeunit 7241 "MDM Cross-Env Source API" { Access = Public; - var - NotYetImplementedErr: Label 'This master data management source action is not yet available.'; - /// /// Wire-version negotiation: returns the API contract version and the action/feature names this source /// supports, so a newer subsidiary only calls actions an older source actually implements. @@ -33,28 +30,438 @@ codeunit 7241 "MDM Cross-Env Source API" end; /// - /// Returns a page of changed source records for the given table. Selector is either a change-feed - /// cursor { modifiedAt, systemId, pageSize } or a targeted { systemIds } list. Response carries the - /// records, nextCursor and hasMore. Only tables/fields in a configured mapping are served. + /// Returns a page of changed source records for the given table. FieldIds is a JSON array of field + /// numbers. Selector is either a change-feed cursor { modifiedAt, systemId } or a targeted + /// { systemIds } list. Response carries records, hasMore and (cursor mode) nextCursor. The table read + /// runs under the CALLER's permission set, so table-level access is enforced by permissions, not here. /// [ServiceEnabled] procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + var + RecRef: RecordRef; + Response: JsonObject; + Records: JsonArray; + UnavailableFields: JsonArray; + SystemIds: JsonArray; + ProjectedFields: List of [Integer]; + CursorModifiedAt: DateTime; + NextModifiedAt: DateTime; + CursorSystemId: Guid; + NextSystemId: Guid; + HasCursor: Boolean; + HasMore: Boolean; + GroupTooLarge: Boolean; + Count: Integer; + ResultText: Text; + begin + Response.Add('tableId', TableId); + + if not TryOpenTable(TableId, RecRef) then + exit(WriteResponse(Response, false, Records, false)); + Response.Add('tableAvailable', true); + + // Missing field => HALT that table's sync (no partial records); subsidiary logs a synch error. + // Fields that exist but are media/blob/flow are skipped silently (media sync is deferred). + ResolveProjection(RecRef, FieldIds, ProjectedFields, UnavailableFields); + if UnavailableFields.Count() > 0 then begin + Response.Add('unavailableFields', UnavailableFields); + exit(WriteResponse(Response, true, Records, false)); + end; + + PageSize := ClampPageSize(PageSize); + + // Targeted mode: caller asked for specific SystemIds (no paging). + if SelectorSystemIds(Selector, SystemIds) then begin + FillBySystemIds(RecRef, SystemIds, ProjectedFields, Records); + exit(WriteResponse(Response, true, Records, false)); + end; + + HasCursor := SelectorCursor(Selector, CursorModifiedAt, CursorSystemId); + + if HasCompositeChangeFeedKey(RecRef) then begin + // Bounded paging: the (SystemModifiedAt, SystemId) key lets us split even a big same-timestamp group. + RecRef.SetView(StrSubstNo('SORTING(Field%1,Field%2)', SystemModifiedAtFieldNo(), SystemIdFieldNo())); + HasMore := FillCursorPage(RecRef, HasCursor, CursorModifiedAt, CursorSystemId, ProjectedFields, PageSize, Records, Count, NextModifiedAt, NextSystemId); + if Count > 0 then + Response.Add('nextCursor', BuildCursor(NextModifiedAt, NextSystemId)); + end else + if HasModifiedAtLeadingKey(RecRef) then begin + // Fallback for tables without the SystemId tiebreak (kept off small/setup tables): drain each + // timestamp group whole so the cursor can advance by SystemModifiedAt alone. Safe while groups + // are small; a group too large to page keylessly asks for the composite key instead. + RecRef.SetView(StrSubstNo('SORTING(Field%1)', SystemModifiedAtFieldNo())); + HasMore := FillDrainPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, PageSize, Records, Count, NextModifiedAt, GroupTooLarge); + if GroupTooLarge then begin + Clear(Records); + Response.Add('indexed', false); + exit(WriteResponse(Response, true, Records, false)); + end; + if Count > 0 then + Response.Add('nextCursor', BuildModifiedAtCursor(NextModifiedAt)); + end else begin + // No SystemModifiedAt index at all: no-code scan fallback. Order by primary key (always indexed), + // filter SystemModifiedAt > watermark, and return the whole changed set in one shot (capped). + // Over the cap, ask for the composite key (only large unindexed tables need it). + HasMore := FillScanPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, Records, Count, NextModifiedAt, GroupTooLarge); + if GroupTooLarge then begin + Clear(Records); + Response.Add('indexed', false); + exit(WriteResponse(Response, true, Records, false)); + end; + if Count > 0 then + Response.Add('nextCursor', BuildModifiedAtCursor(NextModifiedAt)); + end; + + Response.Add('records', Records); + Response.Add('hasMore', HasMore); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + local procedure WriteResponse(var Response: JsonObject; TableAvailable: Boolean; var Records: JsonArray; HasMore: Boolean): Text + var + ResultText: Text; begin - // TODO(cross-env): authorize (calling app + mapping scope), project FieldIds, page by the composite - // cursor (SystemModifiedAt, SystemId), serialize field types, and report unavailable table/fields. - Error(NotYetImplementedErr); + if not Response.Contains('tableAvailable') then + Response.Add('tableAvailable', TableAvailable); + Response.Add('records', Records); + Response.Add('hasMore', HasMore); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + [TryFunction] + local procedure TryOpenTable(TableId: Integer; var RecRef: RecordRef) + begin + RecRef.Open(TableId); + end; + + local procedure ResolveProjection(var RecRef: RecordRef; FieldIds: Text; var ProjectedFields: List of [Integer]; var UnavailableFields: JsonArray) + var + RequestedFields: JsonArray; + Token: JsonToken; + FieldNo: Integer; + begin + if not TryReadJsonArray(FieldIds, RequestedFields) then + exit; + foreach Token in RequestedFields do begin + FieldNo := Token.AsValue().AsInteger(); + if not RecRef.FieldExist(FieldNo) then + UnavailableFields.Add(FieldNo) + else + if IsProjectableField(RecRef.Field(FieldNo)) then + ProjectedFields.Add(FieldNo); + end; + end; + + local procedure IsProjectableField(FieldReference: FieldRef): Boolean + begin + exit((FieldReference.Class() = FieldClass::Normal) and + not (FieldReference.Type() in [FieldType::Blob, FieldType::Media, FieldType::MediaSet])); + end; + + local procedure SelectorSystemIds(Selector: Text; var SystemIds: JsonArray): Boolean + var + SelectorObject: JsonObject; + Token: JsonToken; + begin + if not TryReadJsonObject(Selector, SelectorObject) then + exit(false); + if not SelectorObject.Get('systemIds', Token) then + exit(false); + if not Token.IsArray() then + exit(false); + SystemIds := Token.AsArray(); + exit(SystemIds.Count() > 0); + end; + + local procedure SelectorCursor(Selector: Text; var CursorModifiedAt: DateTime; var CursorSystemId: Guid): Boolean + var + SelectorObject: JsonObject; + Token: JsonToken; + begin + if not TryReadJsonObject(Selector, SelectorObject) then + exit(false); + if not SelectorObject.Get('modifiedAt', Token) then + exit(false); + if not Evaluate(CursorModifiedAt, Token.AsValue().AsText(), 9) then + exit(false); + if SelectorObject.Get('systemId', Token) then + Evaluate(CursorSystemId, Token.AsValue().AsText()); + exit(true); + end; + + local procedure HasCompositeChangeFeedKey(var RecRef: RecordRef): Boolean + var + CurrentKey: KeyRef; + Index: Integer; + begin + for Index := 1 to RecRef.KeyCount() do begin + CurrentKey := RecRef.KeyIndex(Index); + if CurrentKey.FieldCount() >= 2 then + if (CurrentKey.FieldIndex(1).Number() = SystemModifiedAtFieldNo()) and + (CurrentKey.FieldIndex(2).Number() = SystemIdFieldNo()) + then + exit(true); + end; + exit(false); + end; + + local procedure HasModifiedAtLeadingKey(var RecRef: RecordRef): Boolean + var + CurrentKey: KeyRef; + Index: Integer; + begin + for Index := 1 to RecRef.KeyCount() do begin + CurrentKey := RecRef.KeyIndex(Index); + if CurrentKey.FieldCount() >= 1 then + if CurrentKey.FieldIndex(1).Number() = SystemModifiedAtFieldNo() then + exit(true); + end; + exit(false); + end; + + // No SystemId tiebreak available, so never split a timestamp group across pages: fill to PageSize, then + // drain the trailing group whole and advance the cursor by SystemModifiedAt with a strict '>'. + local procedure FillDrainPage(var RecRef: RecordRef; HasCursor: Boolean; CursorModifiedAt: DateTime; ProjectedFields: List of [Integer]; PageSize: Integer; var Records: JsonArray; var Count: Integer; var NextModifiedAt: DateTime; var GroupTooLarge: Boolean): Boolean + var + ModifiedAtRef: FieldRef; + CurrentModifiedAt: DateTime; + LastEmittedAt: DateTime; + IgnoredSystemId: Guid; + MaxKeylessGroup: Integer; + begin + Count := 0; + GroupTooLarge := false; + MaxKeylessGroup := 10000; + ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); + if HasCursor then + ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); + if RecRef.FindSet() then + repeat + CurrentModifiedAt := ModifiedAtRef.Value(); + // Stop only at a clean group boundary once the page is full. + if (Count >= PageSize) and (CurrentModifiedAt <> LastEmittedAt) then + exit(true); + if Count >= MaxKeylessGroup then begin + GroupTooLarge := true; + exit(false); + end; + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, IgnoredSystemId); + LastEmittedAt := NextModifiedAt; + Count += 1; + until RecRef.Next() = 0; + exit(false); + end; + + // No SystemModifiedAt index at all: order by primary key (always indexed) and filter SystemModifiedAt > + // watermark. Single-shot up to a cap; a bigger changed set trips TooLarge so the caller asks for a key. + local procedure FillScanPage(var RecRef: RecordRef; HasCursor: Boolean; CursorModifiedAt: DateTime; ProjectedFields: List of [Integer]; var Records: JsonArray; var Count: Integer; var MaxModifiedAt: DateTime; var TooLarge: Boolean): Boolean + var + ModifiedAtRef: FieldRef; + CurrentModifiedAt: DateTime; + IgnoredModifiedAt: DateTime; + IgnoredSystemId: Guid; + MaxUnindexedRecords: Integer; + begin + Count := 0; + TooLarge := false; + MaxUnindexedRecords := 10000; + ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); + if HasCursor then + ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); + if RecRef.FindSet() then + repeat + if Count >= MaxUnindexedRecords then begin + TooLarge := true; + exit(false); + end; + CurrentModifiedAt := ModifiedAtRef.Value(); + if CurrentModifiedAt > MaxModifiedAt then + MaxModifiedAt := CurrentModifiedAt; + AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId); + Count += 1; + until RecRef.Next() = 0; + exit(false); + end; + + local procedure FillCursorPage(var RecRef: RecordRef; HasCursor: Boolean; CursorModifiedAt: DateTime; CursorSystemId: Guid; ProjectedFields: List of [Integer]; PageSize: Integer; var Records: JsonArray; var Count: Integer; var NextModifiedAt: DateTime; var NextSystemId: Guid): Boolean + var + ModifiedAtRef: FieldRef; + SystemIdRef: FieldRef; + begin + Count := 0; + ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); + SystemIdRef := RecRef.Field(SystemIdFieldNo()); + + // Pass 1: records at exactly the cursor timestamp but a later SystemId (DB uniqueidentifier order). + if HasCursor then begin + ModifiedAtRef.SetRange(CursorModifiedAt); + SystemIdRef.SetFilter('>%1', CursorSystemId); + if RecRef.FindSet() then + repeat + if Count = PageSize then + exit(true); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId); + Count += 1; + until RecRef.Next() = 0; + ModifiedAtRef.SetRange(); + SystemIdRef.SetRange(); + end; + + // Pass 2: records strictly after the cursor timestamp (or all records on the first call). + if HasCursor then + ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); + if RecRef.FindSet() then + repeat + if Count = PageSize then + exit(true); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId); + Count += 1; + until RecRef.Next() = 0; + + exit(false); + end; + + local procedure FillBySystemIds(var RecRef: RecordRef; SystemIds: JsonArray; ProjectedFields: List of [Integer]; var Records: JsonArray) + var + SystemIdRef: FieldRef; + Token: JsonToken; + SystemIdValue: Guid; + IgnoredModifiedAt: DateTime; + IgnoredSystemId: Guid; + FilterText: Text; + begin + foreach Token in SystemIds do + if Evaluate(SystemIdValue, Token.AsValue().AsText()) then begin + if FilterText <> '' then + FilterText += '|'; + FilterText += Format(SystemIdValue); + end; + if FilterText = '' then + exit; + + SystemIdRef := RecRef.Field(SystemIdFieldNo()); + SystemIdRef.SetFilter(FilterText); + if RecRef.FindSet() then + repeat + AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId); + until RecRef.Next() = 0; + end; + + local procedure AppendRecord(var RecRef: RecordRef; ProjectedFields: List of [Integer]; var Records: JsonArray; var LastModifiedAt: DateTime; var LastSystemId: Guid) + var + RecordObject: JsonObject; + FieldsObject: JsonObject; + FieldNo: Integer; + begin + LastModifiedAt := RecRef.Field(SystemModifiedAtFieldNo()).Value(); + LastSystemId := RecRef.Field(SystemIdFieldNo()).Value(); + RecordObject.Add('systemId', Format(LastSystemId)); + RecordObject.Add('systemModifiedAt', FormatFieldValue(RecRef.Field(SystemModifiedAtFieldNo()))); + foreach FieldNo in ProjectedFields do + FieldsObject.Add(Format(FieldNo), FormatFieldValue(RecRef.Field(FieldNo))); + RecordObject.Add('fields', FieldsObject); + Records.Add(RecordObject); + end; + + local procedure BuildCursor(ModifiedAt: DateTime; SystemId: Guid): JsonObject + var + Cursor: JsonObject; + begin + Cursor.Add('modifiedAt', Format(ModifiedAt, 0, 9)); + Cursor.Add('systemId', Format(SystemId)); + exit(Cursor); + end; + + local procedure BuildModifiedAtCursor(ModifiedAt: DateTime): JsonObject + var + Cursor: JsonObject; + begin + Cursor.Add('modifiedAt', Format(ModifiedAt, 0, 9)); + exit(Cursor); + end; + + // Invariant (XML) format so field values round-trip via Evaluate(..., 9) on the subsidiary. + local procedure FormatFieldValue(FieldReference: FieldRef): Text + begin + exit(Format(FieldReference.Value(), 0, 9)); + end; + + [TryFunction] + local procedure TryReadJsonArray(Value: Text; var JsonArrayValue: JsonArray) + begin + JsonArrayValue.ReadFrom(Value); + end; + + [TryFunction] + local procedure TryReadJsonObject(Value: Text; var JsonObjectValue: JsonObject) + begin + JsonObjectValue.ReadFrom(Value); + end; + + local procedure ClampPageSize(PageSize: Integer): Integer + begin + if PageSize <= 0 then + exit(100); + if PageSize > 1000 then + exit(1000); + exit(PageSize); + end; + + local procedure SystemIdFieldNo(): Integer + begin + exit(2000000000); + end; + + local procedure SystemModifiedAtFieldNo(): Integer + begin + exit(2000000003); end; /// /// Change detection: for each requested table id, returns its latest source modification timestamp so - /// the subsidiary detector can decide which per-table sync jobs to reschedule. + /// the subsidiary detector can decide which per-table sync jobs to reschedule. Read from the change-feed + /// index tip (FindLast), so no scan and no summary table to maintain. /// [ServiceEnabled] procedure LastModifiedAtPerTable(TableIds: Text): Text + var + RequestedTables: JsonArray; + Tables: JsonArray; + Response: JsonObject; + Token: JsonToken; + ResultText: Text; + begin + if TryReadJsonArray(TableIds, RequestedTables) then + foreach Token in RequestedTables do + Tables.Add(BuildTableModifiedAt(Token.AsValue().AsInteger())); + Response.Add('tables', Tables); + Response.WriteTo(ResultText); + exit(ResultText); + end; + + local procedure BuildTableModifiedAt(TableId: Integer): JsonObject + var + RecRef: RecordRef; + Entry: JsonObject; begin - // TODO(cross-env): read the trigger-maintained (TableId, LastModifiedAt) summary table (not yet built) - // rather than scanning; return [{ tableId, lastModifiedAt }]. - Error(NotYetImplementedErr); + Entry.Add('tableId', TableId); + if not TryOpenTable(TableId, RecRef) then begin + Entry.Add('tableAvailable', false); + exit(Entry); + end; + Entry.Add('tableAvailable', true); + if not HasModifiedAtLeadingKey(RecRef) then begin + Entry.Add('indexed', false); + exit(Entry); + end; + RecRef.SetView(StrSubstNo('SORTING(Field%1)', SystemModifiedAtFieldNo())); + if RecRef.FindLast() then + Entry.Add('lastModifiedAt', Format(RecRef.Field(SystemModifiedAtFieldNo()).Value(), 0, 9)) + else + Entry.Add('lastModifiedAt', ''); // empty table: no changes to detect + exit(Entry); end; local procedure ApiVersion(): Integer diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al new file mode 100644 index 00000000000..57ef2d95053 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -0,0 +1,31 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Production transport: calls the source environment's ODataV4 web service with an app-only (client +/// credentials) token. The OAuth2 + HttpClient body is implemented in a follow-up; until then it fails loudly +/// so a misconfigured environment is obvious. Tests never hit this — they inject an in-process transport. +/// +codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" +{ + Access = Internal; + + var + NotConfiguredErr: Label 'The cross-environment connection to the source is not configured yet.'; + + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + begin + // TODO(cross-env): OAuth2 client-credentials token (creds from Isolated Storage) + HttpClient POST to + // {sourceUrl}/ODataV4/{service}_GetRecords?company={company}; return the response body. + Error(NotConfiguredErr); + end; + + procedure LastModifiedAtPerTable(TableIds: Text): Text + begin + Error(NotConfiguredErr); + end; + + procedure GetCapabilities(): Text + begin + Error(NotConfiguredErr); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al new file mode 100644 index 00000000000..3b09c63c678 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -0,0 +1,177 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Parses a GetRecords/LastModifiedAtPerTable JSON response from the source and materializes records into a +/// temporary RecordRef, so the existing synchronization engine can read them as if they were local. Also +/// exposes the response's control fields (tableAvailable / indexed / unavailableFields / hasMore / nextCursor) +/// so the caller can raise the right synchronization error. +/// +codeunit 7248 "MDM Source Response" +{ + Access = Internal; + + [TryFunction] + procedure TryParse(ResponseText: Text; var Response: JsonObject) + begin + Response.ReadFrom(ResponseText); + end; + + procedure TableAvailable(var Response: JsonObject): Boolean + var + Token: JsonToken; + begin + if Response.Get('tableAvailable', Token) then + exit(Token.AsValue().AsBoolean()); + exit(true); + end; + + procedure Indexed(var Response: JsonObject): Boolean + var + Token: JsonToken; + begin + // 'indexed' is only emitted when false (a too-large unindexed/keyless table). + if Response.Get('indexed', Token) then + exit(Token.AsValue().AsBoolean()); + exit(true); + end; + + procedure GetUnavailableFields(var Response: JsonObject; var UnavailableFields: JsonArray): Boolean + var + Token: JsonToken; + begin + if not Response.Get('unavailableFields', Token) then + exit(false); + if not Token.IsArray() then + exit(false); + UnavailableFields := Token.AsArray(); + exit(UnavailableFields.Count() > 0); + end; + + procedure HasMore(var Response: JsonObject): Boolean + var + Token: JsonToken; + begin + if Response.Get('hasMore', Token) then + exit(Token.AsValue().AsBoolean()); + exit(false); + end; + + // The nextCursor object, re-serialized so the caller can pass it straight back as the next Selector. + procedure GetNextCursor(var Response: JsonObject): Text + var + Token: JsonToken; + CursorText: Text; + begin + if not Response.Get('nextCursor', Token) then + exit(''); + Token.WriteTo(CursorText); + exit(CursorText); + end; + + // Inserts the response's records into TempSourceRecordRef (already opened temporary on the table). + // Returns the number inserted. Requires the caller to have requested the primary-key fields so the + // temporary inserts don't collide. + procedure InsertRecords(var Response: JsonObject; var TempSourceRecordRef: RecordRef): Integer + var + RecordsToken: JsonToken; + RecordToken: JsonToken; + RecordsArray: JsonArray; + Count: Integer; + begin + if not Response.Get('records', RecordsToken) then + exit(0); + if not RecordsToken.IsArray() then + exit(0); + RecordsArray := RecordsToken.AsArray(); + foreach RecordToken in RecordsArray do begin + InsertRecord(RecordToken.AsObject(), TempSourceRecordRef); + Count += 1; + end; + exit(Count); + end; + + local procedure InsertRecord(RecordObject: JsonObject; var TempSourceRecordRef: RecordRef) + var + FieldsToken: JsonToken; + ValueToken: JsonToken; + FieldsObject: JsonObject; + DestField: FieldRef; + FieldName: Text; + SystemIdValue: Guid; + FieldNo: Integer; + begin + TempSourceRecordRef.Init(); + if RecordObject.Get('fields', FieldsToken) then begin + FieldsObject := FieldsToken.AsObject(); + foreach FieldName in FieldsObject.Keys() do + if Evaluate(FieldNo, FieldName) then + if TempSourceRecordRef.FieldExist(FieldNo) then begin + FieldsObject.Get(FieldName, ValueToken); + DestField := TempSourceRecordRef.Field(FieldNo); + SetFieldFromText(DestField, ValueToken.AsValue().AsText()); + end; + end; + if GetGuid(RecordObject, 'systemId', SystemIdValue) then + TempSourceRecordRef.Field(TempSourceRecordRef.SystemIdNo()).Value := SystemIdValue; + TempSourceRecordRef.Insert(false); + end; + + // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. + local procedure SetFieldFromText(var DestField: FieldRef; ValueText: Text) + var + IntegerValue: Integer; + BigIntegerValue: BigInteger; + DecimalValue: Decimal; + BooleanValue: Boolean; + DateValue: Date; + TimeValue: Time; + DateTimeValue: DateTime; + DurationValue: Duration; + DateFormulaValue: DateFormula; + GuidValue: Guid; + begin + case DestField.Type() of + FieldType::Text, FieldType::Code: + DestField.Value := CopyStr(ValueText, 1, DestField.Length()); + FieldType::Integer, FieldType::Option: + if Evaluate(IntegerValue, ValueText, 9) then + DestField.Value := IntegerValue; + FieldType::BigInteger: + if Evaluate(BigIntegerValue, ValueText, 9) then + DestField.Value := BigIntegerValue; + FieldType::Decimal: + if Evaluate(DecimalValue, ValueText, 9) then + DestField.Value := DecimalValue; + FieldType::Boolean: + if Evaluate(BooleanValue, ValueText, 9) then + DestField.Value := BooleanValue; + FieldType::Date: + if Evaluate(DateValue, ValueText, 9) then + DestField.Value := DateValue; + FieldType::Time: + if Evaluate(TimeValue, ValueText, 9) then + DestField.Value := TimeValue; + FieldType::DateTime: + if Evaluate(DateTimeValue, ValueText, 9) then + DestField.Value := DateTimeValue; + FieldType::Duration: + if Evaluate(DurationValue, ValueText, 9) then + DestField.Value := DurationValue; + FieldType::DateFormula: + if Evaluate(DateFormulaValue, ValueText, 9) then + DestField.Value := DateFormulaValue; + FieldType::Guid: + if Evaluate(GuidValue, ValueText) then + DestField.Value := GuidValue; + end; + end; + + local procedure GetGuid(var Container: JsonObject; PropertyName: Text; var Value: Guid): Boolean + var + Token: JsonToken; + begin + if not Container.Get(PropertyName, Token) then + exit(false); + exit(Evaluate(Value, Token.AsValue().AsText())); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al b/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al index 2128b5affbb..ee50e8e584d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al +++ b/src/Apps/W1/MasterDataManagement/app/src/enums/MDMDataSourceType.Enum.al @@ -14,4 +14,10 @@ enum 7239 "MDM Data Source Type" implements "IMDM Data Source" Caption = 'Local Company'; Implementation = "IMDM Data Source" = "MDM Local Data Source"; } + + value(1; CrossEnvironment) + { + Caption = 'Cross Environment'; + Implementation = "IMDM Data Source" = "MDM Cross-Env Data Source"; + } } diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al new file mode 100644 index 00000000000..1373efe9690 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al @@ -0,0 +1,17 @@ +namespace Microsoft.Integration.MDM; + +/// +/// The wire contract the cross-environment data source calls on the source's ODataV4 web service. One method +/// per unbound action; JSON in, JSON out. Kept as a seam so the HTTP/OAuth transport can be swapped for an +/// in-process transport in tests (the source and subsidiary run in the same environment there). +/// +interface "IMDM Source Transport" +{ + Access = Internal; + + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text; + + procedure LastModifiedAtPerTable(TableIds: Text): Text; + + procedure GetCapabilities(): Text; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al new file mode 100644 index 00000000000..00e1a1f7451 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMContact.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.CRM.Contact; + +tableextension 7246 "MDM Contact" extends Contact +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al new file mode 100644 index 00000000000..c748d9ecef9 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCurrencyExchRate.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Finance.Currency; + +tableextension 7251 "MDM Currency Exch. Rate" extends "Currency Exchange Rate" +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al new file mode 100644 index 00000000000..2910298f215 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMCustomer.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Sales.Customer; + +tableextension 7244 "MDM Customer" extends Customer +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al new file mode 100644 index 00000000000..74459e3dc5e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMPostCode.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Foundation.Address; + +tableextension 7250 "MDM Post Code" extends "Post Code" +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al new file mode 100644 index 00000000000..db87851e6e7 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MDMVendor.TableExt.al @@ -0,0 +1,14 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Purchases.Vendor; + +tableextension 7245 "MDM Vendor" extends Vendor +{ + keys + { + // Change-feed order for cross-environment paging (SystemModifiedAt seek + SystemId tiebreak). + key(MDMChangeFeed; SystemModifiedAt, SystemId) + { + } + } +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 12459e82cee..87b65f63be7 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -253,7 +253,7 @@ table 7230 "Master Data Management Setup" internal procedure GetDataSource(): Interface "IMDM Data Source" begin if "Source Environment Name" <> '' then - Error(CrossEnvNotYetSupportedErr); + exit(Enum::"MDM Data Source Type"::CrossEnvironment); exit(Enum::"MDM Data Source Type"::LocalCompany); end; @@ -264,5 +264,4 @@ table 7230 "Master Data Management Setup" MustNotPickCurrentCompanyErr: label 'You are currently signed into this company. \\Choose a different company to synchronize data with.'; MustPickSourceCompanyErr: label 'You must choose a source company to synchronize data from.'; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; - CrossEnvNotYetSupportedErr: label 'Cross-environment synchronization is not yet available.'; } From 7b0fa7f5a0af5a346edbc08bbb3f094da5b15a3d Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 27 Aug 2026 11:28:42 +0200 Subject: [PATCH 08/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 2 + .../MDMCrossEnvChangeDetector.Codeunit.al | 209 +++++++++++++++ .../MDMCrossEnvDataSource.Codeunit.al | 12 +- .../MDMCrossEnvSourceAPI.Codeunit.al | 3 + .../codeunits/MDMSourceConnection.Codeunit.al | 23 ++ .../MasterDataMgtSetupDefault.Codeunit.al | 47 ++++ .../tables/MasterDataManagementSetup.Table.al | 10 + .../test library/app.json | 10 + .../src/LibraryMasterDataMgt.Codeunit.al | 25 ++ .../src/MDMInProcessTransport.Codeunit.al | 68 +++++ .../src/MDMTestDetectorProbe.Codeunit.al | 44 ++++ .../W1/MasterDataManagement/test/app.json | 14 + .../src/MDMCrossEnvConsumerTests.Codeunit.al | 196 ++++++++++++++ .../src/MDMCrossEnvDetectorTests.Codeunit.al | 243 ++++++++++++++++++ .../src/MDMCrossEnvSourceTests.Codeunit.al | 194 ++++++++++++++ .../src/MasterDataMgtSynchTests.Codeunit.al | 6 +- 16 files changed, 1094 insertions(+), 12 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 1fd896ed926..a57c648fc49 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -22,6 +22,8 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "MDM Source Response" = X, codeunit "MDM Http Source Transport" = X, codeunit "MDM Cross-Env Data Source" = X, + codeunit "MDM Source Connection" = X, + codeunit "MDM Cross-Env Change Detector" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al new file mode 100644 index 00000000000..ab4f8554059 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -0,0 +1,209 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.Integration.SyncEngine; +using System.Environment; +using System.Threading; + +/// +/// Cross-environment change detector. One recurring job on the subsidiary polls the source's +/// LastModifiedAtPerTable for the tables it synchronizes and, for those changed since the mapping's watermark, +/// nudges that table's synchronization job to run now. A job already In Process is left alone. +/// +codeunit 7245 "MDM Cross-Env Change Detector" +{ + Access = Internal; + Permissions = tabledata "Master Data Management Setup" = r, + tabledata "Integration Table Mapping" = r, + tabledata "Job Queue Entry" = rm, + tabledata "Scheduled Task" = r; + + trigger OnRun() + begin + DetectChanges(); + end; + + internal procedure DetectChanges() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + SourceConnection: Codeunit "MDM Source Connection"; + SourceResponse: Codeunit "MDM Source Response"; + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + TableIds: JsonArray; + begin + if not MasterDataManagementSetup.Get() then + exit; + if not MasterDataManagementSetup."Is Enabled" then + exit; + if MasterDataManagementSetup."Source Environment Name" = '' then + exit; // detector is cross-environment only + + if not CollectSynchronizedTableIds(TableIds) then + exit; + + Transport := SourceConnection.GetTransport(); + if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(WriteArray(TableIds)), Response) then + exit; + + ProcessDetectionResponse(Response); + end; + + local procedure CollectSynchronizedTableIds(var TableIds: JsonArray): Boolean + var + IntegrationTableMapping: Record "Integration Table Mapping"; + AddedTables: List of [Integer]; + begin + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); + if not IntegrationTableMapping.FindSet() then + exit(false); + repeat + if not AddedTables.Contains(IntegrationTableMapping."Integration Table ID") then begin + AddedTables.Add(IntegrationTableMapping."Integration Table ID"); + TableIds.Add(IntegrationTableMapping."Integration Table ID"); + end; + until IntegrationTableMapping.Next() = 0; + exit(TableIds.Count() > 0); + end; + + local procedure ProcessDetectionResponse(var Response: JsonObject) + var + Tables: JsonArray; + TablesToken: JsonToken; + EntryToken: JsonToken; + begin + if not Response.Get('tables', TablesToken) then + exit; + if not TablesToken.IsArray() then + exit; + Tables := TablesToken.AsArray(); + foreach EntryToken in Tables do + ProcessTableEntry(EntryToken.AsObject()); + end; + + local procedure ProcessTableEntry(Entry: JsonObject) + var + IntegrationTableMapping: Record "Integration Table Mapping"; + LastModifiedAt: DateTime; + TableId: Integer; + HasTimestamp: Boolean; + begin + TableId := GetInteger(Entry, 'tableId'); + if TableId = 0 then + exit; + if not GetBoolean(Entry, 'tableAvailable', true) then + exit; // the sync job itself will report the unavailability + + // No timestamp (indexed:false or empty table): can't compare cheaply, so let the sync job poll (scan). + HasTimestamp := GetDateTime(Entry, 'lastModifiedAt', LastModifiedAt); + + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); + IntegrationTableMapping.SetRange("Delete After Synchronization", false); + IntegrationTableMapping.SetRange("Integration Table ID", TableId); + IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); + if not IntegrationTableMapping.FindSet() then + exit; + repeat + if (not HasTimestamp) or (LastModifiedAt > IntegrationTableMapping."Synch. Modified On Filter") then + NudgeSynchJob(IntegrationTableMapping); + until IntegrationTableMapping.Next() = 0; + end; + + local procedure NudgeSynchJob(IntegrationTableMapping: Record "Integration Table Mapping") + var + JobQueueEntry: Record "Job Queue Entry"; + IsHandled: Boolean; + begin + // In Process / Error / missing jobs are left alone (FindIdleSynchJob returns only idle jobs). + if not FindIdleSynchJob(IntegrationTableMapping, JobQueueEntry) then + exit; + + // Seam so tests can observe the decision and skip the reschedule (jobs can't be scheduled in the test lab). + OnBeforeRescheduleSynchJob(JobQueueEntry, IntegrationTableMapping, IsHandled); + if IsHandled then + exit; + + RescheduleSynchJobNow(JobQueueEntry); + end; + + local procedure FindIdleSynchJob(IntegrationTableMapping: Record "Integration Table Mapping"; var JobQueueEntry: Record "Job Queue Entry"): Boolean + begin + JobQueueEntry.ReadIsolation := IsolationLevel::ReadUncommitted; + JobQueueEntry.SetLoadFields(Status, "System Task ID"); + JobQueueEntry.SetRange("Object Type to Run", JobQueueEntry."Object Type to Run"::Codeunit); + JobQueueEntry.SetRange("Object ID to Run", Codeunit::"Integration Synch. Job Runner"); + JobQueueEntry.SetRange("Record ID to Process", IntegrationTableMapping.RecordId()); + JobQueueEntry.SetRange("Recurring Job", true); + // Only idle jobs. In Process / Error / plain On Hold are excluded, so a running job is left alone. + JobQueueEntry.SetFilter(Status, '%1|%2', JobQueueEntry.Status::Ready, JobQueueEntry.Status::"On Hold with Inactivity Timeout"); + exit(JobQueueEntry.FindFirst()); + end; + + local procedure RescheduleSynchJobNow(JobQueueEntry: Record "Job Queue Entry") + var + JobQueueEntryUpdate: Record "Job Queue Entry"; + ScheduledTask: Record "Scheduled Task"; + NewEarliestStart: DateTime; + begin + NewEarliestStart := CurrentDateTime(); + ScheduledTask.ReadIsolation := IsolationLevel::ReadUncommitted; + if not ScheduledTask.Get(JobQueueEntry."System Task ID") then + exit; + if ScheduledTask."Not Before" <= NewEarliestStart then + exit; // already due to run + + if not TaskScheduler.SetTaskReady(ScheduledTask.ID, NewEarliestStart) then + exit; + + JobQueueEntryUpdate.ReadIsolation := IsolationLevel::UpdLock; + JobQueueEntryUpdate.ID := JobQueueEntry.ID; + if JobQueueEntryUpdate.GetRecLockedExtendedTimeout() then + if JobQueueEntryUpdate.Status in [JobQueueEntryUpdate.Status::Ready, JobQueueEntryUpdate.Status::"On Hold with Inactivity Timeout"] then begin + JobQueueEntryUpdate.Status := JobQueueEntryUpdate.Status::Ready; + JobQueueEntryUpdate."Earliest Start Date/Time" := NewEarliestStart; + JobQueueEntryUpdate.Modify(); + end; + end; + + [InternalEvent(false)] + local procedure OnBeforeRescheduleSynchJob(var JobQueueEntry: Record "Job Queue Entry"; IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) + begin + end; + + local procedure GetInteger(var Container: JsonObject; PropertyName: Text): Integer + var + Token: JsonToken; + begin + if Container.Get(PropertyName, Token) then + exit(Token.AsValue().AsInteger()); + exit(0); + end; + + local procedure GetBoolean(var Container: JsonObject; PropertyName: Text; DefaultValue: Boolean): Boolean + var + Token: JsonToken; + begin + if Container.Get(PropertyName, Token) then + exit(Token.AsValue().AsBoolean()); + exit(DefaultValue); + end; + + local procedure GetDateTime(var Container: JsonObject; PropertyName: Text; var Value: DateTime): Boolean + var + Token: JsonToken; + ValueText: Text; + begin + if not Container.Get(PropertyName, Token) then + exit(false); + ValueText := Token.AsValue().AsText(); + if ValueText = '' then + exit(false); + exit(Evaluate(Value, ValueText, 9)); + end; + + local procedure WriteArray(JsonArrayValue: JsonArray) ResultText: Text + begin + JsonArrayValue.WriteTo(ResultText); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index f1bf69d0996..fe545bdd8a9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -245,16 +245,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(1000); end; - local procedure GetTransport() Transport: Interface "IMDM Source Transport" + local procedure GetTransport(): Interface "IMDM Source Transport" var - HttpTransport: Codeunit "MDM Http Source Transport"; - begin - Transport := HttpTransport; - OnResolveSourceTransport(Transport); - end; - - [InternalEvent(false)] - local procedure OnResolveSourceTransport(var Transport: Interface "IMDM Source Transport") + SourceConnection: Codeunit "MDM Source Connection"; begin + exit(SourceConnection.GetTransport()); end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index edfaa2c5ad9..8d9c3749200 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -457,6 +457,9 @@ codeunit 7241 "MDM Cross-Env Source API" exit(Entry); end; RecRef.SetView(StrSubstNo('SORTING(Field%1)', SystemModifiedAtFieldNo())); + // Detection only needs an approximate max, so read uncommitted: never takes or waits on a lock, even + // where snapshot isolation is off (OnPrem). Data reads (GetRecords) stay at committed isolation. + RecRef.ReadIsolation := IsolationLevel::ReadUncommitted; if RecRef.FindLast() then Entry.Add('lastModifiedAt', Format(RecRef.Field(SystemModifiedAtFieldNo()).Value(), 0, 9)) else diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al new file mode 100644 index 00000000000..03e18d9aa0e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceConnection.Codeunit.al @@ -0,0 +1,23 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Single resolution point for the source transport, shared by the cross-environment data source and the change +/// detector. Defaults to the HTTP transport; tests inject an in-process transport via OnResolveSourceTransport. +/// +codeunit 7244 "MDM Source Connection" +{ + Access = Internal; + + procedure GetTransport() Transport: Interface "IMDM Source Transport" + var + HttpTransport: Codeunit "MDM Http Source Transport"; + begin + Transport := HttpTransport; + OnResolveSourceTransport(Transport); + end; + + [InternalEvent(false)] + local procedure OnResolveSourceTransport(var Transport: Interface "IMDM Source Transport") + begin + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al index 16ce36cbb3e..32105963dd5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSetupDefault.Codeunit.al @@ -49,6 +49,7 @@ codeunit 7230 "Master Data Mgt. Setup Default" JobQueueEntryNameTok: Label ' %1 - %2 synchronization job.', Comment = '%1 = The Integration Table Name to synchronized (ex. CUSTOMER), %2 = Business Central product name'; UncoupleJobQueueEntryNameTok: Label ' %1 uncouple job.', Comment = '%1 = Integration mapping description, for example, CUSTOMER <-> CUSTOMER'; CoupleJobQueueEntryNameTok: Label ' %1 coupling job.', Comment = '%1 = Integration mapping description, for example, CUSTOMER <-> CUSTOMER'; + ChangeDetectorJobDescriptionTxt: Label 'Master Data Management cross-environment change detection.'; IntegrationTablePrefixTok: Label 'Business Central', Comment = 'Product name', Locked = true; CustomerConfigTemplateCodeTok: Label 'MDMCUST', Comment = 'Customer template code for new customers created from source company data. Max length 10.', Locked = true; VendorConfigTemplateCodeTok: Label 'MDMVEND', Comment = 'Vendor template code for new vendors created from source company data. Max length 10.', Locked = true; @@ -97,6 +98,8 @@ codeunit 7230 "Master Data Mgt. Setup Default" ResetDimensionValueMapping('MDM_DIMENSIONVALUE', (not MasterDataManagementSetup."Delay Job Scheduling")); SetCustomIntegrationsTableMappings(MasterDataManagementSetup); + + UpdateChangeDetectorJob(MasterDataManagementSetup); end; internal procedure ResetSalesPeopleSystemUserMapping(IntegrationTableMappingName: Code[20]; ShouldRecreateJobQueueEntry: Boolean) @@ -1435,6 +1438,50 @@ codeunit 7230 "Master Data Mgt. Setup Default" exit(Codeunit.Run(Codeunit::"Job Queue - Enqueue", JobQueueEntry)) end; + // One recurring detector job per subsidiary; the cross-environment analog of same-env's event-driven reschedule. + internal procedure UpdateChangeDetectorJob(MasterDataManagementSetup: Record "Master Data Management Setup") + begin + if MasterDataManagementSetup."Is Enabled" and (MasterDataManagementSetup."Source Environment Name" <> '') then + EnsureChangeDetectorJob() + else + RemoveChangeDetectorJob(); + end; + + local procedure EnsureChangeDetectorJob() + var + JobQueueEntry: Record "Job Queue Entry"; + begin + JobQueueEntry.SetRange("Object Type to Run", JobQueueEntry."Object Type to Run"::Codeunit); + JobQueueEntry.SetRange("Object ID to Run", Codeunit::"MDM Cross-Env Change Detector"); + if not JobQueueEntry.IsEmpty() then + exit; + + JobQueueEntry.InitRecurringJob(ChangeDetectorIntervalInMinutes()); + JobQueueEntry."Object Type to Run" := JobQueueEntry."Object Type to Run"::Codeunit; + JobQueueEntry."Object ID to Run" := Codeunit::"MDM Cross-Env Change Detector"; + JobQueueEntry."Run in User Session" := false; + JobQueueEntry.Description := CopyStr(ChangeDetectorJobDescriptionTxt, 1, MaxStrLen(JobQueueEntry.Description)); + JobQueueEntry."Maximum No. of Attempts to Run" := 10; + JobQueueEntry.Status := JobQueueEntry.Status::Ready; + JobQueueEntry."Rerun Delay (sec.)" := 30; + JobQueueEntry."Job Queue Category Code" := JobQueueCategoryLbl; + Codeunit.Run(Codeunit::"Job Queue - Enqueue", JobQueueEntry); + end; + + local procedure RemoveChangeDetectorJob() + var + JobQueueEntry: Record "Job Queue Entry"; + begin + JobQueueEntry.SetRange("Object Type to Run", JobQueueEntry."Object Type to Run"::Codeunit); + JobQueueEntry.SetRange("Object ID to Run", Codeunit::"MDM Cross-Env Change Detector"); + JobQueueEntry.DeleteTasks(); + end; + + local procedure ChangeDetectorIntervalInMinutes(): Integer + begin + exit(1); + end; + internal procedure RecreateJobQueueEntryFromIntTableMapping(IntegrationTableMapping: Record "Integration Table Mapping"; IntervalInMinutes: Integer; ShouldRecreateJobQueueEntry: Boolean; InactivityTimeoutPeriod: Integer) begin RecreateJobQueueEntryFromIntTableMapping(IntegrationTableMapping, IntervalInMinutes, ShouldRecreateJobQueueEntry, InactivityTimeoutPeriod, ProductName.Short(), false); diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 87b65f63be7..fa6d980893b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -27,10 +27,13 @@ table 7230 "Master Data Management Setup" DataClassification = SystemMetadata; trigger OnValidate() + var + MasterDataMgtSetupDefault: Codeunit "Master Data Mgt. Setup Default"; begin if "Is Enabled" then if "Company Name" = '' then Error(MustPickSourceCompanyErr); + MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); end; } field(151; "Company Name"; Text[30]) @@ -83,6 +86,13 @@ table 7230 "Master Data Management Setup" { Caption = 'Source Environment'; DataClassification = OrganizationIdentifiableInformation; + + trigger OnValidate() + var + MasterDataMgtSetupDefault: Codeunit "Master Data Mgt. Setup Default"; + begin + MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); + end; } } diff --git a/src/Apps/W1/MasterDataManagement/test library/app.json b/src/Apps/W1/MasterDataManagement/test library/app.json index e385205f59e..04719d40770 100644 --- a/src/Apps/W1/MasterDataManagement/test library/app.json +++ b/src/Apps/W1/MasterDataManagement/test library/app.json @@ -22,6 +22,16 @@ "screenshots": [], "platform": "29.0.0.0", "target": "Cloud", + "idRanges": [ + { + "from": 139757, + "to": 139758 + }, + { + "from": 139929, + "to": 139930 + } + ], "resourceExposurePolicy": { "allowDebugging": false, "allowDownloadingSource": true, diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 1522f9fd53a..a1148636675 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -75,6 +75,31 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef)); end; + procedure DataSourceGetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableId, SystemId, SourceRecordRef)); + end; + + // Setting a Source Environment Name routes GetDataSource() to the cross-environment implementation. + procedure SetSourceEnvironmentName(EnvironmentName: Text) + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Source Environment Name" := CopyStr(EnvironmentName, 1, MaxStrLen(MasterDataManagementSetup."Source Environment Name")); + MasterDataManagementSetup.Modify(false); + end; + + procedure RunChangeDetector() + var + MDMCrossEnvChangeDetector: Codeunit "MDM Cross-Env Change Detector"; + begin + MDMCrossEnvChangeDetector.DetectChanges(); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; MasterDataManagement: Codeunit "Master Data Management"; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al new file mode 100644 index 00000000000..50b032396b7 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al @@ -0,0 +1,68 @@ +#pragma warning disable AA0247 +codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" +{ + // Test transport: injected into the cross-env data source via OnResolveSourceTransport so the subsidiary and + // source run in ONE environment. Pass-through calls the real source API (codeunit 7241) for round-trip tests; + // a canned response lets tests exercise the subsidiary's error handling. SingleInstance so the injected + // instance shares the state the test sets. + SingleInstance = true; + Access = Public; + + var + CannedResponse: Text; + Active: Boolean; + UseCanned: Boolean; + + procedure Activate() + begin + Active := true; + end; + + procedure Deactivate() + begin + Active := false; + UseCanned := false; + Clear(CannedResponse); + end; + + procedure SetCannedResponse(Response: Text) + begin + CannedResponse := Response; + UseCanned := true; + end; + + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + begin + if UseCanned then + exit(CannedResponse); + exit(SourceApi.GetRecords(TableId, FieldIds, Selector, PageSize)); + end; + + procedure LastModifiedAtPerTable(TableIds: Text): Text + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + begin + if UseCanned then + exit(CannedResponse); + exit(SourceApi.LastModifiedAtPerTable(TableIds)); + end; + + procedure GetCapabilities(): Text + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + begin + exit(SourceApi.GetCapabilities()); + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Source Connection", 'OnResolveSourceTransport', '', false, false)] + local procedure InjectTransport(var Transport: Interface "IMDM Source Transport") + var + InProcessTransport: Codeunit "MDM In-Process Transport"; + begin + if not Active then + exit; + Transport := InProcessTransport; // SingleInstance: same stateful instance the test configured + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al new file mode 100644 index 00000000000..79702196cef --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al @@ -0,0 +1,44 @@ +#pragma warning disable AA0247 +codeunit 139930 "MDM Test Detector Probe" +{ + // Captures which tables the change detector decided to nudge and short-circuits the real reschedule, so the + // detector's decision logic can be asserted without scheduling background jobs (not possible in the test lab). + SingleInstance = true; + Access = Public; + + var + NudgedTableIds: List of [Integer]; + Active: Boolean; + + procedure Activate() + begin + Active := true; + Clear(NudgedTableIds); + end; + + procedure Deactivate() + begin + Active := false; + Clear(NudgedTableIds); + end; + + procedure WasNudged(TableId: Integer): Boolean + begin + exit(NudgedTableIds.Contains(TableId)); + end; + + procedure NudgeCount(): Integer + begin + exit(NudgedTableIds.Count()); + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Change Detector", 'OnBeforeRescheduleSynchJob', '', false, false)] + local procedure CaptureNudge(var JobQueueEntry: Record "Job Queue Entry"; IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) + begin + if not Active then + exit; + if not NudgedTableIds.Contains(IntegrationTableMapping."Integration Table ID") then + NudgedTableIds.Add(IntegrationTableMapping."Integration Table ID"); + IsHandled := true; // the test lab cannot reschedule background jobs + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/app.json b/src/Apps/W1/MasterDataManagement/test/app.json index 60e46b1395a..34ff2a4ddfa 100644 --- a/src/Apps/W1/MasterDataManagement/test/app.json +++ b/src/Apps/W1/MasterDataManagement/test/app.json @@ -40,6 +40,20 @@ "screenshots": [], "platform": "29.0.0.0", "target": "Cloud", + "idRanges": [ + { + "from": 139758, + "to": 139758 + }, + { + "from": 139770, + "to": 139770 + }, + { + "from": 139931, + "to": 139933 + } + ], "resourceExposurePolicy": { "allowDebugging": false, "allowDownloadingSource": true, diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al new file mode 100644 index 00000000000..caf8353abb1 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -0,0 +1,196 @@ +#pragma warning disable AA0247 +codeunit 139932 "MDM Cross-Env Consumer Tests" +{ + Subtype = Test; + TestType = IntegrationTest; + TestPermissions = Disabled; + EventSubscriberInstance = Manual; + + var + Assert: Codeunit Assert; + LibrarySalesLib: Codeunit "Library - Sales"; + + [Test] + procedure CrossEnvGetBySystemIdRoundTripsSourceRecord() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] The cross-env data source fetches a record over the transport and materializes it (fields included). + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + Customer.Name := CopyStr(LibraryRandomText(), 1, MaxStrLen(Customer.Name)); + Customer.Modify(); + + // [GIVEN] a subsidiary configured for cross-env with the in-process (pass-through) transport + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [WHEN] the record is fetched by SystemId + Found := LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + + // [THEN] the materialized record carries the same SystemId and field values + Assert.IsTrue(Found, 'Cross-env GetBySystemId should find the source record'); + Assert.AreEqual(Customer.SystemId, SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(), 'Wrong SystemId materialized'); + Assert.AreEqual(Customer.Name, Format(SourceRecordRef.Field(Customer.FieldNo(Name)).Value()), 'Name should round-trip through the wire'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvGetModifiedSetMaterializesSourceChange() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + Found: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Cross-env GetModifiedSet pages the source feed over the transport and materializes the changes. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + CreateMinimalCustomerMapping(IntegrationTableMapping); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Found := LibraryMasterDataMgt.DataSourceGetModifiedSet(IntegrationTableMapping, '', SourceRecordRef); + + Assert.IsTrue(Found, 'Cross-env GetModifiedSet should return source records'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer.SystemId), 'Materialized set should contain the seeded customer'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenSourceTableUnavailable() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A tableAvailable:false response surfaces as a clear synchronization error. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse('{"tableId":18,"tableAvailable":false,"records":[],"hasMore":false}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('not available on the source environment'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenSourceTableNotIndexed() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] An indexed:false response (too-large unindexed table) asks for the composite key. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse('{"tableId":18,"tableAvailable":true,"indexed":false,"records":[],"hasMore":false}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('Add a key on SystemModifiedAt and SystemId'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenFieldsUnavailable() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] An unavailableFields response halts the table with a clear synchronization error. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse('{"tableId":18,"tableAvailable":true,"unavailableFields":[5],"records":[],"hasMore":false}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('do not exist on table'); + + CleanUp(); + end; + + local procedure Initialize() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + InProcessTransport: Codeunit "MDM In-Process Transport"; + begin + InProcessTransport.Deactivate(); + if not MasterDataManagementSetup.Get() then begin + MasterDataManagementSetup.Init(); + MasterDataManagementSetup.Insert(); + end; + MasterDataManagementSetup."Source Environment Name" := ''; + MasterDataManagementSetup.Modify(false); + end; + + local procedure CleanUp() + var + InProcessTransport: Codeunit "MDM In-Process Transport"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + InProcessTransport.Deactivate(); + LibraryMasterDataMgt.SetSourceEnvironmentName(''); + end; + + local procedure CreateMinimalCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") + begin + IntegrationTableMapping.Init(); + IntegrationTableMapping.Name := CopyStr('MDMXENV' + Format(LibraryRandomInt()), 1, MaxStrLen(IntegrationTableMapping.Name)); + IntegrationTableMapping.Type := IntegrationTableMapping.Type::"Master Data Management"; + IntegrationTableMapping."Table ID" := Database::Customer; + IntegrationTableMapping."Integration Table ID" := Database::Customer; + IntegrationTableMapping."Integration Table UID Fld. No." := 2000000000; // SystemId + IntegrationTableMapping."Int. Tbl. Modified On Fld. No." := 2000000003; // SystemModifiedAt + IntegrationTableMapping."Delete After Synchronization" := false; + IntegrationTableMapping.Insert(); + end; + + local procedure ContainsSystemId(var SourceRecordRef: RecordRef; SystemIdValue: Guid): Boolean + begin + if SourceRecordRef.FindSet() then + repeat + if Format(SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value()) = Format(SystemIdValue) then + exit(true); + until SourceRecordRef.Next() = 0; + exit(false); + end; + + local procedure LibraryRandomText(): Text + var + LibraryRandomCu: Codeunit "Library - Random"; + begin + exit(LibraryRandomCu.RandText(20)); + end; + + local procedure LibraryRandomInt(): Integer + var + LibraryRandomCu: Codeunit "Library - Random"; + begin + exit(LibraryRandomCu.RandIntInRange(1, 999999)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al new file mode 100644 index 00000000000..0bf03b72744 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al @@ -0,0 +1,243 @@ +#pragma warning disable AA0247 +codeunit 139933 "MDM Cross-Env Detector Tests" +{ + Subtype = Test; + TestType = IntegrationTest; + TestPermissions = Disabled; + EventSubscriberInstance = Manual; + + var + Assert: Codeunit Assert; + + [Test] + procedure DetectorNudgesChangedEnabledTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] A table changed since its watermark gets its synchronization job nudged. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); // an idle (Ready) job exists + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, ChangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsTrue(DetectorProbe.WasNudged(Database::Customer), 'A changed enabled table should be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorSkipsUnchangedTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A table whose source timestamp is not past the watermark is not nudged. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, UnchangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'An unchanged table should not be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorSkipsDisabledTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A disabled table is never polled or nudged, even if the source reports a change. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), false); // disabled + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, ChangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'A disabled table should not be nudged'); + CleanUp(); + end; + + [Test] + procedure DetectorLeavesInProcessJobAlone() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A changed table whose job is already In Process is left alone (no idle job to nudge). + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, true); // job In Process + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, ChangedDateTime(), true)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'A table whose job is In Process should be left alone'); + CleanUp(); + end; + + [Test] + procedure DetectorSkipsUnavailableTable() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A table the source reports as unavailable is not nudged (the sync job reports the error). + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, false, ChangedDateTime(), false)); + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsFalse(DetectorProbe.WasNudged(Database::Customer), 'An unavailable table should not be nudged'); + CleanUp(); + end; + + local procedure Initialize() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + InProcessTransport.Deactivate(); + DetectorProbe.Deactivate(); + // RunChangeDetector goes through Codeunit.Run, which commits, so prior test data survives rollback. + DeleteTestArtifacts(); + if not MasterDataManagementSetup.Get() then begin + MasterDataManagementSetup.Init(); + MasterDataManagementSetup.Insert(); + end; + // Direct assignment avoids the OnValidate that (de)provisions the detector job. + MasterDataManagementSetup."Is Enabled" := true; + MasterDataManagementSetup."Source Environment Name" := 'PROD'; + MasterDataManagementSetup.Modify(false); + end; + + local procedure CleanUp() + var + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + InProcessTransport.Deactivate(); + DetectorProbe.Deactivate(); + DeleteTestArtifacts(); + end; + + local procedure DeleteTestArtifacts() + var + Mapping: Record "Integration Table Mapping"; + JobQueueEntry: Record "Job Queue Entry"; + begin + Mapping.SetFilter(Name, 'MDMXD*'); + if Mapping.FindSet() then + repeat + JobQueueEntry.SetRange("Record ID to Process", Mapping.RecordId()); + JobQueueEntry.DeleteAll(); + until Mapping.Next() = 0; + Mapping.DeleteAll(); + end; + + local procedure CreateMapping(var Mapping: Record "Integration Table Mapping"; TableId: Integer; Watermark: DateTime; Enabled: Boolean) + var + LibraryRandom: Codeunit "Library - Random"; + begin + Mapping.Init(); + Mapping.Name := CopyStr('MDMXD' + Format(LibraryRandom.RandIntInRange(1, 999999)), 1, MaxStrLen(Mapping.Name)); + Mapping.Type := Mapping.Type::"Master Data Management"; + Mapping."Table ID" := TableId; + Mapping."Integration Table ID" := TableId; + Mapping."Integration Table UID Fld. No." := 2000000000; // SystemId + Mapping."Int. Tbl. Modified On Fld. No." := 2000000003; // SystemModifiedAt + Mapping."Synch. Modified On Filter" := Watermark; + Mapping."Delete After Synchronization" := false; + if Enabled then + Mapping.Status := Mapping.Status::Enabled + else + Mapping.Status := Mapping.Status::Disabled; + Mapping.Insert(); + end; + + // Insert-only (no scheduling): the test lab can't schedule background jobs, and the detector only reads status. + local procedure InsertSynchJob(Mapping: Record "Integration Table Mapping"; InProcess: Boolean) + var + JobQueueEntry: Record "Job Queue Entry"; + begin + JobQueueEntry.Init(); + JobQueueEntry.ID := CreateGuid(); + JobQueueEntry."Object Type to Run" := JobQueueEntry."Object Type to Run"::Codeunit; + JobQueueEntry."Object ID to Run" := Codeunit::"Integration Synch. Job Runner"; + JobQueueEntry."Record ID to Process" := Mapping.RecordId(); + JobQueueEntry."Recurring Job" := true; + if InProcess then + JobQueueEntry.Status := JobQueueEntry.Status::"In Process" + else + JobQueueEntry.Status := JobQueueEntry.Status::Ready; + JobQueueEntry.Insert(false); + end; + + local procedure CannedLastModified(TableId: Integer; TableAvailable: Boolean; LastModifiedAt: DateTime; IncludeTimestamp: Boolean) ResultText: Text + var + Response: JsonObject; + Entry: JsonObject; + Tables: JsonArray; + begin + Entry.Add('tableId', TableId); + Entry.Add('tableAvailable', TableAvailable); + if IncludeTimestamp then + Entry.Add('lastModifiedAt', Format(LastModifiedAt, 0, 9)); + Tables.Add(Entry); + Response.Add('tables', Tables); + Response.WriteTo(ResultText); + end; + + local procedure WatermarkDateTime(): DateTime + begin + exit(CreateDateTime(DMY2Date(1, 1, 2020), 0T)); + end; + + local procedure ChangedDateTime(): DateTime + begin + exit(CreateDateTime(DMY2Date(1, 1, 2030), 0T)); + end; + + local procedure UnchangedDateTime(): DateTime + begin + exit(CreateDateTime(DMY2Date(1, 1, 2019), 0T)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al new file mode 100644 index 00000000000..e8004b4913b --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -0,0 +1,194 @@ +#pragma warning disable AA0247 +codeunit 139931 "MDM Cross-Env Source Tests" +{ + Subtype = Test; + TestType = IntegrationTest; + TestPermissions = Disabled; + + var + Assert: Codeunit Assert; + LibrarySales: Codeunit "Library - Sales"; + LibraryRandom: Codeunit "Library - Random"; + + [Test] + procedure GetCapabilitiesReturnsVersionAndFeatures() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Token: JsonToken; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] GetCapabilities advertises the contract version and supported features. + Response.ReadFrom(SourceApi.GetCapabilities()); + + Response.Get('version', Token); + Assert.AreEqual(1, Token.AsValue().AsInteger(), 'Unexpected capability version'); + Assert.IsTrue(FeaturesContain(Response, 'records'), 'records feature should be advertised'); + Assert.IsTrue(FeaturesContain(Response, 'lastModifiedPerTable'), 'lastModifiedPerTable feature should be advertised'); + end; + + [Test] + procedure GetRecordsBySystemIdReturnsRequestedFields() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Customer: Record Customer; + Response: JsonObject; + RecordObject: JsonObject; + FieldsObject: JsonObject; + RecordsArray: JsonArray; + Token: JsonToken; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] GetRecords with a systemIds selector returns just that record with the requested fields. + LibrarySales.CreateCustomer(Customer); + Customer.Name := CopyStr(LibraryRandom.RandText(20), 1, MaxStrLen(Customer.Name)); + Customer.Modify(); + + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100)); + + Response.Get('records', Token); + RecordsArray := Token.AsArray(); + Assert.AreEqual(1, RecordsArray.Count(), 'Expected exactly the requested record'); + RecordsArray.Get(0, Token); + RecordObject := Token.AsObject(); + RecordObject.Get('systemId', Token); + Assert.AreEqual(Format(Customer.SystemId), Token.AsValue().AsText(), 'Wrong systemId returned'); + RecordObject.Get('fields', Token); + FieldsObject := Token.AsObject(); + FieldsObject.Get(Format(Customer.FieldNo(Name)), Token); + Assert.AreEqual(Customer.Name, Token.AsValue().AsText(), 'Wrong Name value returned'); + end; + + [Test] + procedure GetRecordsCursorModePagesWithHasMore() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Customer: Record Customer; + Watermark: DateTime; + Response: JsonObject; + NextCursor: Text; + Index: Integer; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Cursor mode pages ascending by (SystemModifiedAt, SystemId) and reports hasMore / nextCursor. + Watermark := CurrentDateTime(); + Sleep(50); // ensure the seeded records sort strictly after the watermark + for Index := 1 to 3 do + LibrarySales.CreateCustomer(Customer); + + // [WHEN] the first page of size 2 is requested from the watermark + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), CursorSelector(Watermark), 2)); + + // [THEN] two records come back and hasMore is true + Assert.AreEqual(2, RecordCount(Response), 'First page should hold the page size'); + Assert.IsTrue(GetBoolean(Response, 'hasMore'), 'hasMore should be true while records remain'); + NextCursor := NextCursorText(Response); + + // [WHEN] the next page is requested with the returned cursor + Clear(Response); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), NextCursor, 2)); + + // [THEN] the remaining record comes back and hasMore is false + Assert.AreEqual(1, RecordCount(Response), 'Second page should hold the remaining record'); + Assert.IsFalse(GetBoolean(Response, 'hasMore'), 'hasMore should be false on the last page'); + end; + + [Test] + procedure LastModifiedAtPerTableReturnsLatestTimestamp() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Customer: Record Customer; + Response: JsonObject; + Entry: JsonObject; + Tables: JsonArray; + Token: JsonToken; + LastModified: DateTime; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] LastModifiedAtPerTable returns each table's latest modification timestamp. + LibrarySales.CreateCustomer(Customer); + + Response.ReadFrom(SourceApi.LastModifiedAtPerTable(TableIdsArray(Database::Customer))); + + Response.Get('tables', Token); + Tables := Token.AsArray(); + Assert.AreEqual(1, Tables.Count(), 'Expected one table entry'); + Tables.Get(0, Token); + Entry := Token.AsObject(); + Entry.Get('lastModifiedAt', Token); + Assert.IsTrue(Evaluate(LastModified, Token.AsValue().AsText(), 9), 'lastModifiedAt should be a round-trippable timestamp'); + Assert.IsTrue(LastModified >= Customer.SystemModifiedAt, 'lastModifiedAt should be at least the just-created customer'); + end; + + local procedure FeaturesContain(var Response: JsonObject; Feature: Text): Boolean + var + Features: JsonArray; + FeatureToken: JsonToken; + Token: JsonToken; + begin + Response.Get('features', Token); + Features := Token.AsArray(); + foreach FeatureToken in Features do + if FeatureToken.AsValue().AsText() = Feature then + exit(true); + exit(false); + end; + + local procedure RecordCount(var Response: JsonObject): Integer + var + Token: JsonToken; + begin + Response.Get('records', Token); + exit(Token.AsArray().Count()); + end; + + local procedure GetBoolean(var Response: JsonObject; PropertyName: Text): Boolean + var + Token: JsonToken; + begin + Response.Get(PropertyName, Token); + exit(Token.AsValue().AsBoolean()); + end; + + local procedure NextCursorText(var Response: JsonObject) CursorText: Text + var + Token: JsonToken; + begin + Response.Get('nextCursor', Token); + Token.WriteTo(CursorText); + end; + + local procedure FieldIdsArray(FieldNo: Integer) ResultText: Text + var + FieldIds: JsonArray; + begin + FieldIds.Add(FieldNo); + FieldIds.WriteTo(ResultText); + end; + + local procedure TableIdsArray(TableId: Integer) ResultText: Text + var + TableIds: JsonArray; + begin + TableIds.Add(TableId); + TableIds.WriteTo(ResultText); + end; + + local procedure SystemIdsSelector(SystemId: Guid) ResultText: Text + var + Selector: JsonObject; + SystemIds: JsonArray; + begin + SystemIds.Add(Format(SystemId)); + Selector.Add('systemIds', SystemIds); + Selector.WriteTo(ResultText); + end; + + local procedure CursorSelector(Watermark: DateTime) ResultText: Text + var + Selector: JsonObject; + begin + Selector.Add('modifiedAt', Format(Watermark, 0, 9)); + Selector.WriteTo(ResultText); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al index 3769252fd6c..25fa554b710 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al @@ -848,7 +848,7 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" IntegrationRecordRef: RecordRef; begin // [FEATURE] [AI test 0.4] - // [SCENARIO] Setting Source Environment Name makes the data-source resolver fail (cross-environment not yet supported). + // [SCENARIO] Setting Source Environment Name routes reads to the cross-environment source, which fails until a connection is configured. Initialize(); // [GIVEN] a coupled source customer and a source environment name set on the setup @@ -865,8 +865,8 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" // [WHEN] a read that resolves the data source is invoked asserterror LibraryMasterDataMgt.GetIntegrationRecordRefByCoupling(Database::Customer, MasterDataMgtCoupling, IntegrationRecordRef); - // [THEN] it fails because cross-environment synchronization is not yet available - Assert.ExpectedError('Cross-environment synchronization is not yet available'); + // [THEN] it fails because the cross-environment connection to the source is not configured yet + Assert.ExpectedError('The cross-environment connection to the source is not configured yet'); end; local procedure GetCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") From 9d75e9f80e599f7d6cfaf465692a335261b15a6b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 27 Aug 2026 13:40:17 +0200 Subject: [PATCH 09/64] [Master Data Management] Cross-environment synchonization --- .../IntegrationMasterDataSynch.Codeunit.al | 99 ++++++++- .../MDMCrossEnvDataSource.Codeunit.al | 47 ++++- .../MDMHttpSourceTransport.Codeunit.al | 188 +++++++++++++++++- .../tables/MasterDataManagementSetup.Table.al | 63 ++++++ .../MasterDataMgtTableMapping.TableExt.al | 7 + .../test library/app.json | 4 + .../src/LibraryMasterDataMgt.Codeunit.al | 7 + .../src/MDMTestPagingConfig.Codeunit.al | 31 +++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 72 +++++++ 9 files changed, 501 insertions(+), 17 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index a8f9851ac08..6a804136a8d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -44,6 +44,8 @@ codeunit 7231 "Integration Master Data Synch." SupportedSourceType: Option ,RecordID,GUID; DateType: Option ,Integration,Local; OutOfMapFilter: Boolean; + CrossEnvBatchEndCursor: Text; + CrossEnvBatchHasMore: Boolean; RecordNotFoundErr: Label 'Cannot find %1 record %2.', Comment = '%1 = Source table caption, %2 = The lookup value when searching for the source record'; SourceRecordIsNotInMappingErr: Label 'Cannot find the mapping %2 in table %1.', Comment = '%1 Integration Table Mapping caption, %2 Integration Table Mapping Name'; CannotDetermineSourceOriginErr: Label 'Cannot determine the source origin: %1.', Comment = '%1 the value of the source id'; @@ -83,6 +85,10 @@ codeunit 7231 "Integration Master Data Synch." exit; MasterDataManagementSetup.Get(); + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + FindModifiedCrossEnvironmentRecords(TempIntegrationRecordRef, IntegrationTableMapping, FailedNotSkippedIdDictionary); + exit; + end; DataSource := MasterDataManagementSetup.GetDataSource(); SplitIntegrationTableFilter(IntegrationTableMapping, FilterList); foreach TableFilter in FilterList do begin @@ -97,6 +103,27 @@ codeunit 7231 "Integration Master Data Synch." end; end; + // Cross-environment reads a BOUNDED batch (MaxPagesPerRun) from the persisted resume cursor, so a large + // initial load drains over several job runs instead of one unbounded, possibly-never-completing job. + local procedure FindModifiedCrossEnvironmentRecords(var TempIntegrationRecordRef: RecordRef; IntegrationTableMapping: Record "Integration Table Mapping"; var FailedNotSkippedIdDictionary: Dictionary of [Guid, Boolean]) + var + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + MasterDataManagement: Codeunit "Master Data Management"; + IntegrationRecordRef: RecordRef; + IntegrationRecordID: Guid; + begin + CrossEnvBatchEndCursor := ''; + CrossEnvBatchHasMore := false; + if CrossEnvDataSource.GetModifiedBatch(IntegrationTableMapping, IntegrationTableMapping.GetIntegrationTableFilter(), IntegrationTableMapping."Source Change Cursor", MaxPagesPerRun(), IntegrationRecordRef, CrossEnvBatchEndCursor, CrossEnvBatchHasMore) then + repeat + IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); + if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then + if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then + Session.LogMessage('0000J8Q', StrSubstNo(CopyRecordRefFailedTxt, IntegrationRecordID), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + until IntegrationRecordRef.Next() = 0; + IntegrationRecordRef.Close(); + end; + [TryFunction] local procedure TryCopyRecordReference(var IntegrationTableMapping: Record "Integration Table Mapping"; FromRec: RecordRef; var ToRec: RecordRef; ValidateOnInsert: Boolean) begin @@ -478,6 +505,7 @@ codeunit 7231 "Integration Master Data Synch." SourceRecordRef: RecordRef; JobId: Guid; JobStartDateTime: DateTime; + Drained: Boolean; begin JobStartDateTime := CurrentDateTime(); JobId := @@ -486,11 +514,78 @@ codeunit 7231 "Integration Master Data Synch." if not IsNullGuid(JobId) then begin MasterDataFullSynchRLn.FullSynchStarted(IntegrationTableMapping, JobId, IntegrationTableMapping.Direction::FromIntegrationTable); LatestIntegrationModifiedOn := SynchIntegrationTableToLocalTable(IntegrationTableMapping, IntegrationTableSynch, SourceRecordRef); + Drained := PersistCrossEnvResumeState(IntegrationTableMapping, JobStartDateTime, LatestIntegrationModifiedOn); + IntegrationTableSynch.EndIntegrationSynchJob(); + if Drained then + MasterDataFullSynchRLn.FullSynchFinished(IntegrationTableMapping, IntegrationTableMapping.Direction::FromIntegrationTable); + end; + end; + + // After a cross-environment run, persist the resume cursor and cap the watermark to what was actually + // processed on a partial (page-capped) run, so the next run continues instead of skipping records. Returns + // whether the source was fully drained (same-env and fully-caught-up cross-env both count as drained). + local procedure PersistCrossEnvResumeState(var IntegrationTableMapping: Record "Integration Table Mapping"; JobStartDateTime: DateTime; var LatestIntegrationModifiedOn: DateTime): Boolean + begin + if not IsCrossEnvironmentSynch() then begin if JobStartDateTime > LatestIntegrationModifiedOn then LatestIntegrationModifiedOn := JobStartDateTime; - IntegrationTableSynch.EndIntegrationSynchJob(); - MasterDataFullSynchRLn.FullSynchFinished(IntegrationTableMapping, IntegrationTableMapping.Direction::FromIntegrationTable); + exit(true); + end; + + if CrossEnvBatchHasMore then begin + IntegrationTableMapping."Source Change Cursor" := CopyStr(CrossEnvBatchEndCursor, 1, MaxStrLen(IntegrationTableMapping."Source Change Cursor")); + // Advance the watermark only to the last processed page; the cursor is the authoritative resume point. + LatestIntegrationModifiedOn := CursorModifiedAt(CrossEnvBatchEndCursor); + IntegrationTableMapping.Modify(); + Commit(); + exit(false); end; + + IntegrationTableMapping."Source Change Cursor" := ''; + IntegrationTableMapping.Modify(); + Commit(); + if JobStartDateTime > LatestIntegrationModifiedOn then + LatestIntegrationModifiedOn := JobStartDateTime; + exit(true); + end; + + local procedure IsCrossEnvironmentSynch(): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit(false); + exit(MasterDataManagementSetup."Source Environment Name" <> ''); + end; + + local procedure CursorModifiedAt(CursorText: Text) ModifiedAt: DateTime + var + Cursor: JsonObject; + Token: JsonToken; + begin + if CursorText = '' then + exit(0DT); + if not Cursor.ReadFrom(CursorText) then + exit(0DT); + if Cursor.Get('modifiedAt', Token) then + if Token.IsValue() then + if not Evaluate(ModifiedAt, Token.AsValue().AsText(), 9) then + exit(0DT); + end; + + local procedure MaxPagesPerRun(): Integer + var + MaxPages: Integer; + begin + // Cap pages per run so a large initial load resumes across runs; 0 = unbounded (test override). + MaxPages := 50; + OnGetCrossEnvMaxPagesPerRun(MaxPages); + exit(MaxPages); + end; + + [InternalEvent(false)] + local procedure OnGetCrossEnvMaxPagesPerRun(var MaxPages: Integer) + begin end; internal procedure CreateMasterDataMgtCouplingClone(ForTable: Integer; var TempMasterDataMgtCoupling: Record "Master Data Mgt. Coupling" temporary) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index fe545bdd8a9..14f2d3eb341 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -20,29 +20,51 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + EndCursor: Text; + HasMore: Boolean; + begin + // Interface entry point: unbounded (fetch the whole delta). The scheduled cross-env synch uses the + // bounded GetModifiedBatch instead, so a large initial load is drained across several job runs. + exit(GetModifiedBatch(IntegrationTableMapping, TableFilter, CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"), 0, SourceRecordRef, EndCursor, HasMore)); + end; + + /// + /// Fetches at most MaxPages pages of changed source records starting from StartCursor (a cursor selector). + /// MaxPages = 0 means unbounded. On return EndCursor holds the resume point and HasMore tells the caller + /// whether more pages remain past the cap, so the next job run can continue from where this one stopped. + /// + internal procedure GetModifiedBatch(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; StartCursor: Text; MaxPages: Integer; var SourceRecordRef: RecordRef; var EndCursor: Text; var HasMore: Boolean): Boolean var Transport: Interface "IMDM Source Transport"; Response: JsonObject; FieldIds: Text; Selector: Text; - MoreToFetch: Boolean; + PagesFetched: Integer; begin SourceRecordRef.Close(); SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID", true); Transport := GetTransport(); FieldIds := BuildFieldIds(IntegrationTableMapping); - // Watermark = 'Synchronize Changes Since'; the wire pages the delta, all accumulated into the temp ref. - Selector := CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"); + if StartCursor <> '' then + Selector := StartCursor + else + Selector := CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"); + EndCursor := ''; + HasMore := false; repeat ParseOrError( IntegrationTableMapping."Integration Table ID", Transport.GetRecords(IntegrationTableMapping."Integration Table ID", FieldIds, Selector, PageSize()), Response); SourceResponse.InsertRecords(Response, SourceRecordRef); - MoreToFetch := SourceResponse.HasMore(Response); - if MoreToFetch then - Selector := SourceResponse.GetNextCursor(Response); - until not MoreToFetch; + PagesFetched += 1; + HasMore := SourceResponse.HasMore(Response); + if HasMore then begin + EndCursor := SourceResponse.GetNextCursor(Response); + Selector := EndCursor; + end; + until (not HasMore) or ((MaxPages > 0) and (PagesFetched >= MaxPages)); IntegrationTableMapping.SetIntRecordRefFilter(SourceRecordRef, TableFilter); exit(SourceRecordRef.FindSet()); end; @@ -241,8 +263,17 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; local procedure PageSize(): Integer + var + Size: Integer; + begin + Size := 1000; + OnGetCrossEnvPageSize(Size); + exit(Size); + end; + + [InternalEvent(false)] + local procedure OnGetCrossEnvPageSize(var PageSize: Integer) begin - exit(1000); end; local procedure GetTransport(): Interface "IMDM Source Transport" diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 57ef2d95053..3a7af7821a1 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -1,31 +1,205 @@ namespace Microsoft.Integration.MDM; +using System.Azure.Identity; +using System.Environment; +using System.Security.Authentication; +using System.Reflection; + /// /// Production transport: calls the source environment's ODataV4 web service with an app-only (client -/// credentials) token. The OAuth2 + HttpClient body is implemented in a follow-up; until then it fails loudly -/// so a misconfigured environment is obvious. Tests never hit this — they inject an in-process transport. +/// credentials) token. Same-tenant by construction — the OAuth authority is derived from THIS environment's +/// Entra tenant, so there is no tenant-id setting to point the connection at another tenant. Tests never hit +/// this: they inject an in-process transport that calls the source API directly. /// codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" { Access = Internal; var + CachedToken: SecretText; + TokenExpiresAt: DateTime; + MaxRetriesValue: Integer; NotConfiguredErr: Label 'The cross-environment connection to the source is not configured yet.'; + NonSaaSErr: Label 'Cross-environment synchronization is only available in online environments.'; + NoTokenErr: Label 'Could not acquire an access token for the source environment. Check the client ID and secret.'; + SendFailedErr: Label 'The request to the source environment could not be sent. Check the source environment URL.'; + HttpErr: Label 'The source environment returned HTTP %1. %2', Comment = '%1 = HTTP status code, %2 = response detail'; + ServiceNameTok: Label 'MDMCrossEnvSource', Locked = true; + ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true; + TokenEndpointTok: Label 'https://login.microsoftonline.com/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; + ActionUrlTok: Label '%1/ODataV4/%2_%3?company=%4', Locked = true, Comment = '%1 = base url, %2 = service, %3 = action, %4 = company'; procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + var + Body: JsonObject; + BodyText: Text; begin - // TODO(cross-env): OAuth2 client-credentials token (creds from Isolated Storage) + HttpClient POST to - // {sourceUrl}/ODataV4/{service}_GetRecords?company={company}; return the response body. - Error(NotConfiguredErr); + Body.Add('tableId', TableId); + Body.Add('fieldIds', FieldIds); + Body.Add('selector', Selector); + Body.Add('pageSize', PageSize); + Body.WriteTo(BodyText); + exit(InvokeAction('GetRecords', BodyText)); end; procedure LastModifiedAtPerTable(TableIds: Text): Text + var + Body: JsonObject; + BodyText: Text; begin - Error(NotConfiguredErr); + Body.Add('tableIds', TableIds); + Body.WriteTo(BodyText); + exit(InvokeAction('LastModifiedAtPerTable', BodyText)); end; procedure GetCapabilities(): Text begin - Error(NotConfiguredErr); + exit(InvokeAction('GetCapabilities', '{}')); + end; + + local procedure InvokeAction(ActionName: Text; RequestBody: Text): Text + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + EnvironmentInformation: Codeunit "Environment Information"; + ResponseMessage: HttpResponseMessage; + ResponseBodyText: Text; + RetryAfter: Duration; + Attempt: Integer; + begin + GetConfiguredSetup(MasterDataManagementSetup); + if not EnvironmentInformation.IsSaaSInfrastructure() then + Error(NonSaaSErr); + + for Attempt := 0 to MaxRetries() do begin + Send(MasterDataManagementSetup, ActionName, RequestBody, ResponseMessage); + ResponseMessage.Content().ReadAs(ResponseBodyText); + if ResponseMessage.IsSuccessStatusCode() then + exit(UnwrapODataValue(ResponseBodyText)); + if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then + Error(HttpErr, ResponseMessage.HttpStatusCode(), ResponseBodyText); + Sleep(RetryAfter); + end; + end; + + local procedure Send(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage) + var + HttpClient: HttpClient; + RequestMessage: HttpRequestMessage; + RequestHeaders: HttpHeaders; + HttpContent: HttpContent; + ContentHeaders: HttpHeaders; + begin + RequestMessage.Method('POST'); + RequestMessage.SetRequestUri(BuildActionUrl(MasterDataManagementSetup, ActionName)); + RequestMessage.GetHeaders(RequestHeaders); + RequestHeaders.Add('Accept', 'application/json'); + RequestHeaders.Add('Authorization', SecretStrSubstNo('Bearer %1', GetBearerToken(MasterDataManagementSetup))); + + HttpContent.WriteFrom(RequestBody); + HttpContent.GetHeaders(ContentHeaders); + ContentHeaders.Remove('Content-Type'); + ContentHeaders.Add('Content-Type', 'application/json'); + RequestMessage.Content(HttpContent); + + if not HttpClient.Send(RequestMessage, ResponseMessage) then + Error(SendFailedErr); + end; + + local procedure BuildActionUrl(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text): Text + var + BaseUrl: Text; + begin + BaseUrl := DelChr(MasterDataManagementSetup."Source Environment URL", '>', '/'); + exit(StrSubstNo(ActionUrlTok, BaseUrl, ServiceNameTok, ActionName, UriEncodeCompany(MasterDataManagementSetup."Source Company Name"))); + end; + + // The ODataV4 envelope for an action returning Text is { "@odata.context": "...", "value": "" }; + // the source API's own JSON is that inner value. Fall back to the raw body if the shape is unexpected. + local procedure UnwrapODataValue(ResponseBody: Text): Text + var + Envelope: JsonObject; + ValueToken: JsonToken; + begin + if Envelope.ReadFrom(ResponseBody) then + if Envelope.Get('value', ValueToken) then + if ValueToken.IsValue() then + exit(ValueToken.AsValue().AsText()); + exit(ResponseBody); + end; + + local procedure GetBearerToken(var MasterDataManagementSetup: Record "Master Data Management Setup"): SecretText + begin + // Cached in-instance for the lifetime of a single sync run (the paging loop reuses this transport). + if (TokenExpiresAt <> 0DT) and (TokenExpiresAt > CurrentDateTime()) then + exit(CachedToken); + CachedToken := AcquireToken(MasterDataManagementSetup); + TokenExpiresAt := CurrentDateTime() + (3500 * 1000); // refresh a little before the ~1h token lifetime + exit(CachedToken); + end; + + [NonDebuggable] + local procedure AcquireToken(var MasterDataManagementSetup: Record "Master Data Management Setup") Token: SecretText + var + OAuth2: Codeunit OAuth2; + AzureADTenant: Codeunit "Azure AD Tenant"; + Scopes: List of [Text]; + TokenEndpoint: Text; + begin + Scopes.Add(ScopeTok); + TokenEndpoint := StrSubstNo(TokenEndpointTok, AzureADTenant.GetAadTenantId()); + OAuth2.AcquireTokenWithClientCredentials( + MasterDataManagementSetup."Source OAuth Client Id", + MasterDataManagementSetup.GetSourceClientSecret(), + TokenEndpoint, '', Scopes, Token); + if Token.IsEmpty() then + Error(NoTokenErr); + end; + + local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") + begin + if not MasterDataManagementSetup.Get() then + Error(NotConfiguredErr); + if not MasterDataManagementSetup.IsCrossEnvConnectionConfigured() then + Error(NotConfiguredErr); + end; + + local procedure ShouldRetry(var ResponseMessage: HttpResponseMessage; Attempt: Integer; var RetryAfter: Duration): Boolean + begin + if Attempt >= MaxRetries() then + exit(false); + if not (ResponseMessage.HttpStatusCode() in [429, 503]) then + exit(false); + RetryAfter := RetryAfterDuration(ResponseMessage); + exit(true); + end; + + local procedure RetryAfterDuration(var ResponseMessage: HttpResponseMessage) RetryAfter: Duration + var + ResponseHeaders: HttpHeaders; + Values: array[10] of Text; + Seconds: Integer; + begin + RetryAfter := 5000; // default backoff when the source gives no Retry-After + ResponseHeaders := ResponseMessage.Headers(); + if ResponseHeaders.GetValues('Retry-After', Values) then + if Evaluate(Seconds, Values[1]) then + if Seconds > 0 then + RetryAfter := Seconds * 1000; + if RetryAfter > 60000 then + RetryAfter := 60000; // never wait more than a minute inside a job + end; + + local procedure UriEncodeCompany(CompanyNameText: Text): Text + var + TypeHelper: Codeunit "Type Helper"; + begin + exit(TypeHelper.UrlEncode(CompanyNameText)); + end; + + local procedure MaxRetries(): Integer + begin + if MaxRetriesValue = 0 then + MaxRetriesValue := 2; + exit(MaxRetriesValue); end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index fa6d980893b..c60bb0a622d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -94,6 +94,28 @@ table 7230 "Master Data Management Setup" MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); end; } + field(156; "Source Environment URL"; Text[250]) + { + Caption = 'Source Environment URL'; + DataClassification = OrganizationIdentifiableInformation; + } + field(157; "Source Company Name"; Text[100]) + { + Caption = 'Source Company'; + DataClassification = OrganizationIdentifiableInformation; + } + field(158; "Source OAuth Client Id"; Text[100]) + { + Caption = 'Source Client ID'; + ExtendedDatatype = Masked; + DataClassification = SystemMetadata; + } + field(159; "Source Client Secret Key"; Guid) + { + Caption = 'Source Client Secret Key'; + ExtendedDatatype = Masked; + DataClassification = SystemMetadata; + } } keys @@ -156,6 +178,47 @@ table 7230 "Master Data Management Setup" exit(Result); end; + internal procedure IsCrossEnvConnectionConfigured(): Boolean + begin + exit(("Source Environment URL" <> '') and ("Source Company Name" <> '') and ("Source OAuth Client Id" <> '') and (not IsNullGuid("Source Client Secret Key"))); + end; + + [NonDebuggable] + internal procedure SetSourceClientSecret(ClientSecret: SecretText) + begin + "Source Client Secret Key" := SetSecret("Source Client Secret Key", ClientSecret); + end; + + internal procedure GetSourceClientSecret(): SecretText + begin + exit(GetSecret("Source Client Secret Key")); + end; + + // Mirrors the Intercompany connection pattern: secrets live in module-scoped Isolated Storage, keyed by a Guid. + [NonDebuggable] + local procedure SetSecret(SecretKey: Guid; SecretValue: SecretText): Guid + var + NewSecretKey: Guid; + begin + if not IsNullGuid(SecretKey) then + if not IsolatedStorage.Delete(SecretKey, DataScope::Company) then; + + NewSecretKey := CreateGuid(); + if not EncryptionEnabled() then + IsolatedStorage.Set(NewSecretKey, SecretValue, DataScope::Company) + else + IsolatedStorage.SetEncrypted(NewSecretKey, SecretValue, DataScope::Company); + + exit(NewSecretKey); + end; + + local procedure GetSecret(SecretKey: Guid) SecretValue: SecretText + begin + if IsNullGuid(SecretKey) then + exit; + if not IsolatedStorage.Get(SecretKey, DataScope::Company, SecretValue) then; + end; + local procedure EnableConnection() var MasterDataMgtSubscriber: Record "Master Data Mgt. Subscriber"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al index ced28c054e8..729265fffbd 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al @@ -42,6 +42,13 @@ tableextension 7235 MasterDataMgtTableMapping extends "Integration Table Mapping Commit(); end; } + field(7236; "Source Change Cursor"; Text[150]) + { + // Composite (SystemModifiedAt, SystemId) resume point ({"modifiedAt":...,"systemId":...}, ~93 chars). + // Empty means "caught up": the run drained the source and the watermark is authoritative. + Caption = 'Source Change Cursor'; + DataClassification = SystemMetadata; + } } var diff --git a/src/Apps/W1/MasterDataManagement/test library/app.json b/src/Apps/W1/MasterDataManagement/test library/app.json index 04719d40770..6b890fa7959 100644 --- a/src/Apps/W1/MasterDataManagement/test library/app.json +++ b/src/Apps/W1/MasterDataManagement/test library/app.json @@ -30,6 +30,10 @@ { "from": 139929, "to": 139930 + }, + { + "from": 139934, + "to": 139934 } ], "resourceExposurePolicy": { diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index a1148636675..ee2cd871904 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -83,6 +83,13 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableId, SystemId, SourceRecordRef)); end; + procedure DataSourceGetModifiedBatch(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; StartCursor: Text; MaxPages: Integer; var SourceRecordRef: RecordRef; var EndCursor: Text; var HasMore: Boolean): Boolean + var + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + begin + exit(CrossEnvDataSource.GetModifiedBatch(IntegrationTableMapping, TableFilter, StartCursor, MaxPages, SourceRecordRef, EndCursor, HasMore)); + end; + // Setting a Source Environment Name routes GetDataSource() to the cross-environment implementation. procedure SetSourceEnvironmentName(EnvironmentName: Text) var diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al new file mode 100644 index 00000000000..c3b1b0cc929 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al @@ -0,0 +1,31 @@ +#pragma warning disable AA0247 +codeunit 139934 "MDM Test Paging Config" +{ + // Test hook: forces a small cross-environment page size so paging/resume can be exercised with a few records. + // SingleInstance so the flag the test sets is the one the static subscriber reads. + SingleInstance = true; + Access = Public; + + var + Active: Boolean; + PageSizeValue: Integer; + + procedure Activate(NewPageSize: Integer) + begin + Active := true; + PageSizeValue := NewPageSize; + end; + + procedure Deactivate() + begin + Active := false; + PageSizeValue := 0; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Data Source", 'OnGetCrossEnvPageSize', '', false, false)] + local procedure HandleGetCrossEnvPageSize(var PageSize: Integer) + begin + if Active then + PageSize := PageSizeValue; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index caf8353abb1..5cce489acdd 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -134,6 +134,64 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvGetModifiedBatchResumesAcrossRuns() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + PagingConfig: Codeunit "MDM Test Paging Config"; + SourceRecordRef: RecordRef; + SeededSystemId: Guid; + SeededSystemIds: List of [Guid]; + CollectedSystemIds: List of [Guid]; + Watermark: DateTime; + Cursor: Text; + EndCursor: Text; + Index: Integer; + Runs: Integer; + HasMore: Boolean; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A page-capped batch (MaxPages=1) drains a multi-page change set across several runs, resuming + // from the returned cursor each time, and covers every source record exactly once. + Initialize(); + + // [GIVEN] five source customers created strictly after a captured watermark (so only they are in the feed) + Watermark := CurrentDateTime(); + Sleep(100); + for Index := 1 to 5 do begin + LibrarySalesLib.CreateCustomer(Customer); + SeededSystemIds.Add(Customer.SystemId); + end; + CreateMinimalCustomerMapping(IntegrationTableMapping); + IntegrationTableMapping."Synch. Modified On Filter" := Watermark; + IntegrationTableMapping.Modify(); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + PagingConfig.Activate(2); // two records per page + + // [WHEN] the batch is drained one page per run, resuming from the returned cursor + Cursor := ''; + repeat + LibraryMasterDataMgt.DataSourceGetModifiedBatch(IntegrationTableMapping, '', Cursor, 1, SourceRecordRef, EndCursor, HasMore); + CollectSystemIds(SourceRecordRef, CollectedSystemIds); + Cursor := EndCursor; + Runs += 1; + until not HasMore; + + // [THEN] it took multiple runs and every seeded record was returned exactly once + Assert.IsTrue(Runs >= 3, 'A 5-record set at 2/page and 1 page/run should need at least three runs'); + Assert.AreEqual(5, CollectedSystemIds.Count(), 'Every seeded record should be returned exactly once across runs'); + foreach SeededSystemId in SeededSystemIds do + Assert.IsTrue(CollectedSystemIds.Contains(SeededSystemId), 'Every seeded record should be covered by the resumed batches'); + + PagingConfig.Deactivate(); + CleanUp(); + end; + local procedure Initialize() var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -152,8 +210,10 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" var InProcessTransport: Codeunit "MDM In-Process Transport"; LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + PagingConfig: Codeunit "MDM Test Paging Config"; begin InProcessTransport.Deactivate(); + PagingConfig.Deactivate(); LibraryMasterDataMgt.SetSourceEnvironmentName(''); end; @@ -180,6 +240,18 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" exit(false); end; + local procedure CollectSystemIds(var SourceRecordRef: RecordRef; var CollectedSystemIds: List of [Guid]) + var + SystemIdValue: Guid; + begin + if SourceRecordRef.FindSet() then + repeat + SystemIdValue := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + if not CollectedSystemIds.Contains(SystemIdValue) then + CollectedSystemIds.Add(SystemIdValue); + until SourceRecordRef.Next() = 0; + end; + local procedure LibraryRandomText(): Text var LibraryRandomCu: Codeunit "Library - Random"; From e96281ee93988626c259f72d1034a5e5a89af046 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 27 Aug 2026 14:16:09 +0200 Subject: [PATCH 10/64] [Master Data Management] Cross-environment synchonization --- .../src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 6 +++++- .../test/src/MDMCrossEnvConsumerTests.Codeunit.al | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 14f2d3eb341..83195003e0c 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -65,7 +65,11 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Selector := EndCursor; end; until (not HasMore) or ((MaxPages > 0) and (PagesFetched >= MaxPages)); - IntegrationTableMapping.SetIntRecordRefFilter(SourceRecordRef, TableFilter); + // Apply only the mapping's row filter. The source already filtered by the watermark cursor server-side; + // re-applying the modified-on filter here would drop every record, since the materialized temp rows + // carry no SystemModifiedAt. + if TableFilter <> '' then + SourceRecordRef.SetView(TableFilter); exit(SourceRecordRef.FindSet()); end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 5cce489acdd..c549641e1e9 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -198,6 +198,9 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" InProcessTransport: Codeunit "MDM In-Process Transport"; begin InProcessTransport.Deactivate(); + // SetSourceEnvironmentName validates the setup, which schedules the detector job and commits, so a mapping + // created earlier in a test survives AutoRollback; clear leftovers to keep tests independent. + DeleteTestArtifacts(); if not MasterDataManagementSetup.Get() then begin MasterDataManagementSetup.Init(); MasterDataManagementSetup.Insert(); @@ -215,6 +218,15 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" InProcessTransport.Deactivate(); PagingConfig.Deactivate(); LibraryMasterDataMgt.SetSourceEnvironmentName(''); + DeleteTestArtifacts(); + end; + + local procedure DeleteTestArtifacts() + var + IntegrationTableMapping: Record "Integration Table Mapping"; + begin + IntegrationTableMapping.SetFilter(Name, 'MDMXENV*'); + IntegrationTableMapping.DeleteAll(); end; local procedure CreateMinimalCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") From 78cb258007890a8e05d96ba39ba507b82b871b7c Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 27 Aug 2026 17:59:20 +0200 Subject: [PATCH 11/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 1 + .../MDMCrossEnvChangeDetector.Codeunit.al | 5 + .../MDMCrossEnvDataSource.Codeunit.al | 51 +++ .../MDMHttpSourceTransport.Codeunit.al | 27 +- .../codeunits/MDMLocalDataSource.Codeunit.al | 8 + .../MDMSourceCapabilities.Codeunit.al | 66 ++++ .../MasterDataManagement.Codeunit.al | 47 +-- .../MasterDataMgtSubscribers.Codeunit.al | 58 +-- .../MasterDataMgtTableCouple.Codeunit.al | 11 +- .../MasterDataMgtTblUncouple.Codeunit.al | 36 +- .../interfaces/IMDMDataSource.Interface.al | 7 + .../src/pages/MDMConnectionDetails.Page.al | 330 ++++++++++++++++++ .../pages/MasterDataManagementSetup.Page.al | 12 + .../src/LibraryMasterDataMgt.Codeunit.al | 15 + .../src/MDMInProcessTransport.Codeunit.al | 15 + .../src/MDMCrossEnvConsumerTests.Codeunit.al | 109 ++++++ 16 files changed, 715 insertions(+), 83 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al create mode 100644 src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index a57c648fc49..01b1a3ad3a3 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -24,6 +24,7 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "MDM Cross-Env Data Source" = X, codeunit "MDM Source Connection" = X, codeunit "MDM Cross-Env Change Detector" = X, + codeunit "MDM Source Capabilities" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index ab4f8554059..67e274c52ff 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -27,9 +27,11 @@ codeunit 7245 "MDM Cross-Env Change Detector" MasterDataManagementSetup: Record "Master Data Management Setup"; SourceConnection: Codeunit "MDM Source Connection"; SourceResponse: Codeunit "MDM Source Response"; + SourceCapabilities: Codeunit "MDM Source Capabilities"; Transport: Interface "IMDM Source Transport"; Response: JsonObject; TableIds: JsonArray; + LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; begin if not MasterDataManagementSetup.Get() then exit; @@ -42,6 +44,9 @@ codeunit 7245 "MDM Cross-Env Change Detector" exit; Transport := SourceConnection.GetTransport(); + // Skip (rather than error every run) if an older source doesn't advertise the detection action. + if not SourceCapabilities.IsSupported(Transport, LastModifiedFeatureTok) then + exit; if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(WriteArray(TableIds)), Response) then exit; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 83195003e0c..37a1b2696e5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -14,10 +14,13 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var SourceResponse: Codeunit "MDM Source Response"; + SourceCapabilities: Codeunit "MDM Source Capabilities"; InvalidResponseErr: Label 'The source environment returned an unexpected response for table %1.', Comment = '%1 = table caption'; TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; + RecordsFeatureTok: Label 'records', Locked = true; + LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean var @@ -29,6 +32,15 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(GetModifiedBatch(IntegrationTableMapping, TableFilter, CursorSelector(IntegrationTableMapping."Synch. Modified On Filter"), 0, SourceRecordRef, EndCursor, HasMore)); end; + procedure GetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + EndCursor: Text; + HasMore: Boolean; + begin + // Full read from the start (selector '{}' = no watermark), then apply the row filter - for coupling/uncoupling. + exit(GetModifiedBatch(IntegrationTableMapping, TableFilter, '{}', 0, SourceRecordRef, EndCursor, HasMore)); + end; + /// /// Fetches at most MaxPages pages of changed source records starting from StartCursor (a cursor selector). /// MaxPages = 0 means unbounded. On return EndCursor holds the resume point and HasMore tells the caller @@ -45,6 +57,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceRecordRef.Close(); SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID", true); Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); FieldIds := BuildFieldIds(IntegrationTableMapping); if StartCursor <> '' then Selector := StartCursor @@ -73,6 +86,43 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(SourceRecordRef.FindSet()); end; + /// + /// Cheap existence probe for the full-synch review: the source reports an empty lastModifiedAt for an empty + /// table, so this avoids counting the whole table over the wire. + /// + internal procedure SourceHasRecords(IntegrationTableId: Integer): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + Entry: JsonObject; + Tables: JsonArray; + Token: JsonToken; + TableIds: JsonArray; + TableIdsText: Text; + LastModifiedAtText: Text; + begin + TableIds.Add(IntegrationTableId); + TableIds.WriteTo(TableIdsText); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, LastModifiedFeatureTok); + if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(TableIdsText), Response) then + exit(false); + if not Response.Get('tables', Token) then + exit(false); + Tables := Token.AsArray(); + if Tables.Count() = 0 then + exit(false); + Tables.Get(0, Token); + Entry := Token.AsObject(); + if Entry.Get('tableAvailable', Token) then + if not Token.AsValue().AsBoolean() then + exit(false); + if Entry.Get('lastModifiedAt', Token) then + if Token.IsValue() then + LastModifiedAtText := Token.AsValue().AsText(); + exit(LastModifiedAtText <> ''); + end; + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean var SystemIds: List of [Guid]; @@ -120,6 +170,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" if SystemIds.Count() = 0 then exit; Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); ParseOrError( IntegrationTableId, Transport.GetRecords(IntegrationTableId, FieldIds, SystemIdsSelector(SystemIds), PageSize()), diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 3a7af7821a1..65583903778 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -4,6 +4,7 @@ using System.Azure.Identity; using System.Environment; using System.Security.Authentication; using System.Reflection; +using System.Telemetry; /// /// Production transport: calls the source environment's ODataV4 web service with an app-only (client @@ -28,6 +29,11 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true; TokenEndpointTok: Label 'https://login.microsoftonline.com/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; ActionUrlTok: Label '%1/ODataV4/%2_%3?company=%4', Locked = true, Comment = '%1 = base url, %2 = service, %3 = action, %4 = company'; + TelemetryCategoryTok: Label 'MDM Cross-Environment', Locked = true; + TokenAcquiredAuditTxt: Label 'Acquired an application access token to read master data from source environment %1.', Comment = '%1 = source environment name'; + TokenFailedAuditTxt: Label 'Failed to acquire an application access token for source environment %1.', Comment = '%1 = source environment name'; + AccessDeniedAuditTxt: Label 'Source environment %1 denied the master data request (HTTP %2).', Comment = '%1 = source environment name, %2 = HTTP status code'; + RequestFailedTelemetryTxt: Label 'Cross-environment %1 request failed with HTTP %2.', Locked = true; procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text var @@ -75,12 +81,25 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ResponseMessage.Content().ReadAs(ResponseBodyText); if ResponseMessage.IsSuccessStatusCode() then exit(UnwrapODataValue(ResponseBodyText)); - if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then + if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin + LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); Error(HttpErr, ResponseMessage.HttpStatusCode(), ResponseBodyText); + end; Sleep(RetryAfter); end; end; + local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage) + var + AuditLog: Codeunit "Audit Log"; + begin + // Operational telemetry: action + status only, never record data or credentials. + Session.LogMessage('', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', TelemetryCategoryTok); + // Security audit: an authorization failure crossing the environment boundary. + if ResponseMessage.HttpStatusCode() in [401, 403] then + AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); + end; + local procedure Send(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage) var HttpClient: HttpClient; @@ -142,6 +161,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" var OAuth2: Codeunit OAuth2; AzureADTenant: Codeunit "Azure AD Tenant"; + AuditLog: Codeunit "Audit Log"; Scopes: List of [Text]; TokenEndpoint: Text; begin @@ -151,8 +171,11 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" MasterDataManagementSetup."Source OAuth Client Id", MasterDataManagementSetup.GetSourceClientSecret(), TokenEndpoint, '', Scopes, Token); - if Token.IsEmpty() then + if Token.IsEmpty() then begin + AuditLog.LogAuditMessage(StrSubstNo(TokenFailedAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Failure, AuditCategory::Authentication, 4, 0); Error(NoTokenErr); + end; + AuditLog.LogAuditMessage(StrSubstNo(TokenAcquiredAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Success, AuditCategory::Authentication, 4, 0); end; local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al index 38b6cc8a598..52a4d66278f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -60,6 +60,14 @@ codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" exit(SourceRecordRef.FindSet()); end; + procedure GetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + begin + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + if TableFilter <> '' then + SourceRecordRef.SetView(TableFilter); + exit(SourceRecordRef.FindSet()); + end; + local procedure OpenSourceRecordRef(IntegrationTableId: Integer; var SourceRecordRef: RecordRef) var MasterDataManagementSetup: Record "Master Data Management Setup"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al new file mode 100644 index 00000000000..f62323fb114 --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -0,0 +1,66 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Wire-contract negotiation: caches the source's advertised version and feature names (GetCapabilities) so the +/// subsidiary only calls actions the source actually supports. A newer subsidiary talking to an older source thus +/// fails with a clear message instead of a 404. SingleInstance: one source connection per session. +/// +codeunit 7246 "MDM Source Capabilities" +{ + Access = Internal; + SingleInstance = true; + + var + Negotiated: Boolean; + ContractVersion: Integer; + SupportedFeatures: List of [Text]; + UnsupportedFeatureErr: Label 'The source environment does not support the required ''%1'' capability. Update the Master Data Management app on the source environment.', Comment = '%1 = capability name'; + + procedure EnsureSupported(Transport: Interface "IMDM Source Transport"; Feature: Text) + begin + if not IsSupported(Transport, Feature) then + Error(UnsupportedFeatureErr, Feature); + end; + + procedure IsSupported(Transport: Interface "IMDM Source Transport"; Feature: Text): Boolean + begin + Negotiate(Transport); + exit(SupportedFeatures.Contains(Feature)); + end; + + procedure ContractVersionNo(Transport: Interface "IMDM Source Transport"): Integer + begin + Negotiate(Transport); + exit(ContractVersion); + end; + + // Clears the cached negotiation (used by tests that swap the injected transport). + procedure Reset() + begin + Negotiated := false; + ContractVersion := 0; + Clear(SupportedFeatures); + end; + + local procedure Negotiate(Transport: Interface "IMDM Source Transport") + var + Capabilities: JsonObject; + FeaturesToken: JsonToken; + VersionToken: JsonToken; + FeatureToken: JsonToken; + begin + if Negotiated then + exit; + if Capabilities.ReadFrom(Transport.GetCapabilities()) then begin + if Capabilities.Get('version', VersionToken) then + if VersionToken.IsValue() then + ContractVersion := VersionToken.AsValue().AsInteger(); + if Capabilities.Get('features', FeaturesToken) then + if FeaturesToken.IsArray() then + foreach FeatureToken in FeaturesToken.AsArray() do + if FeatureToken.IsValue() then + SupportedFeatures.Add(FeatureToken.AsValue().AsText()); + end; + Negotiated := true; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index febf099ba23..edd970d98dd 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -655,7 +655,6 @@ codeunit 7233 "Master Data Management" LocalRecordRef: RecordRef; IntegrationRecordRef: RecordRef; CountFailed: Integer; - SourceCompanyName: Text[30]; begin AddIntegrationTableMapping(IntegrationTableMapping); IntegrationTableMapping.SetTableFilter(LocalTableFilter); @@ -670,13 +669,8 @@ codeunit 7233 "Master Data Management" until LocalRecordRef.Next() = 0 end else begin MasterDataManagementSetup.Get(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationRecordRef.SetView(IntegrationTableFilter); - if IntegrationRecordRef.FindSet() then + // Route the source read so uncoupling works against the local company or another environment. + if MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, IntegrationTableFilter, IntegrationRecordRef) then repeat if not PerformUncoupling(IntegrationTableMapping, LocalRecordRef, IntegrationRecordRef) then CountFailed += 1; @@ -2155,23 +2149,15 @@ codeunit 7233 "Master Data Management" local procedure FindCouplingByIntegrationSystemID(var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; IntegrationSystemID: Guid) Found: Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - RecRef: RecordRef; - RecId: RecordId; - SourceCompanyName: Text[30]; + IntegrationRecordRef: RecordRef; begin Clear(MasterDataMgtCoupling."Integration System ID"); MasterDataManagementSetup.Get(); MasterDataMgtCoupling.Reset(); MasterDataMgtCoupling.SetRange("Integration System ID", IntegrationSystemID); if MasterDataMgtCoupling.FindFirst() then - if MasterDataMgtCoupling.FindRecordId(RecId) then begin - RecRef.Open(RecId.TableNo()); - OnSetSourceCompanyName(SourceCompanyName, RecId.TableNo()); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecRef.ChangeCompany(SourceCompanyName); - Found := RecRef.Get(RecId); - end; + // Route through the data source: the source record is in the local company or another environment. + Found := GetIntegrationRecordRef(MasterDataMgtCoupling."Table ID", MasterDataMgtCoupling, IntegrationRecordRef); end; internal procedure GetIntegrationRecRefCount(var IntegrationTableMapping: Record "Integration Table Mapping"): Integer @@ -2180,11 +2166,18 @@ codeunit 7233 "Master Data Management" INtegrationVendor: Record Vendor; IntegrationCustomer: Record Customer; MasterDataManagementSetup: Record "Master Data Management Setup"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; IntegrationRecRef: RecordRef; IntegrationRecRefCount: Integer; SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + // Cross-environment: the review only needs existence; probe the source instead of counting over the wire. + if CrossEnvDataSource.SourceHasRecords(IntegrationTableMapping."Integration Table ID") then + exit(1); + exit(0); + end; OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); if SourceCompanyName = '' then SourceCompanyName := MasterDataManagementSetup."Company Name"; @@ -2293,9 +2286,6 @@ codeunit 7233 "Master Data Management" var MasterDataManagementSetup: Record "Master Data Management Setup"; IntegrationRecordRef: RecordRef; - IntegrationSystemIdFieldRef: FieldRef; - IntegrationTableView: Text; - SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); @@ -2304,17 +2294,8 @@ codeunit 7233 "Master Data Management" IntegrationTableMapping.SetFilter("Integration Table ID", '<>0'); if IntegrationTableMapping.FindSet() then repeat - IntegrationRecordRef.Close(); - IntegrationTableView := IntegrationTableMapping.GetIntegrationTableFilter(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationSystemIdFieldRef := IntegrationRecordRef.Field(IntegrationRecordRef.SystemIdNo); - IntegrationRecordRef.SetView(IntegrationTableView); - IntegrationSystemIdFieldRef.SetRange(MasterDataMgtCoupling."Integration System ID"); - if not IntegrationRecordRef.IsEmpty() then + // Route through the data source: find which enabled mapping's source table holds this record. + if MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableMapping."Integration Table ID", MasterDataMgtCoupling."Integration System ID", IntegrationRecordRef) then exit(true); until IntegrationTableMapping.Next() = 0; exit(false); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 2b1e32cb7c8..6087a21b4de 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -135,18 +135,23 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IsJobQueueEntryDataSynchJob(Sender, IntegrationTableMapping) then begin MasterDataManagementSetup.Get(); if MasterDataManagementSetup."Is Enabled" then begin - MasterDataManagement.OnSetIntegrationTableFilter(IntegrationTableMapping, RecRef, IsHandled); - if not IsHandled then begin - RecRef.Open(IntegrationTableMapping."Integration Table ID", false); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - RecRef.ChangeCompany(SourceCompanyName); - IntegrationTableMapping.SetIntRecordRefFilter(RecRef); + if MasterDataManagementSetup."Source Environment Name" <> '' then + // Cross-environment: the change detector governs when this job is nudged; let it run and fetch the delta. + Result := true + else begin + MasterDataManagement.OnSetIntegrationTableFilter(IntegrationTableMapping, RecRef, IsHandled); + if not IsHandled then begin + RecRef.Open(IntegrationTableMapping."Integration Table ID", false); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Integration Table ID"); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + RecRef.ChangeCompany(SourceCompanyName); + IntegrationTableMapping.SetIntRecordRefFilter(RecRef); + end; + if not RecRef.IsEmpty() then + Result := true; + RecRef.Close(); end; - if not RecRef.IsEmpty() then - Result := true; - RecRef.Close(); end; end; end; @@ -501,7 +506,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagement: Codeunit "Master Data Management"; BeforeRenameDestinationRecordRef: RecordRef; IsHandled: Boolean; - SourceCompanyName: Text[30]; + SourceSystemId: Guid; begin if not MasterDataManagement.IsEnabled() then exit; @@ -512,11 +517,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagementSetup.Get(); MasterDataManagement.OnGetIntegrationRecordRef(IntegrationTableMapping, SourceRecordRef, IsHandled); if not IsHandled then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - SourceRecordRef.ChangeCompany(SourceCompanyName); - SourceRecordRef.GetBySystemId(SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value()); + // Route the source re-fetch: the record lives in the local company or another environment. + SourceSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + if MasterDataManagementSetup.GetDataSource().GetBySystemId(SourceRecordRef.Number(), SourceSystemId, SourceRecordRef) then; end; BeforeRenameDestinationRecordRef.Open(DestinationRecordRef.Number()); BeforeRenameDestinationRecordRef.GetBySystemId(DestinationRecordRef.Field(DestinationRecordRef.SystemIdNo()).Value()); @@ -687,19 +690,14 @@ codeunit 7237 "Master Data Mgt. Subscribers" ModifiedFieldRef: FieldRef; IsHandled: Boolean; IntRecSystemId: Guid; - SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); IntegrationRecordRef.Open(FromRecordRef.Number, false); IntRecSystemId := FromRecordRef.Field(FromRecordRef.SystemIdNo).Value(); MasterDataManagement.OnGetIntegrationRecordRefBySystemId(IntegrationTableMapping, IntegrationRecordRef, IntRecSystemId, IsHandled); - if not IsHandled then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationRecordRef.GetBySystemId(IntRecSystemId); - end; + if not IsHandled then + // Route the source re-fetch: the record lives in the local company or another environment. + if MasterDataManagementSetup.GetDataSource().GetBySystemId(FromRecordRef.Number, IntRecSystemId, IntegrationRecordRef) then; if FromRecordRef.Number() = IntegrationTableMapping."Integration Table ID" then begin ModifiedFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Int. Tbl. Modified On Fld. No."); exit(ModifiedFieldRef.Value()); @@ -913,6 +911,10 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup."Is Enabled" then exit; + // Media synchronization is not supported cross-environment (deferred); skip so pictures are not cleared. + if MasterDataManagementSetup."Source Environment Name" <> '' then + exit; + IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); IntegrationTableMapping.SetRange("Delete After Synchronization", false); @@ -1091,6 +1093,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" exit(false); MasterDataManagementSetup.Get(); + // Cross-environment: related contact/customer resolution reads the source company directly; deferred for now. + if MasterDataManagementSetup."Source Environment Name" <> '' then + exit(false); DestinationRecordRef.SetTable(Contact); IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); SourceRecordRef.SetTable(IntegrationContact); @@ -1198,6 +1203,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" exit; // all contacts have parent company set MasterDataManagementSetup.Get(); + // Cross-environment: related contact resolution reads the source company directly; deferred for now. + if MasterDataManagementSetup."Source Environment Name" <> '' then + exit; IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al index a01934bdc0f..1173abcd072 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al @@ -89,7 +89,6 @@ codeunit 7235 "Master Data Mgt. Table Couple" FilterList: List of [Text]; MatchPriorityList: List of [Integer]; MatchPriority: Integer; - SourceCompanyName: Text[30]; begin // collect the matching criteria fields in a temporary record IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); @@ -117,16 +116,10 @@ codeunit 7235 "Master Data Mgt. Table Couple" // iterate through integration records and for each of them try to find a match in local system MasterDataManagementSetup.Get(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); IntegrationMasterDataSynch.SplitIntegrationTableFilter(IntegrationTableMapping, FilterList); foreach TableFilter in FilterList do begin - if TableFilter <> '' then - IntegrationRecordRef.SetView(TableFilter); - if IntegrationRecordRef.FindSet() then + // Route the source read so match-based coupling works against the local company or another environment. + if MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, TableFilter, IntegrationRecordRef) then repeat if GuiAllowed() then begin RecordNumber += 1; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al index f77882b2867..724d047db0a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTblUncouple.Codeunit.al @@ -88,20 +88,28 @@ codeunit 7236 "Master Data Mgt. Tbl. Uncouple" SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); - IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - IntegrationRecordRef.ChangeCompany(SourceCompanyName); - IntegrationTableMapping.SetIntRecordRefFilter(IntegrationRecordRef); - if IntegrationRecordRef.FindSet() then - repeat - if TempMasterDataMgtCoupling.IsIntegrationRecordRefCoupled(IntegrationRecordRef) then begin - TempMasterDataMgtCoupling.Delete(); - Clear(LocalRecordRef); - IntegrationTableSynch.Uncouple(LocalRecordRef, IntegrationRecordRef); - end; - until IntegrationRecordRef.Next() = 0; + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + // Cross-environment: read the whole filtered source set over the wire (no modified-on watermark). + if not MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, IntegrationTableMapping.GetIntegrationTableFilter(), IntegrationRecordRef) then + exit; + end else begin + IntegrationRecordRef.Open(IntegrationTableMapping."Integration Table ID"); + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableMapping."Table ID"); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + IntegrationRecordRef.ChangeCompany(SourceCompanyName); + IntegrationTableMapping.SetIntRecordRefFilter(IntegrationRecordRef); + if not IntegrationRecordRef.FindSet() then + exit; + end; + + repeat + if TempMasterDataMgtCoupling.IsIntegrationRecordRefCoupled(IntegrationRecordRef) then begin + TempMasterDataMgtCoupling.Delete(); + Clear(LocalRecordRef); + IntegrationTableSynch.Uncouple(LocalRecordRef, IntegrationRecordRef); + end; + until IntegrationRecordRef.Next() = 0; end; local procedure UncoupleAllCoupledRecords(var IntegrationTableMapping: Record "Integration Table Mapping"; var IntegrationTableSynch: Codeunit "Integration Table Synch."; var TempMasterDataMgtCoupling: Record "Master Data Mgt. Coupling" temporary) diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al index cbfb8269f2d..1219fbc96d5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al @@ -34,4 +34,11 @@ interface "IMDM Data Source" /// matches UidFilter (a filter expression, e.g. a list of SystemIds). Returns true if any matched. /// procedure GetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean; + + /// + /// Opens SourceRecordRef on the source integration table and returns ALL records matching TableFilter + /// (the whole set, not just those modified since the watermark) - used by coupling and uncoupling. + /// Returns true if at least one record matches. + /// + procedure GetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al new file mode 100644 index 00000000000..b6936fb8a2e --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -0,0 +1,330 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Guided setup for reading master data from another Business Central ENVIRONMENT (same tenant). Mirrors the +/// Intercompany cross-environment partner wizard: consent, then the source environment/company and the +/// Microsoft Entra client-credentials application, then an optional connection test. The client secret is +/// write-only and stored in module-scoped Isolated Storage; it is never read back to the page. +/// +page 7232 "MDM Connection Details" +{ + Caption = 'Cross-Environment Connection Setup'; + PageType = NavigatePage; + ApplicationArea = Suite; + UsageCategory = None; + Permissions = tabledata "Master Data Management Setup" = imd; + + layout + { + area(Content) + { + group(WelcomeTab) + { + ShowCaption = false; + Visible = Step = Step::Welcome; + group(Introduction) + { + Caption = 'Welcome'; + InstructionalText = 'This guide helps you connect to a company in a different Business Central environment so you can synchronize master data from it. Before you continue, make sure the source environment has this extension installed and exposes its data, and that you have a Microsoft Entra application with a read-only permission set on the source.'; + } + group(TermsAndConditions) + { + Caption = 'Review the terms and conditions'; + InstructionalText = 'By enabling this feature, you consent to your data being shared between Business Central environments. Your privacy is important to us. To learn more, follow the link below.'; + + field(Consent; ConsentState) + { + ApplicationArea = All; + Caption = 'I accept'; + ToolTip = 'Accept the terms and conditions.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + field(LearnMore; LearnMoreTok) + { + ApplicationArea = All; + Editable = false; + ShowCaption = false; + ToolTip = 'View information about privacy.'; + + trigger OnDrillDown() + begin + Hyperlink(PrivacyLinkTxt); + end; + } + } + } + group(ConnectionTab) + { + ShowCaption = false; + Visible = Step = Step::Connection; + group(SourceConnectionDetails) + { + Caption = 'Source environment connection details'; + InstructionalText = 'Provide the environment and company you want to read master data from.'; + + field(SourceEnvironmentName; SourceEnvironmentName) + { + Caption = 'Source Environment'; + ApplicationArea = Suite; + ShowMandatory = true; + ToolTip = 'Specifies the name of the source Business Central environment.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + field(SourceEnvironmentUrl; SourceEnvironmentUrl) + { + Caption = 'Source Environment URL'; + ApplicationArea = Suite; + ExtendedDatatype = URL; + ShowMandatory = true; + ToolTip = 'Specifies the base URL of the source environment''s web services, up to but not including /ODataV4.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + field(SourceCompanyName; SourceCompanyName) + { + Caption = 'Source Company Name'; + ApplicationArea = Suite; + ShowMandatory = true; + ToolTip = 'Specifies the name of the company in the source environment that data is read from.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + } + group(OAuth2ConnectionDetails) + { + Caption = 'Authentication details'; + InstructionalText = 'Provide the Microsoft Entra application that this environment uses to authenticate to the source environment.'; + + field(OAuth2ClientId; OAuth2ClientId) + { + Caption = 'Client ID'; + ApplicationArea = Suite; + ExtendedDatatype = Masked; + ShowMandatory = true; + ToolTip = 'Specifies the application (client) ID of the Microsoft Entra authentication application.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + field(OAuth2ClientSecret; OAuth2ClientSecret) + { + Caption = 'Client Secret'; + ApplicationArea = Suite; + ExtendedDatatype = Masked; + ShowMandatory = not SecretAlreadyStored; + ToolTip = 'Specifies the client secret of the Microsoft Entra authentication application. The secret is stored securely and is not shown again after you enter it.'; + + trigger OnValidate() + begin + SetControls(); + end; + } + } + } + group(TestConnectionTab) + { + ShowCaption = false; + Visible = Step = Step::TestConnection; + group(VerifyConnection) + { + Caption = 'Verify connection'; + InstructionalText = 'Optionally test that the source environment can be reached with the details you entered. Choose Test Connection, or choose Next to continue.'; + } + } + group(FinishTab) + { + ShowCaption = false; + Visible = Step = Step::Finish; + group(AllDone) + { + Caption = 'All done'; + InstructionalText = 'You''re all set. Choose Finish to save your connection settings.'; + } + } + } + } + + actions + { + area(Processing) + { + action(TestConnection) + { + ApplicationArea = Suite; + Caption = 'Test Connection'; + ToolTip = 'Test that the source environment can be reached with the current URL and credentials.'; + Visible = TestConnectionEnabled; + Image = InteractionTemplateSetup; + InFooterBar = true; + + trigger OnAction() + begin + TestConnectionToSource(); + end; + } + action(ActionBack) + { + ApplicationArea = Suite; + Caption = 'Back'; + ToolTip = 'Go to the previous step.'; + Enabled = BackEnabled; + Image = PreviousRecord; + InFooterBar = true; + + trigger OnAction() + begin + PreviousStep(); + end; + } + action(ActionNext) + { + ApplicationArea = Suite; + Caption = 'Next'; + ToolTip = 'Go to the next step.'; + Enabled = NextEnabled; + Image = NextRecord; + InFooterBar = true; + + trigger OnAction() + begin + NextStep(); + end; + } + action(ActionFinish) + { + ApplicationArea = Suite; + Caption = 'Finish'; + ToolTip = 'Save the connection settings.'; + Enabled = FinishEnabled; + Image = Approve; + InFooterBar = true; + + trigger OnAction() + begin + SaveConfiguration(); + CurrPage.Close(); + end; + } + } + } + + trigger OnOpenPage() + begin + LoadConfiguration(); + Step := Step::Welcome; + SetControls(); + end; + + var + Step: Option Welcome,Connection,TestConnection,Finish; + NextEnabled, BackEnabled, FinishEnabled, TestConnectionEnabled : Boolean; + ConsentState, SecretAlreadyStored : Boolean; + SourceEnvironmentName: Text[100]; + SourceEnvironmentUrl: Text[250]; + SourceCompanyName: Text[100]; + OAuth2ClientId: Text[100]; + [NonDebuggable] + OAuth2ClientSecret: Text; + LearnMoreTok: Label 'Privacy and Cookies'; + PrivacyLinkTxt: Label 'https://go.microsoft.com/fwlink/?linkid=521839', Locked = true; + ConnectionOkMsg: Label 'Successfully connected to the source environment (contract version %1).', Comment = '%1 = wire contract version'; + + local procedure LoadConfiguration() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit; + SourceEnvironmentName := MasterDataManagementSetup."Source Environment Name"; + SourceEnvironmentUrl := MasterDataManagementSetup."Source Environment URL"; + SourceCompanyName := MasterDataManagementSetup."Source Company Name"; + OAuth2ClientId := MasterDataManagementSetup."Source OAuth Client Id"; + SecretAlreadyStored := not IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"); + end; + + [NonDebuggable] + local procedure SaveConfiguration() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then begin + MasterDataManagementSetup.Init(); + MasterDataManagementSetup.Insert(); + end; + MasterDataManagementSetup.Validate("Source Environment Name", SourceEnvironmentName); + MasterDataManagementSetup."Source Environment URL" := SourceEnvironmentUrl; + MasterDataManagementSetup."Source Company Name" := SourceCompanyName; + MasterDataManagementSetup."Source OAuth Client Id" := OAuth2ClientId; + if OAuth2ClientSecret <> '' then + MasterDataManagementSetup.SetSourceClientSecret(OAuth2ClientSecret); + MasterDataManagementSetup.Modify(true); + end; + + [NonDebuggable] + local procedure TestConnectionToSource() + var + SourceConnection: Codeunit "MDM Source Connection"; + Transport: Interface "IMDM Source Transport"; + Capabilities: JsonObject; + VersionToken: JsonToken; + VersionText: Text; + begin + // Persist first so the transport reads the details entered in the wizard. + SaveConfiguration(); + Commit(); + Transport := SourceConnection.GetTransport(); + Capabilities.ReadFrom(Transport.GetCapabilities()); + if Capabilities.Get('version', VersionToken) then + VersionText := Format(VersionToken.AsValue().AsInteger()); + Message(ConnectionOkMsg, VersionText); + end; + + local procedure NextStep() + begin + Step += 1; + SetControls(); + end; + + local procedure PreviousStep() + begin + Step -= 1; + SetControls(); + end; + + local procedure SetControls() + begin + BackEnabled := Step > Step::Welcome; + TestConnectionEnabled := Step = Step::TestConnection; + FinishEnabled := Step = Step::Finish; + NextEnabled := (Step < Step::Finish) and StepIsComplete(); + end; + + local procedure StepIsComplete(): Boolean + begin + case Step of + Step::Welcome: + exit(ConsentState); + Step::Connection: + exit((SourceEnvironmentName <> '') and (SourceEnvironmentUrl <> '') and (SourceCompanyName <> '') and + (OAuth2ClientId <> '') and ((OAuth2ClientSecret <> '') or SecretAlreadyStored)); + else + exit(true); + end; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index e73c7cb6008..0f774141385 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -51,6 +51,18 @@ page 7230 "Master Data Management Setup" { area(Processing) { + action(ConnectionDetails) + { + ApplicationArea = Suite; + Caption = 'Connection Details'; + Image = LinkAccount; + ToolTip = 'Set up the connection to a company in a different Business Central environment for cross-environment synchronization.'; + + trigger OnAction() + begin + Page.RunModal(Page::"MDM Connection Details"); + end; + } action(ResetConfiguration) { ApplicationArea = Suite; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index ee2cd871904..610668aa69e 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -75,6 +75,21 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef)); end; + procedure DataSourceGetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, TableFilter, SourceRecordRef)); + end; + + procedure GetIntegrationRecRefCount(IntegrationTableMapping: Record "Integration Table Mapping"): Integer + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + exit(MasterDataManagement.GetIntegrationRecRefCount(IntegrationTableMapping)); + end; + procedure DataSourceGetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al index 50b032396b7..7d8e9cf8a53 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al @@ -10,8 +10,10 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" var CannedResponse: Text; + CannedCapabilities: Text; Active: Boolean; UseCanned: Boolean; + UseCannedCapabilities: Boolean; procedure Activate() begin @@ -19,10 +21,15 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" end; procedure Deactivate() + var + SourceCapabilities: Codeunit "MDM Source Capabilities"; begin Active := false; UseCanned := false; + UseCannedCapabilities := false; Clear(CannedResponse); + Clear(CannedCapabilities); + SourceCapabilities.Reset(); // clear negotiated capabilities between tests end; procedure SetCannedResponse(Response: Text) @@ -31,6 +38,12 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" UseCanned := true; end; + procedure SetCannedCapabilities(Response: Text) + begin + CannedCapabilities := Response; + UseCannedCapabilities := true; + end; + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text var SourceApi: Codeunit "MDM Cross-Env Source API"; @@ -53,6 +66,8 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" var SourceApi: Codeunit "MDM Cross-Env Source API"; begin + if UseCannedCapabilities then + exit(CannedCapabilities); exit(SourceApi.GetCapabilities()); end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index c549641e1e9..f398572619d 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -134,6 +134,115 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvIntegrationRecRefCountReportsExistence() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The full-synch review existence probe reports records for a non-empty cross-env source table. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + CreateMinimalCustomerMapping(IntegrationTableMapping); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.AreEqual(1, LibraryMasterDataMgt.GetIntegrationRecRefCount(IntegrationTableMapping), 'A non-empty source table should report records to the full-synch review'); + + CleanUp(); + end; + + [Test] + procedure ConnectionDetailsWizardSavesConfiguration() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + ConnectionDetails: TestPage "MDM Connection Details"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The Connection Details wizard collects the source connection details and saves them (secret to Isolated Storage). + Initialize(); + + ConnectionDetails.OpenEdit(); + // Welcome step: accept the terms so Next is enabled. + ConnectionDetails.Consent.SetValue(true); + ConnectionDetails.ActionNext.Invoke(); + // Connection step: provide the source environment and credentials. + ConnectionDetails.SourceEnvironmentName.SetValue('CONTOSO-PROD'); + ConnectionDetails.SourceEnvironmentUrl.SetValue('https://api.businesscentral.dynamics.com/v2.0/contoso-prod'); + ConnectionDetails.SourceCompanyName.SetValue('CRONUS'); + ConnectionDetails.OAuth2ClientId.SetValue('11111111-2222-3333-4444-555555555555'); + ConnectionDetails.OAuth2ClientSecret.SetValue('super-secret'); + ConnectionDetails.ActionNext.Invoke(); // -> Test Connection step + ConnectionDetails.ActionNext.Invoke(); // -> Finish step (optional test skipped) + ConnectionDetails.ActionFinish.Invoke(); + + // [THEN] the setup holds the connection and a stored client secret + MasterDataManagementSetup.Get(); + Assert.AreEqual('CONTOSO-PROD', MasterDataManagementSetup."Source Environment Name", 'Source environment not saved'); + Assert.AreEqual('CRONUS', MasterDataManagementSetup."Source Company Name", 'Source company not saved'); + Assert.IsFalse(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Client secret should be stored'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvGetByFilterReturnsRecordsPastTheWatermark() + var + Customer: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + FilterSourceRef: RecordRef; + ModifiedSetSourceRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] GetByFilter returns the whole filtered set (used by coupling/uncoupling), ignoring the mapping's + // watermark, unlike GetModifiedSet which only returns changes after the watermark. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + CreateMinimalCustomerMapping(IntegrationTableMapping); + // [GIVEN] a watermark far in the future, so the seeded customer is not "modified since" + IntegrationTableMapping."Synch. Modified On Filter" := CreateDateTime(DMY2Date(1, 1, 2099), 0T); + IntegrationTableMapping.Modify(); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [THEN] the watermark-based read excludes it, but the full filtered read includes it + Assert.IsFalse( + LibraryMasterDataMgt.DataSourceGetModifiedSet(IntegrationTableMapping, '', ModifiedSetSourceRef) and ContainsSystemId(ModifiedSetSourceRef, Customer.SystemId), + 'GetModifiedSet should not return records at or before the watermark'); + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetByFilter(IntegrationTableMapping, '', FilterSourceRef), 'GetByFilter should return records'); + Assert.IsTrue(ContainsSystemId(FilterSourceRef, Customer.SystemId), 'GetByFilter should return the seeded customer regardless of the watermark'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvErrorsWhenSourceLacksRecordsCapability() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] The subsidiary negotiates capabilities first and fails clearly if the source doesn't advertise 'records'. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + InProcessTransport.SetCannedCapabilities('{"version":1,"features":["lastModifiedPerTable"]}'); + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('does not support the required'); + + CleanUp(); + end; + [Test] procedure CrossEnvGetModifiedBatchResumesAcrossRuns() var From af4ffa792468fcb695900d248ccd24cbd9bcec55 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 27 Aug 2026 19:56:34 +0200 Subject: [PATCH 12/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 1 + .../MDMCrossEnvDataSource.Codeunit.al | 5 +- .../MDMCrossEnvSourceAPI.Codeunit.al | 87 +++++++++++- .../src/codeunits/MDMInlineMedia.Codeunit.al | 67 +++++++++ .../codeunits/MDMSourceResponse.Codeunit.al | 96 ++++++++++++- .../MasterDataMgtSubscribers.Codeunit.al | 60 ++++++++ .../tables/MasterDataManagementSetup.Table.al | 26 +++- .../src/LibraryMasterDataMgt.Codeunit.al | 7 + .../test library/src/MDMTestTableA.Table.al | 8 ++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 134 ++++++++++++++++++ .../src/MasterDataMgtSetupTests.Codeunit.al | 47 ++++++ 11 files changed, 529 insertions(+), 9 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 01b1a3ad3a3..017bc23e6d6 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -25,6 +25,7 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "MDM Source Connection" = X, codeunit "MDM Cross-Env Change Detector" = X, codeunit "MDM Source Capabilities" = X, + codeunit "MDM Inline Media" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 37a1b2696e5..ed7fd5c9a26 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -15,6 +15,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var SourceResponse: Codeunit "MDM Source Response"; SourceCapabilities: Codeunit "MDM Source Capabilities"; + InlineMedia: Codeunit "MDM Inline Media"; InvalidResponseErr: Label 'The source environment returned an unexpected response for table %1.', Comment = '%1 = table caption'; TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; @@ -56,6 +57,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" begin SourceRecordRef.Close(); SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID", true); + InlineMedia.Reset(); // fresh batch: drop the previous page's inline media bytes Transport := GetTransport(); SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); FieldIds := BuildFieldIds(IntegrationTableMapping); @@ -167,6 +169,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" begin SourceRecordRef.Close(); SourceRecordRef.Open(IntegrationTableId, true); + InlineMedia.Reset(); // fresh fetch: drop any prior inline media bytes if SystemIds.Count() = 0 then exit; Transport := GetTransport(); @@ -252,7 +255,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" for Index := 1 to RecRef.FieldCount() do begin CurrentField := RecRef.FieldIndex(Index); if CurrentField.Class() = FieldClass::Normal then - if not (CurrentField.Type() in [FieldType::Blob, FieldType::Media, FieldType::MediaSet]) then + if not (CurrentField.Type() in [FieldType::MediaSet, FieldType::TableFilter]) then AddFieldId(FieldIds, AddedFields, CurrentField.Number()); end; RecRef.Close(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 8d9c3749200..16ea086eae6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -1,5 +1,9 @@ namespace Microsoft.Integration.MDM; +using System.Environment; +using System.Text; +using System.Utilities; + /// /// Source-side generic API, published as an ODataV4 web service. The cross-environment implementation of /// "IMDM Data Source" (running in a subsidiary) calls these unbound actions to read source master data. @@ -156,8 +160,9 @@ codeunit 7241 "MDM Cross-Env Source API" local procedure IsProjectableField(FieldReference: FieldRef): Boolean begin + // Media and Blob are projected inline (base64); MediaSet is deferred and TableFilter carries no data. exit((FieldReference.Class() = FieldClass::Normal) and - not (FieldReference.Type() in [FieldType::Blob, FieldType::Media, FieldType::MediaSet])); + not (FieldReference.Type() in [FieldType::MediaSet, FieldType::TableFilter])); end; local procedure SelectorSystemIds(Selector: Text; var SystemIds: JsonArray): Boolean @@ -351,6 +356,7 @@ codeunit 7241 "MDM Cross-Env Source API" local procedure AppendRecord(var RecRef: RecordRef; ProjectedFields: List of [Integer]; var Records: JsonArray; var LastModifiedAt: DateTime; var LastSystemId: Guid) var + CurrentField: FieldRef; RecordObject: JsonObject; FieldsObject: JsonObject; FieldNo: Integer; @@ -359,8 +365,17 @@ codeunit 7241 "MDM Cross-Env Source API" LastSystemId := RecRef.Field(SystemIdFieldNo()).Value(); RecordObject.Add('systemId', Format(LastSystemId)); RecordObject.Add('systemModifiedAt', FormatFieldValue(RecRef.Field(SystemModifiedAtFieldNo()))); - foreach FieldNo in ProjectedFields do - FieldsObject.Add(Format(FieldNo), FormatFieldValue(RecRef.Field(FieldNo))); + foreach FieldNo in ProjectedFields do begin + CurrentField := RecRef.Field(FieldNo); + case CurrentField.Type() of + FieldType::Media: + FieldsObject.Add(Format(FieldNo), BuildMediaValue(CurrentField)); + FieldType::Blob: + FieldsObject.Add(Format(FieldNo), BuildBlobValue(CurrentField)); + else + FieldsObject.Add(Format(FieldNo), FormatFieldValue(CurrentField)); + end; + end; RecordObject.Add('fields', FieldsObject); Records.Add(RecordObject); end; @@ -388,6 +403,72 @@ codeunit 7241 "MDM Cross-Env Source API" exit(Format(FieldReference.Value(), 0, 9)); end; + // Single Media field: emit { media, name, mimeType, length, content(base64) }, or { media, empty } when the + // source has no picture, or { media, skipped, length } when it exceeds the inline cap. + local procedure BuildMediaValue(FieldReference: FieldRef): JsonObject + var + TenantMedia: Record "Tenant Media"; + Base64Convert: Codeunit "Base64 Convert"; + MediaValue: JsonObject; + MediaId: Guid; + ContentInStream: InStream; + begin + MediaValue.Add('media', true); + MediaId := FieldReference.Value(); + if IsNullGuid(MediaId) then begin + MediaValue.Add('empty', true); + exit(MediaValue); + end; + TenantMedia.SetAutoCalcFields(Content); + if not TenantMedia.Get(MediaId) then begin + MediaValue.Add('empty', true); + exit(MediaValue); + end; + if TenantMedia.Content.Length() > MaxInlineContentSize() then begin + MediaValue.Add('skipped', true); + MediaValue.Add('length', TenantMedia.Content.Length()); + exit(MediaValue); + end; + MediaValue.Add('name', TenantMedia."File Name"); + MediaValue.Add('mimeType', TenantMedia."Mime Type"); + MediaValue.Add('length', TenantMedia.Content.Length()); + TenantMedia.Content.CreateInStream(ContentInStream); + MediaValue.Add('content', Base64Convert.ToBase64(ContentInStream)); + exit(MediaValue); + end; + + // Blob field: emit { blob, length, content(base64) }, or { blob, empty }, or { blob, skipped, length }. + local procedure BuildBlobValue(FieldReference: FieldRef): JsonObject + var + Base64Convert: Codeunit "Base64 Convert"; + TempBlob: Codeunit "Temp Blob"; + BlobValue: JsonObject; + ContentInStream: InStream; + begin + BlobValue.Add('blob', true); + TempBlob.FromFieldRef(FieldReference); + if not TempBlob.HasValue() then begin + BlobValue.Add('empty', true); + exit(BlobValue); + end; + if TempBlob.Length() > MaxInlineContentSize() then begin + BlobValue.Add('skipped', true); + BlobValue.Add('length', TempBlob.Length()); + exit(BlobValue); + end; + BlobValue.Add('length', TempBlob.Length()); + TempBlob.CreateInStream(ContentInStream); + BlobValue.Add('content', Base64Convert.ToBase64(ContentInStream)); + exit(BlobValue); + end; + + // 512 KB raw. Its base64 form (~700 KB) stays under BC's 1,000,000-byte single-stream-read limit; do not + // raise toward 1 MB, where the encoded value would exceed that limit on a single read. + local procedure MaxInlineContentSize(): Integer + begin + exit(512 * 1024); + end; + [TryFunction] local procedure TryReadJsonArray(Value: Text; var JsonArrayValue: JsonArray) begin diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al new file mode 100644 index 00000000000..56e14ddf0ae --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al @@ -0,0 +1,67 @@ +namespace Microsoft.Integration.MDM; + +using System.Text; +using System.Utilities; + +/// +/// Per-batch cache of inline Media bytes carried in a cross-environment GetRecords response. The materializer +/// stashes the decoded content keyed by the source record's SystemId and the Media field number; the field +/// transfer subscriber (UpdateMedia, cross-environment branch) reads it to build the destination Tenant Media. +/// A Media field only holds a Tenant Media GUID, which is meaningless in the subsidiary, so the bytes travel +/// here instead of on the temporary source record. Single instance so the value survives from materialization +/// through the synchronization write; Reset() clears it between batches. +/// +codeunit 7232 "MDM Inline Media" +{ + Access = Internal; + SingleInstance = true; + + var + ContentByKey: Dictionary of [Text, Text]; + NameByKey: Dictionary of [Text, Text]; + MimeByKey: Dictionary of [Text, Text]; + + procedure Reset() + begin + Clear(ContentByKey); + Clear(NameByKey); + Clear(MimeByKey); + end; + + procedure Put(SystemId: Guid; FieldNo: Integer; FileName: Text; MimeType: Text; ContentBase64: Text) + var + MediaKey: Text; + begin + MediaKey := MakeKey(SystemId, FieldNo); + ContentByKey.Set(MediaKey, ContentBase64); + NameByKey.Set(MediaKey, FileName); + MimeByKey.Set(MediaKey, MimeType); + end; + + procedure Contains(SystemId: Guid; FieldNo: Integer): Boolean + begin + exit(ContentByKey.ContainsKey(MakeKey(SystemId, FieldNo))); + end; + + procedure TryGet(SystemId: Guid; FieldNo: Integer; var FileName: Text; var MimeType: Text; var TempBlob: Codeunit "Temp Blob"): Boolean + var + Base64Convert: Codeunit "Base64 Convert"; + MediaKey: Text; + ContentBase64: Text; + ContentOutStream: OutStream; + begin + MediaKey := MakeKey(SystemId, FieldNo); + if not ContentByKey.Get(MediaKey, ContentBase64) then + exit(false); + NameByKey.Get(MediaKey, FileName); + MimeByKey.Get(MediaKey, MimeType); + TempBlob.CreateOutStream(ContentOutStream); + Base64Convert.FromBase64(ContentBase64, ContentOutStream); + exit(true); + end; + + local procedure MakeKey(SystemId: Guid; FieldNo: Integer): Text + begin + exit(Format(SystemId, 0, 4) + '|' + Format(FieldNo)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 3b09c63c678..d14a15aff88 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -1,5 +1,8 @@ namespace Microsoft.Integration.MDM; +using System.Text; +using System.Utilities; + /// /// Parses a GetRecords/LastModifiedAtPerTable JSON response from the source and materializes records into a /// temporary RecordRef, so the existing synchronization engine can read them as if they were local. Also @@ -10,6 +13,9 @@ codeunit 7248 "MDM Source Response" { Access = Internal; + var + SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; + [TryFunction] procedure TryParse(ResponseText: Text; var Response: JsonObject) begin @@ -101,6 +107,7 @@ codeunit 7248 "MDM Source Response" FieldNo: Integer; begin TempSourceRecordRef.Init(); + GetGuid(RecordObject, 'systemId', SystemIdValue); if RecordObject.Get('fields', FieldsToken) then begin FieldsObject := FieldsToken.AsObject(); foreach FieldName in FieldsObject.Keys() do @@ -108,14 +115,99 @@ codeunit 7248 "MDM Source Response" if TempSourceRecordRef.FieldExist(FieldNo) then begin FieldsObject.Get(FieldName, ValueToken); DestField := TempSourceRecordRef.Field(FieldNo); - SetFieldFromText(DestField, ValueToken.AsValue().AsText()); + case DestField.Type() of + FieldType::Media: + ApplyInlineMedia(SystemIdValue, FieldNo, TempSourceRecordRef.Number(), ValueToken); + FieldType::Blob: + ApplyInlineBlob(DestField, SystemIdValue, FieldNo, TempSourceRecordRef.Number(), ValueToken); + else + SetFieldFromText(DestField, ValueToken.AsValue().AsText()); + end; end; end; - if GetGuid(RecordObject, 'systemId', SystemIdValue) then + if not IsNullGuid(SystemIdValue) then TempSourceRecordRef.Field(TempSourceRecordRef.SystemIdNo()).Value := SystemIdValue; TempSourceRecordRef.Insert(false); end; + // Media bytes travel in a per-batch cache keyed by (SystemId, fieldNo); the temp record's Media field only + // holds a GUID that is meaningless in the subsidiary. UpdateMedia (cross-env) reads the cache during transfer. + local procedure ApplyInlineMedia(SystemId: Guid; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) + var + InlineMedia: Codeunit "MDM Inline Media"; + MediaObject: JsonObject; + ContentToken: JsonToken; + NameToken: JsonToken; + MimeToken: JsonToken; + FileName: Text; + MimeType: Text; + begin + if not ValueToken.IsObject() then + exit; + MediaObject := ValueToken.AsObject(); + if IsSkipped(MediaObject) then begin + LogSkippedField(TableId, FieldNo, SystemId, MediaObject); + exit; + end; + if not MediaObject.Get('content', ContentToken) then + exit; // empty source media: leave the destination picture untouched + if MediaObject.Get('name', NameToken) then + FileName := NameToken.AsValue().AsText(); + if MediaObject.Get('mimeType', MimeToken) then + MimeType := MimeToken.AsValue().AsText(); + InlineMedia.Put(SystemId, FieldNo, FileName, MimeType, ContentToken.AsValue().AsText()); + end; + + // Blob bytes are placed directly on the temp source record; the framework's record transfer carries them to + // the destination (the same path same-env uses for mapped blobs), so no destination-side apply is needed. + local procedure ApplyInlineBlob(var DestField: FieldRef; SystemId: Guid; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) + var + Base64Convert: Codeunit "Base64 Convert"; + TempBlob: Codeunit "Temp Blob"; + BlobObject: JsonObject; + ContentToken: JsonToken; + ContentOutStream: OutStream; + begin + if not ValueToken.IsObject() then + exit; + BlobObject := ValueToken.AsObject(); + if IsSkipped(BlobObject) then begin + LogSkippedField(TableId, FieldNo, SystemId, BlobObject); + exit; + end; + if not BlobObject.Get('content', ContentToken) then + exit; // empty source blob: leave the destination untouched + TempBlob.CreateOutStream(ContentOutStream); + Base64Convert.FromBase64(ContentToken.AsValue().AsText(), ContentOutStream); + TempBlob.ToFieldRef(DestField); + end; + + local procedure IsSkipped(FieldObject: JsonObject): Boolean + var + Token: JsonToken; + begin + if FieldObject.Get('skipped', Token) then + exit(Token.AsValue().AsBoolean()); + exit(false); + end; + + // Over-cap media/blob is not synchronized (v1). We can't error (it would retry the record every run) and MDM + // surfaces no synch warnings, so the skip is emitted as telemetry only; the record's other fields still sync. + local procedure LogSkippedField(TableId: Integer; FieldNo: Integer; SystemId: Guid; FieldObject: JsonObject) + var + MasterDataManagement: Codeunit "Master Data Management"; + Dimensions: Dictionary of [Text, Text]; + LengthToken: JsonToken; + begin + Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); + Dimensions.Add('tableId', Format(TableId)); + Dimensions.Add('fieldNo', Format(FieldNo)); + Dimensions.Add('systemId', Format(SystemId, 0, 4)); + if FieldObject.Get('length', LengthToken) then + Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); + Session.LogMessage('', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, Dimensions); + end; + // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. local procedure SetFieldFromText(var DestField: FieldRef; ValueText: Text) var diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 6087a21b4de..1fad2f20458 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -17,6 +17,7 @@ using System.IO; using System.Reflection; using System.Telemetry; using System.Threading; +using System.Utilities; codeunit 7237 "Master Data Mgt. Subscribers" { @@ -321,6 +322,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" MediaUpdated: Boolean; SourceMediaName, DestinationMediaName : Text; begin + if IsCrossEnvironmentSync() then + exit(UpdateMediaCrossEnvironment(SourceFieldRef, DestinationFieldRef, NewValue)); + SourceTenantMedia.SetAutoCalcFields(Content); DestinationTenantMedia.SetAutoCalcFields(Content); @@ -359,6 +363,62 @@ codeunit 7237 "Master Data Mgt. Subscribers" exit(MediaUpdated); end; + local procedure IsCrossEnvironmentSync(): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit(false); + exit(MasterDataManagementSetup.IsCrossEnvironment()); + end; + + // Cross-env: the source Tenant Media lives in another environment, so the bytes arrive inline (per-batch + // cache) rather than via the source field's GUID. Build the destination Tenant Media from them, keeping the + // same length+name change check. Returns the new media id via NewValue during transfer -> single write. + local procedure UpdateMediaCrossEnvironment(var SourceFieldRef: FieldRef; var DestinationFieldRef: FieldRef; var NewValue: Variant): Boolean + var + DestinationTenantMedia: Record "Tenant Media"; + InlineMedia: Codeunit "MDM Inline Media"; + TempBlob: Codeunit "Temp Blob"; + SourceRecordRef: RecordRef; + SourceSystemId, DestinationMediaId, EmptyGuid : Guid; + MediaInStream: InStream; + MediaOutStream: OutStream; + FileName, MimeType, DestinationName : Text; + SourceLength, DestinationLength : Integer; + begin + SourceRecordRef := SourceFieldRef.Record(); + SourceSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + if not InlineMedia.TryGet(SourceSystemId, SourceFieldRef.Number(), FileName, MimeType, TempBlob) then + exit(false); // no inline bytes (empty source or over-cap skip): leave the destination untouched + SourceLength := TempBlob.Length(); + + DestinationMediaId := DestinationFieldRef.Value(); + DestinationTenantMedia.SetAutoCalcFields(Content); + if DestinationTenantMedia.Get(DestinationMediaId) then begin + DestinationLength := DestinationTenantMedia.Content.Length(); + DestinationName := DestinationTenantMedia."File Name"; + end; + if (SourceLength = DestinationLength) and (FileName = DestinationName) then + exit(false); // unchanged + + if DestinationMediaId <> EmptyGuid then + if DestinationTenantMedia.Get(DestinationMediaId) then + DestinationTenantMedia.Delete(); + + Clear(DestinationTenantMedia); + DestinationTenantMedia.ID := CreateGuid(); + DestinationTenantMedia."Company Name" := CopyStr(CompanyName(), 1, MaxStrLen(DestinationTenantMedia."Company Name")); + DestinationTenantMedia."File Name" := CopyStr(FileName, 1, MaxStrLen(DestinationTenantMedia."File Name")); + DestinationTenantMedia."Mime Type" := CopyStr(MimeType, 1, MaxStrLen(DestinationTenantMedia."Mime Type")); + TempBlob.CreateInStream(MediaInStream); + DestinationTenantMedia.Content.CreateOutStream(MediaOutStream); + CopyStream(MediaOutStream, MediaInStream); + DestinationTenantMedia.Insert(); + NewValue := DestinationTenantMedia.ID; + exit(true); + end; + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Table Synch.", 'OnDetermineSynchDirection', '', false, false)] local procedure OnDetermineSynchDirection(var CurrentIntegrationTableMapping: Record "Integration Table Mapping"; var TableID: Integer; var ErrorMessage: Text; var IsHandled: Boolean) begin diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index c60bb0a622d..92902619c17 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -31,8 +31,12 @@ table 7230 "Master Data Management Setup" MasterDataMgtSetupDefault: Codeunit "Master Data Mgt. Setup Default"; begin if "Is Enabled" then - if "Company Name" = '' then - Error(MustPickSourceCompanyErr); + if IsCrossEnvironment() then begin + if not IsCrossEnvConnectionConfigured() then + Error(MustConfigureConnectionErr); + end else + if "Company Name" = '' then + Error(MustPickSourceCompanyErr); MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); end; } @@ -183,6 +187,11 @@ table 7230 "Master Data Management Setup" exit(("Source Environment URL" <> '') and ("Source Company Name" <> '') and ("Source OAuth Client Id" <> '') and (not IsNullGuid("Source Client Secret Key"))); end; + internal procedure IsCrossEnvironment(): Boolean + begin + exit("Source Environment Name" <> ''); + end; + [NonDebuggable] internal procedure SetSourceClientSecret(ClientSecret: SecretText) begin @@ -237,6 +246,14 @@ table 7230 "Master Data Management Setup" ResetConfig := Confirm(ResetConfigQst); if ResetConfig then MasterDataMgtSetupDefault.ResetConfiguration(Rec); + + if IsCrossEnvironment() then begin + // Cross-environment: the source is a different environment; never write to its subscriber table. + Message(StrSubstNo(SynchronizationEnabledMsg, "Source Company Name")); + Session.LogMessage('0000JIM', "Source Environment Name", Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + exit; + end; + CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); MasterDataManagement.AddSubsidiarySubscriptionToMasterCompany(Rec."Company Name", CurrentCompanyName); Message(StrSubstNo(SynchronizationEnabledMsg, Rec."Company Name")); @@ -264,7 +281,9 @@ table 7230 "Master Data Management Setup" begin CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(Rec."Company Name")); - MasterDataManagement.RemoveSubsidiarySubscriptionFromMasterCompany(Rec."Company Name", CurrentCompanyName); + // Cross-environment: the source is a different environment; never touch its subscriber table. + if not IsCrossEnvironment() then + MasterDataManagement.RemoveSubsidiarySubscriptionFromMasterCompany(Rec."Company Name", CurrentCompanyName); UpdateDataSynchJobQueueEntriesStatus(); if not MasterDataMgtCoupling.IsEmpty() then @@ -336,5 +355,6 @@ table 7230 "Master Data Management Setup" KeepTheCouplingsQst: label 'Data synchronization with company %1 is disabled. \\We recommend to keep the table setup and coupling information, especially if you intend to reenable the synchronization with the same company. \\Do you want to keep the table setup and coupling information?', Comment = '%1 - a company name'; MustNotPickCurrentCompanyErr: label 'You are currently signed into this company. \\Choose a different company to synchronize data with.'; MustPickSourceCompanyErr: label 'You must choose a source company to synchronize data from.'; + MustConfigureConnectionErr: label 'Enter the cross-environment connection details before you enable synchronization.'; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; } diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 610668aa69e..40d57df44cb 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -122,6 +122,13 @@ codeunit 139757 "Library - Master Data Mgt." MDMCrossEnvChangeDetector.DetectChanges(); end; + procedure InlineMediaCacheContains(SystemId: Guid; FieldNo: Integer): Boolean + var + InlineMedia: Codeunit "MDM Inline Media"; + begin + exit(InlineMedia.Contains(SystemId, FieldNo)); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; MasterDataManagement: Codeunit "Master Data Management"; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al index 445bfa8f36c..40bbc97702b 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al @@ -19,6 +19,14 @@ table 139757 "MDM Test Table A" Caption = 'TableB Reference'; TableRelation = "MDM Test Table B"."Primary Key"; } + field(4; "Test Blob"; Blob) + { + Caption = 'Test Blob'; + } + field(5; "Test Image"; Media) + { + Caption = 'Test Image'; + } } keys diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index f398572619d..30a5601c2c2 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -301,6 +301,138 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvBlobFieldRoundTrips() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + TempBlob: Codeunit "Temp Blob"; + SourceRecordRef: RecordRef; + BlobField: FieldRef; + BlobInStream: InStream; + BlobText: Text; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] A Blob field is projected inline (base64) and materialized back onto the temp source record. + Initialize(); + CreateTestTableAWithBlob(TestTableA, 'the quick brown fox'); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should be materialized'); + BlobField := SourceRecordRef.Field(TestTableA.FieldNo("Test Blob")); + TempBlob.FromFieldRef(BlobField); + Assert.IsTrue(TempBlob.HasValue(), 'Blob should round-trip onto the materialized record'); + TempBlob.CreateInStream(BlobInStream, TextEncoding::UTF8); + BlobInStream.ReadText(BlobText); + Assert.AreEqual('the quick brown fox', BlobText, 'Blob content should round-trip through the wire'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvOversizeBlobIsSkipped() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + TempBlob: Codeunit "Temp Blob"; + SourceRecordRef: RecordRef; + BlobField: FieldRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A Blob over the 512 KB inline cap is skipped: the record materializes without the blob. + Initialize(); + CreateTestTableAWithBlob(TestTableA, PadStr('', 600000, 'A')); // > 512 KB raw + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should still materialize'); + BlobField := SourceRecordRef.Field(TestTableA.FieldNo("Test Blob")); + TempBlob.FromFieldRef(BlobField); + Assert.IsFalse(TempBlob.HasValue(), 'Over-cap blob must be skipped, not synchronized'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvMediaFieldIsCachedForApply() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A Media field's bytes are cached (keyed by SystemId+fieldNo) for the transfer-time apply. + Initialize(); + CreateTestTableAWithImage(TestTableA, 'small picture bytes'); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should be materialized'); + Assert.IsTrue( + LibraryMasterDataMgt.InlineMediaCacheContains(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'Inline media bytes should be cached for the transfer-time apply'); + + CleanUp(); + end; + + [Test] + procedure CrossEnvOversizeMediaIsSkipped() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A Media over the 512 KB cap is skipped: no bytes are cached (telemetry-only warning). + Initialize(); + CreateTestTableAWithImage(TestTableA, PadStr('', 600000, 'A')); // > 512 KB raw + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should still materialize'); + Assert.IsFalse( + LibraryMasterDataMgt.InlineMediaCacheContains(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'Over-cap media must not be cached (skipped, telemetry only)'); + + CleanUp(); + end; + + local procedure CreateTestTableAWithBlob(var TestTableA: Record "MDM Test Table A"; Content: Text) + var + BlobOutStream: OutStream; + begin + Clear(TestTableA); + TestTableA."Primary Key" := CopyStr('B' + Format(LibraryRandomInt()), 1, MaxStrLen(TestTableA."Primary Key")); + TestTableA."Test Blob".CreateOutStream(BlobOutStream, TextEncoding::UTF8); + BlobOutStream.WriteText(Content); + TestTableA.Insert(); + end; + + local procedure CreateTestTableAWithImage(var TestTableA: Record "MDM Test Table A"; Content: Text) + var + TempBlob: Codeunit "Temp Blob"; + MediaInStream: InStream; + MediaOutStream: OutStream; + begin + Clear(TestTableA); + TestTableA."Primary Key" := CopyStr('M' + Format(LibraryRandomInt()), 1, MaxStrLen(TestTableA."Primary Key")); + TestTableA.Insert(); + TempBlob.CreateOutStream(MediaOutStream, TextEncoding::UTF8); + MediaOutStream.WriteText(Content); + TempBlob.CreateInStream(MediaInStream, TextEncoding::UTF8); + TestTableA."Test Image".ImportStream(MediaInStream, 'pic.bin', 'application/octet-stream'); + TestTableA.Modify(); + end; + local procedure Initialize() var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -333,9 +465,11 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" local procedure DeleteTestArtifacts() var IntegrationTableMapping: Record "Integration Table Mapping"; + TestTableA: Record "MDM Test Table A"; begin IntegrationTableMapping.SetFilter(Name, 'MDMXENV*'); IntegrationTableMapping.DeleteAll(); + TestTableA.DeleteAll(); // deterministic LibraryRandom keys collide across tests; the setup commit survives rollback end; local procedure CreateMinimalCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al index c9f9f776695..e1ebe1aeedd 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al @@ -98,6 +98,53 @@ codeunit 139770 "Master Data Mgt. Setup Tests" asserterror MasterDataManagementSetup.Validate("Company Name", CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Company Name"))); end; + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure EnableCrossEnvironmentAllowsSameSourceCompanyName() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataMgtSubscriber: Record "Master Data Mgt. Subscriber"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Cross-env enables with a source company whose name equals the current company (different environment), + // and never writes to the source subscriber table. + Initialize(); + MasterDataManagementSetup.Init(); + MasterDataManagementSetup."Source Environment Name" := 'CONTOSO-PROD'; + MasterDataManagementSetup."Source Environment URL" := 'https://example/v2.0/contoso-prod'; + MasterDataManagementSetup."Source Company Name" := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Source Company Name")); + MasterDataManagementSetup."Source OAuth Client Id" := '11111111-2222-3333-4444-555555555555'; + MasterDataManagementSetup."Source Client Secret Key" := CreateGuid(); // simulate a stored secret + MasterDataManagementSetup.Insert(); + + MasterDataManagementSetup.Validate("Is Enabled", true); + MasterDataManagementSetup.Modify(true); + + // [THEN] no subscriber row was written for the current company + MasterDataMgtSubscriber.SetRange("Company Name", CompanyName()); + Assert.AreEqual(0, MasterDataMgtSubscriber.Count(), 'Cross-env enable must not write to the source subscriber table'); + + // cleanup: disable to remove the detector job + MasterDataManagementSetup.Validate("Is Enabled", false); + MasterDataManagementSetup.Modify(true); + end; + + [Test] + procedure EnableCrossEnvironmentRequiresConnectionDetails() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Enabling cross-env without a configured connection is blocked with a clear error. + Initialize(); + MasterDataManagementSetup.Init(); + MasterDataManagementSetup."Source Environment Name" := 'CONTOSO-PROD'; // env set, but URL/company/client id/secret missing + MasterDataManagementSetup.Insert(); + + asserterror MasterDataManagementSetup.Validate("Is Enabled", true); + Assert.ExpectedError('connection details'); + end; + [Test] [HandlerFunctions('SynchronizationEnabledMessageHandler,ConfirmHandlerYes')] procedure DisableSetupKeepCouplingTable() From e3879465009fb972279088f9c3a863afd6d563f7 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 27 Aug 2026 20:20:05 +0200 Subject: [PATCH 13/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvSourceAPI.Codeunit.al | 53 ++++++++++---- .../src/MDMTestPagingConfig.Codeunit.al | 17 +++++ .../test library/src/MDMTestTableA.Table.al | 3 + .../src/MDMCrossEnvConsumerTests.Codeunit.al | 71 +++++++++++++++++++ 4 files changed, 131 insertions(+), 13 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 16ea086eae6..f7436f2af93 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -235,6 +235,7 @@ codeunit 7241 "MDM Cross-Env Source API" LastEmittedAt: DateTime; IgnoredSystemId: Guid; MaxKeylessGroup: Integer; + PageBytes: Integer; begin Count := 0; GroupTooLarge := false; @@ -245,14 +246,14 @@ codeunit 7241 "MDM Cross-Env Source API" if RecRef.FindSet() then repeat CurrentModifiedAt := ModifiedAtRef.Value(); - // Stop only at a clean group boundary once the page is full. - if (Count >= PageSize) and (CurrentModifiedAt <> LastEmittedAt) then + // Stop only at a clean group boundary once the page is full (by count or inline bytes). + if ((Count >= PageSize) or (PageBytes >= MaxPageInlineBytes())) and (CurrentModifiedAt <> LastEmittedAt) then exit(true); if Count >= MaxKeylessGroup then begin GroupTooLarge := true; exit(false); end; - AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, IgnoredSystemId); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, IgnoredSystemId, PageBytes); LastEmittedAt := NextModifiedAt; Count += 1; until RecRef.Next() = 0; @@ -268,6 +269,7 @@ codeunit 7241 "MDM Cross-Env Source API" IgnoredModifiedAt: DateTime; IgnoredSystemId: Guid; MaxUnindexedRecords: Integer; + PageBytes: Integer; begin Count := 0; TooLarge := false; @@ -277,14 +279,15 @@ codeunit 7241 "MDM Cross-Env Source API" ModifiedAtRef.SetFilter('>%1', CursorModifiedAt); if RecRef.FindSet() then repeat - if Count >= MaxUnindexedRecords then begin + // Unindexed scan can't resume mid-set, so too many records OR too many inline bytes => ask for a key. + if (Count >= MaxUnindexedRecords) or (PageBytes >= MaxPageInlineBytes()) then begin TooLarge := true; exit(false); end; CurrentModifiedAt := ModifiedAtRef.Value(); if CurrentModifiedAt > MaxModifiedAt then MaxModifiedAt := CurrentModifiedAt; - AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId); + AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId, PageBytes); Count += 1; until RecRef.Next() = 0; exit(false); @@ -294,6 +297,7 @@ codeunit 7241 "MDM Cross-Env Source API" var ModifiedAtRef: FieldRef; SystemIdRef: FieldRef; + PageBytes: Integer; begin Count := 0; ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); @@ -307,8 +311,10 @@ codeunit 7241 "MDM Cross-Env Source API" repeat if Count = PageSize then exit(true); - AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId, PageBytes); Count += 1; + if PageBytes >= MaxPageInlineBytes() then + exit(true); // inline-byte budget: >=1 record emitted; resume from NextModifiedAt/NextSystemId until RecRef.Next() = 0; ModifiedAtRef.SetRange(); SystemIdRef.SetRange(); @@ -321,8 +327,10 @@ codeunit 7241 "MDM Cross-Env Source API" repeat if Count = PageSize then exit(true); - AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId); + AppendRecord(RecRef, ProjectedFields, Records, NextModifiedAt, NextSystemId, PageBytes); Count += 1; + if PageBytes >= MaxPageInlineBytes() then + exit(true); until RecRef.Next() = 0; exit(false); @@ -335,6 +343,7 @@ codeunit 7241 "MDM Cross-Env Source API" SystemIdValue: Guid; IgnoredModifiedAt: DateTime; IgnoredSystemId: Guid; + IgnoredPageBytes: Integer; FilterText: Text; begin foreach Token in SystemIds do @@ -350,11 +359,11 @@ codeunit 7241 "MDM Cross-Env Source API" SystemIdRef.SetFilter(FilterText); if RecRef.FindSet() then repeat - AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId); + AppendRecord(RecRef, ProjectedFields, Records, IgnoredModifiedAt, IgnoredSystemId, IgnoredPageBytes); until RecRef.Next() = 0; end; - local procedure AppendRecord(var RecRef: RecordRef; ProjectedFields: List of [Integer]; var Records: JsonArray; var LastModifiedAt: DateTime; var LastSystemId: Guid) + local procedure AppendRecord(var RecRef: RecordRef; ProjectedFields: List of [Integer]; var Records: JsonArray; var LastModifiedAt: DateTime; var LastSystemId: Guid; var PageBytes: Integer) var CurrentField: FieldRef; RecordObject: JsonObject; @@ -369,9 +378,9 @@ codeunit 7241 "MDM Cross-Env Source API" CurrentField := RecRef.Field(FieldNo); case CurrentField.Type() of FieldType::Media: - FieldsObject.Add(Format(FieldNo), BuildMediaValue(CurrentField)); + FieldsObject.Add(Format(FieldNo), BuildMediaValue(CurrentField, PageBytes)); FieldType::Blob: - FieldsObject.Add(Format(FieldNo), BuildBlobValue(CurrentField)); + FieldsObject.Add(Format(FieldNo), BuildBlobValue(CurrentField, PageBytes)); else FieldsObject.Add(Format(FieldNo), FormatFieldValue(CurrentField)); end; @@ -405,7 +414,7 @@ codeunit 7241 "MDM Cross-Env Source API" // Single Media field: emit { media, name, mimeType, length, content(base64) }, or { media, empty } when the // source has no picture, or { media, skipped, length } when it exceeds the inline cap. - local procedure BuildMediaValue(FieldReference: FieldRef): JsonObject + local procedure BuildMediaValue(FieldReference: FieldRef; var PageBytes: Integer): JsonObject var TenantMedia: Record "Tenant Media"; Base64Convert: Codeunit "Base64 Convert"; @@ -434,11 +443,12 @@ codeunit 7241 "MDM Cross-Env Source API" MediaValue.Add('length', TenantMedia.Content.Length()); TenantMedia.Content.CreateInStream(ContentInStream); MediaValue.Add('content', Base64Convert.ToBase64(ContentInStream)); + PageBytes += TenantMedia.Content.Length(); exit(MediaValue); end; // Blob field: emit { blob, length, content(base64) }, or { blob, empty }, or { blob, skipped, length }. - local procedure BuildBlobValue(FieldReference: FieldRef): JsonObject + local procedure BuildBlobValue(FieldReference: FieldRef; var PageBytes: Integer): JsonObject var Base64Convert: Codeunit "Base64 Convert"; TempBlob: Codeunit "Temp Blob"; @@ -459,6 +469,7 @@ codeunit 7241 "MDM Cross-Env Source API" BlobValue.Add('length', TempBlob.Length()); TempBlob.CreateInStream(ContentInStream); BlobValue.Add('content', Base64Convert.ToBase64(ContentInStream)); + PageBytes += TempBlob.Length(); exit(BlobValue); end; @@ -469,6 +480,22 @@ codeunit 7241 "MDM Cross-Env Source API" exit(512 * 1024); end; + // Cap inline media/blob bytes per page (~3 MB raw, ~4 MB base64) so a media-heavy page stays well under the + // 8-minute operation timeout and memory; the composite cursor resumes the remaining records on the next page. + local procedure MaxPageInlineBytes(): Integer + var + MaxBytes: Integer; + begin + MaxBytes := 3 * 1024 * 1024; + OnGetMaxPageInlineBytes(MaxBytes); + exit(MaxBytes); + end; + + [InternalEvent(false)] + local procedure OnGetMaxPageInlineBytes(var MaxBytes: Integer) + begin + end; + [TryFunction] local procedure TryReadJsonArray(Value: Text; var JsonArrayValue: JsonArray) begin diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al index c3b1b0cc929..c7665c8af19 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al @@ -9,6 +9,8 @@ codeunit 139934 "MDM Test Paging Config" var Active: Boolean; PageSizeValue: Integer; + InlineBytesActive: Boolean; + InlineBytesValue: Integer; procedure Activate(NewPageSize: Integer) begin @@ -16,10 +18,18 @@ codeunit 139934 "MDM Test Paging Config" PageSizeValue := NewPageSize; end; + procedure ActivateInlineBytes(NewMaxBytes: Integer) + begin + InlineBytesActive := true; + InlineBytesValue := NewMaxBytes; + end; + procedure Deactivate() begin Active := false; PageSizeValue := 0; + InlineBytesActive := false; + InlineBytesValue := 0; end; [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Data Source", 'OnGetCrossEnvPageSize', '', false, false)] @@ -28,4 +38,11 @@ codeunit 139934 "MDM Test Paging Config" if Active then PageSize := PageSizeValue; end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"MDM Cross-Env Source API", 'OnGetMaxPageInlineBytes', '', false, false)] + local procedure HandleGetMaxPageInlineBytes(var MaxBytes: Integer) + begin + if InlineBytesActive then + MaxBytes := InlineBytesValue; + end; } diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al index 40bbc97702b..fc223f05f88 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestTableA.Table.al @@ -35,5 +35,8 @@ table 139757 "MDM Test Table A" { Clustered = true; } + key(ChangeFeed; SystemModifiedAt, SystemId) + { + } } } diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 30a5601c2c2..ae0291530da 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -406,6 +406,74 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvPageStopsAtInlineByteBudget() + var + TestTableA: Record "MDM Test Table A"; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + PagingConfig: Codeunit "MDM Test Paging Config"; + SourceRecordRef: RecordRef; + Watermark: DateTime; + EndCursor: Text; + HasMore: Boolean; + Index: Integer; + Count: Integer; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] A page stops at the inline-byte budget: media-heavy records page out even under the record count cap. + Initialize(); + + // [GIVEN] three media records created after a captured watermark, and a byte budget so small any one media fills it + Watermark := CurrentDateTime(); + Sleep(100); + for Index := 1 to 3 do + CreateTestTableAWithImage(TestTableA, 'picture bytes'); + CreateTestTableAMapping(IntegrationTableMapping); + IntegrationTableMapping."Synch. Modified On Filter" := Watermark; + IntegrationTableMapping.Modify(); + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + PagingConfig.ActivateInlineBytes(1); // 1-byte budget: the first inlined media ends the page + + // [WHEN] a single page is fetched + LibraryMasterDataMgt.DataSourceGetModifiedBatch(IntegrationTableMapping, '', '', 1, SourceRecordRef, EndCursor, HasMore); + if SourceRecordRef.FindSet() then + repeat + Count += 1; + until SourceRecordRef.Next() = 0; + + // [THEN] the byte budget capped the page to one record, with more remaining + Assert.AreEqual(1, Count, 'The inline-byte budget should stop the page after the first media record'); + Assert.IsTrue(HasMore, 'More records should remain past the byte-budget cap'); + + PagingConfig.Deactivate(); + CleanUp(); + end; + + local procedure CreateTestTableAMapping(var IntegrationTableMapping: Record "Integration Table Mapping") + var + IntegrationFieldMapping: Record "Integration Field Mapping"; + begin + IntegrationTableMapping.Init(); + IntegrationTableMapping.Name := CopyStr('MDMXENV' + Format(LibraryRandomInt()), 1, MaxStrLen(IntegrationTableMapping.Name)); + IntegrationTableMapping.Type := IntegrationTableMapping.Type::"Master Data Management"; + IntegrationTableMapping."Table ID" := Database::"MDM Test Table A"; + IntegrationTableMapping."Integration Table ID" := Database::"MDM Test Table A"; + IntegrationTableMapping."Integration Table UID Fld. No." := 2000000000; // SystemId + IntegrationTableMapping."Int. Tbl. Modified On Fld. No." := 2000000003; // SystemModifiedAt + IntegrationTableMapping."Delete After Synchronization" := false; + IntegrationTableMapping.Insert(); + // map the Media field (5) so the batch requests it and the page carries inline bytes + IntegrationFieldMapping.Init(); + IntegrationFieldMapping."Integration Table Mapping Name" := IntegrationTableMapping.Name; + IntegrationFieldMapping."Field No." := 5; + IntegrationFieldMapping."Integration Table Field No." := 5; + IntegrationFieldMapping.Insert(true); + end; + local procedure CreateTestTableAWithBlob(var TestTableA: Record "MDM Test Table A"; Content: Text) var BlobOutStream: OutStream; @@ -465,8 +533,11 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" local procedure DeleteTestArtifacts() var IntegrationTableMapping: Record "Integration Table Mapping"; + IntegrationFieldMapping: Record "Integration Field Mapping"; TestTableA: Record "MDM Test Table A"; begin + IntegrationFieldMapping.SetFilter("Integration Table Mapping Name", 'MDMXENV*'); + IntegrationFieldMapping.DeleteAll(); IntegrationTableMapping.SetFilter(Name, 'MDMXENV*'); IntegrationTableMapping.DeleteAll(); TestTableA.DeleteAll(); // deterministic LibraryRandom keys collide across tests; the setup commit survives rollback From 2f5971bf3e2219bfe4042f0cdb80a8f06f6b685c Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 10:21:27 +0200 Subject: [PATCH 14/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvChangeDetector.Codeunit.al | 6 ++++- .../MDMCrossEnvDataSource.Codeunit.al | 5 +++- .../MDMHttpSourceTransport.Codeunit.al | 23 ++++++++++++++++++- .../codeunits/MDMLocalDataSource.Codeunit.al | 2 ++ .../MDMSourceCapabilities.Codeunit.al | 23 +++++++++++-------- .../codeunits/MDMSourceResponse.Codeunit.al | 2 +- .../MasterDataMgtUpgrade.Codeunit.al | 4 ++-- 7 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 67e274c52ff..d10f0fcc4a3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -17,6 +17,9 @@ codeunit 7245 "MDM Cross-Env Change Detector" tabledata "Job Queue Entry" = rm, tabledata "Scheduled Task" = r; + var + LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; + trigger OnRun() begin DetectChanges(); @@ -31,7 +34,6 @@ codeunit 7245 "MDM Cross-Env Change Detector" Transport: Interface "IMDM Source Transport"; Response: JsonObject; TableIds: JsonArray; - LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; begin if not MasterDataManagementSetup.Get() then exit; @@ -61,6 +63,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Delete After Synchronization", false); IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); + IntegrationTableMapping.SetLoadFields("Integration Table ID"); if not IntegrationTableMapping.FindSet() then exit(false); repeat @@ -125,6 +128,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" exit; // Seam so tests can observe the decision and skip the reschedule (jobs can't be scheduled in the test lab). + IsHandled := false; OnBeforeRescheduleSynchJob(JobQueueEntry, IntegrationTableMapping, IsHandled); if IsHandled then exit; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index ed7fd5c9a26..a36262f96d5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -20,6 +20,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; + SourceProbeFailedErr: Label 'Could not read the change probe from the source environment for table %1.', Comment = '%1 = table caption'; RecordsFeatureTok: Label 'records', Locked = true; LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; @@ -107,8 +108,9 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" TableIds.WriteTo(TableIdsText); Transport := GetTransport(); SourceCapabilities.EnsureSupported(Transport, LastModifiedFeatureTok); + // A transport/parse failure is not an empty source; surface it so the full-synch review isn't misled. if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(TableIdsText), Response) then - exit(false); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); if not Response.Get('tables', Token) then exit(false); Tables := Token.AsArray(); @@ -205,6 +207,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" begin AddPrimaryKeyFields(IntegrationTableMapping."Integration Table ID", FieldIds, AddedFields); IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); + IntegrationFieldMapping.SetLoadFields("Integration Table Field No."); if IntegrationFieldMapping.FindSet() then repeat if IntegrationFieldMapping."Integration Table Field No." <> 0 then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 65583903778..fe41b97cbbb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -5,6 +5,7 @@ using System.Environment; using System.Security.Authentication; using System.Reflection; using System.Telemetry; +using System.Utilities; /// /// Production transport: calls the source environment's ODataV4 web service with an app-only (client @@ -24,6 +25,8 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" NonSaaSErr: Label 'Cross-environment synchronization is only available in online environments.'; NoTokenErr: Label 'Could not acquire an access token for the source environment. Check the client ID and secret.'; SendFailedErr: Label 'The request to the source environment could not be sent. Check the source environment URL.'; + InvalidSourceUrlErr: Label 'The source environment URL is not a valid Business Central endpoint.'; + InvalidSourceUrlAuditTxt: Label 'Blocked a cross-environment request: the configured source environment URL host ''%1'' is not a valid Business Central endpoint.', Comment = '%1 = the rejected host'; HttpErr: Label 'The source environment returned HTTP %1. %2', Comment = '%1 = HTTP status code, %2 = response detail'; ServiceNameTok: Label 'MDMCrossEnvSource', Locked = true; ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true; @@ -94,7 +97,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" AuditLog: Codeunit "Audit Log"; begin // Operational telemetry: action + status only, never record data or credentials. - Session.LogMessage('', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', TelemetryCategoryTok); + Session.LogMessage('0000QF1', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', TelemetryCategoryTok); // Security audit: an authorization failure crossing the environment boundary. if ResponseMessage.HttpStatusCode() in [401, 403] then AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); @@ -129,9 +132,27 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" BaseUrl: Text; begin BaseUrl := DelChr(MasterDataManagementSetup."Source Environment URL", '>', '/'); + ValidateSourceHost(BaseUrl); exit(StrSubstNo(ActionUrlTok, BaseUrl, ServiceNameTok, ActionName, UriEncodeCompany(MasterDataManagementSetup."Source Company Name"))); end; + // The source must be a Business Central SaaS endpoint over HTTPS. Validating the configured URL before the + // bearer token is attached stops the setup field from redirecting the authenticated call to an arbitrary host (SSRF). + local procedure ValidateSourceHost(BaseUrl: Text) + var + AuditLog: Codeunit "Audit Log"; + Uri: Codeunit Uri; + Host: Text; + begin + Uri.Init(BaseUrl); + // Embed/ISV clusters vary in hostname but always end with dynamics.com; dynamics-tie.com is the test (TIE) ring. + Host := LowerCase(Uri.GetHost()); + if (Uri.GetScheme() = 'https') and (Host.EndsWith('.dynamics.com') or Host.EndsWith('.dynamics-tie.com')) then + exit; + AuditLog.LogAuditMessage(StrSubstNo(InvalidSourceUrlAuditTxt, Host), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); + Error(InvalidSourceUrlErr); + end; + // The ODataV4 envelope for an action returning Text is { "@odata.context": "...", "value": "" }; // the source API's own JSON is that inner value. Fall back to the raw body if the shape is unexpected. local procedure UnwrapODataValue(ResponseBody: Text): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al index 52a4d66278f..96ce24d5d73 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -75,6 +75,8 @@ codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" SourceCompanyName: Text[30]; begin MasterDataManagementSetup.Get(); + if SourceRecordRef.Number() <> 0 then + SourceRecordRef.Close(); // a re-fetch may pass an already-open ref; start from a clean handle SourceRecordRef.Open(IntegrationTableId); MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, IntegrationTableId); if SourceCompanyName = '' then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index f62323fb114..46e4eae82fa 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -15,6 +15,7 @@ codeunit 7246 "MDM Source Capabilities" ContractVersion: Integer; SupportedFeatures: List of [Text]; UnsupportedFeatureErr: Label 'The source environment does not support the required ''%1'' capability. Update the Master Data Management app on the source environment.', Comment = '%1 = capability name'; + CapabilitiesParseErr: Label 'The source environment returned an invalid capabilities response.'; procedure EnsureSupported(Transport: Interface "IMDM Source Transport"; Feature: Text) begin @@ -51,16 +52,18 @@ codeunit 7246 "MDM Source Capabilities" begin if Negotiated then exit; - if Capabilities.ReadFrom(Transport.GetCapabilities()) then begin - if Capabilities.Get('version', VersionToken) then - if VersionToken.IsValue() then - ContractVersion := VersionToken.AsValue().AsInteger(); - if Capabilities.Get('features', FeaturesToken) then - if FeaturesToken.IsArray() then - foreach FeatureToken in FeaturesToken.AsArray() do - if FeatureToken.IsValue() then - SupportedFeatures.Add(FeatureToken.AsValue().AsText()); - end; + // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading + // "capability unsupported / update the source app" error instead of the real bad-response problem. + if not Capabilities.ReadFrom(Transport.GetCapabilities()) then + Error(CapabilitiesParseErr); + if Capabilities.Get('version', VersionToken) then + if VersionToken.IsValue() then + ContractVersion := VersionToken.AsValue().AsInteger(); + if Capabilities.Get('features', FeaturesToken) then + if FeaturesToken.IsArray() then + foreach FeatureToken in FeaturesToken.AsArray() do + if FeatureToken.IsValue() then + SupportedFeatures.Add(FeatureToken.AsValue().AsText()); Negotiated := true; end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index d14a15aff88..fab7a19d10e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -205,7 +205,7 @@ codeunit 7248 "MDM Source Response" Dimensions.Add('systemId', Format(SystemId, 0, 4)); if FieldObject.Get('length', LengthToken) then Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); - Session.LogMessage('', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, Dimensions); + Session.LogMessage('0000QF2', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, Dimensions); end; // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al index 10336b6aecc..d34c63a4e2c 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al @@ -34,13 +34,13 @@ codeunit 7238 "Master Data Mgt. Upgrade" TenantWebService: Record "Tenant Web Service"; UpgradeTag: Codeunit "Upgrade Tag"; begin - if UpgradeTag.HasUpgradeTag(GetCrossEnvWebServiceUpgradeTag()) then + if UpgradeTag.HasDatabaseUpgradeTag(GetCrossEnvWebServiceUpgradeTag()) then exit; // Idempotent: creates or updates the service. Access stays gated by the dedicated "Cross Env" permission set, not by publishing. WebServiceManagement.CreateTenantWebService(TenantWebService."Object Type"::Codeunit, Codeunit::"MDM Cross-Env Source API", CrossEnvSourceWebServiceName(), true); - UpgradeTag.SetUpgradeTag(GetCrossEnvWebServiceUpgradeTag()); + UpgradeTag.SetDatabaseUpgradeTag(GetCrossEnvWebServiceUpgradeTag()); end; internal procedure CrossEnvSourceWebServiceName(): Text[240] From 8b5eeaa6d32d7c0e1ff4877ea7e4cf16cbb758d9 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 10:34:10 +0200 Subject: [PATCH 15/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al | 3 ++- .../app/src/pages/MDMConnectionDetails.Page.al | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index f7436f2af93..b8e648a88ad 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -249,7 +249,8 @@ codeunit 7241 "MDM Cross-Env Source API" // Stop only at a clean group boundary once the page is full (by count or inline bytes). if ((Count >= PageSize) or (PageBytes >= MaxPageInlineBytes())) and (CurrentModifiedAt <> LastEmittedAt) then exit(true); - if Count >= MaxKeylessGroup then begin + // A single same-timestamp group that can't be paged cleanly (too many rows OR too many inline bytes) => ask for a key. + if (Count >= MaxKeylessGroup) or (PageBytes >= MaxPageInlineBytes()) then begin GroupTooLarge := true; exit(false); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index b6936fb8a2e..a1321aabb49 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -244,6 +244,7 @@ page 7232 "MDM Connection Details" LearnMoreTok: Label 'Privacy and Cookies'; PrivacyLinkTxt: Label 'https://go.microsoft.com/fwlink/?linkid=521839', Locked = true; ConnectionOkMsg: Label 'Successfully connected to the source environment (contract version %1).', Comment = '%1 = wire contract version'; + ConnectionFailedErr: Label 'Could not connect to the source environment. Check the URL, company, and credentials, then try again.'; local procedure LoadConfiguration() var @@ -289,7 +290,8 @@ page 7232 "MDM Connection Details" SaveConfiguration(); Commit(); Transport := SourceConnection.GetTransport(); - Capabilities.ReadFrom(Transport.GetCapabilities()); + if not Capabilities.ReadFrom(Transport.GetCapabilities()) then + Error(ConnectionFailedErr); if Capabilities.Get('version', VersionToken) then VersionText := Format(VersionToken.AsValue().AsInteger()); Message(ConnectionOkMsg, VersionText); From 4f8f31525aa834cd3e1248fcdce4d2003c7b80fb Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 12:39:51 +0200 Subject: [PATCH 16/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 1 + .../MDMHttpSourceTransport.Codeunit.al | 2 + .../codeunits/MDMPrivacyNotice.Codeunit.al | 56 +++++++++++++++++++ .../src/pages/MDMConnectionDetails.Page.al | 1 + .../src/LibraryMasterDataMgt.Codeunit.al | 24 ++++++++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 29 ++++++++++ 6 files changed, 113 insertions(+) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 017bc23e6d6..0dc93d1a9d2 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -26,6 +26,7 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "MDM Cross-Env Change Detector" = X, codeunit "MDM Source Capabilities" = X, codeunit "MDM Inline Media" = X, + codeunit "MDM Privacy Notice" = X, page * = X, table * = X, xmlport * = X; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index fe41b97cbbb..7f8fc5e3238 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -70,6 +70,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" var MasterDataManagementSetup: Record "Master Data Management Setup"; EnvironmentInformation: Codeunit "Environment Information"; + PrivacyNotice: Codeunit "MDM Privacy Notice"; ResponseMessage: HttpResponseMessage; ResponseBodyText: Text; RetryAfter: Duration; @@ -78,6 +79,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" GetConfiguredSetup(MasterDataManagementSetup); if not EnvironmentInformation.IsSaaSInfrastructure() then Error(NonSaaSErr); + PrivacyNotice.CheckApproved(); for Attempt := 0 to MaxRetries() do begin Send(MasterDataManagementSetup, ActionName, RequestBody, ResponseMessage); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al new file mode 100644 index 00000000000..a1711324c5c --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al @@ -0,0 +1,56 @@ +namespace Microsoft.Integration.MDM; + +using System.Privacy; + +/// +/// Registers the privacy notice for cross-environment master data synchronization and gates data transfer on +/// its approval. The setup wizard records the durable platform approval; the HTTP transport verifies it before +/// every outbound call, so no master data leaves the environment without recorded per-integration consent. +/// +codeunit 7242 "MDM Privacy Notice" +{ + Access = Internal; + InherentEntitlements = X; + InherentPermissions = X; + + var + PrivacyNoticeIdTok: Label 'MDMCrossEnvSync', Locked = true; + IntegrationServiceNameTxt: Label 'Master Data Management - cross-environment synchronization'; + NotApprovedErr: Label 'Cross-environment master data synchronization requires the privacy notice to be approved. Open Master Data Management Setup and approve sharing data between Business Central environments.'; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Privacy Notice", OnRegisterPrivacyNotices, '', false, false)] + local procedure RegisterPrivacyNotice(var TempPrivacyNotice: Record "Privacy Notice" temporary) + begin + TempPrivacyNotice.Init(); + TempPrivacyNotice.ID := PrivacyNoticeIdTok; + TempPrivacyNotice."Integration Service Name" := IntegrationServiceNameTxt; + if not TempPrivacyNotice.Insert() then; + end; + + procedure GetPrivacyNoticeId(): Code[50] + begin + exit(PrivacyNoticeIdTok); + end; + + procedure IsApproved(): Boolean + var + PrivacyNotice: Codeunit "Privacy Notice"; + begin + exit(PrivacyNotice.GetPrivacyNoticeApprovalState(PrivacyNoticeIdTok, false) = "Privacy Notice Approval State"::Agreed); + end; + + // Interactive: shows the platform notice and records the admin's decision (used from the setup wizard). + procedure ConfirmApproval(): Boolean + var + PrivacyNotice: Codeunit "Privacy Notice"; + begin + exit(PrivacyNotice.ConfirmPrivacyNoticeApproval(PrivacyNoticeIdTok, false)); + end; + + // Non-interactive gate for background/transport paths: fail closed if consent isn't recorded. + procedure CheckApproved() + begin + if not IsApproved() then + Error(NotApprovedErr); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index a1321aabb49..8ecb19f9166 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -235,6 +235,7 @@ page 7232 "MDM Connection Details" Step: Option Welcome,Connection,TestConnection,Finish; NextEnabled, BackEnabled, FinishEnabled, TestConnectionEnabled : Boolean; ConsentState, SecretAlreadyStored : Boolean; + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; SourceEnvironmentName: Text[100]; SourceEnvironmentUrl: Text[250]; SourceCompanyName: Text[100]; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 40d57df44cb..0aaf0cbe6f1 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -129,6 +129,30 @@ codeunit 139757 "Library - Master Data Mgt." exit(InlineMedia.Contains(SystemId, FieldNo)); end; + /// Returns the registered privacy-notice ID that gates cross-environment synchronization. + procedure PrivacyNoticeId(): Code[50] + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + exit(MDMPrivacyNotice.GetPrivacyNoticeId()); + end; + + /// Returns whether the cross-environment privacy notice is currently approved. + procedure PrivacyNoticeIsApproved(): Boolean + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + exit(MDMPrivacyNotice.IsApproved()); + end; + + /// Runs the fail-closed transport gate; errors when the privacy notice is not approved. + procedure PrivacyNoticeCheckApproved() + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + MDMPrivacyNotice.CheckApproved(); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; MasterDataManagement: Codeunit "Master Data Management"; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index ae0291530da..eeacd449c64 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -41,6 +41,35 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvTransferBlockedUntilPrivacyNoticeApproved() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + PrivacyNotice: Codeunit "Privacy Notice"; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] Cross-env data transfer is gated on the privacy notice: blocked until approved, allowed after. + Initialize(); + + // [GIVEN] the cross-environment privacy notice is not approved + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Disagreed); + + // [THEN] the gate reports not approved and the transport check fails closed + Assert.IsFalse(LibraryMasterDataMgt.PrivacyNoticeIsApproved(), 'Gate should report not approved before consent'); + asserterror LibraryMasterDataMgt.PrivacyNoticeCheckApproved(); + + // [WHEN] the admin approves the notice + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); + + // [THEN] the gate reports approved and the transport check passes + Assert.IsTrue(LibraryMasterDataMgt.PrivacyNoticeIsApproved(), 'Gate should report approved after consent'); + LibraryMasterDataMgt.PrivacyNoticeCheckApproved(); + + // reset approval so it does not leak into later tests + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Disagreed); + CleanUp(); + end; + [Test] procedure CrossEnvGetModifiedSetMaterializesSourceChange() var From 0e6019ff91139497d56e045bbfbc126f0d9a17e9 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 12:53:48 +0200 Subject: [PATCH 17/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtCrossEnv.PermissionSet.al | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al index f325430e9a7..2fdf2ee2380 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al @@ -4,8 +4,9 @@ namespace Microsoft.Integration.MDM; /// Assigned to the customer-registered Entra app on the SOURCE (Microsoft Entra Application Card) to grant /// cross-environment, read-only access to master data. Deliberately NOT part of "Master Data Mgt. - Objects": /// only this set grants execute on the ODataV4 source API, so local users cannot invoke it. -/// The per-synchronized-table read permissions are added dynamically as a tenant permission set as the user -/// edits Synchronization Tables (see design doc); this static set covers only the API surface. +/// Grants read on the master-data tables synchronized by the default configuration, so cross-environment sync +/// works read-only out of the box. For custom or additional tables, a tenant admin extends this set (or assigns +/// a second, narrowly scoped set alongside it) - which keeps least privilege instead of reaching for SUPER. /// permissionset 7242 "MDM Cross-Env Read" { @@ -13,5 +14,36 @@ permissionset 7242 "MDM Cross-Env Read" Access = Public; Caption = 'Master Data Mgt. - Cross Environment'; - Permissions = codeunit "MDM Cross-Env Source API" = X; + Permissions = codeunit "MDM Cross-Env Source API" = X, + tabledata "Salesperson/Purchaser" = R, + tabledata Customer = R, + tabledata Vendor = R, + tabledata Contact = R, + tabledata "Business Relation" = R, + tabledata "Country/Region" = R, + tabledata "Post Code" = R, + tabledata Currency = R, + tabledata "Currency Exchange Rate" = R, + tabledata "Payment Terms" = R, + tabledata "Shipment Method" = R, + tabledata "Shipping Agent" = R, + tabledata "Sales & Receivables Setup" = R, + tabledata "Purchases & Payables Setup" = R, + tabledata "Marketing Setup" = R, + tabledata "No. Series" = R, + tabledata "No. Series Line" = R, + tabledata "G/L Account" = R, + tabledata Dimension = R, + tabledata "Dimension Value" = R, + tabledata "Gen. Business Posting Group" = R, + tabledata "Gen. Product Posting Group" = R, + tabledata "Customer Posting Group" = R, + tabledata "Vendor Posting Group" = R, + tabledata "VAT Business Posting Group" = R, + tabledata "VAT Product Posting Group" = R, + tabledata "VAT Posting Setup" = R, + tabledata "Tax Area" = R, + tabledata "Tax Group" = R, + tabledata "Tax Jurisdiction" = R, + tabledata "Tenant Media" = R; } From 308b52610b5add2cf5463b464caccaae4d6e85e2 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 13:27:29 +0200 Subject: [PATCH 18/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtCrossEnv.PermissionSet.al | 20 +++++++++++ .../MDMCrossEnvChangeDetector.Codeunit.al | 6 +++- .../MDMCrossEnvDataSource.Codeunit.al | 9 +++-- .../MDMCrossEnvSourceAPI.Codeunit.al | 11 ++++++ .../MDMHttpSourceTransport.Codeunit.al | 35 ++++++++++++++----- .../codeunits/MDMSourceResponse.Codeunit.al | 2 +- .../tables/MasterDataManagementSetup.Table.al | 32 +++++++++++++---- 7 files changed, 96 insertions(+), 19 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al index 2fdf2ee2380..f826696425c 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al @@ -1,5 +1,25 @@ namespace Microsoft.Integration.MDM; +using Microsoft.CRM.BusinessRelation; +using Microsoft.CRM.Contact; +using Microsoft.CRM.Setup; +using Microsoft.CRM.Team; +using Microsoft.Finance.Currency; +using Microsoft.Finance.Dimension; +using Microsoft.Finance.GeneralLedger.Account; +using Microsoft.Finance.GeneralLedger.Setup; +using Microsoft.Finance.SalesTax; +using Microsoft.Finance.VAT.Setup; +using Microsoft.Foundation.Address; +using Microsoft.Foundation.NoSeries; +using Microsoft.Foundation.PaymentTerms; +using Microsoft.Foundation.Shipping; +using Microsoft.Purchases.Setup; +using Microsoft.Purchases.Vendor; +using Microsoft.Sales.Customer; +using Microsoft.Sales.Setup; +using System.Environment; + /// /// Assigned to the customer-registered Entra app on the SOURCE (Microsoft Entra Application Card) to grant /// cross-environment, read-only access to master data. Deliberately NOT part of "Master Data Mgt. - Objects": diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index d10f0fcc4a3..77587213890 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -19,6 +19,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" var LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; + DetectorParseFailedTxt: Label 'The cross-environment change detector received an invalid response from the source and skipped this run.', Locked = true; trigger OnRun() begin @@ -31,6 +32,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" SourceConnection: Codeunit "MDM Source Connection"; SourceResponse: Codeunit "MDM Source Response"; SourceCapabilities: Codeunit "MDM Source Capabilities"; + MasterDataManagement: Codeunit "Master Data Management"; Transport: Interface "IMDM Source Transport"; Response: JsonObject; TableIds: JsonArray; @@ -49,8 +51,10 @@ codeunit 7245 "MDM Cross-Env Change Detector" // Skip (rather than error every run) if an older source doesn't advertise the detection action. if not SourceCapabilities.IsSupported(Transport, LastModifiedFeatureTok) then exit; - if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(WriteArray(TableIds)), Response) then + if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(WriteArray(TableIds)), Response) then begin + Session.LogMessage('0000QF4', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; + end; ProcessDetectionResponse(Response); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index a36262f96d5..9ffbb68369a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -112,15 +112,20 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(TableIdsText), Response) then Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); if not Response.Get('tables', Token) then - exit(false); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); Tables := Token.AsArray(); if Tables.Count() = 0 then - exit(false); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); Tables.Get(0, Token); Entry := Token.AsObject(); if Entry.Get('tableAvailable', Token) then if not Token.AsValue().AsBoolean() then exit(false); + // Unindexed source table: LastModifiedAtPerTable reports indexed:false and no timestamp, so we can't prove + // emptiness cheaply - assume records may exist so the full-synch review isn't wrongly suppressed. + if Entry.Get('indexed', Token) then + if Token.IsValue() and (not Token.AsValue().AsBoolean()) then + exit(true); if Entry.Get('lastModifiedAt', Token) then if Token.IsValue() then LastModifiedAtText := Token.AsValue().AsText(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index b8e648a88ad..07dcfa3c399 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -73,6 +73,7 @@ codeunit 7241 "MDM Cross-Env Source API" end; PageSize := ClampPageSize(PageSize); + ApplyProjectionLoadFields(RecRef, ProjectedFields); // Targeted mode: caller asked for specific SystemIds (no paging). if SelectorSystemIds(Selector, SystemIds) then begin @@ -165,6 +166,16 @@ codeunit 7241 "MDM Cross-Env Source API" not (FieldReference.Type() in [FieldType::MediaSet, FieldType::TableFilter])); end; + // Load only the projected fields (plus the change-feed keys) so wide source tables aren't fully materialized. + local procedure ApplyProjectionLoadFields(var RecRef: RecordRef; ProjectedFields: List of [Integer]) + var + FieldNo: Integer; + begin + RecRef.SetLoadFields(SystemModifiedAtFieldNo(), SystemIdFieldNo()); + foreach FieldNo in ProjectedFields do + RecRef.AddLoadFields(FieldNo); + end; + local procedure SelectorSystemIds(Selector: Text; var SystemIds: JsonArray): Boolean var SelectorObject: JsonObject; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 7f8fc5e3238..d1d49c7b572 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -27,7 +27,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" SendFailedErr: Label 'The request to the source environment could not be sent. Check the source environment URL.'; InvalidSourceUrlErr: Label 'The source environment URL is not a valid Business Central endpoint.'; InvalidSourceUrlAuditTxt: Label 'Blocked a cross-environment request: the configured source environment URL host ''%1'' is not a valid Business Central endpoint.', Comment = '%1 = the rejected host'; - HttpErr: Label 'The source environment returned HTTP %1. %2', Comment = '%1 = HTTP status code, %2 = response detail'; + HttpErr: Label 'The source environment returned HTTP %1.', Comment = '%1 = HTTP status code'; ServiceNameTok: Label 'MDMCrossEnvSource', Locked = true; ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true; TokenEndpointTok: Label 'https://login.microsoftonline.com/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; @@ -36,7 +36,8 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" TokenAcquiredAuditTxt: Label 'Acquired an application access token to read master data from source environment %1.', Comment = '%1 = source environment name'; TokenFailedAuditTxt: Label 'Failed to acquire an application access token for source environment %1.', Comment = '%1 = source environment name'; AccessDeniedAuditTxt: Label 'Source environment %1 denied the master data request (HTTP %2).', Comment = '%1 = source environment name, %2 = HTTP status code'; - RequestFailedTelemetryTxt: Label 'Cross-environment %1 request failed with HTTP %2.', Locked = true; + RequestFailedTelemetryTxt: Label 'Cross-environment %1 request failed with HTTP %2.', Locked = true, Comment = '%1 = action name, %2 = HTTP status code'; + TransportFailedTelemetryTxt: Label 'Cross-environment %1 request could not be sent to the source environment.', Locked = true, Comment = '%1 = action name'; procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text var @@ -87,24 +88,39 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" if ResponseMessage.IsSuccessStatusCode() then exit(UnwrapODataValue(ResponseBodyText)); if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin - LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); - Error(HttpErr, ResponseMessage.HttpStatusCode(), ResponseBodyText); + LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage, ResponseBodyText); + Error(HttpErr, ResponseMessage.HttpStatusCode()); end; Sleep(RetryAfter); end; end; - local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage) + local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage; ResponseBodyText: Text) var AuditLog: Codeunit "Audit Log"; + Dimensions: Dictionary of [Text, Text]; begin - // Operational telemetry: action + status only, never record data or credentials. - Session.LogMessage('0000QF1', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', TelemetryCategoryTok); + // The raw response body stays in structured telemetry context, never in a user-facing error. + Dimensions.Add('Category', TelemetryCategoryTok); + Dimensions.Add('action', ActionName); + Dimensions.Add('httpStatusCode', Format(ResponseMessage.HttpStatusCode())); + Dimensions.Add('responseBody', CopyStr(ResponseBodyText, 1, 2048)); + Session.LogMessage('0000QF1', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); // Security audit: an authorization failure crossing the environment boundary. if ResponseMessage.HttpStatusCode() in [401, 403] then AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); end; + local procedure LogTransportFailure(ActionName: Text; SendErrorText: Text) + var + Dimensions: Dictionary of [Text, Text]; + begin + Dimensions.Add('Category', TelemetryCategoryTok); + Dimensions.Add('action', ActionName); + Dimensions.Add('transportError', CopyStr(SendErrorText, 1, 2048)); + Session.LogMessage('0000QF3', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + end; + local procedure Send(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage) var HttpClient: HttpClient; @@ -113,6 +129,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" HttpContent: HttpContent; ContentHeaders: HttpHeaders; begin + HttpClient.Timeout := 100000; // explicit 100s cap so a stalled connection can't hang a background sync run RequestMessage.Method('POST'); RequestMessage.SetRequestUri(BuildActionUrl(MasterDataManagementSetup, ActionName)); RequestMessage.GetHeaders(RequestHeaders); @@ -125,8 +142,10 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ContentHeaders.Add('Content-Type', 'application/json'); RequestMessage.Content(HttpContent); - if not HttpClient.Send(RequestMessage, ResponseMessage) then + if not HttpClient.Send(RequestMessage, ResponseMessage) then begin + LogTransportFailure(ActionName, GetLastErrorText()); Error(SendFailedErr); + end; end; local procedure BuildActionUrl(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index fab7a19d10e..2e9e7a611ca 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -205,7 +205,7 @@ codeunit 7248 "MDM Source Response" Dimensions.Add('systemId', Format(SystemId, 0, 4)); if FieldObject.Get('length', LengthToken) then Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); - Session.LogMessage('0000QF2', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, Dimensions); + Session.LogMessage('0000QF2', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 92902619c17..eba7007dee9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -207,16 +207,22 @@ table 7230 "Master Data Management Setup" [NonDebuggable] local procedure SetSecret(SecretKey: Guid; SecretValue: SecretText): Guid var + EnvironmentInformation: Codeunit "Environment Information"; NewSecretKey: Guid; begin if not IsNullGuid(SecretKey) then if not IsolatedStorage.Delete(SecretKey, DataScope::Company) then; NewSecretKey := CreateGuid(); - if not EncryptionEnabled() then - IsolatedStorage.Set(NewSecretKey, SecretValue, DataScope::Company) - else - IsolatedStorage.SetEncrypted(NewSecretKey, SecretValue, DataScope::Company); + if EncryptionEnabled() then + IsolatedStorage.SetEncrypted(NewSecretKey, SecretValue, DataScope::Company) + else begin + // On SaaS (the only runtime for cross-env) encryption is always available, so refuse to store the + // secret unencrypted there; off-SaaS dev/test/on-prem may lack encryption, so fall back as Intercompany does. + if EnvironmentInformation.IsSaaSInfrastructure() then + Error(EncryptionRequiredErr); + IsolatedStorage.Set(NewSecretKey, SecretValue, DataScope::Company); + end; exit(NewSecretKey); end; @@ -249,18 +255,28 @@ table 7230 "Master Data Management Setup" if IsCrossEnvironment() then begin // Cross-environment: the source is a different environment; never write to its subscriber table. - Message(StrSubstNo(SynchronizationEnabledMsg, "Source Company Name")); - Session.LogMessage('0000JIM', "Source Environment Name", Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Message(SynchronizationEnabledMsg, "Source Company Name"); + LogCrossEnvironmentEnabled(MasterDataManagement.GetTelemetryCategory()); exit; end; CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); MasterDataManagement.AddSubsidiarySubscriptionToMasterCompany(Rec."Company Name", CurrentCompanyName); - Message(StrSubstNo(SynchronizationEnabledMsg, Rec."Company Name")); + Message(SynchronizationEnabledMsg, Rec."Company Name"); Session.LogMessage('0000JIM', Rec."Company Name", Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); Session.LogMessage('0000JIN', CurrentCompanyName, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); end; + // Env name is organization-identifiable: keep it out of the free-text message and in a structured dimension. + local procedure LogCrossEnvironmentEnabled(TelemetryCategory: Text) + var + Dimensions: Dictionary of [Text, Text]; + begin + Dimensions.Add('Category', TelemetryCategory); + Dimensions.Add('sourceEnvironment', "Source Environment Name"); + Session.LogMessage('0000JIM', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); + end; + local procedure GetConfigurationUpdates(var IsEnabledChanged: Boolean) var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -356,5 +372,7 @@ table 7230 "Master Data Management Setup" MustNotPickCurrentCompanyErr: label 'You are currently signed into this company. \\Choose a different company to synchronize data with.'; MustPickSourceCompanyErr: label 'You must choose a source company to synchronize data from.'; MustConfigureConnectionErr: label 'Enter the cross-environment connection details before you enable synchronization.'; + EncryptionRequiredErr: label 'Enable data encryption before saving the source connection secret. Cross-environment credentials are never stored unencrypted.'; + CrossEnvEnabledTelemetryTxt: label 'Cross-environment master data synchronization was enabled.', Locked = true; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; } From 624a65212aaeecaf890c8a25b8ea33ef78b0cf4b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 14:19:02 +0200 Subject: [PATCH 19/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvDataSource.Codeunit.al | 12 +++++- .../MDMCrossEnvSourceAPI.Codeunit.al | 1 + .../MDMHttpSourceTransport.Codeunit.al | 6 +-- .../codeunits/MDMPrivacyNotice.Codeunit.al | 14 ++++++- .../src/pages/MDMConnectionDetails.Page.al | 5 ++- .../tables/MasterDataManagementSetup.Table.al | 2 +- .../src/LibraryMasterDataMgt.Codeunit.al | 10 +++++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 41 +++++++++++++++++++ 8 files changed, 83 insertions(+), 8 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 9ffbb68369a..4db5c6342dc 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -188,13 +188,23 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceResponse.InsertRecords(Response, SourceRecordRef); end; + // A malformed wire response is an internal integration defect, not something the user can act on. + local procedure InternalError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) var UnavailableFields: JsonArray; begin Clear(Response); if not SourceResponse.TryParse(ResponseText, Response) then - Error(InvalidResponseErr, TableCaption(IntegrationTableId)); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); if not SourceResponse.TableAvailable(Response) then Error(TableUnavailableErr, TableCaption(IntegrationTableId)); if not SourceResponse.Indexed(Response) then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 07dcfa3c399..14fa597d5bb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -139,6 +139,7 @@ codeunit 7241 "MDM Cross-Env Source API" local procedure TryOpenTable(TableId: Integer; var RecRef: RecordRef) begin RecRef.Open(TableId); + RecRef.ReadIsolation := IsolationLevel::ReadCommitted; end; local procedure ResolveProjection(var RecRef: RecordRef; FieldIds: Text; var ProjectedFields: List of [Integer]; var UnavailableFields: JsonArray) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index d1d49c7b572..a6dc47c289b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -209,11 +209,11 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" begin Scopes.Add(ScopeTok); TokenEndpoint := StrSubstNo(TokenEndpointTok, AzureADTenant.GetAadTenantId()); - OAuth2.AcquireTokenWithClientCredentials( + if not OAuth2.AcquireTokenWithClientCredentials( MasterDataManagementSetup."Source OAuth Client Id", MasterDataManagementSetup.GetSourceClientSecret(), - TokenEndpoint, '', Scopes, Token); - if Token.IsEmpty() then begin + TokenEndpoint, '', Scopes, Token) or Token.IsEmpty() + then begin AuditLog.LogAuditMessage(StrSubstNo(TokenFailedAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Failure, AuditCategory::Authentication, 4, 0); Error(NoTokenErr); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al index a1711324c5c..a7b06847fe4 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al @@ -17,6 +17,7 @@ codeunit 7242 "MDM Privacy Notice" PrivacyNoticeIdTok: Label 'MDMCrossEnvSync', Locked = true; IntegrationServiceNameTxt: Label 'Master Data Management - cross-environment synchronization'; NotApprovedErr: Label 'Cross-environment master data synchronization requires the privacy notice to be approved. Open Master Data Management Setup and approve sharing data between Business Central environments.'; + OpenSetupActionTxt: Label 'Open Master Data Management Setup'; [EventSubscriber(ObjectType::Codeunit, Codeunit::"Privacy Notice", OnRegisterPrivacyNotices, '', false, false)] local procedure RegisterPrivacyNotice(var TempPrivacyNotice: Record "Privacy Notice" temporary) @@ -49,8 +50,17 @@ codeunit 7242 "MDM Privacy Notice" // Non-interactive gate for background/transport paths: fail closed if consent isn't recorded. procedure CheckApproved() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + ErrInfo: ErrorInfo; begin - if not IsApproved() then - Error(NotApprovedErr); + if IsApproved() then + exit; + ErrInfo.Message := NotApprovedErr; + if MasterDataManagementSetup.Get() then begin + ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); + ErrInfo.AddNavigationAction(OpenSetupActionTxt); + end; + Error(ErrInfo); end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 8ecb19f9166..b7fa4ee034b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -30,7 +30,7 @@ page 7232 "MDM Connection Details" group(TermsAndConditions) { Caption = 'Review the terms and conditions'; - InstructionalText = 'By enabling this feature, you consent to your data being shared between Business Central environments. Your privacy is important to us. To learn more, follow the link below.'; + InstructionalText = 'By enabling this, you consent that this environment will read data from the source environment that you configure. Your privacy is important to us. To learn more, follow the link below.'; field(Consent; ConsentState) { @@ -40,6 +40,9 @@ page 7232 "MDM Connection Details" trigger OnValidate() begin + // Ticking "I accept" records the durable platform privacy-notice approval. + if ConsentState then + ConsentState := MDMPrivacyNotice.ConfirmApproval(); SetControls(); end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index eba7007dee9..25f350e55e9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -274,7 +274,7 @@ table 7230 "Master Data Management Setup" begin Dimensions.Add('Category', TelemetryCategory); Dimensions.Add('sourceEnvironment', "Source Environment Name"); - Session.LogMessage('0000JIM', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); + Session.LogMessage('0000QF5', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); end; local procedure GetConfigurationUpdates(var IsEnabledChanged: Boolean) diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 0aaf0cbe6f1..fa3681c3294 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -153,6 +153,16 @@ codeunit 139757 "Library - Master Data Mgt." MDMPrivacyNotice.CheckApproved(); end; + /// Removes any recorded approval for the cross-env privacy notice, resetting it to Not set. + procedure PrivacyNoticeResetApproval() + var + PrivacyNoticeApproval: Record "Privacy Notice Approval"; + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + PrivacyNoticeApproval.SetRange(ID, MDMPrivacyNotice.GetPrivacyNoticeId()); + PrivacyNoticeApproval.DeleteAll(); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; MasterDataManagement: Codeunit "Master Data Management"; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index eeacd449c64..a7ded7c6f7d 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -9,6 +9,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" var Assert: Codeunit Assert; LibrarySalesLib: Codeunit "Library - Sales"; + WizardOpenedPrivacyNotice: Boolean; [Test] procedure CrossEnvGetBySystemIdRoundTripsSourceRecord() @@ -57,6 +58,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [THEN] the gate reports not approved and the transport check fails closed Assert.IsFalse(LibraryMasterDataMgt.PrivacyNoticeIsApproved(), 'Gate should report not approved before consent'); asserterror LibraryMasterDataMgt.PrivacyNoticeCheckApproved(); + Assert.ExpectedError('privacy notice to be approved'); // [WHEN] the admin approves the notice PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); @@ -70,6 +72,41 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + [HandlerFunctions('PrivacyNoticeModalHandler')] + procedure WizardConsentOpensPrivacyNotice() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + ConnectionWizard: TestPage "MDM Connection Details"; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] Ticking consent in the wizard opens the platform privacy notice - a regression guard that the + // wizard actually calls ConfirmApproval (the handler below fires only if the notice dialog is shown). + Initialize(); + + // [GIVEN] the privacy notice has no recorded decision, so accepting consent must prompt it + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + WizardOpenedPrivacyNotice := false; + + // [WHEN] the admin ticks consent in the connection wizard + ConnectionWizard.OpenEdit(); + ConnectionWizard.Consent.SetValue(true); + ConnectionWizard.Close(); + + // [THEN] the privacy-notice dialog was shown - proving the wizard invoked ConfirmApproval + Assert.IsTrue(WizardOpenedPrivacyNotice, 'Ticking consent should open the privacy notice (call ConfirmApproval)'); + + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + CleanUp(); + end; + + [ModalPageHandler] + procedure PrivacyNoticeModalHandler(var PrivacyNoticePage: TestPage "Privacy Notice") + begin + // Reached only if the wizard actually opened the notice. + WizardOpenedPrivacyNotice := true; + end; + [Test] procedure CrossEnvGetModifiedSetMaterializesSourceChange() var @@ -188,11 +225,15 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" procedure ConnectionDetailsWizardSavesConfiguration() var MasterDataManagementSetup: Record "Master Data Management Setup"; + PrivacyNotice: Codeunit "Privacy Notice"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; ConnectionDetails: TestPage "MDM Connection Details"; begin // [FEATURE] [AI test 0.4] // [SCENARIO] The Connection Details wizard collects the source connection details and saves them (secret to Isolated Storage). Initialize(); + // Pre-approve so ticking consent doesn't open the notice dialog in this configuration-focused test. + PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); ConnectionDetails.OpenEdit(); // Welcome step: accept the terms so Next is enabled. From 59e2cd6c6df5786c8a6474484b69c26ca6859f7d Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 15:18:55 +0200 Subject: [PATCH 20/64] [Master Data Management] Cross-environment synchonization --- ...et.al => MDMCrossEnvRead.PermissionSet.al} | 0 .../MDMCrossEnvChangeDetector.Codeunit.al | 1 + .../MDMHttpSourceTransport.Codeunit.al | 21 ++++--- .../codeunits/MDMSourceResponse.Codeunit.al | 9 +-- .../MasterDataMgtSubscribers.Codeunit.al | 21 +++++-- .../tables/MasterDataManagementSetup.Table.al | 22 ++++++- .../src/LibraryMasterDataMgt.Codeunit.al | 59 +++++++++++++++++++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 28 +++++++++ .../src/MasterDataMgtSetupTests.Codeunit.al | 1 + 9 files changed, 144 insertions(+), 18 deletions(-) rename src/Apps/W1/MasterDataManagement/app/permissions/{MasterDataMgtCrossEnv.PermissionSet.al => MDMCrossEnvRead.PermissionSet.al} (100%) diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al similarity index 100% rename from src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtCrossEnv.PermissionSet.al rename to src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 77587213890..3b740bdc801 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -114,6 +114,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" IntegrationTableMapping.SetRange("Delete After Synchronization", false); IntegrationTableMapping.SetRange("Integration Table ID", TableId); IntegrationTableMapping.SetRange(Status, IntegrationTableMapping.Status::Enabled); + IntegrationTableMapping.SetLoadFields("Synch. Modified On Filter"); if not IntegrationTableMapping.FindSet() then exit; repeat diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index a6dc47c289b..7a2fa239a7a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -88,36 +88,37 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" if ResponseMessage.IsSuccessStatusCode() then exit(UnwrapODataValue(ResponseBodyText)); if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin - LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage, ResponseBodyText); + LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); Error(HttpErr, ResponseMessage.HttpStatusCode()); end; Sleep(RetryAfter); end; end; - local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage; ResponseBodyText: Text) + local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage) var AuditLog: Codeunit "Audit Log"; Dimensions: Dictionary of [Text, Text]; begin - // The raw response body stays in structured telemetry context, never in a user-facing error. + // The response body can echo source-environment record content, so it is never emitted to telemetry; only + // the action and HTTP status (non-content diagnostics) are logged. Dimensions.Add('Category', TelemetryCategoryTok); Dimensions.Add('action', ActionName); Dimensions.Add('httpStatusCode', Format(ResponseMessage.HttpStatusCode())); - Dimensions.Add('responseBody', CopyStr(ResponseBodyText, 1, 2048)); Session.LogMessage('0000QF1', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); // Security audit: an authorization failure crossing the environment boundary. if ResponseMessage.HttpStatusCode() in [401, 403] then AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); end; - local procedure LogTransportFailure(ActionName: Text; SendErrorText: Text) + local procedure LogTransportFailure(ActionName: Text) var Dimensions: Dictionary of [Text, Text]; begin + // GetLastErrorText() can contain record keys or file names, so the raw error is never emitted to telemetry; + // only the action is logged. Dimensions.Add('Category', TelemetryCategoryTok); Dimensions.Add('action', ActionName); - Dimensions.Add('transportError', CopyStr(SendErrorText, 1, 2048)); Session.LogMessage('0000QF3', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; @@ -143,7 +144,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" RequestMessage.Content(HttpContent); if not HttpClient.Send(RequestMessage, ResponseMessage) then begin - LogTransportFailure(ActionName, GetLastErrorText()); + LogTransportFailure(ActionName); Error(SendFailedErr); end; end; @@ -174,6 +175,12 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Error(InvalidSourceUrlErr); end; + // Test seam: exercise the source-host allow-list without a live environment or the SaaS gate. + internal procedure ValidateSourceHostUrl(BaseUrl: Text) + begin + ValidateSourceHost(BaseUrl); + end; + // The ODataV4 envelope for an action returning Text is { "@odata.context": "...", "value": "" }; // the source API's own JSON is that inner value. Fall back to the raw body if the shape is unexpected. local procedure UnwrapODataValue(ResponseBody: Text): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 2e9e7a611ca..a055dd8cd78 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -146,7 +146,7 @@ codeunit 7248 "MDM Source Response" exit; MediaObject := ValueToken.AsObject(); if IsSkipped(MediaObject) then begin - LogSkippedField(TableId, FieldNo, SystemId, MediaObject); + LogSkippedField(TableId, FieldNo, MediaObject); exit; end; if not MediaObject.Get('content', ContentToken) then @@ -172,7 +172,7 @@ codeunit 7248 "MDM Source Response" exit; BlobObject := ValueToken.AsObject(); if IsSkipped(BlobObject) then begin - LogSkippedField(TableId, FieldNo, SystemId, BlobObject); + LogSkippedField(TableId, FieldNo, BlobObject); exit; end; if not BlobObject.Get('content', ContentToken) then @@ -193,16 +193,17 @@ codeunit 7248 "MDM Source Response" // Over-cap media/blob is not synchronized (v1). We can't error (it would retry the record every run) and MDM // surfaces no synch warnings, so the skip is emitted as telemetry only; the record's other fields still sync. - local procedure LogSkippedField(TableId: Integer; FieldNo: Integer; SystemId: Guid; FieldObject: JsonObject) + local procedure LogSkippedField(TableId: Integer; FieldNo: Integer; FieldObject: JsonObject) var MasterDataManagement: Codeunit "Master Data Management"; Dimensions: Dictionary of [Text, Text]; LengthToken: JsonToken; begin + // The source record identifier (systemId) is customer data, so it is not emitted; only the table, field, + // and length (non-identifying diagnostics) are logged. Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); Dimensions.Add('tableId', Format(TableId)); Dimensions.Add('fieldNo', Format(FieldNo)); - Dimensions.Add('systemId', Format(SystemId, 0, 4)); if FieldObject.Get('length', LengthToken) then Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); Session.LogMessage('0000QF2', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 1fad2f20458..edd956cbac4 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -747,23 +747,32 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; IntegrationRecordRef: RecordRef; - ModifiedFieldRef: FieldRef; IsHandled: Boolean; IntRecSystemId: Guid; begin MasterDataManagementSetup.Get(); + // Cross-environment: FromRecordRef is already the source row materialized from the fetched batch, so read + // the watermark from it instead of issuing another per-record OData round-trip (avoids an N+1 fetch). + if MasterDataManagementSetup."Source Environment Name" <> '' then + exit(ModifiedOnFromRecordRef(IntegrationTableMapping, FromRecordRef)); + IntegrationRecordRef.Open(FromRecordRef.Number, false); IntRecSystemId := FromRecordRef.Field(FromRecordRef.SystemIdNo).Value(); MasterDataManagement.OnGetIntegrationRecordRefBySystemId(IntegrationTableMapping, IntegrationRecordRef, IntRecSystemId, IsHandled); if not IsHandled then // Route the source re-fetch: the record lives in the local company or another environment. if MasterDataManagementSetup.GetDataSource().GetBySystemId(FromRecordRef.Number, IntRecSystemId, IntegrationRecordRef) then; - if FromRecordRef.Number() = IntegrationTableMapping."Integration Table ID" then begin - ModifiedFieldRef := IntegrationRecordRef.Field(IntegrationTableMapping."Int. Tbl. Modified On Fld. No."); - exit(ModifiedFieldRef.Value()); - end; + exit(ModifiedOnFromRecordRef(IntegrationTableMapping, IntegrationRecordRef)); + end; - ModifiedFieldRef := IntegrationRecordRef.Field(IntegrationRecordRef.SystemModifiedAtNo()); + local procedure ModifiedOnFromRecordRef(IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef): DateTime + var + ModifiedFieldRef: FieldRef; + begin + if SourceRecordRef.Number() = IntegrationTableMapping."Integration Table ID" then + ModifiedFieldRef := SourceRecordRef.Field(IntegrationTableMapping."Int. Tbl. Modified On Fld. No.") + else + ModifiedFieldRef := SourceRecordRef.Field(SourceRecordRef.SystemModifiedAtNo()); exit(ModifiedFieldRef.Value()); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 25f350e55e9..b6fedc9b19d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -33,7 +33,7 @@ table 7230 "Master Data Management Setup" if "Is Enabled" then if IsCrossEnvironment() then begin if not IsCrossEnvConnectionConfigured() then - Error(MustConfigureConnectionErr); + Error(BuildConfigureConnectionError()); end else if "Company Name" = '' then Error(MustPickSourceCompanyErr); @@ -95,6 +95,9 @@ table 7230 "Master Data Management Setup" var MasterDataMgtSetupDefault: Codeunit "Master Data Mgt. Setup Default"; begin + // Re-pointing an enabled setup would leave stale couplings/cursors against the old source; force a disable/rebuild. + if "Is Enabled" and ("Source Environment Name" <> xRec."Source Environment Name") then + Error(CannotChangeSourceWhileEnabledErr); MasterDataMgtSetupDefault.UpdateChangeDetectorJob(Rec); end; } @@ -255,6 +258,11 @@ table 7230 "Master Data Management Setup" if IsCrossEnvironment() then begin // Cross-environment: the source is a different environment; never write to its subscriber table. + // Drop any stale local-company subscription left from a prior local-source setup. + if "Company Name" <> '' then begin + CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); + MasterDataManagement.RemoveSubsidiarySubscriptionFromMasterCompany("Company Name", CurrentCompanyName); + end; Message(SynchronizationEnabledMsg, "Source Company Name"); LogCrossEnvironmentEnabled(MasterDataManagement.GetTelemetryCategory()); exit; @@ -365,6 +373,16 @@ table 7230 "Master Data Management Setup" exit(Enum::"MDM Data Source Type"::LocalCompany); end; + local procedure BuildConfigureConnectionError(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MustConfigureConnectionErr; + ErrInfo.RecordId := Rec.RecordId(); + ErrInfo.AddNavigationAction(OpenSetupNavigationTxt); + exit(ErrInfo); + end; + var SynchronizationEnabledMsg: label 'The synchronization of data from company %1 is enabled. \\To review the tables and fields that will be synchronized, choose action Synchronization Tables. \\To perform the initial synchronization of data from %1, choose Start Initial Synchronization. \\After the initial synchronization is done, job queue entries will continue to synchronize modifications.', Comment = '%1 - a company name'; CouplingsWillBeDeletedQst: label 'All the couplings with records from previous source company %1 will be deleted. Do you want to continue?', Comment = '%1 - a company name'; @@ -372,6 +390,8 @@ table 7230 "Master Data Management Setup" MustNotPickCurrentCompanyErr: label 'You are currently signed into this company. \\Choose a different company to synchronize data with.'; MustPickSourceCompanyErr: label 'You must choose a source company to synchronize data from.'; MustConfigureConnectionErr: label 'Enter the cross-environment connection details before you enable synchronization.'; + OpenSetupNavigationTxt: label 'Open Master Data Management Setup'; + CannotChangeSourceWhileEnabledErr: label 'You cannot change the source environment while synchronization is enabled. Disable synchronization first, then change the source.'; EncryptionRequiredErr: label 'Enable data encryption before saving the source connection secret. Cross-environment credentials are never stored unencrypted.'; CrossEnvEnabledTelemetryTxt: label 'Cross-environment master data synchronization was enabled.', Locked = true; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index fa3681c3294..6eb983b7269 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -40,6 +40,7 @@ codeunit 139757 "Library - Master Data Mgt." MasterDataSynchTables.FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, TableId); end; + /// Sets the source company on Master Data Management Setup to the current company. procedure SetSourceCompanyToCurrent() var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -49,16 +50,31 @@ codeunit 139757 "Library - Master Data Mgt." MasterDataManagementSetup.Modify(false); end; + /// Gets the integration record reference for a coupling. + /// The integration table ID to resolve. + /// The coupling whose integration record is requested. + /// Returns the resolved integration record reference. + /// True if the integration record was found; otherwise false. procedure GetIntegrationRecordRefByCoupling(IntegrationTableID: Integer; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; var RecRef: RecordRef): Boolean begin exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableID, MasterDataMgtCoupling, RecRef)); end; + /// Gets the integration record reference identified by a coupling ID. + /// The integration table mapping to resolve against. + /// The coupling ID to resolve. + /// Returns the resolved integration record reference. + /// True if the integration record was found; otherwise false. procedure GetIntegrationRecordRefById(var IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var RecRef: RecordRef): Boolean begin exit(MasterDataManagement.GetIntegrationRecordRef(IntegrationTableMapping, ID, RecRef)); end; + /// Gets the set of modified source records for a table mapping from the configured data source. + /// The integration table mapping to read. + /// The source table filter to apply. + /// Returns the record reference positioned on the modified set. + /// True if any modified records were found; otherwise false. procedure DataSourceGetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -67,6 +83,11 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetModifiedSet(IntegrationTableMapping, TableFilter, SourceRecordRef)); end; + /// Gets source records matching a UID filter from the configured data source. + /// The integration table mapping to read. + /// The UID filter to apply. + /// Returns the record reference positioned on the matching set. + /// True if any matching records were found; otherwise false. procedure DataSourceGetByUidFilter(IntegrationTableMapping: Record "Integration Table Mapping"; UidFilter: Text; var SourceRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -75,6 +96,11 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef)); end; + /// Gets source records matching a table filter from the configured data source. + /// The integration table mapping to read. + /// The source table filter to apply. + /// Returns the record reference positioned on the matching set. + /// True if any matching records were found; otherwise false. procedure DataSourceGetByFilter(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -83,6 +109,9 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, TableFilter, SourceRecordRef)); end; + /// Gets the count of integration records for a table mapping. + /// The integration table mapping to count. + /// The number of integration records. procedure GetIntegrationRecRefCount(IntegrationTableMapping: Record "Integration Table Mapping"): Integer var MasterDataManagement: Codeunit "Master Data Management"; @@ -90,6 +119,11 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagement.GetIntegrationRecRefCount(IntegrationTableMapping)); end; + /// Gets a single source record by its SystemId from the configured data source. + /// The integration table ID to read. + /// The SystemId of the source record. + /// Returns the record reference positioned on the found record. + /// True if the record was found; otherwise false. procedure DataSourceGetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -98,6 +132,15 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableId, SystemId, SourceRecordRef)); end; + /// Gets a cursor-paged batch of modified source records from the cross-environment data source. + /// The integration table mapping to read. + /// The source table filter to apply. + /// The cursor to resume from; empty starts a new scan. + /// The maximum number of pages to fetch in this call. + /// Returns the record reference positioned on the fetched batch. + /// Returns the cursor to resume from on the next call. + /// Returns true if more records remain beyond this batch. + /// True if any records were fetched; otherwise false. procedure DataSourceGetModifiedBatch(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; StartCursor: Text; MaxPages: Integer; var SourceRecordRef: RecordRef; var EndCursor: Text; var HasMore: Boolean): Boolean var CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; @@ -106,6 +149,8 @@ codeunit 139757 "Library - Master Data Mgt." end; // Setting a Source Environment Name routes GetDataSource() to the cross-environment implementation. + /// Sets the source environment name, routing the data source to the cross-environment implementation. + /// The source environment name to set. procedure SetSourceEnvironmentName(EnvironmentName: Text) var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -115,6 +160,7 @@ codeunit 139757 "Library - Master Data Mgt." MasterDataManagementSetup.Modify(false); end; + /// Runs the cross-environment change detector once. procedure RunChangeDetector() var MDMCrossEnvChangeDetector: Codeunit "MDM Cross-Env Change Detector"; @@ -122,6 +168,19 @@ codeunit 139757 "Library - Master Data Mgt." MDMCrossEnvChangeDetector.DetectChanges(); end; + /// Validates a source environment URL against the HTTP transport's host allow-list; errors if it is not a valid Business Central endpoint. + /// The source environment base URL to validate. + procedure ValidateHttpTransportSourceHost(BaseUrl: Text) + var + MDMHttpSourceTransport: Codeunit "MDM Http Source Transport"; + begin + MDMHttpSourceTransport.ValidateSourceHostUrl(BaseUrl); + end; + + /// Checks whether the inline media cache holds an entry for a record field. + /// The SystemId of the source record. + /// The field number of the media/blob field. + /// True if the cache contains the entry; otherwise false. procedure InlineMediaCacheContains(SystemId: Guid; FieldNo: Integer): Boolean var InlineMedia: Codeunit "MDM Inline Media"; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index a7ded7c6f7d..03a8016ec7e 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -42,6 +42,34 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure HttpTransportRejectsNonBusinessCentralHosts() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Security] + // [SCENARIO] The HTTP transport's source-host allow-list accepts only HTTPS Business Central endpoints (SSRF guard). + Initialize(); + + // [GIVEN] valid Business Central SaaS and TIE endpoints over HTTPS [THEN] validation passes + LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.api.bc.dynamics.com'); + LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.api.bc.dynamics-tie.com'); + + // [GIVEN] a non-HTTPS scheme [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('http://myenv.api.bc.dynamics.com'); + Assert.ExpectedError('not a valid Business Central endpoint'); + + // [GIVEN] a host outside the dynamics.com allow-list [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://evil.example.com'); + Assert.ExpectedError('not a valid Business Central endpoint'); + + // [GIVEN] a look-alike host that only embeds dynamics.com as a non-final label [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.dynamics.com.evil.example.com'); + Assert.ExpectedError('not a valid Business Central endpoint'); + + CleanUp(); + end; + [Test] procedure CrossEnvTransferBlockedUntilPrivacyNoticeApproved() var diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al index e1ebe1aeedd..d8a10d179b9 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al @@ -96,6 +96,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" Initialize(); MasterDataManagementSetup.Init(); asserterror MasterDataManagementSetup.Validate("Company Name", CopyStr(CompanyName(), 1, MaxStrLen(MasterDataManagementSetup."Company Name"))); + Assert.ExpectedError('You are currently signed into this company'); end; [Test] From 807630e0583620ce238aae5f81ae6e4c9311ef11 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 28 Aug 2026 15:55:14 +0200 Subject: [PATCH 21/64] [Master Data Management] Cross-environment synchonization --- .../IntegrationMasterDataSynch.Codeunit.al | 3 +- .../MDMCrossEnvChangeDetector.Codeunit.al | 8 ++++-- .../MDMCrossEnvDataSource.Codeunit.al | 28 ++++++++++++++++--- .../MDMHttpSourceTransport.Codeunit.al | 20 +++++++++++-- .../src/MDMInProcessTransport.Codeunit.al | 17 +++++++++++ .../src/MDMTestDetectorProbe.Codeunit.al | 7 +++++ .../src/MDMTestPagingConfig.Codeunit.al | 5 ++++ 7 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index 6a804136a8d..ec60ae05fd7 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -50,6 +50,7 @@ codeunit 7231 "Integration Master Data Synch." SourceRecordIsNotInMappingErr: Label 'Cannot find the mapping %2 in table %1.', Comment = '%1 Integration Table Mapping caption, %2 Integration Table Mapping Name'; CannotDetermineSourceOriginErr: Label 'Cannot determine the source origin: %1.', Comment = '%1 the value of the source id'; CopyRecordRefFailedTxt: Label 'Copy record reference failed. Integration Record ID: %1', Locked = true, Comment = '%1 - Business Central record id'; + CrossEnvCopyFailedTelemetryTxt: Label 'Cross-environment copy of a source record failed for table %1.', Locked = true, Comment = '%1 - integration table id'; UnableToFindMappingErr: Label 'Unable to find Integration Table Mapping %1', Locked = true, Comment = '%1 - Mapping name'; FieldKeyTxt: Label '%1-%2', Locked = true; @@ -119,7 +120,7 @@ codeunit 7231 "Integration Master Data Synch." IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then - Session.LogMessage('0000J8Q', StrSubstNo(CopyRecordRefFailedTxt, IntegrationRecordID), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000QF6', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); until IntegrationRecordRef.Next() = 0; IntegrationRecordRef.Close(); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 3b740bdc801..278d77d49a3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -20,6 +20,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" var LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; DetectorParseFailedTxt: Label 'The cross-environment change detector received an invalid response from the source and skipped this run.', Locked = true; + DetectionContractFailedTxt: Label 'The cross-environment change detector received a response without a valid tables array and skipped this run.', Locked = true; trigger OnRun() begin @@ -81,14 +82,15 @@ codeunit 7245 "MDM Cross-Env Change Detector" local procedure ProcessDetectionResponse(var Response: JsonObject) var + MasterDataManagement: Codeunit "Master Data Management"; Tables: JsonArray; TablesToken: JsonToken; EntryToken: JsonToken; begin - if not Response.Get('tables', TablesToken) then - exit; - if not TablesToken.IsArray() then + if (not Response.Get('tables', TablesToken)) or (not TablesToken.IsArray()) then begin + Session.LogMessage('0000QF7', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; + end; Tables := TablesToken.AsArray(); foreach EntryToken in Tables do ProcessTableEntry(EntryToken.AsObject()); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 4db5c6342dc..7a6720f41ee 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -23,6 +23,11 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceProbeFailedErr: Label 'Could not read the change probe from the source environment for table %1.', Comment = '%1 = table caption'; RecordsFeatureTok: Label 'records', Locked = true; LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; + ParseFailureTelemetryTxt: Label 'The cross-environment sync received an unusable response for table %1 (reason: %2).', Locked = true, Comment = '%1 = table id, %2 = reason'; + InvalidResponseReasonTok: Label 'InvalidResponse', Locked = true; + TableUnavailableReasonTok: Label 'TableUnavailable', Locked = true; + NotIndexedReasonTok: Label 'NotIndexed', Locked = true; + FieldsUnavailableReasonTok: Label 'FieldsUnavailable', Locked = true; procedure GetModifiedSet(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text; var SourceRecordRef: RecordRef): Boolean var @@ -198,19 +203,34 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(ErrInfo); end; + local procedure LogParseFailure(IntegrationTableId: Integer; Reason: Text) + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + Session.LogMessage('0000QF8', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) var UnavailableFields: JsonArray; begin Clear(Response); - if not SourceResponse.TryParse(ResponseText, Response) then + if not SourceResponse.TryParse(ResponseText, Response) then begin + LogParseFailure(IntegrationTableId, InvalidResponseReasonTok); Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); - if not SourceResponse.TableAvailable(Response) then + end; + if not SourceResponse.TableAvailable(Response) then begin + LogParseFailure(IntegrationTableId, TableUnavailableReasonTok); Error(TableUnavailableErr, TableCaption(IntegrationTableId)); - if not SourceResponse.Indexed(Response) then + end; + if not SourceResponse.Indexed(Response) then begin + LogParseFailure(IntegrationTableId, NotIndexedReasonTok); Error(NotIndexedErr, TableCaption(IntegrationTableId)); - if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then + end; + if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then begin + LogParseFailure(IntegrationTableId, FieldsUnavailableReasonTok); Error(FieldsUnavailableErr, TableCaption(IntegrationTableId)); + end; end; // FieldIds = the mapping's integration-side fields + the table's primary-key fields (so temp inserts don't collide). diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 7a2fa239a7a..43e3cb9445e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -22,6 +22,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" TokenExpiresAt: DateTime; MaxRetriesValue: Integer; NotConfiguredErr: Label 'The cross-environment connection to the source is not configured yet.'; + OpenSetupActionTxt: Label 'Open Master Data Management Setup'; NonSaaSErr: Label 'Cross-environment synchronization is only available in online environments.'; NoTokenErr: Label 'Could not acquire an access token for the source environment. Check the client ID and secret.'; SendFailedErr: Label 'The request to the source environment could not be sent. Check the source environment URL.'; @@ -89,7 +90,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" exit(UnwrapODataValue(ResponseBodyText)); if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); - Error(HttpErr, ResponseMessage.HttpStatusCode()); + Error(SetupNavigationError(StrSubstNo(HttpErr, ResponseMessage.HttpStatusCode()))); end; Sleep(RetryAfter); end; @@ -230,9 +231,22 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") begin if not MasterDataManagementSetup.Get() then - Error(NotConfiguredErr); + Error(SetupNavigationError(NotConfiguredErr)); if not MasterDataManagementSetup.IsCrossEnvConnectionConfigured() then - Error(NotConfiguredErr); + Error(SetupNavigationError(NotConfiguredErr)); + end; + + local procedure SetupNavigationError(MessageText: Text): ErrorInfo + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + if MasterDataManagementSetup.Get() then begin + ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); + ErrInfo.AddNavigationAction(OpenSetupActionTxt); + end; + exit(ErrInfo); end; local procedure ShouldRetry(var ResponseMessage: HttpResponseMessage; Attempt: Integer; var RetryAfter: Duration): Boolean diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al index 7d8e9cf8a53..3ac03d96fcc 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al @@ -15,11 +15,13 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" UseCanned: Boolean; UseCannedCapabilities: Boolean; + /// Activates the in-process transport so it is injected in place of the HTTP transport. procedure Activate() begin Active := true; end; + /// Deactivates the transport and clears all canned state and negotiated capabilities. procedure Deactivate() var SourceCapabilities: Codeunit "MDM Source Capabilities"; @@ -32,18 +34,28 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" SourceCapabilities.Reset(); // clear negotiated capabilities between tests end; + /// Sets a canned records/last-modified response returned instead of calling the real source API. + /// The raw JSON response to return. procedure SetCannedResponse(Response: Text) begin CannedResponse := Response; UseCanned := true; end; + /// Sets a canned capabilities response returned instead of calling the real source API. + /// The raw JSON capabilities response to return. procedure SetCannedCapabilities(Response: Text) begin CannedCapabilities := Response; UseCannedCapabilities := true; end; + /// Returns records for a table, using the canned response if one is set, otherwise the real source API. + /// The source table ID to read. + /// The projected field IDs. + /// The cursor/systemId selector. + /// The page size. + /// The raw JSON records response. procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text var SourceApi: Codeunit "MDM Cross-Env Source API"; @@ -53,6 +65,9 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" exit(SourceApi.GetRecords(TableId, FieldIds, Selector, PageSize)); end; + /// Returns the last-modified-per-table probe, using the canned response if set, otherwise the real source API. + /// The JSON array of table IDs to probe. + /// The raw JSON last-modified response. procedure LastModifiedAtPerTable(TableIds: Text): Text var SourceApi: Codeunit "MDM Cross-Env Source API"; @@ -62,6 +77,8 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" exit(SourceApi.LastModifiedAtPerTable(TableIds)); end; + /// Returns the source capabilities, using the canned capabilities if set, otherwise the real source API. + /// The raw JSON capabilities response. procedure GetCapabilities(): Text var SourceApi: Codeunit "MDM Cross-Env Source API"; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al index 79702196cef..e40b1f8868d 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestDetectorProbe.Codeunit.al @@ -10,23 +10,30 @@ codeunit 139930 "MDM Test Detector Probe" NudgedTableIds: List of [Integer]; Active: Boolean; + /// Activates the probe so it captures detector nudges and clears any previously captured table IDs. procedure Activate() begin Active := true; Clear(NudgedTableIds); end; + /// Deactivates the probe and clears the captured table IDs. procedure Deactivate() begin Active := false; Clear(NudgedTableIds); end; + /// Checks whether the change detector nudged the sync job for a given table. + /// The integration table ID to check. + /// True if the table was nudged since activation; otherwise false. procedure WasNudged(TableId: Integer): Boolean begin exit(NudgedTableIds.Contains(TableId)); end; + /// Returns the number of distinct tables the detector nudged since activation. + /// The nudge count. procedure NudgeCount(): Integer begin exit(NudgedTableIds.Count()); diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al index c7665c8af19..f092e41491f 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMTestPagingConfig.Codeunit.al @@ -12,18 +12,23 @@ codeunit 139934 "MDM Test Paging Config" InlineBytesActive: Boolean; InlineBytesValue: Integer; + /// Activates a forced cross-environment page size for paging/resume tests. + /// The page size to force. procedure Activate(NewPageSize: Integer) begin Active := true; PageSizeValue := NewPageSize; end; + /// Activates a forced maximum inline-bytes cap so the over-cap media skip path can be exercised. + /// The maximum inline bytes to force. procedure ActivateInlineBytes(NewMaxBytes: Integer) begin InlineBytesActive := true; InlineBytesValue := NewMaxBytes; end; + /// Deactivates all forced paging and inline-bytes overrides. procedure Deactivate() begin Active := false; From ff729c8c863dba8c4c73a8de97648274cc90648f Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Sat, 29 Aug 2026 09:40:16 +0200 Subject: [PATCH 22/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvDataSource.Codeunit.al | 2 +- .../MDMHttpSourceTransport.Codeunit.al | 41 +++++++++--- .../MDMSourceCapabilities.Codeunit.al | 14 ++++- .../codeunits/MDMSourceResponse.Codeunit.al | 9 +++ .../src/LibraryMasterDataMgt.Codeunit.al | 23 +++++++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 63 +++++++++++++++++-- 6 files changed, 136 insertions(+), 16 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 7a6720f41ee..e5c726ff64f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -207,7 +207,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('0000QF8', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000QF8', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 43e3cb9445e..5705a3b3c32 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -146,7 +146,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" if not HttpClient.Send(RequestMessage, ResponseMessage) then begin LogTransportFailure(ActionName); - Error(SendFailedErr); + Error(SetupNavigationError(SendFailedErr)); end; end; @@ -196,6 +196,12 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" exit(ResponseBody); end; + // Test seam: exercise OData envelope unwrapping without a live transport. + internal procedure UnwrapODataValueForTest(ResponseBody: Text): Text + begin + exit(UnwrapODataValue(ResponseBody)); + end; + local procedure GetBearerToken(var MasterDataManagementSetup: Record "Master Data Management Setup"): SecretText begin // Cached in-instance for the lifetime of a single sync run (the paging loop reuses this transport). @@ -217,17 +223,34 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" begin Scopes.Add(ScopeTok); TokenEndpoint := StrSubstNo(TokenEndpointTok, AzureADTenant.GetAadTenantId()); - if not OAuth2.AcquireTokenWithClientCredentials( - MasterDataManagementSetup."Source OAuth Client Id", - MasterDataManagementSetup.GetSourceClientSecret(), - TokenEndpoint, '', Scopes, Token) or Token.IsEmpty() - then begin - AuditLog.LogAuditMessage(StrSubstNo(TokenFailedAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Failure, AuditCategory::Authentication, 4, 0); - Error(NoTokenErr); - end; + // Prefer a cached/refreshed token (no round-trip when a valid one exists); fall back to a fresh + // client-credentials grant if the cache misses, errors, or returns an empty token. + if not TryAcquireTokenFromCache(MasterDataManagementSetup, TokenEndpoint, Scopes, Token) then + Clear(Token); + if Token.IsEmpty() then + if not OAuth2.AcquireTokenWithClientCredentials( + MasterDataManagementSetup."Source OAuth Client Id", + MasterDataManagementSetup.GetSourceClientSecret(), + TokenEndpoint, '', Scopes, Token) or Token.IsEmpty() + then begin + AuditLog.LogAuditMessage(StrSubstNo(TokenFailedAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Failure, AuditCategory::Authentication, 4, 0); + Error(SetupNavigationError(NoTokenErr)); + end; AuditLog.LogAuditMessage(StrSubstNo(TokenAcquiredAuditTxt, MasterDataManagementSetup."Source Environment Name"), SecurityOperationResult::Success, AuditCategory::Authentication, 4, 0); end; + // Reuses a token from the platform (MSAL) cache when one is valid; a cache miss or error is treated as "no token" + // so the caller falls back to a fresh client-credentials grant. + [TryFunction] + [NonDebuggable] + local procedure TryAcquireTokenFromCache(MasterDataManagementSetup: Record "Master Data Management Setup"; TokenEndpoint: Text; Scopes: List of [Text]; var Token: SecretText) + var + OAuth2: Codeunit OAuth2; + begin + if not OAuth2.AcquireAuthorizationCodeTokenFromCache(MasterDataManagementSetup."Source OAuth Client Id", MasterDataManagementSetup.GetSourceClientSecret(), '', TokenEndpoint, Scopes, Token) then + Clear(Token); + end; + local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") begin if not MasterDataManagementSetup.Get() then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index 46e4eae82fa..4dd1b00ed98 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -12,6 +12,7 @@ codeunit 7246 "MDM Source Capabilities" var Negotiated: Boolean; + NegotiatedForUrl: Text; ContractVersion: Integer; SupportedFeatures: List of [Text]; UnsupportedFeatureErr: Label 'The source environment does not support the required ''%1'' capability. Update the Master Data Management app on the source environment.', Comment = '%1 = capability name'; @@ -39,19 +40,29 @@ codeunit 7246 "MDM Source Capabilities" procedure Reset() begin Negotiated := false; + NegotiatedForUrl := ''; ContractVersion := 0; Clear(SupportedFeatures); end; local procedure Negotiate(Transport: Interface "IMDM Source Transport") var + MasterDataManagementSetup: Record "Master Data Management Setup"; Capabilities: JsonObject; FeaturesToken: JsonToken; VersionToken: JsonToken; FeatureToken: JsonToken; + CurrentSource: Text; begin - if Negotiated then + if MasterDataManagementSetup.Get() then + CurrentSource := MasterDataManagementSetup."Source Environment URL"; + // Re-negotiate when the configured source changes so a switched environment can't reuse the previous + // source's cached feature/version data. + if Negotiated and (NegotiatedForUrl = CurrentSource) then exit; + Negotiated := false; + ContractVersion := 0; + Clear(SupportedFeatures); // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading // "capability unsupported / update the source app" error instead of the real bad-response problem. if not Capabilities.ReadFrom(Transport.GetCapabilities()) then @@ -65,5 +76,6 @@ codeunit 7246 "MDM Source Capabilities" if FeatureToken.IsValue() then SupportedFeatures.Add(FeatureToken.AsValue().AsText()); Negotiated := true; + NegotiatedForUrl := CurrentSource; end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index a055dd8cd78..1003bc7a785 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -127,6 +127,15 @@ codeunit 7248 "MDM Source Response" end; if not IsNullGuid(SystemIdValue) then TempSourceRecordRef.Field(TempSourceRecordRef.SystemIdNo()).Value := SystemIdValue; + // A record re-modified between page fetches can arrive on two pages under the advancing cursor; keep the + // newest copy instead of aborting the whole batch on the duplicate primary key. + if not TryInsertTempRecord(TempSourceRecordRef) then + TempSourceRecordRef.Modify(false); + end; + + [TryFunction] + local procedure TryInsertTempRecord(var TempSourceRecordRef: RecordRef) + begin TempSourceRecordRef.Insert(false); end; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 6eb983b7269..70718857d67 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -96,6 +96,19 @@ codeunit 139757 "Library - Master Data Mgt." exit(MasterDataManagementSetup.GetDataSource().GetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef)); end; + /// Gets a single source record by its coupling ID (SystemId as GUID or text) from the configured data source. + /// The integration table mapping to read. + /// The record ID (SystemId as a GUID or its text form). + /// Returns the record reference positioned on the found record. + /// True if the record was found; otherwise false. + procedure DataSourceGetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + MasterDataManagementSetup.Get(); + exit(MasterDataManagementSetup.GetDataSource().GetById(IntegrationTableMapping, ID, SourceRecordRef)); + end; + /// Gets source records matching a table filter from the configured data source. /// The integration table mapping to read. /// The source table filter to apply. @@ -177,6 +190,16 @@ codeunit 139757 "Library - Master Data Mgt." MDMHttpSourceTransport.ValidateSourceHostUrl(BaseUrl); end; + /// Unwraps the ODataV4 action envelope the HTTP transport receives, returning the inner value (or the raw body). + /// The raw response body to unwrap. + /// The inner OData value, or the body unchanged if it is not a value-envelope. + procedure UnwrapHttpTransportODataValue(ResponseBody: Text): Text + var + MDMHttpSourceTransport: Codeunit "MDM Http Source Transport"; + begin + exit(MDMHttpSourceTransport.UnwrapODataValueForTest(ResponseBody)); + end; + /// Checks whether the inline media cache holds an entry for a record field. /// The SystemId of the source record. /// The field number of the media/blob field. diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 03a8016ec7e..c6efe73e59d 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -9,7 +9,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" var Assert: Codeunit Assert; LibrarySalesLib: Codeunit "Library - Sales"; - WizardOpenedPrivacyNotice: Boolean; + WizardPrivacyNoticeOpenCount: Integer; [Test] procedure CrossEnvGetBySystemIdRoundTripsSourceRecord() @@ -114,25 +114,78 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [GIVEN] the privacy notice has no recorded decision, so accepting consent must prompt it LibraryMasterDataMgt.PrivacyNoticeResetApproval(); - WizardOpenedPrivacyNotice := false; + WizardPrivacyNoticeOpenCount := 0; // [WHEN] the admin ticks consent in the connection wizard ConnectionWizard.OpenEdit(); ConnectionWizard.Consent.SetValue(true); ConnectionWizard.Close(); - // [THEN] the privacy-notice dialog was shown - proving the wizard invoked ConfirmApproval - Assert.IsTrue(WizardOpenedPrivacyNotice, 'Ticking consent should open the privacy notice (call ConfirmApproval)'); + // [THEN] the privacy-notice dialog was shown exactly once - proving the wizard invoked ConfirmApproval + Assert.AreEqual(1, WizardPrivacyNoticeOpenCount, 'Ticking consent should open the privacy notice exactly once (call ConfirmApproval)'); LibraryMasterDataMgt.PrivacyNoticeResetApproval(); CleanUp(); end; + [Test] + procedure CrossEnvGetByUidFilterAndGetByIdMaterializeSourceRecords() + var + Customer1: Record Customer; + Customer2: Record Customer; + IntegrationTableMapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + UidFilter: Text; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] Cross-env GetByUidFilter (pipe-split selector) and GetById (variant->SystemId) fetch source records. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer1); + LibrarySalesLib.CreateCustomer(Customer2); + CreateMinimalCustomerMapping(IntegrationTableMapping); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [GIVEN] a pipe-delimited UID filter of two source SystemIds + UidFilter := Format(Customer1.SystemId) + '|' + Format(Customer2.SystemId); + + // [WHEN] fetched via the cross-env UID filter [THEN] both records materialize (exercises ParseSystemIds pipe-split) + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetByUidFilter(IntegrationTableMapping, UidFilter, SourceRecordRef), 'GetByUidFilter should return records'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer1.SystemId), 'UID filter should include the first customer'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer2.SystemId), 'UID filter should include the second customer'); + + // [WHEN] fetched via GetById with a text SystemId [THEN] the record materializes (exercises the variant->SystemId conversion) + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetById(IntegrationTableMapping, Format(Customer1.SystemId), SourceRecordRef), 'GetById should return the record'); + Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer1.SystemId), 'GetById should include the requested customer'); + + CleanUp(); + end; + + [Test] + procedure HttpTransportUnwrapsODataValueEnvelope() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] The HTTP transport unwraps the ODataV4 { value: } envelope and passes other bodies through. + Initialize(); + + // [GIVEN] an OData action envelope [THEN] the inner value is returned + Assert.AreEqual('{"records":[]}', LibraryMasterDataMgt.UnwrapHttpTransportODataValue('{"@odata.context":"x","value":"{\"records\":[]}"}'), 'Envelope value should be unwrapped'); + + // [GIVEN] a body that is not a value-envelope [THEN] it is returned unchanged + Assert.AreEqual('{"records":[]}', LibraryMasterDataMgt.UnwrapHttpTransportODataValue('{"records":[]}'), 'A non-envelope body should pass through unchanged'); + + CleanUp(); + end; + [ModalPageHandler] procedure PrivacyNoticeModalHandler(var PrivacyNoticePage: TestPage "Privacy Notice") begin // Reached only if the wizard actually opened the notice. - WizardOpenedPrivacyNotice := true; + WizardPrivacyNoticeOpenCount += 1; end; [Test] From c8ffc0409ae5ff4b1ace7480e98e443a7885ec37 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Sat, 29 Aug 2026 19:15:15 +0200 Subject: [PATCH 23/64] [Master Data Management] Cross-environment synchonization --- .../MDMHttpSourceTransport.Codeunit.al | 4 +-- .../MDMSourceCapabilities.Codeunit.al | 12 +++++++- .../codeunits/MDMSourceResponse.Codeunit.al | 4 +-- .../MasterDataMgtSubscribers.Codeunit.al | 3 +- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 4 +-- .../src/MDMCrossEnvSourceTests.Codeunit.al | 29 ++++++++++++++++++- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 5705a3b3c32..794287540dd 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -2,8 +2,8 @@ namespace Microsoft.Integration.MDM; using System.Azure.Identity; using System.Environment; -using System.Security.Authentication; using System.Reflection; +using System.Security.Authentication; using System.Telemetry; using System.Utilities; @@ -173,7 +173,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" if (Uri.GetScheme() = 'https') and (Host.EndsWith('.dynamics.com') or Host.EndsWith('.dynamics-tie.com')) then exit; AuditLog.LogAuditMessage(StrSubstNo(InvalidSourceUrlAuditTxt, Host), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); - Error(InvalidSourceUrlErr); + Error(SetupNavigationError(InvalidSourceUrlErr)); end; // Test seam: exercise the source-host allow-list without a live environment or the SaaS gate. diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index 4dd1b00ed98..478fe6c80c9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -45,6 +45,16 @@ codeunit 7246 "MDM Source Capabilities" Clear(SupportedFeatures); end; + // A malformed capabilities response is an internal contract failure; keep the detail in telemetry and show a generic error. + local procedure InternalError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + local procedure Negotiate(Transport: Interface "IMDM Source Transport") var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -66,7 +76,7 @@ codeunit 7246 "MDM Source Capabilities" // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading // "capability unsupported / update the source app" error instead of the real bad-response problem. if not Capabilities.ReadFrom(Transport.GetCapabilities()) then - Error(CapabilitiesParseErr); + Error(InternalError(CapabilitiesParseErr)); if Capabilities.Get('version', VersionToken) then if VersionToken.IsValue() then ContractVersion := VersionToken.AsValue().AsInteger(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 1003bc7a785..d114d76022b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -119,7 +119,7 @@ codeunit 7248 "MDM Source Response" FieldType::Media: ApplyInlineMedia(SystemIdValue, FieldNo, TempSourceRecordRef.Number(), ValueToken); FieldType::Blob: - ApplyInlineBlob(DestField, SystemIdValue, FieldNo, TempSourceRecordRef.Number(), ValueToken); + ApplyInlineBlob(DestField, FieldNo, TempSourceRecordRef.Number(), ValueToken); else SetFieldFromText(DestField, ValueToken.AsValue().AsText()); end; @@ -169,7 +169,7 @@ codeunit 7248 "MDM Source Response" // Blob bytes are placed directly on the temp source record; the framework's record transfer carries them to // the destination (the same path same-env uses for mapped blobs), so no destination-side apply is needed. - local procedure ApplyInlineBlob(var DestField: FieldRef; SystemId: Guid; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) + local procedure ApplyInlineBlob(var DestField: FieldRef; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) var Base64Convert: Codeunit "Base64 Convert"; TempBlob: Codeunit "Temp Blob"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index edd956cbac4..533f48e804c 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -135,7 +135,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IsJobQueueEntryDataSynchJob(Sender, IntegrationTableMapping) then begin MasterDataManagementSetup.Get(); - if MasterDataManagementSetup."Is Enabled" then begin + if MasterDataManagementSetup."Is Enabled" then if MasterDataManagementSetup."Source Environment Name" <> '' then // Cross-environment: the change detector governs when this job is nudged; let it run and fetch the delta. Result := true @@ -153,7 +153,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" Result := true; RecRef.Close(); end; - end; end; end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index c6efe73e59d..e2e0050448f 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -721,11 +721,11 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" var SystemIdValue: Guid; begin + // Record every occurrence (no de-dup) so a record returned on two pages makes the count exceed the seeded set. if SourceRecordRef.FindSet() then repeat SystemIdValue := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); - if not CollectedSystemIds.Contains(SystemIdValue) then - CollectedSystemIds.Add(SystemIdValue); + CollectedSystemIds.Add(SystemIdValue); until SourceRecordRef.Next() = 0; end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al index e8004b4913b..3b13d01ba44 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -68,13 +68,18 @@ codeunit 139931 "MDM Cross-Env Source Tests" Response: JsonObject; NextCursor: Text; Index: Integer; + SeededSystemIds: List of [Guid]; + PagedSystemIds: List of [Guid]; + SeededSystemId: Guid; begin // [FEATURE] [AI test 0.4] // [SCENARIO] Cursor mode pages ascending by (SystemModifiedAt, SystemId) and reports hasMore / nextCursor. Watermark := CurrentDateTime(); Sleep(50); // ensure the seeded records sort strictly after the watermark - for Index := 1 to 3 do + for Index := 1 to 3 do begin LibrarySales.CreateCustomer(Customer); + SeededSystemIds.Add(Customer.SystemId); + end; // [WHEN] the first page of size 2 is requested from the watermark Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), CursorSelector(Watermark), 2)); @@ -82,6 +87,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [THEN] two records come back and hasMore is true Assert.AreEqual(2, RecordCount(Response), 'First page should hold the page size'); Assert.IsTrue(GetBoolean(Response, 'hasMore'), 'hasMore should be true while records remain'); + CollectResponseSystemIds(Response, PagedSystemIds); NextCursor := NextCursorText(Response); // [WHEN] the next page is requested with the returned cursor @@ -91,6 +97,12 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [THEN] the remaining record comes back and hasMore is false Assert.AreEqual(1, RecordCount(Response), 'Second page should hold the remaining record'); Assert.IsFalse(GetBoolean(Response, 'hasMore'), 'hasMore should be false on the last page'); + CollectResponseSystemIds(Response, PagedSystemIds); + + // [THEN] the two pages together returned every seeded record exactly once - no repeats, no dropped/reordered records + Assert.AreEqual(3, PagedSystemIds.Count(), 'Paging should return each record exactly once across the two pages'); + foreach SeededSystemId in SeededSystemIds do + Assert.IsTrue(PagedSystemIds.Contains(SeededSystemId), 'Every seeded record should appear in the paged results'); end; [Test] @@ -142,6 +154,21 @@ codeunit 139931 "MDM Cross-Env Source Tests" exit(Token.AsArray().Count()); end; + local procedure CollectResponseSystemIds(var Response: JsonObject; var SystemIds: List of [Guid]) + var + RecordsToken: JsonToken; + RecordToken: JsonToken; + SystemIdToken: JsonToken; + SystemIdValue: Guid; + begin + // Record every returned systemId (no de-dup) so a repeat across pages is caught by the count assertion. + Response.Get('records', RecordsToken); + foreach RecordToken in RecordsToken.AsArray() do + if RecordToken.AsObject().Get('systemId', SystemIdToken) then + if Evaluate(SystemIdValue, SystemIdToken.AsValue().AsText()) then + SystemIds.Add(SystemIdValue); + end; + local procedure GetBoolean(var Response: JsonObject; PropertyName: Text): Boolean var Token: JsonToken; From 5096dd1749a83c2e0b1b71d57ca4fa5bd31fc72b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Sat, 29 Aug 2026 19:22:20 +0200 Subject: [PATCH 24/64] [Master Data Management] Cross-environment synchonization --- .../test/src/MasterDataMgtSetupTests.Codeunit.al | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al index d8a10d179b9..cee4038605d 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al @@ -17,6 +17,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" LibraryVariableStorage: Codeunit "Library - Variable Storage"; LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; InitializeHandled: Boolean; + IncorrectTablesListErr: Label 'Synchronization tables list is incorrect.'; [Test] [HandlerFunctions('SynchronizationEnabledMessageHandler')] @@ -379,7 +380,6 @@ codeunit 139770 "Master Data Mgt. Setup Tests" SynchTables: List of [Integer]; RelatedTablesToAdd: List of [Integer]; TablesToAddText: Text; - IncorrectTablesListErr: Label 'Synchronization tables list is incorrect.'; begin // [SCENARIO] When selecting a table that has a self-reference or other reference that create a cycle, duplicate records are not added to the setup list From d72a1bc9f138ba5d4ba9ba54e72381a2fa099e8f Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Sat, 29 Aug 2026 19:54:43 +0200 Subject: [PATCH 25/64] [Master Data Management] Cross-environment synchonization --- .../IntegrationMasterDataSynch.Codeunit.al | 2 +- .../MDMCrossEnvChangeDetector.Codeunit.al | 35 ++++++++++++++++--- .../MDMCrossEnvDataSource.Codeunit.al | 20 +++++++++-- .../MDMCrossEnvSourceAPI.Codeunit.al | 1 + .../MDMHttpSourceTransport.Codeunit.al | 1 + .../codeunits/MDMPrivacyNotice.Codeunit.al | 3 ++ .../MDMSourceCapabilities.Codeunit.al | 6 +++- .../src/pages/MDMConnectionDetails.Page.al | 6 +++- .../tables/MasterDataManagementSetup.Table.al | 1 + .../MasterDataMgtTableMapping.TableExt.al | 2 +- 10 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index ec60ae05fd7..f80e7a68eb9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -120,7 +120,7 @@ codeunit 7231 "Integration Master Data Synch." IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then - Session.LogMessage('0000QF6', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000QF6', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); until IntegrationRecordRef.Next() = 0; IntegrationRecordRef.Close(); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 278d77d49a3..ba798425ac9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -21,6 +21,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; DetectorParseFailedTxt: Label 'The cross-environment change detector received an invalid response from the source and skipped this run.', Locked = true; DetectionContractFailedTxt: Label 'The cross-environment change detector received a response without a valid tables array and skipped this run.', Locked = true; + DetectorTransportFailedTxt: Label 'The cross-environment change detector could not reach the source and skipped this run; the next scheduled run will retry.', Locked = true; trigger OnRun() begin @@ -32,11 +33,13 @@ codeunit 7245 "MDM Cross-Env Change Detector" MasterDataManagementSetup: Record "Master Data Management Setup"; SourceConnection: Codeunit "MDM Source Connection"; SourceResponse: Codeunit "MDM Source Response"; - SourceCapabilities: Codeunit "MDM Source Capabilities"; MasterDataManagement: Codeunit "Master Data Management"; Transport: Interface "IMDM Source Transport"; Response: JsonObject; TableIds: JsonArray; + Dimensions: Dictionary of [Text, Text]; + ResponseText: Text; + Supported: Boolean; begin if not MasterDataManagementSetup.Get() then exit; @@ -49,10 +52,21 @@ codeunit 7245 "MDM Cross-Env Change Detector" exit; Transport := SourceConnection.GetTransport(); - // Skip (rather than error every run) if an older source doesn't advertise the detection action. - if not SourceCapabilities.IsSupported(Transport, LastModifiedFeatureTok) then + // An operational transport failure (source outage, auth, bad connection state) must NOT error this recurring + // detector job - that would burn its retry budget and could stop change detection. Skip this poll instead; + // the next scheduled run recovers. + if not TryFetchDetection(Transport, TableIds, Supported, ResponseText) then begin + // The detector's transport errors are operational (HTTP status, connection, auth, config) with no record + // content, so the caught message - which carries the HTTP status code for HttpErr - is safe as system metadata. + Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); + Dimensions.Add('failure', CopyStr(GetLastErrorText(), 1, 2048)); + Session.LogMessage('0000QF9', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + exit; + end; + // Older source doesn't advertise the detection action: skip rather than error every run. + if not Supported then exit; - if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(WriteArray(TableIds)), Response) then begin + if not SourceResponse.TryParse(ResponseText, Response) then begin Session.LogMessage('0000QF4', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; @@ -60,6 +74,19 @@ codeunit 7245 "MDM Cross-Env Change Detector" ProcessDetectionResponse(Response); end; + // Contains the source calls (capability negotiation + LastModifiedAtPerTable) so an operational transport error + // is caught by the caller (log + skip) instead of escaping and erroring the recurring detector job. + [TryFunction] + local procedure TryFetchDetection(Transport: Interface "IMDM Source Transport"; TableIds: JsonArray; var Supported: Boolean; var ResponseText: Text) + var + SourceCapabilities: Codeunit "MDM Source Capabilities"; + begin + Supported := SourceCapabilities.IsSupported(Transport, LastModifiedFeatureTok); + if not Supported then + exit; + ResponseText := Transport.LastModifiedAtPerTable(WriteArray(TableIds)); + end; + local procedure CollectSynchronizedTableIds(var TableIds: JsonArray): Boolean var IntegrationTableMapping: Record "Integration Table Mapping"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index e5c726ff64f..200a87889c1 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -23,6 +23,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceProbeFailedErr: Label 'Could not read the change probe from the source environment for table %1.', Comment = '%1 = table caption'; RecordsFeatureTok: Label 'records', Locked = true; LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; + SourceProbeTelemetryTxt: Label 'The cross-environment source-record probe failed for table %1.', Locked = true, Comment = '%1 = table id'; ParseFailureTelemetryTxt: Label 'The cross-environment sync received an unusable response for table %1 (reason: %2).', Locked = true, Comment = '%1 = table id, %2 = reason'; InvalidResponseReasonTok: Label 'InvalidResponse', Locked = true; TableUnavailableReasonTok: Label 'TableUnavailable', Locked = true; @@ -114,13 +115,19 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Transport := GetTransport(); SourceCapabilities.EnsureSupported(Transport, LastModifiedFeatureTok); // A transport/parse failure is not an empty source; surface it so the full-synch review isn't misled. - if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(TableIdsText), Response) then + if not SourceResponse.TryParse(Transport.LastModifiedAtPerTable(TableIdsText), Response) then begin + LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); - if not Response.Get('tables', Token) then + end; + if not Response.Get('tables', Token) then begin + LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + end; Tables := Token.AsArray(); - if Tables.Count() = 0 then + if Tables.Count() = 0 then begin + LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + end; Tables.Get(0, Token); Entry := Token.AsObject(); if Entry.Get('tableAvailable', Token) then @@ -210,6 +217,13 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Session.LogMessage('0000QF8', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; + local procedure LogProbeFailure(IntegrationTableId: Integer) + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + Session.LogMessage('0000QFB', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) var UnavailableFields: JsonArray; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 14fa597d5bb..81a785c2b2b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -581,6 +581,7 @@ codeunit 7241 "MDM Cross-Env Source API" // Detection only needs an approximate max, so read uncommitted: never takes or waits on a lock, even // where snapshot isolation is off (OnPrem). Data reads (GetRecords) stay at committed isolation. RecRef.ReadIsolation := IsolationLevel::ReadUncommitted; + RecRef.SetLoadFields(SystemModifiedAtFieldNo()); if RecRef.FindLast() then Entry.Add('lastModifiedAt', Format(RecRef.Field(SystemModifiedAtFieldNo()).Value(), 0, 9)) else diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 794287540dd..e8ef6324988 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -267,6 +267,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ErrInfo.Message := MessageText; if MasterDataManagementSetup.Get() then begin ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); + ErrInfo.PageNo := Page::"Master Data Management Setup"; ErrInfo.AddNavigationAction(OpenSetupActionTxt); end; exit(ErrInfo); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al index a7b06847fe4..192b1eb2173 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al @@ -18,6 +18,7 @@ codeunit 7242 "MDM Privacy Notice" IntegrationServiceNameTxt: Label 'Master Data Management - cross-environment synchronization'; NotApprovedErr: Label 'Cross-environment master data synchronization requires the privacy notice to be approved. Open Master Data Management Setup and approve sharing data between Business Central environments.'; OpenSetupActionTxt: Label 'Open Master Data Management Setup'; + PrivacyLinkTok: Label 'https://go.microsoft.com/fwlink/?linkid=521839', Locked = true; [EventSubscriber(ObjectType::Codeunit, Codeunit::"Privacy Notice", OnRegisterPrivacyNotices, '', false, false)] local procedure RegisterPrivacyNotice(var TempPrivacyNotice: Record "Privacy Notice" temporary) @@ -25,6 +26,7 @@ codeunit 7242 "MDM Privacy Notice" TempPrivacyNotice.Init(); TempPrivacyNotice.ID := PrivacyNoticeIdTok; TempPrivacyNotice."Integration Service Name" := IntegrationServiceNameTxt; + TempPrivacyNotice.Link := PrivacyLinkTok; if not TempPrivacyNotice.Insert() then; end; @@ -59,6 +61,7 @@ codeunit 7242 "MDM Privacy Notice" ErrInfo.Message := NotApprovedErr; if MasterDataManagementSetup.Get() then begin ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); + ErrInfo.PageNo := Page::"Master Data Management Setup"; ErrInfo.AddNavigationAction(OpenSetupActionTxt); end; Error(ErrInfo); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index 478fe6c80c9..76dd00f009c 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -17,6 +17,7 @@ codeunit 7246 "MDM Source Capabilities" SupportedFeatures: List of [Text]; UnsupportedFeatureErr: Label 'The source environment does not support the required ''%1'' capability. Update the Master Data Management app on the source environment.', Comment = '%1 = capability name'; CapabilitiesParseErr: Label 'The source environment returned an invalid capabilities response.'; + CapabilitiesParseTelemetryTxt: Label 'The source environment returned a malformed capabilities response during cross-environment negotiation.', Locked = true; procedure EnsureSupported(Transport: Interface "IMDM Source Transport"; Feature: Text) begin @@ -58,6 +59,7 @@ codeunit 7246 "MDM Source Capabilities" local procedure Negotiate(Transport: Interface "IMDM Source Transport") var MasterDataManagementSetup: Record "Master Data Management Setup"; + MasterDataManagement: Codeunit "Master Data Management"; Capabilities: JsonObject; FeaturesToken: JsonToken; VersionToken: JsonToken; @@ -75,8 +77,10 @@ codeunit 7246 "MDM Source Capabilities" Clear(SupportedFeatures); // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading // "capability unsupported / update the source app" error instead of the real bad-response problem. - if not Capabilities.ReadFrom(Transport.GetCapabilities()) then + if not Capabilities.ReadFrom(Transport.GetCapabilities()) then begin + Session.LogMessage('0000QFA', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); Error(InternalError(CapabilitiesParseErr)); + end; if Capabilities.Get('version', VersionToken) then if VersionToken.IsValue() then ContractVersion := VersionToken.AsValue().AsInteger(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index b7fa4ee034b..661e590dede 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -276,8 +276,12 @@ page 7232 "MDM Connection Details" MasterDataManagementSetup."Source Environment URL" := SourceEnvironmentUrl; MasterDataManagementSetup."Source Company Name" := SourceCompanyName; MasterDataManagementSetup."Source OAuth Client Id" := OAuth2ClientId; - if OAuth2ClientSecret <> '' then + if OAuth2ClientSecret <> '' then begin MasterDataManagementSetup.SetSourceClientSecret(OAuth2ClientSecret); + // Minimize the plain-text window: the secret is now encrypted at rest, so drop the wizard copy. + Clear(OAuth2ClientSecret); + SecretAlreadyStored := true; + end; MasterDataManagementSetup.Modify(true); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index b6fedc9b19d..26e1f391a50 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -379,6 +379,7 @@ table 7230 "Master Data Management Setup" begin ErrInfo.Message := MustConfigureConnectionErr; ErrInfo.RecordId := Rec.RecordId(); + ErrInfo.PageNo := Page::"Master Data Management Setup"; ErrInfo.AddNavigationAction(OpenSetupNavigationTxt); exit(ErrInfo); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al index 729265fffbd..956c499eefd 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al @@ -47,7 +47,7 @@ tableextension 7235 MasterDataMgtTableMapping extends "Integration Table Mapping // Composite (SystemModifiedAt, SystemId) resume point ({"modifiedAt":...,"systemId":...}, ~93 chars). // Empty means "caught up": the run drained the source and the watermark is authoritative. Caption = 'Source Change Cursor'; - DataClassification = SystemMetadata; + DataClassification = CustomerContent; } } From e61e0d0f1c73195e653ffef6d04a13e993161add Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Sat, 29 Aug 2026 20:26:46 +0200 Subject: [PATCH 26/64] [Master Data Management] Cross-environment synchonization --- .../codeunits/MDMSourceResponse.Codeunit.al | 48 ++++++++++++++----- .../src/LibraryMasterDataMgt.Codeunit.al | 31 ++++++++++++ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index d114d76022b..86b515439c8 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -15,6 +15,7 @@ codeunit 7248 "MDM Source Response" var SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; + BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; [TryFunction] procedure TryParse(ResponseText: Text; var Response: JsonObject) @@ -218,7 +219,8 @@ codeunit 7248 "MDM Source Response" Session.LogMessage('0000QF2', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; - // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. + // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. A + // failed conversion means a drifted/malformed source value; error rather than silently syncing a defaulted one. local procedure SetFieldFromText(var DestField: FieldRef; ValueText: Text) var IntegerValue: Integer; @@ -231,41 +233,65 @@ codeunit 7248 "MDM Source Response" DurationValue: Duration; DateFormulaValue: DateFormula; GuidValue: Guid; + Converted: Boolean; begin + Converted := true; case DestField.Type() of FieldType::Text, FieldType::Code: DestField.Value := CopyStr(ValueText, 1, DestField.Length()); FieldType::Integer, FieldType::Option: if Evaluate(IntegerValue, ValueText, 9) then - DestField.Value := IntegerValue; + DestField.Value := IntegerValue + else + Converted := false; FieldType::BigInteger: if Evaluate(BigIntegerValue, ValueText, 9) then - DestField.Value := BigIntegerValue; + DestField.Value := BigIntegerValue + else + Converted := false; FieldType::Decimal: if Evaluate(DecimalValue, ValueText, 9) then - DestField.Value := DecimalValue; + DestField.Value := DecimalValue + else + Converted := false; FieldType::Boolean: if Evaluate(BooleanValue, ValueText, 9) then - DestField.Value := BooleanValue; + DestField.Value := BooleanValue + else + Converted := false; FieldType::Date: if Evaluate(DateValue, ValueText, 9) then - DestField.Value := DateValue; + DestField.Value := DateValue + else + Converted := false; FieldType::Time: if Evaluate(TimeValue, ValueText, 9) then - DestField.Value := TimeValue; + DestField.Value := TimeValue + else + Converted := false; FieldType::DateTime: if Evaluate(DateTimeValue, ValueText, 9) then - DestField.Value := DateTimeValue; + DestField.Value := DateTimeValue + else + Converted := false; FieldType::Duration: if Evaluate(DurationValue, ValueText, 9) then - DestField.Value := DurationValue; + DestField.Value := DurationValue + else + Converted := false; FieldType::DateFormula: if Evaluate(DateFormulaValue, ValueText, 9) then - DestField.Value := DateFormulaValue; + DestField.Value := DateFormulaValue + else + Converted := false; FieldType::Guid: if Evaluate(GuidValue, ValueText) then - DestField.Value := GuidValue; + DestField.Value := GuidValue + else + Converted := false; end; + if not Converted then + Error(BadFieldValueErr, DestField.Caption(), Format(DestField.Type())); end; local procedure GetGuid(var Container: JsonObject; PropertyName: Text; var Value: Guid): Boolean diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 70718857d67..f9268fa8f4a 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -3,36 +3,67 @@ codeunit 139757 "Library - Master Data Mgt." { Access = Public; + /// Invokes the field-transfer subscriber logic that resolves the value to synchronize for a field. + /// The source field being transferred. + /// The destination field receiving the value. + /// Returns the value to apply to the destination field. + /// Returns true when the subscriber resolved the value. + /// Returns whether the resolved value still needs type conversion. procedure HandleOnTransferFieldData(SourceFieldRef: FieldRef; DestinationFieldRef: FieldRef; var NewValue: Variant; var IsValueFound: Boolean; var NeedsConversion: Boolean) begin MasterDataMgtSubscribers.HandleOnTransferFieldData(SourceFieldRef, DestinationFieldRef, NewValue, IsValueFound, NeedsConversion); end; + /// Renames the destination record before modification when the source primary key has changed. + /// The integration table mapping being synchronized. + /// The source record providing the new primary key. + /// The destination record to rename in place. procedure RenameIfNeededOnBeforeModifyRecord(IntegrationTableMapping: Record "Integration Table Mapping"; SourceRecordRef: RecordRef; var DestinationRecordRef: RecordRef) begin MasterDataMgtSubscribers.RenameIfNeededOnBeforeModifyRecord(IntegrationTableMapping, SourceRecordRef, DestinationRecordRef); end; + /// Determines whether the source record was modified after the last synchronization. + /// The connection type of the integration table. + /// The integration table mapping being synchronized. + /// The source record to evaluate. + /// Returns true when the source record changed since the last synch. + /// Returns true when the subscriber handled the evaluation. procedure HandleOnWasModifiedAfterLastSynch(IntegrationTableConnectionType: TableConnectionType; IntegrationTableMapping: Record "Integration Table Mapping"; var SourceRecordRef: RecordRef; var SourceWasChanged: Boolean; var IsHandled: Boolean) begin MasterDataMgtSubscribers.HandleOnWasModifiedAfterLastSynch(IntegrationTableConnectionType, IntegrationTableMapping, SourceRecordRef, SourceWasChanged, IsHandled); end; + /// Resolves the local record ID coupled to an integration SystemId, synchronizing it first if needed. + /// The SystemId of the integration record. + /// The table ID to resolve against. + /// Returns the coupled local record ID. + /// Returns true when the subscriber handled the resolution. procedure HandleOnFindAndSynchRecordIDFromIntegrationSystemId(IntegrationSystemId: Guid; TableId: Integer; var LocalRecordID: RecordID; var IsHandled: Boolean) begin MasterDataMgtSubscribers.HandleOnFindAndSynchRecordIDFromIntegrationSystemId(IntegrationSystemId, TableId, LocalRecordID, IsHandled); end; + /// Evaluates whether a data-synchronization job queue entry needs to run. + /// The job queue entry being evaluated. + /// Returns true when the job needs to run. procedure HandleOnFindingIfJobNeedsToBeRun(var Sender: Record "Job Queue Entry"; var Result: Boolean) begin MasterDataMgtSubscribers.HandleOnFindingIfJobNeedsToBeRun(Sender, Result); end; + /// Runs the post-run handling for a data-synchronization job queue entry. + /// The job queue entry that finished running. procedure HandleOnAfterJobQueueEntryRun(var JobQueueEntry: Record "Job Queue Entry") begin MasterDataMgtSubscribers.HandleOnAfterJobQueueEntryRun(JobQueueEntry); end; + /// Finds the tables related to a synchronization table, as offered when adding it to the setup. + /// The tables already in the synchronization setup. + /// Returns the related table IDs proposed for adding. + /// Returns the display text for the proposed related tables. + /// The table whose related tables are resolved. procedure FindRelatedTables(var ExistingSynchTableNos: List of [Integer]; var RelatedTablesToAdd: List of [Integer]; var RelatedTablesToAddText: Text; TableId: Integer) var MasterDataSynchTables: Page "Master Data Synch. Tables"; From 5678c949657292076ecbde1da29b68ab1e638dd8 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 11:22:24 +0200 Subject: [PATCH 27/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 1 + .../IntegrationMasterDataSynch.Codeunit.al | 13 ++++--- .../MDMCrossEnvDataSource.Codeunit.al | 7 +++- .../MDMCrossEnvSourceAPI.Codeunit.al | 13 +++++-- .../codeunits/MDMSourceResponse.Codeunit.al | 28 ++++++++++++-- .../codeunits/MDMSourceWatermark.Codeunit.al | 38 +++++++++++++++++++ .../MasterDataManagement.Codeunit.al | 9 +++-- .../MasterDataMgtSubscribers.Codeunit.al | 14 +++++-- .../MasterDataMgtUpgrade.Codeunit.al | 2 +- .../src/pages/MDMConnectionDetails.Page.al | 2 +- .../src/LibraryMasterDataMgt.Codeunit.al | 13 ++++++- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 6 ++- .../src/MDMCrossEnvSourceTests.Codeunit.al | 6 +-- .../src/MasterDataMgtSetupTests.Codeunit.al | 11 ++++++ 14 files changed, 133 insertions(+), 30 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 0dc93d1a9d2..1e5e94d5e1f 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -26,6 +26,7 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "MDM Cross-Env Change Detector" = X, codeunit "MDM Source Capabilities" = X, codeunit "MDM Inline Media" = X, + codeunit "MDM Source Watermark" = X, codeunit "MDM Privacy Notice" = X, page * = X, table * = X, diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index f80e7a68eb9..ca4025fe0a1 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -74,8 +74,8 @@ codeunit 7231 "Integration Master Data Synch." var MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; - DataSource: Interface "IMDM Data Source"; IntegrationRecordRef: RecordRef; + DataSource: Interface "IMDM Data Source"; IntegrationRecordID: Guid; TableFilter: Text; FilterList: List of [Text]; @@ -230,8 +230,8 @@ codeunit 7231 "Integration Master Data Synch." local procedure CacheFilteredIntegrationRecords(var IntegrationSystemIDFilterList: List of [Text]; IntegrationTableMapping: Record "Integration Table Mapping"; var TempIntegrationRecordRef: RecordRef): Boolean var MasterDataManagementSetup: Record "Master Data Management Setup"; - DataSource: Interface "IMDM Data Source"; IntegrationRecordRef: RecordRef; + DataSource: Interface "IMDM Data Source"; IntegrationSystemIDFilter: Text; Cached: Boolean; IsHandled: Boolean; @@ -245,10 +245,11 @@ codeunit 7231 "Integration Master Data Synch." foreach IntegrationSystemIDFilter in IntegrationSystemIDFilterList do if IntegrationSystemIDFilter <> '' then begin if DataSource.GetByUidFilter(IntegrationTableMapping, IntegrationSystemIDFilter, IntegrationRecordRef) then - repeat - CopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false); - Cached := true; - until IntegrationRecordRef.Next() = 0; + if IntegrationRecordRef.FindSet() then + repeat + CopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false); + Cached := true; + until IntegrationRecordRef.Next() = 0; IntegrationRecordRef.Close(); end; exit(Cached); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 200a87889c1..0d81aab7f8f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -16,6 +16,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceResponse: Codeunit "MDM Source Response"; SourceCapabilities: Codeunit "MDM Source Capabilities"; InlineMedia: Codeunit "MDM Inline Media"; + SourceWatermark: Codeunit "MDM Source Watermark"; InvalidResponseErr: Label 'The source environment returned an unexpected response for table %1.', Comment = '%1 = table caption'; TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; @@ -65,6 +66,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceRecordRef.Close(); SourceRecordRef.Open(IntegrationTableMapping."Integration Table ID", true); InlineMedia.Reset(); // fresh batch: drop the previous page's inline media bytes + SourceWatermark.Reset(); Transport := GetTransport(); SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); FieldIds := BuildFieldIds(IntegrationTableMapping); @@ -189,6 +191,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceRecordRef.Close(); SourceRecordRef.Open(IntegrationTableId, true); InlineMedia.Reset(); // fresh fetch: drop any prior inline media bytes + SourceWatermark.Reset(); if SystemIds.Count() = 0 then exit; Transport := GetTransport(); @@ -221,7 +224,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('0000QFB', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000QFB', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) @@ -298,9 +301,9 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" local procedure AllNormalFields(IntegrationTableId: Integer): Text var RecRef: RecordRef; + CurrentField: FieldRef; FieldIds: JsonArray; AddedFields: List of [Integer]; - CurrentField: FieldRef; Index: Integer; begin RecRef.Open(IntegrationTableId, true); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 81a785c2b2b..73f1a2da7bb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -14,6 +14,10 @@ codeunit 7241 "MDM Cross-Env Source API" { Access = Public; + var + SortByChangeFeedKeyTok: Label 'SORTING(Field%1,Field%2)', Locked = true; + SortByModifiedAtTok: Label 'SORTING(Field%1)', Locked = true; + /// /// Wire-version negotiation: returns the API contract version and the action/feature names this source /// supports, so a newer subsidiary only calls actions an older source actually implements. @@ -85,7 +89,7 @@ codeunit 7241 "MDM Cross-Env Source API" if HasCompositeChangeFeedKey(RecRef) then begin // Bounded paging: the (SystemModifiedAt, SystemId) key lets us split even a big same-timestamp group. - RecRef.SetView(StrSubstNo('SORTING(Field%1,Field%2)', SystemModifiedAtFieldNo(), SystemIdFieldNo())); + RecRef.SetView(StrSubstNo(SortByChangeFeedKeyTok, SystemModifiedAtFieldNo(), SystemIdFieldNo())); HasMore := FillCursorPage(RecRef, HasCursor, CursorModifiedAt, CursorSystemId, ProjectedFields, PageSize, Records, Count, NextModifiedAt, NextSystemId); if Count > 0 then Response.Add('nextCursor', BuildCursor(NextModifiedAt, NextSystemId)); @@ -94,7 +98,7 @@ codeunit 7241 "MDM Cross-Env Source API" // Fallback for tables without the SystemId tiebreak (kept off small/setup tables): drain each // timestamp group whole so the cursor can advance by SystemModifiedAt alone. Safe while groups // are small; a group too large to page keylessly asks for the composite key instead. - RecRef.SetView(StrSubstNo('SORTING(Field%1)', SystemModifiedAtFieldNo())); + RecRef.SetView(StrSubstNo(SortByModifiedAtTok, SystemModifiedAtFieldNo())); HasMore := FillDrainPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, PageSize, Records, Count, NextModifiedAt, GroupTooLarge); if GroupTooLarge then begin Clear(Records); @@ -250,6 +254,8 @@ codeunit 7241 "MDM Cross-Env Source API" PageBytes: Integer; begin Count := 0; + PageBytes := 0; + LastEmittedAt := 0DT; GroupTooLarge := false; MaxKeylessGroup := 10000; ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); @@ -285,6 +291,7 @@ codeunit 7241 "MDM Cross-Env Source API" PageBytes: Integer; begin Count := 0; + PageBytes := 0; TooLarge := false; MaxUnindexedRecords := 10000; ModifiedAtRef := RecRef.Field(SystemModifiedAtFieldNo()); @@ -577,7 +584,7 @@ codeunit 7241 "MDM Cross-Env Source API" Entry.Add('indexed', false); exit(Entry); end; - RecRef.SetView(StrSubstNo('SORTING(Field%1)', SystemModifiedAtFieldNo())); + RecRef.SetView(StrSubstNo(SortByModifiedAtTok, SystemModifiedAtFieldNo())); // Detection only needs an approximate max, so read uncommitted: never takes or waits on a lock, even // where snapshot isolation is off (OnPrem). Data reads (GetRecords) stay at committed isolation. RecRef.ReadIsolation := IsolationLevel::ReadUncommitted; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 86b515439c8..7422f8a7046 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -16,6 +16,7 @@ codeunit 7248 "MDM Source Response" var SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; + SourceWatermark: Codeunit "MDM Source Watermark"; [TryFunction] procedure TryParse(ResponseText: Text; var Response: JsonObject) @@ -99,12 +100,13 @@ codeunit 7248 "MDM Source Response" local procedure InsertRecord(RecordObject: JsonObject; var TempSourceRecordRef: RecordRef) var + DestField: FieldRef; FieldsToken: JsonToken; ValueToken: JsonToken; FieldsObject: JsonObject; - DestField: FieldRef; FieldName: Text; SystemIdValue: Guid; + SystemModifiedAtValue: DateTime; FieldNo: Integer; begin TempSourceRecordRef.Init(); @@ -132,6 +134,10 @@ codeunit 7248 "MDM Source Response" // newest copy instead of aborting the whole batch on the duplicate primary key. if not TryInsertTempRecord(TempSourceRecordRef) then TempSourceRecordRef.Modify(false); + // The temp row can't hold SystemModifiedAt (the platform ignores the write), so stash the source watermark + // in a side cache the sync loop reads back via GetRowLastModifiedOn. + if GetDateTime(RecordObject, 'systemModifiedAt', SystemModifiedAtValue) then + SourceWatermark.Put(SystemIdValue, SystemModifiedAtValue); end; [TryFunction] @@ -223,6 +229,7 @@ codeunit 7248 "MDM Source Response" // failed conversion means a drifted/malformed source value; error rather than silently syncing a defaulted one. local procedure SetFieldFromText(var DestField: FieldRef; ValueText: Text) var + DateFormulaValue: DateFormula; IntegerValue: Integer; BigIntegerValue: BigInteger; DecimalValue: Decimal; @@ -231,9 +238,9 @@ codeunit 7248 "MDM Source Response" TimeValue: Time; DateTimeValue: DateTime; DurationValue: Duration; - DateFormulaValue: DateFormula; GuidValue: Guid; Converted: Boolean; + ErrInfo: ErrorInfo; begin Converted := true; case DestField.Type() of @@ -290,8 +297,12 @@ codeunit 7248 "MDM Source Response" else Converted := false; end; - if not Converted then - Error(BadFieldValueErr, DestField.Caption(), Format(DestField.Type())); + if not Converted then begin + // Internal contract failure the user can't fix: detail goes to telemetry, user sees a generic dialog. + ErrInfo.Message := StrSubstNo(BadFieldValueErr, DestField.Caption(), Format(DestField.Type())); + ErrInfo.ErrorType := ErrorType::Internal; + Error(ErrInfo); + end; end; local procedure GetGuid(var Container: JsonObject; PropertyName: Text; var Value: Guid): Boolean @@ -302,4 +313,13 @@ codeunit 7248 "MDM Source Response" exit(false); exit(Evaluate(Value, Token.AsValue().AsText())); end; + + local procedure GetDateTime(var Container: JsonObject; PropertyName: Text; var Value: DateTime): Boolean + var + Token: JsonToken; + begin + if not Container.Get(PropertyName, Token) then + exit(false); + exit(Evaluate(Value, Token.AsValue().AsText(), 9)); + end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al new file mode 100644 index 00000000000..584e1a3867f --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceWatermark.Codeunit.al @@ -0,0 +1,38 @@ +namespace Microsoft.Integration.MDM; + +/// +/// Per-batch cache of the source SystemModifiedAt watermark carried in a cross-environment GetRecords response. +/// The materializer stashes it keyed by the source record's SystemId; the synchronization engine reads it back +/// (GetRowLastModifiedOn) to obtain the real source change time. The platform does not let a temporary record +/// hold SystemModifiedAt (writes are ignored), so the watermark travels here instead of on the temp source row. +/// Single instance so the value survives from materialization through the synchronization write; Reset() clears +/// it between batches. +/// +codeunit 7239 "MDM Source Watermark" +{ + Access = Internal; + SingleInstance = true; + + var + ModifiedAtByKey: Dictionary of [Text, DateTime]; + + procedure Reset() + begin + Clear(ModifiedAtByKey); + end; + + procedure Put(SystemId: Guid; ModifiedAt: DateTime) + begin + ModifiedAtByKey.Set(MakeKey(SystemId), ModifiedAt); + end; + + procedure TryGet(SystemId: Guid; var ModifiedAt: DateTime): Boolean + begin + exit(ModifiedAtByKey.Get(MakeKey(SystemId), ModifiedAt)); + end; + + local procedure MakeKey(SystemId: Guid): Text + begin + exit(Format(SystemId, 0, 4)); + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index edd970d98dd..cc42ba29fa7 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -671,10 +671,11 @@ codeunit 7233 "Master Data Management" MasterDataManagementSetup.Get(); // Route the source read so uncoupling works against the local company or another environment. if MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, IntegrationTableFilter, IntegrationRecordRef) then - repeat - if not PerformUncoupling(IntegrationTableMapping, LocalRecordRef, IntegrationRecordRef) then - CountFailed += 1; - until IntegrationRecordRef.Next() = 0; + if IntegrationRecordRef.FindSet() then + repeat + if not PerformUncoupling(IntegrationTableMapping, LocalRecordRef, IntegrationRecordRef) then + CountFailed += 1; + until IntegrationRecordRef.Next() = 0; end; IntegrationTableMapping.Delete(true); exit(CountFailed = 0); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 533f48e804c..e07c3f30bc3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -745,15 +745,23 @@ codeunit 7237 "Master Data Mgt. Subscribers" var MasterDataManagementSetup: Record "Master Data Management Setup"; MasterDataManagement: Codeunit "Master Data Management"; + SourceWatermark: Codeunit "MDM Source Watermark"; IntegrationRecordRef: RecordRef; IsHandled: Boolean; IntRecSystemId: Guid; + SourceSystemId: Guid; + SourceModifiedAt: DateTime; begin MasterDataManagementSetup.Get(); - // Cross-environment: FromRecordRef is already the source row materialized from the fetched batch, so read - // the watermark from it instead of issuing another per-record OData round-trip (avoids an N+1 fetch). - if MasterDataManagementSetup."Source Environment Name" <> '' then + // Cross-environment: FromRecordRef is the source row materialized from the fetched batch. Its SystemModifiedAt + // can't be carried on a temp row (the platform ignores the write), so the real watermark rides a side cache; + // fall back to the row's own value on a cache miss. + if MasterDataManagementSetup."Source Environment Name" <> '' then begin + SourceSystemId := FromRecordRef.Field(FromRecordRef.SystemIdNo()).Value(); + if SourceWatermark.TryGet(SourceSystemId, SourceModifiedAt) then + exit(SourceModifiedAt); exit(ModifiedOnFromRecordRef(IntegrationTableMapping, FromRecordRef)); + end; IntegrationRecordRef.Open(FromRecordRef.Number, false); IntRecSystemId := FromRecordRef.Field(FromRecordRef.SystemIdNo).Value(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al index d34c63a4e2c..8f8041e4c32 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al @@ -30,8 +30,8 @@ codeunit 7238 "Master Data Mgt. Upgrade" // Guaranteed provisioning path: install codeunits are skipped when BC is pre-baked into a package and mounted per tenant. internal procedure RegisterCrossEnvSourceWebService() var - WebServiceManagement: Codeunit "Web Service Management"; TenantWebService: Record "Tenant Web Service"; + WebServiceManagement: Codeunit "Web Service Management"; UpgradeTag: Codeunit "Upgrade Tag"; begin if UpgradeTag.HasDatabaseUpgradeTag(GetCrossEnvWebServiceUpgradeTag()) then diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 661e590dede..b873b734766 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -235,10 +235,10 @@ page 7232 "MDM Connection Details" end; var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; Step: Option Welcome,Connection,TestConnection,Finish; NextEnabled, BackEnabled, FinishEnabled, TestConnectionEnabled : Boolean; ConsentState, SecretAlreadyStored : Boolean; - MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; SourceEnvironmentName: Text[100]; SourceEnvironmentUrl: Text[250]; SourceCompanyName: Text[100]; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index f9268fa8f4a..b8e96dc3d5b 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -157,8 +157,6 @@ codeunit 139757 "Library - Master Data Mgt." /// The integration table mapping to count. /// The number of integration records. procedure GetIntegrationRecRefCount(IntegrationTableMapping: Record "Integration Table Mapping"): Integer - var - MasterDataManagement: Codeunit "Master Data Management"; begin exit(MasterDataManagement.GetIntegrationRecRefCount(IntegrationTableMapping)); end; @@ -242,6 +240,17 @@ codeunit 139757 "Library - Master Data Mgt." exit(InlineMedia.Contains(SystemId, FieldNo)); end; + /// Reads the source SystemModifiedAt watermark cached during cross-environment materialization. + /// The SystemId of the source record. + /// Returns the cached source SystemModifiedAt. + /// True if the watermark was cached; otherwise false. + procedure TryGetSourceWatermark(SystemId: Guid; var ModifiedAt: DateTime): Boolean + var + SourceWatermark: Codeunit "MDM Source Watermark"; + begin + exit(SourceWatermark.TryGet(SystemId, ModifiedAt)); + end; + /// Returns the registered privacy-notice ID that gates cross-environment synchronization. procedure PrivacyNoticeId(): Code[50] var diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index e2e0050448f..a2db81a65ce 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -18,6 +18,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; InProcessTransport: Codeunit "MDM In-Process Transport"; SourceRecordRef: RecordRef; + MaterializedModifiedAt: DateTime; Found: Boolean; begin // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] @@ -34,10 +35,13 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [WHEN] the record is fetched by SystemId Found := LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); - // [THEN] the materialized record carries the same SystemId and field values + // [THEN] the materialized record carries the same SystemId and field values, and the source SystemModifiedAt + // watermark survives materialization (the sync loop reads it via the side cache, not the temp row) Assert.IsTrue(Found, 'Cross-env GetBySystemId should find the source record'); Assert.AreEqual(Customer.SystemId, SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(), 'Wrong SystemId materialized'); Assert.AreEqual(Customer.Name, Format(SourceRecordRef.Field(Customer.FieldNo(Name)).Value()), 'Name should round-trip through the wire'); + Assert.IsTrue(LibraryMasterDataMgt.TryGetSourceWatermark(Customer.SystemId, MaterializedModifiedAt), 'Source watermark should be cached during materialization'); + Assert.AreEqual(Customer.SystemModifiedAt, MaterializedModifiedAt, 'Source SystemModifiedAt should survive materialization'); CleanUp(); end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al index 3b13d01ba44..2455fe910e4 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -30,8 +30,8 @@ codeunit 139931 "MDM Cross-Env Source Tests" [Test] procedure GetRecordsBySystemIdReturnsRequestedFields() var - SourceApi: Codeunit "MDM Cross-Env Source API"; Customer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; Response: JsonObject; RecordObject: JsonObject; FieldsObject: JsonObject; @@ -62,8 +62,8 @@ codeunit 139931 "MDM Cross-Env Source Tests" [Test] procedure GetRecordsCursorModePagesWithHasMore() var - SourceApi: Codeunit "MDM Cross-Env Source API"; Customer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; Watermark: DateTime; Response: JsonObject; NextCursor: Text; @@ -108,8 +108,8 @@ codeunit 139931 "MDM Cross-Env Source Tests" [Test] procedure LastModifiedAtPerTableReturnsLatestTimestamp() var - SourceApi: Codeunit "MDM Cross-Env Source API"; Customer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; Response: JsonObject; Entry: JsonObject; Tables: JsonArray; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al index cee4038605d..da471ad5052 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSetupTests.Codeunit.al @@ -18,6 +18,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; InitializeHandled: Boolean; IncorrectTablesListErr: Label 'Synchronization tables list is incorrect.'; + UnexpectedConfirmErr: Label 'Unexpected confirmation dialog: %1', Locked = true; [Test] [HandlerFunctions('SynchronizationEnabledMessageHandler')] @@ -175,6 +176,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" MasterDataMgtCoupling."Local System ID" := EmptyGuid; MasterDataMgtCoupling.Insert(); + LibraryVariableStorage.Enqueue('keep the table setup and coupling'); MasterDataManagementSetup.Validate("Is Enabled", false); BindSubscription(MasterDataMgtSetupTests); MasterDataManagementSetup.Modify(true); @@ -189,6 +191,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" Assert.IsTrue(IntegrationTableMapping.Count() > 0, ''); Assert.IsTrue(IntegrationFieldMapping.Count() > 0, ''); Assert.AreEqual(1, MasterDataMgtCoupling.Count(), ''); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -220,8 +223,10 @@ codeunit 139770 "Master Data Mgt. Setup Tests" // reset configuration MasterDataManagementSetupPage.OpenEdit(); + LibraryVariableStorage.Enqueue('restore the default synchronization table setup'); MasterDataManagementSetupPage.ResetConfiguration.Invoke(); VerifyDefaultSetup(); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -326,6 +331,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" IntegrationFieldMapping.SetRange(Status); IntegrationFieldMapping.SetRange("Field Caption", ''); Assert.IsTrue(IntegrationFieldMapping.Count() = 0, 'All synchronization fields for the added table should have a caption.'); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -358,6 +364,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" MasterDataMgtCoupling."Local System ID" := EmptyGuid; MasterDataMgtCoupling.Insert(); + LibraryVariableStorage.Enqueue('keep the table setup and coupling'); MasterDataManagementSetup.Validate("Is Enabled", false); BindSubscription(MasterDataMgtSetupTests); MasterDataManagementSetup.Modify(true); @@ -372,6 +379,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" Assert.AreEqual(0, IntegrationTableMapping.Count(), ''); Assert.AreEqual(0, IntegrationFieldMapping.Count(), ''); Assert.AreEqual(0, MasterDataMgtCoupling.Count(), ''); + LibraryVariableStorage.AssertEmpty(); end; [Test] @@ -404,6 +412,7 @@ codeunit 139770 "Master Data Mgt. Setup Tests" exit; LibrarySetupStorage.Restore(); + LibraryVariableStorage.Clear(); BindSubscription(MasterDataMgtSynchTests); IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); @@ -457,12 +466,14 @@ codeunit 139770 "Master Data Mgt. Setup Tests" [ConfirmHandler] internal procedure ConfirmHandlerYes(Question: Text; var Reply: Boolean) begin + Assert.IsTrue(StrPos(Question, LibraryVariableStorage.DequeueText()) > 0, StrSubstNo(UnexpectedConfirmErr, Question)); Reply := true; end; [ConfirmHandler] internal procedure ConfirmHandlerNo(Question: Text; var Reply: Boolean) begin + Assert.IsTrue(StrPos(Question, LibraryVariableStorage.DequeueText()) > 0, StrSubstNo(UnexpectedConfirmErr, Question)); Reply := false; end; From 7bd650e5e8a552277cb151923393b7d10f15f69d Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 14:06:26 +0200 Subject: [PATCH 28/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMSourceResponse.Codeunit.al | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 7422f8a7046..a4b77fc6fca 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -14,9 +14,9 @@ codeunit 7248 "MDM Source Response" Access = Internal; var + SourceWatermark: Codeunit "MDM Source Watermark"; SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; - SourceWatermark: Codeunit "MDM Source Watermark"; [TryFunction] procedure TryParse(ResponseText: Text; var Response: JsonObject) From 205a84d9a2c992e8ad34a20e75c035c2348e552c Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 14:43:50 +0200 Subject: [PATCH 29/64] [Master Data Management] Cross-environment synchonization --- .../codeunits/IntegrationMasterDataSynch.Codeunit.al | 2 +- .../codeunits/MDMCrossEnvChangeDetector.Codeunit.al | 11 +++++------ .../src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 4 ++-- .../src/codeunits/MDMHttpSourceTransport.Codeunit.al | 4 ++-- .../src/codeunits/MDMSourceCapabilities.Codeunit.al | 2 +- .../app/src/codeunits/MDMSourceResponse.Codeunit.al | 2 +- .../app/src/tables/MasterDataManagementSetup.Table.al | 2 +- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index ca4025fe0a1..15682b9aa90 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -120,7 +120,7 @@ codeunit 7231 "Integration Master Data Synch." IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then - Session.LogMessage('0000QF6', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); until IntegrationRecordRef.Next() = 0; IntegrationRecordRef.Close(); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index ba798425ac9..58da7d3dfbc 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -56,18 +56,17 @@ codeunit 7245 "MDM Cross-Env Change Detector" // detector job - that would burn its retry budget and could stop change detection. Skip this poll instead; // the next scheduled run recovers. if not TryFetchDetection(Transport, TableIds, Supported, ResponseText) then begin - // The detector's transport errors are operational (HTTP status, connection, auth, config) with no record - // content, so the caught message - which carries the HTTP status code for HttpErr - is safe as system metadata. + // A transport failure here is operational (source outage, auth, bad connection state); skip this poll. The + // raw error text is not emitted: GetLastErrorText can carry customer content and this event is All-scope. Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); - Dimensions.Add('failure', CopyStr(GetLastErrorText(), 1, 2048)); - Session.LogMessage('0000QF9', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); exit; end; // Older source doesn't advertise the detection action: skip rather than error every run. if not Supported then exit; if not SourceResponse.TryParse(ResponseText, Response) then begin - Session.LogMessage('0000QF4', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; @@ -115,7 +114,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" EntryToken: JsonToken; begin if (not Response.Get('tables', TablesToken)) or (not TablesToken.IsArray()) then begin - Session.LogMessage('0000QF7', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; Tables := TablesToken.AsArray(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 0d81aab7f8f..40f46d9e6e9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -217,14 +217,14 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('0000QF8', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure LogProbeFailure(IntegrationTableId: Integer) var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('0000QFB', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index e8ef6324988..6a1bda26ef3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -106,7 +106,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Dimensions.Add('Category', TelemetryCategoryTok); Dimensions.Add('action', ActionName); Dimensions.Add('httpStatusCode', Format(ResponseMessage.HttpStatusCode())); - Session.LogMessage('0000QF1', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); // Security audit: an authorization failure crossing the environment boundary. if ResponseMessage.HttpStatusCode() in [401, 403] then AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); @@ -120,7 +120,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" // only the action is logged. Dimensions.Add('Category', TelemetryCategoryTok); Dimensions.Add('action', ActionName); - Session.LogMessage('0000QF3', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; local procedure Send(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index 76dd00f009c..47d36f26439 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -78,7 +78,7 @@ codeunit 7246 "MDM Source Capabilities" // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading // "capability unsupported / update the source app" error instead of the real bad-response problem. if not Capabilities.ReadFrom(Transport.GetCapabilities()) then begin - Session.LogMessage('0000QFA', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); Error(InternalError(CapabilitiesParseErr)); end; if Capabilities.Get('version', VersionToken) then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index a4b77fc6fca..0ab6e994c1b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -222,7 +222,7 @@ codeunit 7248 "MDM Source Response" Dimensions.Add('fieldNo', Format(FieldNo)); if FieldObject.Get('length', LengthToken) then Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); - Session.LogMessage('0000QF2', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. A diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 26e1f391a50..adc31e60f16 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -282,7 +282,7 @@ table 7230 "Master Data Management Setup" begin Dimensions.Add('Category', TelemetryCategory); Dimensions.Add('sourceEnvironment', "Source Environment Name"); - Session.LogMessage('0000QF5', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); + Session.LogMessage('', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); end; local procedure GetConfigurationUpdates(var IsEnabledChanged: Boolean) From 5f5f72a42748742f408a035f11f247eb94475108 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 14:47:08 +0200 Subject: [PATCH 30/64] [Master Data Management] Cross-environment synchonization --- .../src/codeunits/IntegrationMasterDataSynch.Codeunit.al | 2 +- .../app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al | 6 +++--- .../app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 4 ++-- .../app/src/codeunits/MDMHttpSourceTransport.Codeunit.al | 4 ++-- .../app/src/codeunits/MDMSourceCapabilities.Codeunit.al | 2 +- .../app/src/codeunits/MDMSourceResponse.Codeunit.al | 2 +- .../app/src/tables/MasterDataManagementSetup.Table.al | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index 15682b9aa90..a79c515dda3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -120,7 +120,7 @@ codeunit 7231 "Integration Master Data Synch." IntegrationRecordID := IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").Value(); if not FailedNotSkippedIdDictionary.ContainsKey(IntegrationRecordID) then if not TryCopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false) then - Session.LogMessage('', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAN', StrSubstNo(CrossEnvCopyFailedTelemetryTxt, IntegrationTableMapping."Integration Table ID"), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); until IntegrationRecordRef.Next() = 0; IntegrationRecordRef.Close(); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 58da7d3dfbc..6e2ed0cab50 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -59,14 +59,14 @@ codeunit 7245 "MDM Cross-Env Change Detector" // A transport failure here is operational (source outage, auth, bad connection state); skip this poll. The // raw error text is not emitted: GetLastErrorText can carry customer content and this event is All-scope. Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); - Session.LogMessage('', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('0000VAO', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); exit; end; // Older source doesn't advertise the detection action: skip rather than error every run. if not Supported then exit; if not SourceResponse.TryParse(ResponseText, Response) then begin - Session.LogMessage('', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAP', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; @@ -114,7 +114,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" EntryToken: JsonToken; begin if (not Response.Get('tables', TablesToken)) or (not TablesToken.IsArray()) then begin - Session.LogMessage('', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAQ', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; Tables := TablesToken.AsArray(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 40f46d9e6e9..6e32a66ee52 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -217,14 +217,14 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAR', StrSubstNo(ParseFailureTelemetryTxt, IntegrationTableId, Reason), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure LogProbeFailure(IntegrationTableId: Integer) var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAS', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 6a1bda26ef3..4ebf886fe6a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -106,7 +106,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Dimensions.Add('Category', TelemetryCategoryTok); Dimensions.Add('action', ActionName); Dimensions.Add('httpStatusCode', Format(ResponseMessage.HttpStatusCode())); - Session.LogMessage('', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('0000VAT', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); // Security audit: an authorization failure crossing the environment boundary. if ResponseMessage.HttpStatusCode() in [401, 403] then AuditLog.LogAuditMessage(StrSubstNo(AccessDeniedAuditTxt, MasterDataManagementSetup."Source Environment Name", ResponseMessage.HttpStatusCode()), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); @@ -120,7 +120,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" // only the action is logged. Dimensions.Add('Category', TelemetryCategoryTok); Dimensions.Add('action', ActionName); - Session.LogMessage('', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('0000VAU', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; local procedure Send(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index 47d36f26439..d3409a1efc3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -78,7 +78,7 @@ codeunit 7246 "MDM Source Capabilities" // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading // "capability unsupported / update the source app" error instead of the real bad-response problem. if not Capabilities.ReadFrom(Transport.GetCapabilities()) then begin - Session.LogMessage('', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAV', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); Error(InternalError(CapabilitiesParseErr)); end; if Capabilities.Get('version', VersionToken) then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 0ab6e994c1b..77531f26f9e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -222,7 +222,7 @@ codeunit 7248 "MDM Source Response" Dimensions.Add('fieldNo', Format(FieldNo)); if FieldObject.Get('length', LengthToken) then Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); - Session.LogMessage('', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('0000VAW', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; // Round-trips a value serialized with Format(v, 0, 9) on the source back into the destination field's type. A diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index adc31e60f16..84ad15e22bc 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -282,7 +282,7 @@ table 7230 "Master Data Management Setup" begin Dimensions.Add('Category', TelemetryCategory); Dimensions.Add('sourceEnvironment', "Source Environment Name"); - Session.LogMessage('', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); + Session.LogMessage('0000VAX', CrossEnvEnabledTelemetryTxt, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, Dimensions); end; local procedure GetConfigurationUpdates(var IsEnabledChanged: Boolean) From 3c2ecd41260c3cc7dad17e0443194eb17d0c8516 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 15:18:52 +0200 Subject: [PATCH 31/64] [Master Data Management] Cross-environment synchonization --- .../codeunits/MDMCrossEnvSourceAPI.Codeunit.al | 8 +++++--- .../codeunits/MDMHttpSourceTransport.Codeunit.al | 8 ++++---- .../codeunits/MasterDataManagement.Codeunit.al | 15 +++++++++++++-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 73f1a2da7bb..2a043d930b2 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -359,6 +359,7 @@ codeunit 7241 "MDM Cross-Env Source API" local procedure FillBySystemIds(var RecRef: RecordRef; SystemIds: JsonArray; ProjectedFields: List of [Integer]; var Records: JsonArray) var SystemIdRef: FieldRef; + FilterBuilder: TextBuilder; Token: JsonToken; SystemIdValue: Guid; IgnoredModifiedAt: DateTime; @@ -368,10 +369,11 @@ codeunit 7241 "MDM Cross-Env Source API" begin foreach Token in SystemIds do if Evaluate(SystemIdValue, Token.AsValue().AsText()) then begin - if FilterText <> '' then - FilterText += '|'; - FilterText += Format(SystemIdValue); + if FilterBuilder.Length() > 0 then + FilterBuilder.Append('|'); + FilterBuilder.Append(Format(SystemIdValue)); end; + FilterText := FilterBuilder.ToText(); if FilterText = '' then exit; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 4ebf886fe6a..96b80c5a8cd 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -265,11 +265,11 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; - if MasterDataManagementSetup.Get() then begin + // The remedy is always the setup page, even before the record exists; only RecordId needs an existing record. + ErrInfo.PageNo := Page::"Master Data Management Setup"; + ErrInfo.AddNavigationAction(OpenSetupActionTxt); + if MasterDataManagementSetup.Get() then ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); - ErrInfo.PageNo := Page::"Master Data Management Setup"; - ErrInfo.AddNavigationAction(OpenSetupActionTxt); - end; exit(ErrInfo); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index cc42ba29fa7..95839f714e6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -70,6 +70,8 @@ codeunit 7233 "Master Data Management" BothRecordsModifiedToNAVQst: Label 'Both %1 and the %4 %2 record have been changed since the last synchronization, or synchronization has never been performed. If you continue with synchronization, data in %3 will be overwritten with data from %4. Are you sure you want to synchronize?', Comment = '%1 is a formatted RecordID, such as ''Customer 1234''. %2 is the caption of a Business Central table. %3 - product name, %4 = Business Central product name'; NoOf: Option ,Scheduled,Failed,Skipped,Total; CategoryTok: Label 'AL Master Data Management', Locked = true; + InvalidIntegrationRecordSystemIdErr: Label 'Invalid integration record system id.', Locked = true; + EmptyIntegrationRecordSystemIdErr: Label 'Empty integration record system id.', Locked = true; DeletionConflictHandledRemoveCouplingTxt: Label 'Deletion conflict handled by removing the coupling to the deleted record.', Locked = true; DeletionConflictHandledRestoreRecordTxt: Label 'Deletion conflict handled by restoring the deleted record.', Locked = true; ResetAllCustomIntegrationTableMappingsLbl: Label 'One or more of the selected integration table mappings is custom. \\To restore a custom table mapping, you must subscribe to the event OnBeforeResetTableMapping in codeunit "Master Data Mgt. Setup Default" and implement the defaults for each custom table mapping. \\Do you want to continue?'; @@ -1995,6 +1997,15 @@ codeunit 7233 "Master Data Management" end; end; + local procedure InternalError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Record Management", 'OnUpdateIntegrationTableCouplingForRecordRef', '', false, false)] local procedure HandleOnUpdateIntegrationTableCouplingForRecordRef(IntegrationTableConnectionType: TableConnectionType; IntegrationTableUid: Variant; RecordRef: RecordRef; var IsHandled: Boolean) var @@ -2011,11 +2022,11 @@ codeunit 7233 "Master Data Management" exit; if not IntegrationTableUid.IsGuid() then - Error('Invalid integration record system id.'); + Error(InternalError(InvalidIntegrationRecordSystemIdErr)); IntegrationSystemId := IntegrationTableUid; if IntegrationSystemId = SysId then - Error('Empty integration record system id.'); + Error(InternalError(EmptyIntegrationRecordSystemIdErr)); if not MasterDataMgtCoupling.FindSystemIdByRecordRef(SysId, RecordRef) then Error(IntegrationRecordNotFoundErr, Format(RecordRef.RecordId(), 0, 1)); From 0679e5565b40714a78fe5e56ccfc39c39c61b89f Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 15:27:54 +0200 Subject: [PATCH 32/64] [Master Data Management] Cross-environment synchonization --- .../IntegrationMasterDataSynch.Codeunit.al | 12 +++++----- .../MDMCrossEnvDataSource.Codeunit.al | 22 ++++++++++++++++--- .../MDMHttpSourceTransport.Codeunit.al | 16 ++++++++++---- .../MasterDataManagement.Codeunit.al | 12 +++++----- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index a79c515dda3..c882c95ee34 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -244,12 +244,14 @@ codeunit 7231 "Integration Master Data Synch." DataSource := MasterDataManagementSetup.GetDataSource(); foreach IntegrationSystemIDFilter in IntegrationSystemIDFilterList do if IntegrationSystemIDFilter <> '' then begin + // GetByUidFilter returns the ref already positioned on the matching set; a separate FindSet would re-read it. +#pragma warning disable AA0181 if DataSource.GetByUidFilter(IntegrationTableMapping, IntegrationSystemIDFilter, IntegrationRecordRef) then - if IntegrationRecordRef.FindSet() then - repeat - CopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false); - Cached := true; - until IntegrationRecordRef.Next() = 0; + repeat + CopyRecordReference(IntegrationTableMapping, IntegrationRecordRef, TempIntegrationRecordRef, false); + Cached := true; + until IntegrationRecordRef.Next() = 0; +#pragma warning restore AA0181 IntegrationRecordRef.Close(); end; exit(Cached); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 6e32a66ee52..d7785f0782a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -22,6 +22,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; SourceProbeFailedErr: Label 'Could not read the change probe from the source environment for table %1.', Comment = '%1 = table caption'; + OpenSynchTablesActionTxt: Label 'Open Synchronization Tables'; RecordsFeatureTok: Label 'records', Locked = true; LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; SourceProbeTelemetryTxt: Label 'The cross-environment source-record probe failed for table %1.', Locked = true, Comment = '%1 = table id'; @@ -121,9 +122,9 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; - if not Response.Get('tables', Token) then begin + if (not Response.Get('tables', Token)) or (not Token.IsArray()) then begin LogProbeFailure(IntegrationTableId); - Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); end; Tables := Token.AsArray(); if Tables.Count() = 0 then begin @@ -131,6 +132,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; Tables.Get(0, Token); + if not Token.IsObject() then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; Entry := Token.AsObject(); if Entry.Get('tableAvailable', Token) then if not Token.AsValue().AsBoolean() then @@ -213,6 +218,17 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(ErrInfo); end; + // The table isn't exposed on the source: a recoverable setup issue, so point the user at Synchronization Tables. + local procedure SynchTablesNavigationError(MessageText: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MessageText; + ErrInfo.PageNo := Page::"Master Data Synch. Tables"; + ErrInfo.AddNavigationAction(OpenSynchTablesActionTxt); + exit(ErrInfo); + end; + local procedure LogParseFailure(IntegrationTableId: Integer; Reason: Text) var MasterDataManagement: Codeunit "Master Data Management"; @@ -238,7 +254,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; if not SourceResponse.TableAvailable(Response) then begin LogParseFailure(IntegrationTableId, TableUnavailableReasonTok); - Error(TableUnavailableErr, TableCaption(IntegrationTableId)); + Error(SynchTablesNavigationError(StrSubstNo(TableUnavailableErr, TableCaption(IntegrationTableId)))); end; if not SourceResponse.Indexed(Response) then begin LogParseFailure(IntegrationTableId, NotIndexedReasonTok); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 96b80c5a8cd..d1986555932 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -167,15 +167,23 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Uri: Codeunit Uri; Host: Text; begin - Uri.Init(BaseUrl); // Embed/ISV clusters vary in hostname but always end with dynamics.com; dynamics-tie.com is the test (TIE) ring. - Host := LowerCase(Uri.GetHost()); - if (Uri.GetScheme() = 'https') and (Host.EndsWith('.dynamics.com') or Host.EndsWith('.dynamics-tie.com')) then - exit; + // A malformed URL must surface the actionable setup error, not the URI parser's raw exception. + if TryInitUri(Uri, BaseUrl) then begin + Host := LowerCase(Uri.GetHost()); + if (Uri.GetScheme() = 'https') and (Host.EndsWith('.dynamics.com') or Host.EndsWith('.dynamics-tie.com')) then + exit; + end; AuditLog.LogAuditMessage(StrSubstNo(InvalidSourceUrlAuditTxt, Host), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); Error(SetupNavigationError(InvalidSourceUrlErr)); end; + [TryFunction] + local procedure TryInitUri(var Uri: Codeunit Uri; BaseUrl: Text) + begin + Uri.Init(BaseUrl); + end; + // Test seam: exercise the source-host allow-list without a live environment or the SaaS gate. internal procedure ValidateSourceHostUrl(BaseUrl: Text) begin diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index 95839f714e6..4f476123db5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -672,12 +672,14 @@ codeunit 7233 "Master Data Management" end else begin MasterDataManagementSetup.Get(); // Route the source read so uncoupling works against the local company or another environment. + // GetByFilter returns the ref already positioned on the matching set; a separate FindSet would re-read it. +#pragma warning disable AA0181 if MasterDataManagementSetup.GetDataSource().GetByFilter(IntegrationTableMapping, IntegrationTableFilter, IntegrationRecordRef) then - if IntegrationRecordRef.FindSet() then - repeat - if not PerformUncoupling(IntegrationTableMapping, LocalRecordRef, IntegrationRecordRef) then - CountFailed += 1; - until IntegrationRecordRef.Next() = 0; + repeat + if not PerformUncoupling(IntegrationTableMapping, LocalRecordRef, IntegrationRecordRef) then + CountFailed += 1; + until IntegrationRecordRef.Next() = 0; +#pragma warning restore AA0181 end; IntegrationTableMapping.Delete(true); exit(CountFailed = 0); From 2cd6a5f9275528082ff19a96e1534bb6f6f7de43 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 16:37:16 +0200 Subject: [PATCH 33/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvChangeDetector.Codeunit.al | 35 +++++++----- .../MDMCrossEnvDataSource.Codeunit.al | 44 ++++++++++++--- .../MDMCrossEnvSourceAPI.Codeunit.al | 52 ++++++++++++++---- .../MDMHttpSourceTransport.Codeunit.al | 3 +- .../src/codeunits/MDMInlineMedia.Codeunit.al | 14 +++++ .../codeunits/MDMSourceResponse.Codeunit.al | 27 ++++++++-- .../MasterDataManagement.Codeunit.al | 2 +- .../MasterDataMgtSubscribers.Codeunit.al | 10 +++- .../IMDMSourceTransport.Interface.al | 2 +- .../MasterDataMgtTableMapping.TableExt.al | 3 +- .../src/LibraryMasterDataMgt.Codeunit.al | 11 ++++ .../src/MDMInProcessTransport.Codeunit.al | 4 +- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 30 +++++++++++ .../src/MDMCrossEnvSourceTests.Codeunit.al | 53 +++++++++++++++++-- 14 files changed, 246 insertions(+), 44 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 6e2ed0cab50..68e41a907a6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -22,6 +22,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" DetectorParseFailedTxt: Label 'The cross-environment change detector received an invalid response from the source and skipped this run.', Locked = true; DetectionContractFailedTxt: Label 'The cross-environment change detector received a response without a valid tables array and skipped this run.', Locked = true; DetectorTransportFailedTxt: Label 'The cross-environment change detector could not reach the source and skipped this run; the next scheduled run will retry.', Locked = true; + DetectorCapabilitiesFailedTxt: Label 'The cross-environment change detector could not negotiate capabilities with the source (malformed or unsupported response) and skipped this run.', Locked = true; trigger OnRun() begin @@ -52,19 +53,24 @@ codeunit 7245 "MDM Cross-Env Change Detector" exit; Transport := SourceConnection.GetTransport(); - // An operational transport failure (source outage, auth, bad connection state) must NOT error this recurring - // detector job - that would burn its retry budget and could stop change detection. Skip this poll instead; - // the next scheduled run recovers. - if not TryFetchDetection(Transport, TableIds, Supported, ResponseText) then begin - // A transport failure here is operational (source outage, auth, bad connection state); skip this poll. The - // raw error text is not emitted: GetLastErrorText can carry customer content and this event is All-scope. + // Capability negotiation is a deterministic contract exchange; a failure here (e.g. malformed capabilities) is + // NOT a transport outage, so classify it distinctly instead of masking it as "could not reach the source". + // Still non-fatal to this recurring job: skip the poll, the next scheduled run retries. + if not TryNegotiateDetectionSupport(Transport, Supported) then begin Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); - Session.LogMessage('0000VAO', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('', DetectorCapabilitiesFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); exit; end; // Older source doesn't advertise the detection action: skip rather than error every run. if not Supported then exit; + // An operational transport failure (source outage, auth, bad connection state) must NOT error this recurring + // detector job - that would burn its retry budget. Skip this poll; the next scheduled run recovers. + if not TryFetchDetection(Transport, TableIds, ResponseText) then begin + Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAO', DetectorTransportFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + exit; + end; if not SourceResponse.TryParse(ResponseText, Response) then begin Session.LogMessage('0000VAP', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; @@ -73,16 +79,21 @@ codeunit 7245 "MDM Cross-Env Change Detector" ProcessDetectionResponse(Response); end; - // Contains the source calls (capability negotiation + LastModifiedAtPerTable) so an operational transport error - // is caught by the caller (log + skip) instead of escaping and erroring the recurring detector job. + // Capability negotiation is a deterministic contract exchange; isolate it so a malformed-capabilities failure is + // classified distinctly from a transport outage. Both are caught by the caller (log + skip), never erroring the job. [TryFunction] - local procedure TryFetchDetection(Transport: Interface "IMDM Source Transport"; TableIds: JsonArray; var Supported: Boolean; var ResponseText: Text) + local procedure TryNegotiateDetectionSupport(Transport: Interface "IMDM Source Transport"; var Supported: Boolean) var SourceCapabilities: Codeunit "MDM Source Capabilities"; begin Supported := SourceCapabilities.IsSupported(Transport, LastModifiedFeatureTok); - if not Supported then - exit; + end; + + // Isolates the LastModifiedAtPerTable transport call so an operational transport error skips the poll instead of + // escaping and erroring the recurring detector job. + [TryFunction] + local procedure TryFetchDetection(Transport: Interface "IMDM Source Transport"; TableIds: JsonArray; var ResponseText: Text) + begin ResponseText := Transport.LastModifiedAtPerTable(WriteArray(TableIds)); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index d7785f0782a..6eda4b1f2a3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -80,7 +80,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" repeat ParseOrError( IntegrationTableMapping."Integration Table ID", - Transport.GetRecords(IntegrationTableMapping."Integration Table ID", FieldIds, Selector, PageSize()), + Transport.GetRecords(IntegrationTableMapping."Integration Table ID", FieldIds, Selector, PageSize(), TableFilter), Response); SourceResponse.InsertRecords(Response, SourceRecordRef); PagesFetched += 1; @@ -90,11 +90,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Selector := EndCursor; end; until (not HasMore) or ((MaxPages > 0) and (PagesFetched >= MaxPages)); - // Apply only the mapping's row filter. The source already filtered by the watermark cursor server-side; - // re-applying the modified-on filter here would drop every record, since the materialized temp rows - // carry no SystemModifiedAt. - if TableFilter <> '' then - SourceRecordRef.SetView(TableFilter); + // The mapping row filter is applied server-side in GetRecords now (parity with same-env), so the materialized + // set is already narrowed; no client-side re-filtering, which would misfire on fields outside the projection. exit(SourceRecordRef.FindSet()); end; @@ -102,7 +99,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" /// Cheap existence probe for the full-synch review: the source reports an empty lastModifiedAt for an empty /// table, so this avoids counting the whole table over the wire. /// - internal procedure SourceHasRecords(IntegrationTableId: Integer): Boolean + internal procedure SourceHasRecords(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text): Boolean var Transport: Interface "IMDM Source Transport"; Response: JsonObject; @@ -112,7 +109,13 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" TableIds: JsonArray; TableIdsText: Text; LastModifiedAtText: Text; + IntegrationTableId: Integer; begin + IntegrationTableId := IntegrationTableMapping."Integration Table ID"; + // With a row filter the cheap per-table timestamp can't tell whether any MATCHING record exists (parity with + // same-env, which counts filtered rows), so fetch a single filtered record and report on its presence. + if TableFilter <> '' then + exit(SourceHasFilteredRecords(IntegrationTableMapping, TableFilter)); TableIds.Add(IntegrationTableId); TableIds.WriteTo(TableIdsText); Transport := GetTransport(); @@ -151,6 +154,31 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(LastModifiedAtText <> ''); end; + /// + /// Existence probe honoring the mapping row filter: fetches a single matching record from the source so the + /// full-synch review reflects the filtered set, matching how the same-env path counts filtered rows. + /// + local procedure SourceHasFilteredRecords(IntegrationTableMapping: Record "Integration Table Mapping"; TableFilter: Text): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + Token: JsonToken; + IntegrationTableId: Integer; + begin + IntegrationTableId := IntegrationTableMapping."Integration Table ID"; + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + // '{}' selector = read from the start (no watermark); PageSize 1 keeps this an existence check, not a count. + if not SourceResponse.TryParse(Transport.GetRecords(IntegrationTableId, BuildFieldIds(IntegrationTableMapping), '{}', 1, TableFilter), Response) then begin + LogProbeFailure(IntegrationTableId); + Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); + end; + // An unavailable table yields no records array; treat as no matching records for the review. + if Response.Get('records', Token) and Token.IsArray() then + exit(Token.AsArray().Count() > 0); + exit(false); + end; + procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean var SystemIds: List of [Guid]; @@ -203,7 +231,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); ParseOrError( IntegrationTableId, - Transport.GetRecords(IntegrationTableId, FieldIds, SystemIdsSelector(SystemIds), PageSize()), + Transport.GetRecords(IntegrationTableId, FieldIds, SystemIdsSelector(SystemIds), PageSize(), ''), Response); SourceResponse.InsertRecords(Response, SourceRecordRef); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 2a043d930b2..816ea8f8ca9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -44,7 +44,7 @@ codeunit 7241 "MDM Cross-Env Source API" /// runs under the CALLER's permission set, so table-level access is enforced by permissions, not here. /// [ServiceEnabled] - procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text var RecRef: RecordRef; Response: JsonObject; @@ -64,6 +64,8 @@ codeunit 7241 "MDM Cross-Env Source API" begin Response.Add('tableId', TableId); + if IsBlockedSourceTable(TableId) then + exit(WriteResponse(Response, false, Records, false)); if not TryOpenTable(TableId, RecRef) then exit(WriteResponse(Response, false, Records, false)); Response.Add('tableAvailable', true); @@ -79,7 +81,8 @@ codeunit 7241 "MDM Cross-Env Source API" PageSize := ClampPageSize(PageSize); ApplyProjectionLoadFields(RecRef, ProjectedFields); - // Targeted mode: caller asked for specific SystemIds (no paging). + // Targeted mode: caller asked for specific SystemIds (no paging). The mapping row filter is not applied here, + // matching same-env GetById/GetByUidFilter (targeted fetches return the requested records directly). if SelectorSystemIds(Selector, SystemIds) then begin FillBySystemIds(RecRef, SystemIds, ProjectedFields, Records); exit(WriteResponse(Response, true, Records, false)); @@ -90,6 +93,7 @@ codeunit 7241 "MDM Cross-Env Source API" if HasCompositeChangeFeedKey(RecRef) then begin // Bounded paging: the (SystemModifiedAt, SystemId) key lets us split even a big same-timestamp group. RecRef.SetView(StrSubstNo(SortByChangeFeedKeyTok, SystemModifiedAtFieldNo(), SystemIdFieldNo())); + ApplyRowFilter(RecRef, Filter); HasMore := FillCursorPage(RecRef, HasCursor, CursorModifiedAt, CursorSystemId, ProjectedFields, PageSize, Records, Count, NextModifiedAt, NextSystemId); if Count > 0 then Response.Add('nextCursor', BuildCursor(NextModifiedAt, NextSystemId)); @@ -99,6 +103,7 @@ codeunit 7241 "MDM Cross-Env Source API" // timestamp group whole so the cursor can advance by SystemModifiedAt alone. Safe while groups // are small; a group too large to page keylessly asks for the composite key instead. RecRef.SetView(StrSubstNo(SortByModifiedAtTok, SystemModifiedAtFieldNo())); + ApplyRowFilter(RecRef, Filter); HasMore := FillDrainPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, PageSize, Records, Count, NextModifiedAt, GroupTooLarge); if GroupTooLarge then begin Clear(Records); @@ -111,6 +116,7 @@ codeunit 7241 "MDM Cross-Env Source API" // No SystemModifiedAt index at all: no-code scan fallback. Order by primary key (always indexed), // filter SystemModifiedAt > watermark, and return the whole changed set in one shot (capped). // Over the cap, ask for the composite key (only large unindexed tables need it). + ApplyRowFilter(RecRef, Filter); HasMore := FillScanPage(RecRef, HasCursor, CursorModifiedAt, ProjectedFields, Records, Count, NextModifiedAt, GroupTooLarge); if GroupTooLarge then begin Clear(Records); @@ -146,6 +152,36 @@ codeunit 7241 "MDM Cross-Env Source API" RecRef.ReadIsolation := IsolationLevel::ReadCommitted; end; + // Media/infrastructure tables are only reachable inline (BuildMediaValue, tied to a record's media field). Never + // serve them as a top-level read, or a caller holding the Tenant Media read grant could enumerate every blob. + local procedure IsBlockedSourceTable(TableId: Integer): Boolean + begin + exit(TableId in [Database::"Tenant Media", Database::"Tenant Media Set", Database::"Tenant Media Thumbnails"]); + end; + + // Applies the mapping's row filter server-side (parity with same-env, which filters the source record directly). + // Re-hydrated as field-level filters so it composes with the change-feed SORTING view already set for paging, + // and works even when the filter references a field outside the projection (which the temp buffer would default). + local procedure ApplyRowFilter(var RecRef: RecordRef; Filter: Text) + var + FilterSource: RecordRef; + SourceField: FieldRef; + FieldFilter: Text; + Index: Integer; + begin + if Filter = '' then + exit; + FilterSource.Open(RecRef.Number()); + FilterSource.SetView(Filter); + for Index := 1 to FilterSource.FieldCount() do begin + SourceField := FilterSource.FieldIndex(Index); + FieldFilter := SourceField.GetFilter(); + if FieldFilter <> '' then + RecRef.Field(SourceField.Number()).SetFilter(FieldFilter); + end; + FilterSource.Close(); + end; + local procedure ResolveProjection(var RecRef: RecordRef; FieldIds: Text; var ProjectedFields: List of [Integer]; var UnavailableFields: JsonArray) var RequestedFields: JsonArray; @@ -518,16 +554,14 @@ codeunit 7241 "MDM Cross-Env Source API" begin end; - [TryFunction] - local procedure TryReadJsonArray(Value: Text; var JsonArrayValue: JsonArray) + local procedure TryReadJsonArray(Value: Text; var JsonArrayValue: JsonArray): Boolean begin - JsonArrayValue.ReadFrom(Value); + exit(JsonArrayValue.ReadFrom(Value)); end; - [TryFunction] - local procedure TryReadJsonObject(Value: Text; var JsonObjectValue: JsonObject) + local procedure TryReadJsonObject(Value: Text; var JsonObjectValue: JsonObject): Boolean begin - JsonObjectValue.ReadFrom(Value); + exit(JsonObjectValue.ReadFrom(Value)); end; local procedure ClampPageSize(PageSize: Integer): Integer @@ -577,7 +611,7 @@ codeunit 7241 "MDM Cross-Env Source API" Entry: JsonObject; begin Entry.Add('tableId', TableId); - if not TryOpenTable(TableId, RecRef) then begin + if IsBlockedSourceTable(TableId) or (not TryOpenTable(TableId, RecRef)) then begin Entry.Add('tableAvailable', false); exit(Entry); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index d1986555932..e967008b355 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -40,7 +40,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" RequestFailedTelemetryTxt: Label 'Cross-environment %1 request failed with HTTP %2.', Locked = true, Comment = '%1 = action name, %2 = HTTP status code'; TransportFailedTelemetryTxt: Label 'Cross-environment %1 request could not be sent to the source environment.', Locked = true, Comment = '%1 = action name'; - procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text var Body: JsonObject; BodyText: Text; @@ -49,6 +49,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Body.Add('fieldIds', FieldIds); Body.Add('selector', Selector); Body.Add('pageSize', PageSize); + Body.Add('filter', Filter); Body.WriteTo(BodyText); exit(InvokeAction('GetRecords', BodyText)); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al index 56e14ddf0ae..c8d76c8f30d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al @@ -20,12 +20,14 @@ codeunit 7232 "MDM Inline Media" ContentByKey: Dictionary of [Text, Text]; NameByKey: Dictionary of [Text, Text]; MimeByKey: Dictionary of [Text, Text]; + ClearedByKey: Dictionary of [Text, Boolean]; procedure Reset() begin Clear(ContentByKey); Clear(NameByKey); Clear(MimeByKey); + Clear(ClearedByKey); end; procedure Put(SystemId: Guid; FieldNo: Integer; FileName: Text; MimeType: Text; ContentBase64: Text) @@ -38,6 +40,18 @@ codeunit 7232 "MDM Inline Media" MimeByKey.Set(MediaKey, MimeType); end; + // The source reported the media field empty (cleared): record it so the transfer clears the destination media + // instead of leaving stale bytes. Distinct from an absent entry, which means "not projected / leave untouched". + procedure PutCleared(SystemId: Guid; FieldNo: Integer) + begin + ClearedByKey.Set(MakeKey(SystemId, FieldNo), true); + end; + + procedure IsCleared(SystemId: Guid; FieldNo: Integer): Boolean + begin + exit(ClearedByKey.ContainsKey(MakeKey(SystemId, FieldNo))); + end; + procedure Contains(SystemId: Guid; FieldNo: Integer): Boolean begin exit(ContentByKey.ContainsKey(MakeKey(SystemId, FieldNo))); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 77531f26f9e..0445d632da7 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -18,10 +18,9 @@ codeunit 7248 "MDM Source Response" SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; - [TryFunction] - procedure TryParse(ResponseText: Text; var Response: JsonObject) + procedure TryParse(ResponseText: Text; var Response: JsonObject): Boolean begin - Response.ReadFrom(ResponseText); + exit(Response.ReadFrom(ResponseText)); end; procedure TableAvailable(var Response: JsonObject): Boolean @@ -165,8 +164,12 @@ codeunit 7248 "MDM Source Response" LogSkippedField(TableId, FieldNo, MediaObject); exit; end; + if IsEmptyField(MediaObject) then begin + InlineMedia.PutCleared(SystemId, FieldNo); // source cleared the picture: mirror it on the destination + exit; + end; if not MediaObject.Get('content', ContentToken) then - exit; // empty source media: leave the destination picture untouched + exit; // no content and not flagged empty: leave the destination picture untouched if MediaObject.Get('name', NameToken) then FileName := NameToken.AsValue().AsText(); if MediaObject.Get('mimeType', MimeToken) then @@ -191,8 +194,13 @@ codeunit 7248 "MDM Source Response" LogSkippedField(TableId, FieldNo, BlobObject); exit; end; + if IsEmptyField(BlobObject) then begin + Clear(TempBlob); + TempBlob.ToFieldRef(DestField); // source cleared the blob: write empty so the transfer clears the destination + exit; + end; if not BlobObject.Get('content', ContentToken) then - exit; // empty source blob: leave the destination untouched + exit; // no content and not flagged empty: leave the destination untouched TempBlob.CreateOutStream(ContentOutStream); Base64Convert.FromBase64(ContentToken.AsValue().AsText(), ContentOutStream); TempBlob.ToFieldRef(DestField); @@ -207,6 +215,15 @@ codeunit 7248 "MDM Source Response" exit(false); end; + local procedure IsEmptyField(FieldObject: JsonObject): Boolean + var + Token: JsonToken; + begin + if FieldObject.Get('empty', Token) then + exit(Token.AsValue().AsBoolean()); + exit(false); + end; + // Over-cap media/blob is not synchronized (v1). We can't error (it would retry the record every run) and MDM // surfaces no synch warnings, so the skip is emitted as telemetry only; the record's other fields still sync. local procedure LogSkippedField(TableId: Integer; FieldNo: Integer; FieldObject: JsonObject) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index 4f476123db5..6a1b958678f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -2188,7 +2188,7 @@ codeunit 7233 "Master Data Management" MasterDataManagementSetup.Get(); if MasterDataManagementSetup."Source Environment Name" <> '' then begin // Cross-environment: the review only needs existence; probe the source instead of counting over the wire. - if CrossEnvDataSource.SourceHasRecords(IntegrationTableMapping."Integration Table ID") then + if CrossEnvDataSource.SourceHasRecords(IntegrationTableMapping, IntegrationTableMapping.GetIntegrationTableFilter()) then exit(1); exit(0); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index e07c3f30bc3..2353cc5c232 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -388,8 +388,16 @@ codeunit 7237 "Master Data Mgt. Subscribers" begin SourceRecordRef := SourceFieldRef.Record(); SourceSystemId := SourceRecordRef.Field(SourceRecordRef.SystemIdNo()).Value(); + // Source cleared the picture: mirror it by deleting the destination media and emptying the field. + if InlineMedia.IsCleared(SourceSystemId, SourceFieldRef.Number()) then begin + DestinationMediaId := DestinationFieldRef.Value(); + if (DestinationMediaId <> EmptyGuid) and DestinationTenantMedia.Get(DestinationMediaId) then + DestinationTenantMedia.Delete(); + NewValue := EmptyGuid; + exit(true); + end; if not InlineMedia.TryGet(SourceSystemId, SourceFieldRef.Number(), FileName, MimeType, TempBlob) then - exit(false); // no inline bytes (empty source or over-cap skip): leave the destination untouched + exit(false); // no inline bytes (over-cap skip or field not projected): leave the destination untouched SourceLength := TempBlob.Length(); DestinationMediaId := DestinationFieldRef.Value(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al index 1373efe9690..a8bea45fc13 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMSourceTransport.Interface.al @@ -9,7 +9,7 @@ interface "IMDM Source Transport" { Access = Internal; - procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text; + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text; procedure LastModifiedAtPerTable(TableIds: Text): Text; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al index 956c499eefd..b504722605e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataMgtTableMapping.TableExt.al @@ -46,8 +46,9 @@ tableextension 7235 MasterDataMgtTableMapping extends "Integration Table Mapping { // Composite (SystemModifiedAt, SystemId) resume point ({"modifiedAt":...,"systemId":...}, ~93 chars). // Empty means "caught up": the run drained the source and the watermark is authoritative. + // Internal sync resume state (the systemId is a pagination pointer, not business data), so SystemMetadata. Caption = 'Source Change Cursor'; - DataClassification = CustomerContent; + DataClassification = SystemMetadata; } } diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index b8e96dc3d5b..daa2103ff42 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -240,6 +240,17 @@ codeunit 139757 "Library - Master Data Mgt." exit(InlineMedia.Contains(SystemId, FieldNo)); end; + /// Checks whether an inline media field was marked cleared (empty on the source) for destination removal. + /// The SystemId of the source record. + /// The field number of the media field. + /// True if the field was marked cleared; otherwise false. + procedure InlineMediaIsCleared(SystemId: Guid; FieldNo: Integer): Boolean + var + InlineMedia: Codeunit "MDM Inline Media"; + begin + exit(InlineMedia.IsCleared(SystemId, FieldNo)); + end; + /// Reads the source SystemModifiedAt watermark cached during cross-environment materialization. /// The SystemId of the source record. /// Returns the cached source SystemModifiedAt. diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al index 3ac03d96fcc..0f7107f462a 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al @@ -56,13 +56,13 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" /// The cursor/systemId selector. /// The page size. /// The raw JSON records response. - procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer): Text + procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text var SourceApi: Codeunit "MDM Cross-Env Source API"; begin if UseCanned then exit(CannedResponse); - exit(SourceApi.GetRecords(TableId, FieldIds, Selector, PageSize)); + exit(SourceApi.GetRecords(TableId, FieldIds, Selector, PageSize, Filter)); end; /// Returns the last-modified-per-table probe, using the canned response if set, otherwise the real source API. diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index a2db81a65ce..1931b3c8913 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -537,6 +537,36 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvEmptyMediaFieldMarksDestinationCleared() + var + TestTableA: Record "MDM Test Table A"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] A source record whose Media field is empty is marked cleared so the destination picture is + // removed during transfer (mirroring a source deletion), instead of leaving a stale image behind. + Initialize(); + Clear(TestTableA); + TestTableA."Primary Key" := CopyStr('M' + Format(LibraryRandomInt()), 1, MaxStrLen(TestTableA."Primary Key")); + TestTableA.Insert(); // no Test Image: the source serializes the media field as empty + + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetBySystemId(Database::"MDM Test Table A", TestTableA.SystemId, SourceRecordRef), 'Record should be materialized'); + Assert.IsTrue( + LibraryMasterDataMgt.InlineMediaIsCleared(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'An empty source media field must be marked cleared so the destination picture is removed'); + Assert.IsFalse( + LibraryMasterDataMgt.InlineMediaCacheContains(TestTableA.SystemId, TestTableA.FieldNo("Test Image")), + 'An empty source media field must not cache any bytes'); + + CleanUp(); + end; + [Test] procedure CrossEnvOversizeMediaIsSkipped() var diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al index 2455fe910e4..66ff9f1ff1a 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -44,7 +44,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" Customer.Name := CopyStr(LibraryRandom.RandText(20), 1, MaxStrLen(Customer.Name)); Customer.Modify(); - Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100)); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100, '')); Response.Get('records', Token); RecordsArray := Token.AsArray(); @@ -82,7 +82,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" end; // [WHEN] the first page of size 2 is requested from the watermark - Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), CursorSelector(Watermark), 2)); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), CursorSelector(Watermark), 2, '')); // [THEN] two records come back and hasMore is true Assert.AreEqual(2, RecordCount(Response), 'First page should hold the page size'); @@ -92,7 +92,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [WHEN] the next page is requested with the returned cursor Clear(Response); - Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), NextCursor, 2)); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), NextCursor, 2, '')); // [THEN] the remaining record comes back and hasMore is false Assert.AreEqual(1, RecordCount(Response), 'Second page should hold the remaining record'); @@ -132,6 +132,53 @@ codeunit 139931 "MDM Cross-Env Source Tests" Assert.IsTrue(LastModified >= Customer.SystemModifiedAt, 'lastModifiedAt should be at least the just-created customer'); end; + [Test] + procedure GetRecordsRejectsTenantMediaInfrastructureTable() + var + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Security] + // [SCENARIO] The source API refuses to serve Tenant Media as a top-level table, so a caller holding the media + // read grant cannot enumerate blobs directly; media stays reachable only inline via a record's media field. + Response.ReadFrom(SourceApi.GetRecords(Database::"Tenant Media", FieldIdsArray(1), CursorSelector(CurrentDateTime()), 10, '')); + + // [THEN] the table is reported unavailable and no records are returned + Assert.IsFalse(GetBoolean(Response, 'tableAvailable'), 'Tenant Media must not be served as a top-level table'); + Assert.AreEqual(0, RecordCount(Response), 'A blocked table must return no records'); + end; + + [Test] + procedure GetRecordsAppliesRowFilterOnUnprojectedField() + var + MatchCustomer: Record Customer; + OtherCustomer: Record Customer; + FilterCustomer: Record Customer; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Watermark: DateTime; + SystemIds: List of [Guid]; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] The source applies the mapping row filter server-side even when it references a field outside the + // projection (parity with same-env), so only matching records are returned rather than filtered post-hoc. + Watermark := CurrentDateTime(); + Sleep(50); // ensure the seeded records sort strictly after the watermark + LibrarySales.CreateCustomer(MatchCustomer); + MatchCustomer.Blocked := MatchCustomer.Blocked::All; + MatchCustomer.Modify(); + LibrarySales.CreateCustomer(OtherCustomer); // Blocked = " ": excluded by the filter + + // [WHEN] records are fetched projecting only Name, with a filter on the (unprojected) Blocked field + FilterCustomer.SetRange(Blocked, MatchCustomer.Blocked::All); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(MatchCustomer.FieldNo(Name)), CursorSelector(Watermark), 100, FilterCustomer.GetView(false))); + + // [THEN] only the customer matching the row filter comes back + CollectResponseSystemIds(Response, SystemIds); + Assert.IsTrue(SystemIds.Contains(MatchCustomer.SystemId), 'The customer matching the row filter should be returned'); + Assert.IsFalse(SystemIds.Contains(OtherCustomer.SystemId), 'A customer outside the row filter must be excluded server-side'); + end; + local procedure FeaturesContain(var Response: JsonObject; Feature: Text): Boolean var Features: JsonArray; From e7e4948f2a314ad22b9c06c9a92b440aaa99e705 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 16:38:07 +0200 Subject: [PATCH 34/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 68e41a907a6..8231a8832c3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -58,7 +58,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" // Still non-fatal to this recurring job: skip the poll, the next scheduled run retries. if not TryNegotiateDetectionSupport(Transport, Supported) then begin Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); - Session.LogMessage('', DetectorCapabilitiesFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('0000VAZ', DetectorCapabilitiesFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); exit; end; // Older source doesn't advertise the detection action: skip rather than error every run. From 851774c63bc3e38303d986164b51feeab70313d1 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 18:13:52 +0200 Subject: [PATCH 35/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 2 ++ .../app/src/codeunits/MDMHttpSourceTransport.Codeunit.al | 1 + .../app/src/codeunits/MDMPrivacyNotice.Codeunit.al | 1 + .../app/src/codeunits/MDMSourceCapabilities.Codeunit.al | 1 + .../app/src/codeunits/MDMSourceResponse.Codeunit.al | 1 + .../app/src/codeunits/MasterDataManagement.Codeunit.al | 1 + .../app/src/tables/MasterDataManagementSetup.Table.al | 1 + 7 files changed, 8 insertions(+) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 6eda4b1f2a3..b461355446d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -242,6 +242,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.ErrorType := ErrorType::Internal; exit(ErrInfo); end; @@ -252,6 +253,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.PageNo := Page::"Master Data Synch. Tables"; ErrInfo.AddNavigationAction(OpenSynchTablesActionTxt); exit(ErrInfo); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index e967008b355..76b50509944 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -274,6 +274,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry // The remedy is always the setup page, even before the record exists; only RecordId needs an existing record. ErrInfo.PageNo := Page::"Master Data Management Setup"; ErrInfo.AddNavigationAction(OpenSetupActionTxt); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al index 192b1eb2173..ce16acf7723 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al @@ -59,6 +59,7 @@ codeunit 7242 "MDM Privacy Notice" if IsApproved() then exit; ErrInfo.Message := NotApprovedErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry if MasterDataManagementSetup.Get() then begin ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); ErrInfo.PageNo := Page::"Master Data Management Setup"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index d3409a1efc3..ae863ba2ba2 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -52,6 +52,7 @@ codeunit 7246 "MDM Source Capabilities" ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.ErrorType := ErrorType::Internal; exit(ErrInfo); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 0445d632da7..f637c1470cb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -317,6 +317,7 @@ codeunit 7248 "MDM Source Response" if not Converted then begin // Internal contract failure the user can't fix: detail goes to telemetry, user sees a generic dialog. ErrInfo.Message := StrSubstNo(BadFieldValueErr, DestField.Caption(), Format(DestField.Type())); + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.ErrorType := ErrorType::Internal; Error(ErrInfo); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index 6a1b958678f..467363b396e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -2004,6 +2004,7 @@ codeunit 7233 "Master Data Management" ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.ErrorType := ErrorType::Internal; exit(ErrInfo); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 84ad15e22bc..f24efeeb99a 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -378,6 +378,7 @@ table 7230 "Master Data Management Setup" ErrInfo: ErrorInfo; begin ErrInfo.Message := MustConfigureConnectionErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.RecordId := Rec.RecordId(); ErrInfo.PageNo := Page::"Master Data Management Setup"; ErrInfo.AddNavigationAction(OpenSetupNavigationTxt); From c69360f0d62f8f378cd9063f038d5e77969ca4a7 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 19:23:49 +0200 Subject: [PATCH 36/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index b461355446d..5cb2d08c1e8 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -292,7 +292,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then begin LogParseFailure(IntegrationTableId, FieldsUnavailableReasonTok); - Error(FieldsUnavailableErr, TableCaption(IntegrationTableId)); + Error(SynchTablesNavigationError(StrSubstNo(FieldsUnavailableErr, TableCaption(IntegrationTableId)))); end; end; From 794484ba8620dd330326c047a684ca2ac53923c9 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Mon, 31 Aug 2026 21:47:37 +0200 Subject: [PATCH 37/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvChangeDetector.Codeunit.al | 23 +++++++++- .../MDMCrossEnvDataSource.Codeunit.al | 21 ++++++++- .../codeunits/MDMSourceResponse.Codeunit.al | 45 +++++++++++++------ 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 8231a8832c3..28398d181d0 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -227,21 +227,38 @@ codeunit 7245 "MDM Cross-Env Change Detector" local procedure GetInteger(var Container: JsonObject; PropertyName: Text): Integer var Token: JsonToken; + Value: Integer; begin + // A parseable but malformed entry must not throw and kill the recurring job: skip it (returns 0). if Container.Get(PropertyName, Token) then - exit(Token.AsValue().AsInteger()); + if Token.IsValue() and TryReadInteger(Token, Value) then + exit(Value); exit(0); end; + [TryFunction] + local procedure TryReadInteger(Token: JsonToken; var Value: Integer) + begin + Value := Token.AsValue().AsInteger(); + end; + local procedure GetBoolean(var Container: JsonObject; PropertyName: Text; DefaultValue: Boolean): Boolean var Token: JsonToken; + Value: Boolean; begin if Container.Get(PropertyName, Token) then - exit(Token.AsValue().AsBoolean()); + if Token.IsValue() and TryReadBoolean(Token, Value) then + exit(Value); exit(DefaultValue); end; + [TryFunction] + local procedure TryReadBoolean(Token: JsonToken; var Value: Boolean) + begin + Value := Token.AsValue().AsBoolean(); + end; + local procedure GetDateTime(var Container: JsonObject; PropertyName: Text; var Value: DateTime): Boolean var Token: JsonToken; @@ -249,6 +266,8 @@ codeunit 7245 "MDM Cross-Env Change Detector" begin if not Container.Get(PropertyName, Token) then exit(false); + if not Token.IsValue() then + exit(false); ValueText := Token.AsValue().AsText(); if ValueText = '' then exit(false); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 5cb2d08c1e8..5fd77ed3881 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -140,9 +140,14 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); end; Entry := Token.AsObject(); - if Entry.Get('tableAvailable', Token) then + if Entry.Get('tableAvailable', Token) then begin + if not Token.IsValue() then begin // malformed contract, not an empty source: keep it in the internal-error path + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; if not Token.AsValue().AsBoolean() then exit(false); + end; // Unindexed source table: LastModifiedAtPerTable reports indexed:false and no timestamp, so we can't prove // emptiness cheaply - assume records may exist so the full-synch review isn't wrongly suppressed. if Entry.Get('indexed', Token) then @@ -169,7 +174,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Transport := GetTransport(); SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); // '{}' selector = read from the start (no watermark); PageSize 1 keeps this an existence check, not a count. - if not SourceResponse.TryParse(Transport.GetRecords(IntegrationTableId, BuildFieldIds(IntegrationTableMapping), '{}', 1, TableFilter), Response) then begin + // Project only the primary key: the row filter is applied server-side, so no mapped media/blob need materialize. + if not SourceResponse.TryParse(Transport.GetRecords(IntegrationTableId, BuildPrimaryKeyFieldIds(IntegrationTableId), '{}', 1, TableFilter), Response) then begin LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; @@ -314,6 +320,17 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(WriteArray(FieldIds)); end; + // Primary-key-only projection for the existence probe: the server-side filter still evaluates against every field, + // so no mapped fields (in particular media/blob) need to be projected just to learn whether a record exists. + local procedure BuildPrimaryKeyFieldIds(IntegrationTableId: Integer): Text + var + FieldIds: JsonArray; + AddedFields: List of [Integer]; + begin + AddPrimaryKeyFields(IntegrationTableId, FieldIds, AddedFields); + exit(WriteArray(FieldIds)); + end; + local procedure BuildFieldIdsForTable(IntegrationTableId: Integer): Text var IntegrationTableMapping: Record "Integration Table Mapping"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index f637c1470cb..931b53e2fdb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -17,6 +17,7 @@ codeunit 7248 "MDM Source Response" SourceWatermark: Codeunit "MDM Source Watermark"; SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; + MalformedControlFieldErr: Label 'The source returned a malformed value for the response control field ''%1''.', Comment = '%1 = response control field name'; procedure TryParse(ResponseText: Text; var Response: JsonObject): Boolean begin @@ -24,22 +25,14 @@ codeunit 7248 "MDM Source Response" end; procedure TableAvailable(var Response: JsonObject): Boolean - var - Token: JsonToken; begin - if Response.Get('tableAvailable', Token) then - exit(Token.AsValue().AsBoolean()); - exit(true); + exit(ReadControlBoolean(Response, 'tableAvailable', true)); end; procedure Indexed(var Response: JsonObject): Boolean - var - Token: JsonToken; begin // 'indexed' is only emitted when false (a too-large unindexed/keyless table). - if Response.Get('indexed', Token) then - exit(Token.AsValue().AsBoolean()); - exit(true); + exit(ReadControlBoolean(Response, 'indexed', true)); end; procedure GetUnavailableFields(var Response: JsonObject; var UnavailableFields: JsonArray): Boolean @@ -55,12 +48,38 @@ codeunit 7248 "MDM Source Response" end; procedure HasMore(var Response: JsonObject): Boolean + begin + exit(ReadControlBoolean(Response, 'hasMore', false)); + end; + + // Control fields (tableAvailable/indexed/hasMore) are always booleans in the contract; a present-but-malformed + // token is an internal contract failure, so surface it as an internal diagnostic instead of a raw runtime throw. + local procedure ReadControlBoolean(var Response: JsonObject; PropertyName: Text; DefaultValue: Boolean): Boolean var Token: JsonToken; + Value: Boolean; begin - if Response.Get('hasMore', Token) then - exit(Token.AsValue().AsBoolean()); - exit(false); + if not Response.Get(PropertyName, Token) then + exit(DefaultValue); + if not (Token.IsValue() and TryReadBoolean(Token, Value)) then + Error(MalformedControlField(PropertyName)); + exit(Value); + end; + + [TryFunction] + local procedure TryReadBoolean(Token: JsonToken; var Value: Boolean) + begin + Value := Token.AsValue().AsBoolean(); + end; + + local procedure MalformedControlField(PropertyName: Text): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := StrSubstNo(MalformedControlFieldErr, PropertyName); + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); end; // The nextCursor object, re-serialized so the caller can pass it straight back as the next Selector. From 90a47de67aca774aa59f398b385e6dfd6078b41b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 06:33:51 +0200 Subject: [PATCH 38/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvDataSource.Codeunit.al | 6 +++ .../MDMSourceCapabilities.Codeunit.al | 38 +++++++++++++------ .../interfaces/IMDMDataSource.Interface.al | 7 +++- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 23 ++++++++--- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 5fd77ed3881..2f5b815ddc0 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -180,6 +180,12 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; // An unavailable table yields no records array; treat as no matching records for the review. + if not SourceResponse.TableAvailable(Response) then + exit(false); + // An unindexed source past the change cap returns indexed:false with an empty page; like the unfiltered probe + // we can't prove emptiness cheaply, so assume records may exist rather than under-reporting to the review. + if not SourceResponse.Indexed(Response) then + exit(true); if Response.Get('records', Token) and Token.IsArray() then exit(Token.AsArray().Count() > 0); exit(false); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index ae863ba2ba2..bd29f364a5b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -78,19 +78,35 @@ codeunit 7246 "MDM Source Capabilities" Clear(SupportedFeatures); // Don't cache a failed parse as a successful (empty) negotiation - that would surface as a misleading // "capability unsupported / update the source app" error instead of the real bad-response problem. - if not Capabilities.ReadFrom(Transport.GetCapabilities()) then begin - Session.LogMessage('0000VAV', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); - Error(InternalError(CapabilitiesParseErr)); - end; + if not Capabilities.ReadFrom(Transport.GetCapabilities()) then + RaiseCapabilitiesParseError(MasterDataManagement); + // A present-but-malformed version/features (wrong token kind or non-integer version) is a contract failure, + // not a valid negotiation: keep it on the internal-diagnostic path instead of throwing a raw runtime error. if Capabilities.Get('version', VersionToken) then - if VersionToken.IsValue() then - ContractVersion := VersionToken.AsValue().AsInteger(); - if Capabilities.Get('features', FeaturesToken) then - if FeaturesToken.IsArray() then - foreach FeatureToken in FeaturesToken.AsArray() do - if FeatureToken.IsValue() then - SupportedFeatures.Add(FeatureToken.AsValue().AsText()); + if not (VersionToken.IsValue() and TryReadInteger(VersionToken, ContractVersion)) then + RaiseCapabilitiesParseError(MasterDataManagement); + if Capabilities.Get('features', FeaturesToken) then begin + if not FeaturesToken.IsArray() then + RaiseCapabilitiesParseError(MasterDataManagement); + foreach FeatureToken in FeaturesToken.AsArray() do begin + if not FeatureToken.IsValue() then + RaiseCapabilitiesParseError(MasterDataManagement); + SupportedFeatures.Add(FeatureToken.AsValue().AsText()); + end; + end; Negotiated := true; NegotiatedForUrl := CurrentSource; end; + + local procedure RaiseCapabilitiesParseError(MasterDataManagement: Codeunit "Master Data Management") + begin + Session.LogMessage('0000VAV', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Error(InternalError(CapabilitiesParseErr)); + end; + + [TryFunction] + local procedure TryReadInteger(Token: JsonToken; var Value: Integer) + begin + Value := Token.AsValue().AsInteger(); + end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al index 1219fbc96d5..2753cc5a4db 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al +++ b/src/Apps/W1/MasterDataManagement/app/src/interfaces/IMDMDataSource.Interface.al @@ -24,8 +24,11 @@ interface "IMDM Data Source" procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean; /// - /// Fetches a single source integration-table record by its id (the integration UID field value, - /// a RecordId, or a business-key text) into SourceRecordRef. Returns true if found. + /// Fetches a single source integration-table record by its identifier into SourceRecordRef. + /// The identifier is the integration UID field value - for Master Data Management the SystemId - + /// passed as a Guid or its text form. A RecordId is environment-specific and only resolvable by the + /// local same-environment implementation; the cross-environment feed keys on SystemId, so it returns + /// false for a RecordId. Returns true if found. /// procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 1931b3c8913..f145fcfc259 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -10,6 +10,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" Assert: Codeunit Assert; LibrarySalesLib: Codeunit "Library - Sales"; WizardPrivacyNoticeOpenCount: Integer; + InvalidSourceHostErr: Label 'not a valid Business Central endpoint', Locked = true; [Test] procedure CrossEnvGetBySystemIdRoundTripsSourceRecord() @@ -61,15 +62,15 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [GIVEN] a non-HTTPS scheme [THEN] validation is rejected asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('http://myenv.api.bc.dynamics.com'); - Assert.ExpectedError('not a valid Business Central endpoint'); + Assert.ExpectedError(InvalidSourceHostErr); // [GIVEN] a host outside the dynamics.com allow-list [THEN] validation is rejected asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://evil.example.com'); - Assert.ExpectedError('not a valid Business Central endpoint'); + Assert.ExpectedError(InvalidSourceHostErr); // [GIVEN] a look-alike host that only embeds dynamics.com as a non-final label [THEN] validation is rejected asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.dynamics.com.evil.example.com'); - Assert.ExpectedError('not a valid Business Central endpoint'); + Assert.ExpectedError(InvalidSourceHostErr); CleanUp(); end; @@ -340,6 +341,8 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" Assert.AreEqual('CRONUS', MasterDataManagementSetup."Source Company Name", 'Source company not saved'); Assert.IsFalse(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Client secret should be stored'); + // Restore privacy state so this configuration test doesn't leak consent into later privacy-notice scenarios. + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); CleanUp(); end; @@ -410,9 +413,11 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" SeededSystemId: Guid; SeededSystemIds: List of [Guid]; CollectedSystemIds: List of [Guid]; + PerRunCounts: List of [Integer]; Watermark: DateTime; Cursor: Text; EndCursor: Text; + PreviousCursor: Text; Index: Integer; Runs: Integer; HasMore: Boolean; @@ -440,14 +445,22 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [WHEN] the batch is drained one page per run, resuming from the returned cursor Cursor := ''; repeat + PreviousCursor := Cursor; LibraryMasterDataMgt.DataSourceGetModifiedBatch(IntegrationTableMapping, '', Cursor, 1, SourceRecordRef, EndCursor, HasMore); + // Each resumed run must advance from the cursor it was given - never restart from the start or stall. + Assert.AreNotEqual(PreviousCursor, EndCursor, 'Each resumed run must advance the cursor'); + PerRunCounts.Add(SourceRecordRef.Count()); CollectSystemIds(SourceRecordRef, CollectedSystemIds); Cursor := EndCursor; Runs += 1; until not HasMore; - // [THEN] it took multiple runs and every seeded record was returned exactly once - Assert.IsTrue(Runs >= 3, 'A 5-record set at 2/page and 1 page/run should need at least three runs'); + // [THEN] the set drained in a stable 2 + 2 + 1 sequence across exactly three resumed runs + Assert.AreEqual(3, Runs, 'A 5-record set at 2/page and 1 page/run must take exactly three runs'); + Assert.AreEqual(2, PerRunCounts.Get(1), 'First run should return a full page of two records'); + Assert.AreEqual(2, PerRunCounts.Get(2), 'Second run should return a full page of two records'); + Assert.AreEqual(1, PerRunCounts.Get(3), 'Final run should return the remaining single record'); + // [THEN] every seeded record was returned exactly once (no overlap, no gaps) Assert.AreEqual(5, CollectedSystemIds.Count(), 'Every seeded record should be returned exactly once across runs'); foreach SeededSystemId in SeededSystemIds do Assert.IsTrue(CollectedSystemIds.Contains(SeededSystemId), 'Every seeded record should be covered by the resumed batches'); From 76ec8cac82248008efcb82ab56dcaac1c06d5ea5 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 06:46:22 +0200 Subject: [PATCH 39/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtTableCouple.Codeunit.al | 22 +++++++++---------- .../pages/MasterDataManagementSetup.Page.al | 2 +- .../src/pages/MasterDataSynchTables.Page.al | 2 +- .../tables/MasterDataManagementSetup.Table.al | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al index 1173abcd072..cbd2f49efe8 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al @@ -283,27 +283,27 @@ codeunit 7235 "Master Data Mgt. Table Couple" exit(StrSubstNo(NoMatchFoundErr, GetIntegrationOrgCompanyName(), ErrorCount)); end; - local procedure GetNoMatchFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text + local procedure GetNoMatchFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text var MatchingFieldNameList: Text; begin - MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, MatchIntegrationFieldMapping); + MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); exit(StrSubstNo(NoMatchFoundTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); end; - local procedure GetMultipleMatchesFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text + local procedure GetMultipleMatchesFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text var MatchingFieldNameList: Text; begin - MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, MatchIntegrationFieldMapping); + MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); exit(StrSubstNo(MultipleMatchesFoundTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); end; - local procedure GetSingleMatchAlreadyCoupledTelemetryErrorMessage(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text + local procedure GetSingleMatchAlreadyCoupledTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text var MatchingFieldNameList: Text; begin - MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, MatchIntegrationFieldMapping); + MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); exit(StrSubstNo(SingleMatchAlreadyCoupledTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); end; @@ -314,15 +314,15 @@ codeunit 7235 "Master Data Mgt. Table Couple" exit(IntegrationTableMapping.Name); end; - local procedure GetMatchingFieldNameList(var LocalRecordRef: RecordRef; var MatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary) MatchingFieldNameList: Text + local procedure GetMatchingFieldNameList(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary) MatchingFieldNameList: Text begin - MatchIntegrationFieldMapping.FindSet(); + TempMatchIntegrationFieldMapping.FindSet(); repeat if MatchingFieldNameList = '' then - MatchingFieldNameList := LocalRecordRef.Field(MatchIntegrationFieldMapping."Field No.").Name() + MatchingFieldNameList := LocalRecordRef.Field(TempMatchIntegrationFieldMapping."Field No.").Name() else - MatchingFieldNameList += ', ' + LocalRecordRef.Field(MatchIntegrationFieldMapping."Field No.").Name() - until MatchIntegrationFieldMapping.Next() = 0; + MatchingFieldNameList += ', ' + LocalRecordRef.Field(TempMatchIntegrationFieldMapping."Field No.").Name() + until TempMatchIntegrationFieldMapping.Next() = 0; end; local procedure GetIntegrationOrgCompanyName(): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index 0f774141385..37edc6d7582 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -238,7 +238,7 @@ page 7230 "Master Data Management Setup" trigger OnQueryClosePage(CloseAction: Action): Boolean begin if not Rec."Is Enabled" then - if not Confirm(StrSubstNo(EnableServiceQst, CurrPage.Caption()), true) then + if not Confirm(EnableServiceQst, true, CurrPage.Caption()) then exit(false); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al index d437aac1c68..6e7f859550d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al @@ -95,7 +95,7 @@ page 7233 "Master Data Synch. Tables" IntegrationFieldMapping.SetRange("Integration Table Mapping Name", IntegrationTableMapping.Name); if RelatedTablesToAdd.Count() > 0 then - if Confirm(StrSubstno(RelatedTablesQst, RelatedTablesToAddText)) then begin + if Confirm(RelatedTablesQst, false, RelatedTablesToAddText) then begin IntegrationTableMapping.Validate(Status, IntegrationTableMapping.Status::Disabled); IntegrationTableMapping.Modify(); foreach RelatedTableNo in RelatedTablesToAdd do diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index f24efeeb99a..246b42835bc 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -72,7 +72,7 @@ table 7230 "Master Data Management Setup" if (xRec."Company Name" <> '') and (xRec."Company Name" <> Rec."Company Name") then if not MasterDataMgtCoupling.IsEmpty() then - if not Confirm(StrSubstNo(CouplingsWillBeDeletedQst, xRec."Company Name")) then + if not Confirm(CouplingsWillBeDeletedQst, false, xRec."Company Name") then Error(''); CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); @@ -311,7 +311,7 @@ table 7230 "Master Data Management Setup" UpdateDataSynchJobQueueEntriesStatus(); if not MasterDataMgtCoupling.IsEmpty() then - if Confirm(StrSubstNo(KeepTheCouplingsQst, Rec."Company Name")) then + if Confirm(KeepTheCouplingsQst, false, Rec."Company Name") then exit else begin IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); From b34df3d8fb08d095a73b968c261e183edbd4f72d Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 10:47:41 +0200 Subject: [PATCH 40/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvChangeDetector.Codeunit.al | 3 ++- .../MDMCrossEnvDataSource.Codeunit.al | 10 +++++++--- .../MDMSourceCapabilities.Codeunit.al | 2 +- .../codeunits/MDMSourceResponse.Codeunit.al | 19 ++++++++++++++++--- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 1 + .../src/MDMCrossEnvSourceTests.Codeunit.al | 19 +++++++++++-------- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 28398d181d0..6121a7e2deb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -130,7 +130,8 @@ codeunit 7245 "MDM Cross-Env Change Detector" end; Tables := TablesToken.AsArray(); foreach EntryToken in Tables do - ProcessTableEntry(EntryToken.AsObject()); + if EntryToken.IsObject() then // skip a malformed non-object entry instead of aborting the recurring job + ProcessTableEntry(EntryToken.AsObject()); end; local procedure ProcessTableEntry(Entry: JsonObject) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 2f5b815ddc0..d3dd0c753b9 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -260,13 +260,17 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; // The table isn't exposed on the source: a recoverable setup issue, so point the user at Synchronization Tables. - local procedure SynchTablesNavigationError(MessageText: Text): ErrorInfo + local procedure SynchTablesNavigationError(IntegrationTableId: Integer; MessageText: Text): ErrorInfo var + IntegrationTableMapping: Record "Integration Table Mapping"; ErrInfo: ErrorInfo; begin ErrInfo.Message := MessageText; ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry ErrInfo.PageNo := Page::"Master Data Synch. Tables"; + // Land on the specific mapping row the user must fix, when it resolves to one. + if GetMappingByIntegrationTableId(IntegrationTableId, IntegrationTableMapping) then + ErrInfo.RecordId := IntegrationTableMapping.RecordId(); ErrInfo.AddNavigationAction(OpenSynchTablesActionTxt); exit(ErrInfo); end; @@ -296,7 +300,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; if not SourceResponse.TableAvailable(Response) then begin LogParseFailure(IntegrationTableId, TableUnavailableReasonTok); - Error(SynchTablesNavigationError(StrSubstNo(TableUnavailableErr, TableCaption(IntegrationTableId)))); + Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(TableUnavailableErr, TableCaption(IntegrationTableId)))); end; if not SourceResponse.Indexed(Response) then begin LogParseFailure(IntegrationTableId, NotIndexedReasonTok); @@ -304,7 +308,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then begin LogParseFailure(IntegrationTableId, FieldsUnavailableReasonTok); - Error(SynchTablesNavigationError(StrSubstNo(FieldsUnavailableErr, TableCaption(IntegrationTableId)))); + Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(FieldsUnavailableErr, TableCaption(IntegrationTableId)))); end; end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al index bd29f364a5b..e0f0f724f56 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceCapabilities.Codeunit.al @@ -100,7 +100,7 @@ codeunit 7246 "MDM Source Capabilities" local procedure RaiseCapabilitiesParseError(MasterDataManagement: Codeunit "Master Data Management") begin - Session.LogMessage('0000VAV', CapabilitiesParseTelemetryTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAV', CapabilitiesParseTelemetryTxt, Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); Error(InternalError(CapabilitiesParseErr)); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 931b53e2fdb..41b337726e5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -18,6 +18,7 @@ codeunit 7248 "MDM Source Response" SkippedFieldTxt: Label 'Cross-environment media or blob field exceeds the inline size cap and was not synchronized.', Locked = true; BadFieldValueErr: Label 'The source returned a value for field %1 that could not be converted to the expected type %2.', Comment = '%1 - a field caption, %2 - a field type'; MalformedControlFieldErr: Label 'The source returned a malformed value for the response control field ''%1''.', Comment = '%1 = response control field name'; + MalformedRecordErr: Label 'The source returned a malformed record entry.', Locked = true; procedure TryParse(ResponseText: Text; var Response: JsonObject): Boolean begin @@ -82,6 +83,16 @@ codeunit 7248 "MDM Source Response" exit(ErrInfo); end; + local procedure MalformedRecordEntry(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MalformedRecordErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); + end; + // The nextCursor object, re-serialized so the caller can pass it straight back as the next Selector. procedure GetNextCursor(var Response: JsonObject): Text var @@ -110,6 +121,8 @@ codeunit 7248 "MDM Source Response" exit(0); RecordsArray := RecordsToken.AsArray(); foreach RecordToken in RecordsArray do begin + if not RecordToken.IsObject() then + Error(MalformedRecordEntry()); InsertRecord(RecordToken.AsObject(), TempSourceRecordRef); Count += 1; end; @@ -254,10 +267,10 @@ codeunit 7248 "MDM Source Response" // The source record identifier (systemId) is customer data, so it is not emitted; only the table, field, // and length (non-identifying diagnostics) are logged. Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); - Dimensions.Add('tableId', Format(TableId)); - Dimensions.Add('fieldNo', Format(FieldNo)); + Dimensions.Add('TableId', Format(TableId)); + Dimensions.Add('FieldNo', Format(FieldNo)); if FieldObject.Get('length', LengthToken) then - Dimensions.Add('length', Format(LengthToken.AsValue().AsBigInteger())); + Dimensions.Add('Length', Format(LengthToken.AsValue().AsBigInteger())); Session.LogMessage('0000VAW', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index f145fcfc259..525af86334d 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -163,6 +163,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [WHEN] fetched via GetById with a text SystemId [THEN] the record materializes (exercises the variant->SystemId conversion) Assert.IsTrue(LibraryMasterDataMgt.DataSourceGetById(IntegrationTableMapping, Format(Customer1.SystemId), SourceRecordRef), 'GetById should return the record'); + Assert.AreEqual(1, SourceRecordRef.Count(), 'GetById should return exactly one record'); Assert.IsTrue(ContainsSystemId(SourceRecordRef, Customer1.SystemId), 'GetById should include the requested customer'); CleanUp(); diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al index 66ff9f1ff1a..0899d69419c 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -69,14 +69,15 @@ codeunit 139931 "MDM Cross-Env Source Tests" NextCursor: Text; Index: Integer; SeededSystemIds: List of [Guid]; - PagedSystemIds: List of [Guid]; - SeededSystemId: Guid; + Page1SystemIds: List of [Guid]; + Page2SystemIds: List of [Guid]; begin // [FEATURE] [AI test 0.4] // [SCENARIO] Cursor mode pages ascending by (SystemModifiedAt, SystemId) and reports hasMore / nextCursor. Watermark := CurrentDateTime(); Sleep(50); // ensure the seeded records sort strictly after the watermark for Index := 1 to 3 do begin + Sleep(20); // distinct, strictly increasing SystemModifiedAt so creation order == cursor order LibrarySales.CreateCustomer(Customer); SeededSystemIds.Add(Customer.SystemId); end; @@ -87,7 +88,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [THEN] two records come back and hasMore is true Assert.AreEqual(2, RecordCount(Response), 'First page should hold the page size'); Assert.IsTrue(GetBoolean(Response, 'hasMore'), 'hasMore should be true while records remain'); - CollectResponseSystemIds(Response, PagedSystemIds); + CollectResponseSystemIds(Response, Page1SystemIds); NextCursor := NextCursorText(Response); // [WHEN] the next page is requested with the returned cursor @@ -97,12 +98,14 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [THEN] the remaining record comes back and hasMore is false Assert.AreEqual(1, RecordCount(Response), 'Second page should hold the remaining record'); Assert.IsFalse(GetBoolean(Response, 'hasMore'), 'hasMore should be false on the last page'); - CollectResponseSystemIds(Response, PagedSystemIds); + CollectResponseSystemIds(Response, Page2SystemIds); - // [THEN] the two pages together returned every seeded record exactly once - no repeats, no dropped/reordered records - Assert.AreEqual(3, PagedSystemIds.Count(), 'Paging should return each record exactly once across the two pages'); - foreach SeededSystemId in SeededSystemIds do - Assert.IsTrue(PagedSystemIds.Contains(SeededSystemId), 'Every seeded record should appear in the paged results'); + // [THEN] the pages follow the ascending (SystemModifiedAt, SystemId) order: first two seeded on page one, last on page two + Assert.AreEqual(2, Page1SystemIds.Count(), 'First page should contain exactly two records'); + Assert.AreEqual(SeededSystemIds.Get(1), Page1SystemIds.Get(1), 'First page, first record should be the earliest-modified customer'); + Assert.AreEqual(SeededSystemIds.Get(2), Page1SystemIds.Get(2), 'First page, second record should be the second-earliest customer'); + Assert.AreEqual(1, Page2SystemIds.Count(), 'Second page should contain exactly one record'); + Assert.AreEqual(SeededSystemIds.Get(3), Page2SystemIds.Get(1), 'Second page should contain the latest-modified customer'); end; [Test] From b2d2b53f3c1d7368fdbf4f4016f3d2b8bf5c72a8 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 10:53:35 +0200 Subject: [PATCH 41/64] [Master Data Management] Cross-environment synchonization --- .../app/src/pages/MasterDataSynchTables.Page.al | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al index 6e7f859550d..7a41a42c71e 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al @@ -80,14 +80,14 @@ page 7233 "Master Data Synch. Tables" Error(TableMetadataNotFoundErr, AllObjWithCaption."Object ID"); if not TableMetadata.DataPerCompany then - Error(TableNotPerCompanyErr, AllObjWithCaption."Object Name"); + Error(TableNotPerCompanyErr, AllObjWithCaption."Object Caption"); if TableMetadata.TableType <> TableMetadata.TableType::Normal then - Error(TableNotOfTypeNormalErr, AllObjWithCaption."Object Name"); + Error(TableNotOfTypeNormalErr, AllObjWithCaption."Object Caption"); RecRef.Open(AllObjWithCaption."Object ID"); if not RecRef.WritePermission() then - Error(TablePermissionMissingErr, AllObjWithCaption."Object Name"); + Error(TablePermissionMissingErr, AllObjWithCaption."Object Caption"); RecRef.Close(); FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, AllObjWithCaption."Object ID"); @@ -105,7 +105,7 @@ page 7233 "Master Data Synch. Tables" IntegrationTableMapping.Validate(Status, IntegrationTableMapping.Status::Disabled); IntegrationTableMapping.Modify(); end; - Message(StrSubstNo(RelatedTablesAddedMsg, AllObjWithCaption."Object Name", RelatedTablesToAddText)); + Message(StrSubstNo(RelatedTablesAddedMsg, AllObjWithCaption."Object Caption", RelatedTablesToAddText)); exit; end; @@ -543,9 +543,9 @@ page 7233 "Master Data Synch. Tables" if RecRef.WritePermission() then begin RelatedTablesToAdd.Add(Field.RelationTableNo); if RelatedTablesToAddText = '' then - RelatedTablesToAddText := TableMetadata.Name + RelatedTablesToAddText := RecRef.Caption() else - RelatedTablesToAddText += ', ' + TableMetadata.Name; + RelatedTablesToAddText += ', ' + RecRef.Caption(); FindRelatedTables(ExistingSynchTableNos, RelatedTablesToAdd, RelatedTablesToAddText, Field.RelationTableNo, TopLevelTableId); end; RecRef.Close(); @@ -655,8 +655,8 @@ page 7233 "Master Data Synch. Tables" UserEditedIntegrationTableFilterTxt: Label 'The user edited the Integration Table Filter on %1 mapping.', Locked = true; EditIntegrationTableFilterTxt: Label ''; NoCoupledRecordsMsg: label 'No records of this table are currently coupled to records from the source company. \\Choose the action Run Full Synchronization.'; - RelatedTablesQst: label 'The chosen table has a relation to the following tables that are currently not included in the synchronization: %1. \\Do you want to synchronize these tables too?', Comment = '%1 - comma-separated list of table names'; - RelatedTablesAddedMsg: label 'Table %1 and related tables: %2 are added to the synchronization with state set to Disabled. \\Open Synchronization Tables page, choose synchronization fields for each of the added tables and then set their status to Enabled.', Comment = '%1 - a table name, %2 - comma-separated list of table names'; + RelatedTablesQst: label 'The chosen table has a relation to the following tables that are currently not included in the synchronization: %1. \\Do you want to synchronize these tables too?', Comment = '%1 - comma-separated list of table captions'; + RelatedTablesAddedMsg: label 'Table %1 and related tables: %2 are added to the synchronization with state set to Disabled. \\Open Synchronization Tables page, choose synchronization fields for each of the added tables and then set their status to Enabled.', Comment = '%1 - a table caption, %2 - comma-separated list of table captions'; TableMetadataNotFoundErr: label 'Metadata for table %1 cannot be loaded. Choose another table.', Comment = '%1 - a table name'; TableNotPerCompanyErr: label 'Table %1 is shared across all companies of this environment. Choose another table.', Comment = '%1 - a table name'; TableNotOfTypeNormalErr: label 'Table %1 is either declared as temporary, a query or as an interface for accessing an external entity. Choose another table.', Comment = '%1 - a table name'; From f5cb39566e6932e244fd60659bc1bbf336c0885b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 12:42:05 +0200 Subject: [PATCH 42/64] [Master Data Management] Cross-environment synchonization --- .../MDMHttpSourceTransport.Codeunit.al | 42 +++++++++++++++---- .../src/pages/MDMConnectionDetails.Page.al | 2 +- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 24 +++++++---- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 76b50509944..22573476564 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -9,8 +9,9 @@ using System.Utilities; /// /// Production transport: calls the source environment's ODataV4 web service with an app-only (client -/// credentials) token. Same-tenant by construction — the OAuth authority is derived from THIS environment's -/// Entra tenant, so there is no tenant-id setting to point the connection at another tenant. Tests never hit +/// credentials) token. Same-tenant by construction — the OAuth authority (tenant + ring) is derived from THIS +/// environment, so there is no tenant-id setting to point the connection at another tenant; production and the +/// TIE/PPE ring use different Entra authorities and Business Central resource audiences. Tests never hit /// this: they inject an in-process transport that calls the source API directly. /// codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" @@ -31,7 +32,11 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" HttpErr: Label 'The source environment returned HTTP %1.', Comment = '%1 = HTTP status code'; ServiceNameTok: Label 'MDMCrossEnvSource', Locked = true; ScopeTok: Label 'https://api.businesscentral.dynamics.com/.default', Locked = true; + ScopePPETok: Label 'https://api.businesscentral.dynamics-tie.com/.default', Locked = true; TokenEndpointTok: Label 'https://login.microsoftonline.com/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; + TokenEndpointPPETok: Label 'https://login.windows-ppe.net/%1/oauth2/v2.0/token', Locked = true, Comment = '%1 = Entra tenant id'; + SourceHostProdTok: Label 'api.businesscentral.dynamics.com', Locked = true; + SourceHostPPETok: Label 'api.businesscentral.dynamics-tie.com', Locked = true; ActionUrlTok: Label '%1/ODataV4/%2_%3?company=%4', Locked = true, Comment = '%1 = base url, %2 = service, %3 = action, %4 = company'; TelemetryCategoryTok: Label 'MDM Cross-Environment', Locked = true; TokenAcquiredAuditTxt: Label 'Acquired an application access token to read master data from source environment %1.', Comment = '%1 = source environment name'; @@ -168,11 +173,13 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Uri: Codeunit Uri; Host: Text; begin - // Embed/ISV clusters vary in hostname but always end with dynamics.com; dynamics-tie.com is the test (TIE) ring. - // A malformed URL must surface the actionable setup error, not the URI parser's raw exception. + // Only the standard Business Central API hosts are allowed - production and the TIE ring. Embed/ISV + // deployments must be on a normal cluster that serves api.businesscentral.dynamics.com. Matching the host + // EXACTLY (not a dynamics.com suffix) blocks SSRF via look-alike hosts; a malformed URL surfaces the + // actionable setup error instead of the URI parser's raw exception. if TryInitUri(Uri, BaseUrl) then begin Host := LowerCase(Uri.GetHost()); - if (Uri.GetScheme() = 'https') and (Host.EndsWith('.dynamics.com') or Host.EndsWith('.dynamics-tie.com')) then + if (Uri.GetScheme() = 'https') and ((Host = SourceHostProdTok) or (Host = SourceHostPPETok)) then exit; end; AuditLog.LogAuditMessage(StrSubstNo(InvalidSourceUrlAuditTxt, Host), SecurityOperationResult::Failure, AuditCategory::Authorization, 4, 0); @@ -230,8 +237,8 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Scopes: List of [Text]; TokenEndpoint: Text; begin - Scopes.Add(ScopeTok); - TokenEndpoint := StrSubstNo(TokenEndpointTok, AzureADTenant.GetAadTenantId()); + Scopes.Add(GetBcResourceScope()); + TokenEndpoint := StrSubstNo(GetTokenEndpointTemplate(), AzureADTenant.GetAadTenantId()); // Prefer a cached/refreshed token (no round-trip when a valid one exists); fall back to a fresh // client-credentials grant if the cache misses, errors, or returns an empty token. if not TryAcquireTokenFromCache(MasterDataManagementSetup, TokenEndpoint, Scopes, Token) then @@ -260,6 +267,27 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Clear(Token); end; + // Production and the TIE/PPE ring use different Entra authorities and BC resource audiences. Cross-env is same + // tenant and same ring, so detect the ring from this environment's web URL and mint the token for the right one. + local procedure IsPPE(): Boolean + begin + exit(StrPos(LowerCase(GetUrl(ClientType::Web)), 'businesscentral.dynamics-tie.com') <> 0); + end; + + local procedure GetBcResourceScope(): Text + begin + if IsPPE() then + exit(ScopePPETok); + exit(ScopeTok); + end; + + local procedure GetTokenEndpointTemplate(): Text + begin + if IsPPE() then + exit(TokenEndpointPPETok); + exit(TokenEndpointTok); + end; + local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") begin if not MasterDataManagementSetup.Get() then diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index b873b734766..52d3106c8be 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -110,7 +110,7 @@ page 7232 "MDM Connection Details" group(OAuth2ConnectionDetails) { Caption = 'Authentication details'; - InstructionalText = 'Provide the Microsoft Entra application that this environment uses to authenticate to the source environment.'; + InstructionalText = 'Provide the Microsoft Entra application that this environment uses to authenticate to the source environment. Register it as a single-tenant application, because synchronization only connects to environments in the same Microsoft Entra tenant.'; field(OAuth2ClientId; OAuth2ClientId) { diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 525af86334d..1da0b48c9a4 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -53,23 +53,31 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; begin // [FEATURE] [Master Data Management] [Cross-Environment] [Security] - // [SCENARIO] The HTTP transport's source-host allow-list accepts only HTTPS Business Central endpoints (SSRF guard). + // [SCENARIO] The source-host allow-list accepts only the exact HTTPS Business Central API hosts (SSRF guard). Initialize(); - // [GIVEN] valid Business Central SaaS and TIE endpoints over HTTPS [THEN] validation passes - LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.api.bc.dynamics.com'); - LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.api.bc.dynamics-tie.com'); + // [GIVEN] the exact Business Central API hosts (production and TIE) over HTTPS [THEN] validation passes + LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://api.businesscentral.dynamics.com/v2.0/CRONUS/Production'); + LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://api.businesscentral.dynamics-tie.com/v2.0/CRONUS/Sandbox'); // [GIVEN] a non-HTTPS scheme [THEN] validation is rejected - asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('http://myenv.api.bc.dynamics.com'); + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('http://api.businesscentral.dynamics.com/v2.0/CRONUS/Production'); Assert.ExpectedError(InvalidSourceHostErr); - // [GIVEN] a host outside the dynamics.com allow-list [THEN] validation is rejected + // [GIVEN] any non-standard dynamics.com host (e.g. an Embed/ISV per-cluster hostname) [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.api.bc.dynamics.com'); + Assert.ExpectedError(InvalidSourceHostErr); + + // [GIVEN] a subdomain of the allowed host [THEN] validation is rejected (exact host only) + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://sub.api.businesscentral.dynamics.com'); + Assert.ExpectedError(InvalidSourceHostErr); + + // [GIVEN] a host outside the allow-list [THEN] validation is rejected asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://evil.example.com'); Assert.ExpectedError(InvalidSourceHostErr); - // [GIVEN] a look-alike host that only embeds dynamics.com as a non-final label [THEN] validation is rejected - asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://myenv.dynamics.com.evil.example.com'); + // [GIVEN] a look-alike host that only embeds the allowed host as a non-final label [THEN] validation is rejected + asserterror LibraryMasterDataMgt.ValidateHttpTransportSourceHost('https://api.businesscentral.dynamics.com.evil.example.com'); Assert.ExpectedError(InvalidSourceHostErr); CleanUp(); From b67d8095533e2c19cf8f8f8f1b548bf26bd2f902 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 13:37:36 +0200 Subject: [PATCH 43/64] [Master Data Management] Cross-environment synchonization --- .../MDMHttpSourceTransport.Codeunit.al | 43 ++++++++++++------- .../codeunits/MDMSourceResponse.Codeunit.al | 5 ++- .../src/MDMCrossEnvDetectorTests.Codeunit.al | 25 +++++++++++ 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 22573476564..092615cabfe 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -89,17 +89,25 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Error(NonSaaSErr); PrivacyNotice.CheckApproved(); - for Attempt := 0 to MaxRetries() do begin - Send(MasterDataManagementSetup, ActionName, RequestBody, ResponseMessage); - ResponseMessage.Content().ReadAs(ResponseBodyText); - if ResponseMessage.IsSuccessStatusCode() then - exit(UnwrapODataValue(ResponseBodyText)); - if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin - LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); - Error(SetupNavigationError(StrSubstNo(HttpErr, ResponseMessage.HttpStatusCode()))); + for Attempt := 0 to MaxRetries() do + if TrySend(MasterDataManagementSetup, ActionName, RequestBody, ResponseMessage) then begin + ResponseMessage.Content().ReadAs(ResponseBodyText); + if ResponseMessage.IsSuccessStatusCode() then + exit(UnwrapODataValue(ResponseBodyText)); + if not ShouldRetry(ResponseMessage, Attempt, RetryAfter) then begin + LogRequestFailure(MasterDataManagementSetup, ActionName, ResponseMessage); + Error(SetupNavigationError(StrSubstNo(HttpErr, ResponseMessage.HttpStatusCode()))); + end; + Sleep(RetryAfter); + end else begin + // Transport-level failure (dropped connection, timeout): these reads are idempotent, so retry while + // attempts remain instead of aborting the sync run on a transient network blip. + if Attempt >= MaxRetries() then begin + LogTransportFailure(ActionName); + Error(SetupNavigationError(SendFailedErr)); + end; + Sleep(TransportRetryBackoff()); end; - Sleep(RetryAfter); - end; end; local procedure LogRequestFailure(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; var ResponseMessage: HttpResponseMessage) @@ -129,7 +137,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" Session.LogMessage('0000VAU', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; - local procedure Send(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage) + local procedure TrySend(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text; RequestBody: Text; var ResponseMessage: HttpResponseMessage): Boolean var HttpClient: HttpClient; RequestMessage: HttpRequestMessage; @@ -150,10 +158,8 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" ContentHeaders.Add('Content-Type', 'application/json'); RequestMessage.Content(HttpContent); - if not HttpClient.Send(RequestMessage, ResponseMessage) then begin - LogTransportFailure(ActionName); - Error(SetupNavigationError(SendFailedErr)); - end; + // A transport-level failure returns false so the caller can retry the read-only call. + exit(HttpClient.Send(RequestMessage, ResponseMessage)); end; local procedure BuildActionUrl(var MasterDataManagementSetup: Record "Master Data Management Setup"; ActionName: Text): Text @@ -315,7 +321,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" begin if Attempt >= MaxRetries() then exit(false); - if not (ResponseMessage.HttpStatusCode() in [429, 503]) then + if not (ResponseMessage.HttpStatusCode() in [408, 429, 502, 503, 504]) then exit(false); RetryAfter := RetryAfterDuration(ResponseMessage); exit(true); @@ -350,4 +356,9 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" MaxRetriesValue := 2; exit(MaxRetriesValue); end; + + local procedure TransportRetryBackoff(): Duration + begin + exit(5000); // fixed backoff for transport failures, where no Retry-After header is available + end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 41b337726e5..27c3c1ca156 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -154,8 +154,11 @@ codeunit 7248 "MDM Source Response" ApplyInlineMedia(SystemIdValue, FieldNo, TempSourceRecordRef.Number(), ValueToken); FieldType::Blob: ApplyInlineBlob(DestField, FieldNo, TempSourceRecordRef.Number(), ValueToken); - else + else begin + if not ValueToken.IsValue() then // an object/array for a scalar field is a broken contract + Error(MalformedRecordEntry()); SetFieldFromText(DestField, ValueToken.AsValue().AsText()); + end; end; end; end; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al index 0bf03b72744..638abc14cf4 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al @@ -129,6 +129,31 @@ codeunit 139933 "MDM Cross-Env Detector Tests" CleanUp(); end; + [Test] + procedure DetectorNudgesAvailableTableWithoutTimestamp() + var + Mapping: Record "Integration Table Mapping"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + DetectorProbe: Codeunit "MDM Test Detector Probe"; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] + // [SCENARIO] A table the source reports available but WITHOUT a lastModifiedAt (keyless/unindexed) is nudged + // so the sync job scans it, rather than being silently skipped. + Initialize(); + CreateMapping(Mapping, Database::Customer, WatermarkDateTime(), true); + InsertSynchJob(Mapping, false); + + InProcessTransport.Activate(); + InProcessTransport.SetCannedResponse(CannedLastModified(Database::Customer, true, 0DT, false)); // available, no timestamp + DetectorProbe.Activate(); + + LibraryMasterDataMgt.RunChangeDetector(); + + Assert.IsTrue(DetectorProbe.WasNudged(Database::Customer), 'An available table without a timestamp should be nudged to scan'); + CleanUp(); + end; + local procedure Initialize() var MasterDataManagementSetup: Record "Master Data Management Setup"; From eba6f26a4398d8531cd63e927896a6b38f9a5bd2 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 15:23:08 +0200 Subject: [PATCH 44/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvChangeDetector.Codeunit.al | 3 ++ .../MDMCrossEnvDataSource.Codeunit.al | 18 +++++++++ .../MDMCrossEnvSourceAPI.Codeunit.al | 23 +++++++++++ .../codeunits/MDMSourceResponse.Codeunit.al | 8 ++++ .../MasterDataMgtInstall.Codeunit.al | 11 ++++++ .../src/pages/MDMConnectionDetails.Page.al | 7 +++- .../src/pages/MasterDataSynchFields.Page.al | 6 +-- .../src/pages/MasterDataSynchTables.Page.al | 2 +- .../src/LibraryMasterDataMgt.Codeunit.al | 9 +++++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 25 ++++++++++++ .../src/MDMCrossEnvSourceTests.Codeunit.al | 39 +++++++++++++++++++ 11 files changed, 145 insertions(+), 6 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 6121a7e2deb..048a4bc8705 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -120,10 +120,13 @@ codeunit 7245 "MDM Cross-Env Change Detector" local procedure ProcessDetectionResponse(var Response: JsonObject) var MasterDataManagement: Codeunit "Master Data Management"; + SourceResponse: Codeunit "MDM Source Response"; Tables: JsonArray; TablesToken: JsonToken; EntryToken: JsonToken; begin + if SourceResponse.ConsentRequired(Response) then + exit; // source hasn't consented to sharing; the sync job surfaces the actionable error, the detector skips if (not Response.Get('tables', TablesToken)) or (not TablesToken.IsArray()) then begin Session.LogMessage('0000VAQ', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index d3dd0c753b9..bfabc6a5a0f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -21,6 +21,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" TableUnavailableErr: Label 'Table %1 is not available on the source environment. Expose it there or remove it from Synchronization Tables.', Comment = '%1 = table caption'; NotIndexedErr: Label 'Table %1 on the source has too many same-timestamp changes to synchronize without an index. Add a key on SystemModifiedAt and SystemId to that table on the source environment.', Comment = '%1 = table caption'; FieldsUnavailableErr: Label 'One or more fields set up for synchronization do not exist on table %1 on the source environment.', Comment = '%1 = table caption'; + SourceConsentRequiredErr: Label 'The source environment has not approved sharing its master data with this environment. Ask an administrator of the source environment to approve the Master Data Management cross-environment privacy notice.'; SourceProbeFailedErr: Label 'Could not read the change probe from the source environment for table %1.', Comment = '%1 = table caption'; OpenSynchTablesActionTxt: Label 'Open Synchronization Tables'; RecordsFeatureTok: Label 'records', Locked = true; @@ -125,6 +126,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; + if SourceResponse.ConsentRequired(Response) then + Error(SourceConsentError()); if (not Response.Get('tables', Token)) or (not Token.IsArray()) then begin LogProbeFailure(IntegrationTableId); Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); @@ -179,6 +182,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; + if SourceResponse.ConsentRequired(Response) then + Error(SourceConsentError()); // An unavailable table yields no records array; treat as no matching records for the review. if not SourceResponse.TableAvailable(Response) then exit(false); @@ -259,6 +264,17 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(ErrInfo); end; + // The source declined to share (its cross-environment privacy notice isn't approved); actionable by the SOURCE + // environment's admin, so there is no local navigation action to add. + local procedure SourceConsentError(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := SourceConsentRequiredErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + exit(ErrInfo); + end; + // The table isn't exposed on the source: a recoverable setup issue, so point the user at Synchronization Tables. local procedure SynchTablesNavigationError(IntegrationTableId: Integer; MessageText: Text): ErrorInfo var @@ -298,6 +314,8 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogParseFailure(IntegrationTableId, InvalidResponseReasonTok); Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); end; + if SourceResponse.ConsentRequired(Response) then + Error(SourceConsentError()); if not SourceResponse.TableAvailable(Response) then begin LogParseFailure(IntegrationTableId, TableUnavailableReasonTok); Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(TableUnavailableErr, TableCaption(IntegrationTableId)))); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 816ea8f8ca9..3ed124b0f95 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -62,6 +62,8 @@ codeunit 7241 "MDM Cross-Env Source API" Count: Integer; ResultText: Text; begin + if not IsSourceConsented() then + exit(ConsentRequiredResponse()); Response.Add('tableId', TableId); if IsBlockedSourceTable(TableId) then @@ -154,6 +156,25 @@ codeunit 7241 "MDM Cross-Env Source API" // Media/infrastructure tables are only reachable inline (BuildMediaValue, tied to a record's media field). Never // serve them as a top-level read, or a caller holding the Tenant Media read grant could enumerate every blob. + // The environment serving its data must have consented. If not, return a structured signal (not an error) so the + // subsidiary surfaces a clear, actionable message instead of a raw HTTP failure. + local procedure IsSourceConsented(): Boolean + var + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + exit(MDMPrivacyNotice.IsApproved()); + end; + + local procedure ConsentRequiredResponse(): Text + var + Response: JsonObject; + ResultText: Text; + begin + Response.Add('consentRequired', true); + Response.WriteTo(ResultText); + exit(ResultText); + end; + local procedure IsBlockedSourceTable(TableId: Integer): Boolean begin exit(TableId in [Database::"Tenant Media", Database::"Tenant Media Set", Database::"Tenant Media Thumbnails"]); @@ -597,6 +618,8 @@ codeunit 7241 "MDM Cross-Env Source API" Token: JsonToken; ResultText: Text; begin + if not IsSourceConsented() then + exit(ConsentRequiredResponse()); if TryReadJsonArray(TableIds, RequestedTables) then foreach Token in RequestedTables do Tables.Add(BuildTableModifiedAt(Token.AsValue().AsInteger())); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 27c3c1ca156..03ece861654 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -53,6 +53,12 @@ codeunit 7248 "MDM Source Response" exit(ReadControlBoolean(Response, 'hasMore', false)); end; + // True when the source declined to share because its cross-environment privacy notice isn't approved. + procedure ConsentRequired(var Response: JsonObject): Boolean + begin + exit(ReadControlBoolean(Response, 'consentRequired', false)); + end; + // Control fields (tableAvailable/indexed/hasMore) are always booleans in the contract; a present-but-malformed // token is an internal contract failure, so surface it as an internal diagnostic instead of a raw runtime throw. local procedure ReadControlBoolean(var Response: JsonObject; PropertyName: Text; DefaultValue: Boolean): Boolean @@ -143,6 +149,8 @@ codeunit 7248 "MDM Source Response" TempSourceRecordRef.Init(); GetGuid(RecordObject, 'systemId', SystemIdValue); if RecordObject.Get('fields', FieldsToken) then begin + if not FieldsToken.IsObject() then // a non-object 'fields' is a broken contract, routed as an internal error + Error(MalformedRecordEntry()); FieldsObject := FieldsToken.AsObject(); foreach FieldName in FieldsObject.Keys() do if Evaluate(FieldNo, FieldName) then diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al index 799809e2c97..b235fb6a426 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al @@ -1,5 +1,7 @@ namespace Microsoft.Integration.MDM; +using System.Upgrade; + /// /// Codeunit Master Data Mgt. Install (ID 7243). /// @@ -15,4 +17,13 @@ codeunit 7243 "Master Data Mgt. Install" // Fresh install (incl. package/base-image build) never fires the upgrade trigger, so publish the source endpoint here too. MasterDataMgtUpgrade.RegisterCrossEnvSourceWebService(); end; + + trigger OnInstallAppPerCompany() + var + UpgradeTag: Codeunit "Upgrade Tag"; + begin + // This trigger only fires on a fresh install (never on upgrade), and a fresh company carries no legacy data: + // mark the historical per-company migrations as done so a later app upgrade never re-runs them here. + UpgradeTag.SetAllUpgradeTags(); + end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 52d3106c8be..b1bd7f10924 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -30,7 +30,7 @@ page 7232 "MDM Connection Details" group(TermsAndConditions) { Caption = 'Review the terms and conditions'; - InstructionalText = 'By enabling this, you consent that this environment will read data from the source environment that you configure. Your privacy is important to us. To learn more, follow the link below.'; + InstructionalText = 'By enabling this, you consent to sharing data between Business Central environments that may be in different geographies: this environment reads master data from the source environment you configure, and the filters it sends there may include your data. Your privacy is important to us. To learn more, follow the link below.'; field(Consent; ConsentState) { @@ -300,8 +300,11 @@ page 7232 "MDM Connection Details" Transport := SourceConnection.GetTransport(); if not Capabilities.ReadFrom(Transport.GetCapabilities()) then Error(ConnectionFailedErr); - if Capabilities.Get('version', VersionToken) then + if Capabilities.Get('version', VersionToken) then begin + if not VersionToken.IsValue() then // a non-scalar version is a broken capabilities contract, not user-actionable + Error(ConnectionFailedErr); VersionText := Format(VersionToken.AsValue().AsInteger()); + end; Message(ConnectionOkMsg, VersionText); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al index ac51870b1bb..603c49183c6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al @@ -147,15 +147,15 @@ page 7236 "Master Data Synch. Fields" until IntegrationFieldMapping.Next() = 0; if FieldsAdded * FieldsRemoved > 0 then begin - Message(StrSubstNo(FieldsAddedAndRemovedTxt, FieldsAdded, FieldsRemoved)); + Message(FieldsAddedAndRemovedTxt, FieldsAdded, FieldsRemoved); exit; end; if FieldsAdded > 0 then - Message(StrSubstNo(FieldsAddedTxt, FieldsAdded)); + Message(FieldsAddedTxt, FieldsAdded); if FieldsRemoved > 0 then - Message(StrSubstNo(FieldsRemovedTxt, FieldsRemoved)); + Message(FieldsRemovedTxt, FieldsRemoved); end; } action(Enable) diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al index 7a41a42c71e..3b7058f9531 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al @@ -105,7 +105,7 @@ page 7233 "Master Data Synch. Tables" IntegrationTableMapping.Validate(Status, IntegrationTableMapping.Status::Disabled); IntegrationTableMapping.Modify(); end; - Message(StrSubstNo(RelatedTablesAddedMsg, AllObjWithCaption."Object Caption", RelatedTablesToAddText)); + Message(RelatedTablesAddedMsg, AllObjWithCaption."Object Caption", RelatedTablesToAddText); exit; end; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index daa2103ff42..8a3e0ae3ec0 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -296,6 +296,15 @@ codeunit 139757 "Library - Master Data Mgt." PrivacyNoticeApproval.DeleteAll(); end; + /// Records approval for the cross-env privacy notice (for tests exercising the consent-gated source API). + procedure ApproveCrossEnvPrivacyNotice() + var + PrivacyNotice: Codeunit "Privacy Notice"; + MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; + begin + PrivacyNotice.SetApprovalState(MDMPrivacyNotice.GetPrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); + end; + var MasterDataMgtSubscribers: Codeunit "Master Data Mgt. Subscribers"; MasterDataManagement: Codeunit "Master Data Management"; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 1da0b48c9a4..2dee62e6399 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -113,6 +113,29 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvSyncSurfacesSourceConsentRequired() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] When the source hasn't approved sharing, the subsidiary sync surfaces a clear, actionable error + // (pointing at the source admin) rather than a raw transport failure. + Initialize(); + LibrarySalesLib.CreateCustomer(Customer); + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); // the (in-process) source has not consented + + asserterror LibraryMasterDataMgt.DataSourceGetBySystemId(Database::Customer, Customer.SystemId, SourceRecordRef); + Assert.ExpectedError('has not approved sharing its master data'); + + CleanUp(); + end; + [Test] [HandlerFunctions('PrivacyNoticeModalHandler')] procedure WizardConsentOpensPrivacyNotice() @@ -712,8 +735,10 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" var MasterDataManagementSetup: Record "Master Data Management Setup"; InProcessTransport: Codeunit "MDM In-Process Transport"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; begin InProcessTransport.Deactivate(); + LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); // the source API is consent-gated; approve for the gated paths // SetSourceEnvironmentName validates the setup, which schedules the detector job and commits, so a mapping // created earlier in a test survives AutoRollback; clear leftovers to keep tests independent. DeleteTestArtifacts(); diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al index 0899d69419c..78d95d5218e 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvSourceTests.Codeunit.al @@ -27,6 +27,40 @@ codeunit 139931 "MDM Cross-Env Source Tests" Assert.IsTrue(FeaturesContain(Response, 'lastModifiedPerTable'), 'lastModifiedPerTable feature should be advertised'); end; + [Test] + procedure GetRecordsSignalsConsentRequiredWhenNotApproved() + var + Customer: Record Customer; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + SourceApi: Codeunit "MDM Cross-Env Source API"; + Response: JsonObject; + Token: JsonToken; + begin + // [FEATURE] [Master Data Management] [Cross-Environment] [Privacy] + // [SCENARIO] Without source consent the API returns a structured consentRequired signal (not data, not a raw + // error), so the subsidiary can surface a clear message; after approval it serves data. + LibrarySales.CreateCustomer(Customer); + + // [GIVEN] this environment has not approved sharing [THEN] the source signals consentRequired and no data + LibraryMasterDataMgt.PrivacyNoticeResetApproval(); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100, '')); + Assert.IsTrue(Response.Get('consentRequired', Token) and Token.AsValue().AsBoolean(), 'Source should signal consentRequired when not approved'); + Assert.IsFalse(Response.Contains('records'), 'No records should be served without consent'); + + // [WHEN] the environment approves sharing [THEN] the source serves data with no consent signal + Clear(Response); + LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); + Response.ReadFrom(SourceApi.GetRecords(Database::Customer, FieldIdsArray(Customer.FieldNo(Name)), SystemIdsSelector(Customer.SystemId), 100, '')); + Assert.IsFalse(Response.Contains('consentRequired'), 'The consent signal should be absent once approved'); + end; + + local procedure ApproveSourceConsent() + var + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + begin + LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); + end; + [Test] procedure GetRecordsBySystemIdReturnsRequestedFields() var @@ -40,6 +74,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" begin // [FEATURE] [AI test 0.4] // [SCENARIO] GetRecords with a systemIds selector returns just that record with the requested fields. + ApproveSourceConsent(); LibrarySales.CreateCustomer(Customer); Customer.Name := CopyStr(LibraryRandom.RandText(20), 1, MaxStrLen(Customer.Name)); Customer.Modify(); @@ -74,6 +109,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" begin // [FEATURE] [AI test 0.4] // [SCENARIO] Cursor mode pages ascending by (SystemModifiedAt, SystemId) and reports hasMore / nextCursor. + ApproveSourceConsent(); Watermark := CurrentDateTime(); Sleep(50); // ensure the seeded records sort strictly after the watermark for Index := 1 to 3 do begin @@ -121,6 +157,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" begin // [FEATURE] [AI test 0.4] // [SCENARIO] LastModifiedAtPerTable returns each table's latest modification timestamp. + ApproveSourceConsent(); LibrarySales.CreateCustomer(Customer); Response.ReadFrom(SourceApi.LastModifiedAtPerTable(TableIdsArray(Database::Customer))); @@ -144,6 +181,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [FEATURE] [Master Data Management] [Cross-Environment] [Security] // [SCENARIO] The source API refuses to serve Tenant Media as a top-level table, so a caller holding the media // read grant cannot enumerate blobs directly; media stays reachable only inline via a record's media field. + ApproveSourceConsent(); Response.ReadFrom(SourceApi.GetRecords(Database::"Tenant Media", FieldIdsArray(1), CursorSelector(CurrentDateTime()), 10, '')); // [THEN] the table is reported unavailable and no records are returned @@ -165,6 +203,7 @@ codeunit 139931 "MDM Cross-Env Source Tests" // [FEATURE] [Master Data Management] [Cross-Environment] // [SCENARIO] The source applies the mapping row filter server-side even when it references a field outside the // projection (parity with same-env), so only matching records are returned rather than filtered post-hoc. + ApproveSourceConsent(); Watermark := CurrentDateTime(); Sleep(50); // ensure the seeded records sort strictly after the watermark LibrarySales.CreateCustomer(MatchCustomer); From 1359ae9b7fd727a903c067a331012374520bade0 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 15:41:24 +0200 Subject: [PATCH 45/64] [Master Data Management] Cross-environment synchonization --- .../app/src/pages/MasterDataManagementSetup.Page.al | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index 37edc6d7582..cbfc6d5b3c8 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -54,7 +54,7 @@ page 7230 "Master Data Management Setup" action(ConnectionDetails) { ApplicationArea = Suite; - Caption = 'Connection Details'; + Caption = 'Cross-environment Setup'; Image = LinkAccount; ToolTip = 'Set up the connection to a company in a different Business Central environment for cross-environment synchronization.'; @@ -194,6 +194,14 @@ page 7230 "Master Data Management Setup" } area(Promoted) { + group(Category_CrossEnvironment) + { + Caption = 'Cross-Environment Setup'; + + actionref(ConnectionDetails_Promoted; ConnectionDetails) + { + } + } group(Category_Process) { Caption = 'Synchronization', Comment = 'Generated from the PromotedActionCategories property index 5.'; From b4dd4147a879667061bc4fb7f417d855251002e8 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 16:08:36 +0200 Subject: [PATCH 46/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvDataSource.Codeunit.al | 20 ++++++++--- .../MDMCrossEnvSourceAPI.Codeunit.al | 31 ++++++++++------ .../codeunits/MDMSourceResponse.Codeunit.al | 36 +++++++++++++------ .../pages/MasterDataManagementSetup.Page.al | 2 +- 4 files changed, 62 insertions(+), 27 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index bfabc6a5a0f..c53aaf653a5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -111,6 +111,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" TableIdsText: Text; LastModifiedAtText: Text; IntegrationTableId: Integer; + BoolValue: Boolean; begin IntegrationTableId := IntegrationTableMapping."Integration Table ID"; // With a row filter the cheap per-table timestamp can't tell whether any MATCHING record exists (parity with @@ -144,18 +145,23 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; Entry := Token.AsObject(); if Entry.Get('tableAvailable', Token) then begin - if not Token.IsValue() then begin // malformed contract, not an empty source: keep it in the internal-error path + if not (Token.IsValue() and TryGetBoolean(Token, BoolValue)) then begin // malformed contract, not an empty source: keep it in the internal-error path LogProbeFailure(IntegrationTableId); Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); end; - if not Token.AsValue().AsBoolean() then + if not BoolValue then exit(false); end; // Unindexed source table: LastModifiedAtPerTable reports indexed:false and no timestamp, so we can't prove // emptiness cheaply - assume records may exist so the full-synch review isn't wrongly suppressed. - if Entry.Get('indexed', Token) then - if Token.IsValue() and (not Token.AsValue().AsBoolean()) then + if Entry.Get('indexed', Token) then begin + if not (Token.IsValue() and TryGetBoolean(Token, BoolValue)) then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + if not BoolValue then exit(true); + end; if Entry.Get('lastModifiedAt', Token) then if Token.IsValue() then LastModifiedAtText := Token.AsValue().AsText(); @@ -264,6 +270,12 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" exit(ErrInfo); end; + [TryFunction] + local procedure TryGetBoolean(Token: JsonToken; var Value: Boolean) + begin + Value := Token.AsValue().AsBoolean(); + end; + // The source declined to share (its cross-environment privacy notice isn't approved); actionable by the SOURCE // environment's admin, so there is no local navigation action to add. local procedure SourceConsentError(): ErrorInfo diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index 3ed124b0f95..b16f2f937a6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -211,14 +211,13 @@ codeunit 7241 "MDM Cross-Env Source API" begin if not TryReadJsonArray(FieldIds, RequestedFields) then exit; - foreach Token in RequestedFields do begin - FieldNo := Token.AsValue().AsInteger(); - if not RecRef.FieldExist(FieldNo) then - UnavailableFields.Add(FieldNo) - else - if IsProjectableField(RecRef.Field(FieldNo)) then - ProjectedFields.Add(FieldNo); - end; + foreach Token in RequestedFields do + if Token.IsValue() and TryReadInteger(Token, FieldNo) then // a malformed field id is skipped, not served as an error + if not RecRef.FieldExist(FieldNo) then + UnavailableFields.Add(FieldNo) + else + if IsProjectableField(RecRef.Field(FieldNo)) then + ProjectedFields.Add(FieldNo); end; local procedure IsProjectableField(FieldReference: FieldRef): Boolean @@ -262,9 +261,11 @@ codeunit 7241 "MDM Cross-Env Source API" exit(false); if not SelectorObject.Get('modifiedAt', Token) then exit(false); + if not Token.IsValue() then + exit(false); if not Evaluate(CursorModifiedAt, Token.AsValue().AsText(), 9) then exit(false); - if SelectorObject.Get('systemId', Token) then + if SelectorObject.Get('systemId', Token) and Token.IsValue() then Evaluate(CursorSystemId, Token.AsValue().AsText()); exit(true); end; @@ -425,7 +426,7 @@ codeunit 7241 "MDM Cross-Env Source API" FilterText: Text; begin foreach Token in SystemIds do - if Evaluate(SystemIdValue, Token.AsValue().AsText()) then begin + if Token.IsValue() and Evaluate(SystemIdValue, Token.AsValue().AsText()) then begin if FilterBuilder.Length() > 0 then FilterBuilder.Append('|'); FilterBuilder.Append(Format(SystemIdValue)); @@ -585,6 +586,12 @@ codeunit 7241 "MDM Cross-Env Source API" exit(JsonObjectValue.ReadFrom(Value)); end; + [TryFunction] + local procedure TryReadInteger(Token: JsonToken; var Value: Integer) + begin + Value := Token.AsValue().AsInteger(); + end; + local procedure ClampPageSize(PageSize: Integer): Integer begin if PageSize <= 0 then @@ -617,12 +624,14 @@ codeunit 7241 "MDM Cross-Env Source API" Response: JsonObject; Token: JsonToken; ResultText: Text; + TableId: Integer; begin if not IsSourceConsented() then exit(ConsentRequiredResponse()); if TryReadJsonArray(TableIds, RequestedTables) then foreach Token in RequestedTables do - Tables.Add(BuildTableModifiedAt(Token.AsValue().AsInteger())); + if Token.IsValue() and TryReadInteger(Token, TableId) then // a malformed table id is skipped, not served as an error + Tables.Add(BuildTableModifiedAt(TableId)); Response.Add('tables', Tables); Response.WriteTo(ResultText); exit(ResultText); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 03ece861654..0ba16a5277d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -213,9 +213,11 @@ codeunit 7248 "MDM Source Response" end; if not MediaObject.Get('content', ContentToken) then exit; // no content and not flagged empty: leave the destination picture untouched - if MediaObject.Get('name', NameToken) then + if not ContentToken.IsValue() then // a non-scalar content payload is a broken record entry + Error(MalformedRecordEntry()); + if MediaObject.Get('name', NameToken) and NameToken.IsValue() then FileName := NameToken.AsValue().AsText(); - if MediaObject.Get('mimeType', MimeToken) then + if MediaObject.Get('mimeType', MimeToken) and MimeToken.IsValue() then MimeType := MimeToken.AsValue().AsText(); InlineMedia.Put(SystemId, FieldNo, FileName, MimeType, ContentToken.AsValue().AsText()); end; @@ -244,27 +246,35 @@ codeunit 7248 "MDM Source Response" end; if not BlobObject.Get('content', ContentToken) then exit; // no content and not flagged empty: leave the destination untouched + if not ContentToken.IsValue() then // a non-scalar content payload is a broken record entry + Error(MalformedRecordEntry()); TempBlob.CreateOutStream(ContentOutStream); Base64Convert.FromBase64(ContentToken.AsValue().AsText(), ContentOutStream); TempBlob.ToFieldRef(DestField); end; local procedure IsSkipped(FieldObject: JsonObject): Boolean - var - Token: JsonToken; begin - if FieldObject.Get('skipped', Token) then - exit(Token.AsValue().AsBoolean()); - exit(false); + exit(ReadRecordBoolean(FieldObject, 'skipped')); end; local procedure IsEmptyField(FieldObject: JsonObject): Boolean + begin + exit(ReadRecordBoolean(FieldObject, 'empty')); + end; + + // Per-record media/blob flags are booleans in the contract; a present-but-malformed token is a broken record + // entry, so route it through the same internal malformed-record path as the rest of record materialization. + local procedure ReadRecordBoolean(FieldObject: JsonObject; PropertyName: Text): Boolean var Token: JsonToken; + Value: Boolean; begin - if FieldObject.Get('empty', Token) then - exit(Token.AsValue().AsBoolean()); - exit(false); + if not FieldObject.Get(PropertyName, Token) then + exit(false); + if not (Token.IsValue() and TryReadBoolean(Token, Value)) then + Error(MalformedRecordEntry()); + exit(Value); end; // Over-cap media/blob is not synchronized (v1). We can't error (it would retry the record every run) and MDM @@ -280,7 +290,7 @@ codeunit 7248 "MDM Source Response" Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); Dimensions.Add('TableId', Format(TableId)); Dimensions.Add('FieldNo', Format(FieldNo)); - if FieldObject.Get('length', LengthToken) then + if FieldObject.Get('length', LengthToken) and LengthToken.IsValue() then Dimensions.Add('Length', Format(LengthToken.AsValue().AsBigInteger())); Session.LogMessage('0000VAW', SkippedFieldTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; @@ -372,6 +382,8 @@ codeunit 7248 "MDM Source Response" begin if not Container.Get(PropertyName, Token) then exit(false); + if not Token.IsValue() then + exit(false); exit(Evaluate(Value, Token.AsValue().AsText())); end; @@ -381,6 +393,8 @@ codeunit 7248 "MDM Source Response" begin if not Container.Get(PropertyName, Token) then exit(false); + if not Token.IsValue() then + exit(false); exit(Evaluate(Value, Token.AsValue().AsText(), 9)); end; } diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index cbfc6d5b3c8..665da3abc74 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -54,7 +54,7 @@ page 7230 "Master Data Management Setup" action(ConnectionDetails) { ApplicationArea = Suite; - Caption = 'Cross-environment Setup'; + Caption = 'Cross-Environment Setup'; Image = LinkAccount; ToolTip = 'Set up the connection to a company in a different Business Central environment for cross-environment synchronization.'; From b1bcd82db0796e01f3a0e3d8034da38419ad58ac Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 16:18:51 +0200 Subject: [PATCH 47/64] [Master Data Management] Cross-environment synchonization --- .../app/src/pages/MDMConnectionDetails.Page.al | 4 ++-- .../app/src/pages/MasterDataManagementSetup.Page.al | 11 +++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index b1bd7f10924..6c3d64425d3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -46,7 +46,7 @@ page 7232 "MDM Connection Details" SetControls(); end; } - field(LearnMore; LearnMoreTok) + field(LearnMore; LearnMoreLbl) { ApplicationArea = All; Editable = false; @@ -245,7 +245,7 @@ page 7232 "MDM Connection Details" OAuth2ClientId: Text[100]; [NonDebuggable] OAuth2ClientSecret: Text; - LearnMoreTok: Label 'Privacy and Cookies'; + LearnMoreLbl: Label 'Privacy and Cookies'; PrivacyLinkTxt: Label 'https://go.microsoft.com/fwlink/?linkid=521839', Locked = true; ConnectionOkMsg: Label 'Successfully connected to the source environment (contract version %1).', Comment = '%1 = wire contract version'; ConnectionFailedErr: Label 'Could not connect to the source environment. Check the URL, company, and credentials, then try again.'; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index 665da3abc74..7946ba13487 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -194,14 +194,6 @@ page 7230 "Master Data Management Setup" } area(Promoted) { - group(Category_CrossEnvironment) - { - Caption = 'Cross-Environment Setup'; - - actionref(ConnectionDetails_Promoted; ConnectionDetails) - { - } - } group(Category_Process) { Caption = 'Synchronization', Comment = 'Generated from the PromotedActionCategories property index 5.'; @@ -212,6 +204,9 @@ page 7230 "Master Data Management Setup" actionref(IntegrationTableMappings_Promoted; IntegrationTableMappings) { } + actionref(ConnectionDetails_Promoted; ConnectionDetails) + { + } actionref("Synch. Job Queue Entries_Promoted"; "Synch. Job Queue Entries") { } From 75911445ffc672d5abef974352be40afc923bb48 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 16:41:16 +0200 Subject: [PATCH 48/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MasterDataMgtInstall.Codeunit.al | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al index b235fb6a426..9face0a2edf 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtInstall.Codeunit.al @@ -21,9 +21,12 @@ codeunit 7243 "Master Data Mgt. Install" trigger OnInstallAppPerCompany() var UpgradeTag: Codeunit "Upgrade Tag"; + AppInfo: ModuleInfo; begin - // This trigger only fires on a fresh install (never on upgrade), and a fresh company carries no legacy data: - // mark the historical per-company migrations as done so a later app upgrade never re-runs them here. - UpgradeTag.SetAllUpgradeTags(); + NavApp.GetCurrentModuleInfo(AppInfo); + // Only a genuine first install (no preserved data) may skip the historical per-company migrations. A reinstall + // over preserved data keeps DataVersion non-zero and must let those migrations run against the existing data. + if AppInfo.DataVersion() = Version.Create(0, 0, 0, 0) then + UpgradeTag.SetAllUpgradeTags(); end; } From 47eb373db60d29a8e397a0dfd00ba1fac5117b34 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 1 Sep 2026 16:46:08 +0200 Subject: [PATCH 49/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al | 9 +++++++++ .../app/src/pages/MDMConnectionDetails.Page.al | 5 +++-- .../test library/src/LibraryMasterDataMgt.Codeunit.al | 2 ++ .../test library/src/MDMInProcessTransport.Codeunit.al | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al index b16f2f937a6..d2c158bd417 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvSourceAPI.Codeunit.al @@ -22,6 +22,7 @@ codeunit 7241 "MDM Cross-Env Source API" /// Wire-version negotiation: returns the API contract version and the action/feature names this source /// supports, so a newer subsidiary only calls actions an older source actually implements. /// + /// A JSON object with the numeric contract 'version' and a 'features' array of supported action names. [ServiceEnabled] procedure GetCapabilities(): Text var @@ -43,6 +44,12 @@ codeunit 7241 "MDM Cross-Env Source API" /// { systemIds } list. Response carries records, hasMore and (cursor mode) nextCursor. The table read /// runs under the CALLER's permission set, so table-level access is enforced by permissions, not here. /// + /// The source table ID to read. + /// A JSON array of the field numbers to project. + /// A JSON object: a change-feed cursor { modifiedAt, systemId } or a targeted { systemIds } list. + /// The maximum number of records to return in this page. + /// An optional source row filter (view) restricting which records are returned. + /// A JSON object with 'records', 'hasMore', 'unavailableFields' and, in cursor mode, 'nextCursor'. [ServiceEnabled] procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text var @@ -616,6 +623,8 @@ codeunit 7241 "MDM Cross-Env Source API" /// the subsidiary detector can decide which per-table sync jobs to reschedule. Read from the change-feed /// index tip (FindLast), so no scan and no summary table to maintain. /// + /// A JSON array of the source table IDs to probe. + /// A JSON object with a 'tables' array of { tableId, tableAvailable, indexed, lastModifiedAt } entries. [ServiceEnabled] procedure LastModifiedAtPerTable(TableIds: Text): Text var diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 6c3d64425d3..5c797529de4 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -292,6 +292,7 @@ page 7232 "MDM Connection Details" Transport: Interface "IMDM Source Transport"; Capabilities: JsonObject; VersionToken: JsonToken; + VersionValue: Integer; VersionText: Text; begin // Persist first so the transport reads the details entered in the wizard. @@ -301,9 +302,9 @@ page 7232 "MDM Connection Details" if not Capabilities.ReadFrom(Transport.GetCapabilities()) then Error(ConnectionFailedErr); if Capabilities.Get('version', VersionToken) then begin - if not VersionToken.IsValue() then // a non-scalar version is a broken capabilities contract, not user-actionable + if not (VersionToken.IsValue() and Evaluate(VersionValue, VersionToken.AsValue().AsText())) then // malformed version is a broken capabilities contract, not user-actionable Error(ConnectionFailedErr); - VersionText := Format(VersionToken.AsValue().AsInteger()); + VersionText := Format(VersionValue); end; Message(ConnectionOkMsg, VersionText); end; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 8a3e0ae3ec0..3bce0f514b1 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -263,6 +263,7 @@ codeunit 139757 "Library - Master Data Mgt." end; /// Returns the registered privacy-notice ID that gates cross-environment synchronization. + /// The privacy-notice ID code. procedure PrivacyNoticeId(): Code[50] var MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; @@ -271,6 +272,7 @@ codeunit 139757 "Library - Master Data Mgt." end; /// Returns whether the cross-environment privacy notice is currently approved. + /// True if the notice is approved; otherwise false. procedure PrivacyNoticeIsApproved(): Boolean var MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al index 0f7107f462a..7588116c26e 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/MDMInProcessTransport.Codeunit.al @@ -55,6 +55,7 @@ codeunit 139929 "MDM In-Process Transport" implements "IMDM Source Transport" /// The projected field IDs. /// The cursor/systemId selector. /// The page size. + /// The optional source row filter (view). /// The raw JSON records response. procedure GetRecords(TableId: Integer; FieldIds: Text; Selector: Text; PageSize: Integer; Filter: Text): Text var From 5b80a8cbead05a97ca154db5e12677d49c348c86 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 2 Sep 2026 14:33:57 +0200 Subject: [PATCH 50/64] [Master Data Management] Cross-environment synchonization --- .../codeunits/MDMCrossEnvDataSource.Codeunit.al | 9 ++++++--- .../src/codeunits/MDMLocalDataSource.Codeunit.al | 16 ++++++++-------- .../src/codeunits/MDMSourceResponse.Codeunit.al | 9 +++++---- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index c53aaf653a5..190fe30a780 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -197,9 +197,12 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" // we can't prove emptiness cheaply, so assume records may exist rather than under-reporting to the review. if not SourceResponse.Indexed(Response) then exit(true); - if Response.Get('records', Token) and Token.IsArray() then - exit(Token.AsArray().Count() > 0); - exit(false); + // Available and indexed, so a records array is contractually present; its absence is a malformed response, not "no records". + if not (Response.Get('records', Token) and Token.IsArray()) then begin + LogProbeFailure(IntegrationTableId); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); + end; + exit(Token.AsArray().Count() > 0); end; procedure GetBySystemId(IntegrationTableId: Integer; SystemId: Guid; var SourceRecordRef: RecordRef): Boolean diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al index 96ce24d5d73..3133c5cb65b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMLocalDataSource.Codeunit.al @@ -25,16 +25,16 @@ codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" procedure GetById(IntegrationTableMapping: Record "Integration Table Mapping"; ID: Variant; var SourceRecordRef: RecordRef): Boolean var - IDFieldRef: FieldRef; RecId: RecordID; + SystemId: Guid; TextKey: Text; begin SourceRecordRef.Close(); + // MDM always maps the integration UID to the SystemId field, so exact-key lookups seek by SystemId instead of a filtered find. if ID.IsGuid then begin OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); - IDFieldRef := SourceRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); - IDFieldRef.SetFilter(ID); - exit(SourceRecordRef.FindFirst()); + SystemId := ID; + exit(SourceRecordRef.GetBySystemId(SystemId)); end; if ID.IsRecordId then begin @@ -45,11 +45,11 @@ codeunit 7240 "MDM Local Data Source" implements "IMDM Data Source" end; if ID.IsText then begin - OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); - IDFieldRef := SourceRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No."); TextKey := ID; - IDFieldRef.SetFilter('%1', TextKey); - exit(SourceRecordRef.FindFirst()); + if not Evaluate(SystemId, TextKey) then + exit(false); + OpenSourceRecordRef(IntegrationTableMapping."Integration Table ID", SourceRecordRef); + exit(SourceRecordRef.GetBySystemId(SystemId)); end; end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index 0ba16a5277d..ff183fcd1e4 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -121,10 +121,10 @@ codeunit 7248 "MDM Source Response" RecordsArray: JsonArray; Count: Integer; begin - if not Response.Get('records', RecordsToken) then - exit(0); + if not Response.Get('records', RecordsToken) then // available+indexed responses always carry a records array; absence is a broken contract + Error(MalformedRecordEntry()); if not RecordsToken.IsArray() then - exit(0); + Error(MalformedRecordEntry()); RecordsArray := RecordsToken.AsArray(); foreach RecordToken in RecordsArray do begin if not RecordToken.IsObject() then @@ -147,7 +147,8 @@ codeunit 7248 "MDM Source Response" FieldNo: Integer; begin TempSourceRecordRef.Init(); - GetGuid(RecordObject, 'systemId', SystemIdValue); + if not GetGuid(RecordObject, 'systemId', SystemIdValue) then // systemId is the record identity and dedup key; a missing/unparsable one is a broken record + Error(MalformedRecordEntry()); if RecordObject.Get('fields', FieldsToken) then begin if not FieldsToken.IsObject() then // a non-object 'fields' is a broken contract, routed as an internal error Error(MalformedRecordEntry()); From 2e325bc83a16fa10dea5f512fb6533a6558f9eca Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 2 Sep 2026 16:40:14 +0200 Subject: [PATCH 51/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMPrivacyNotice.Codeunit.al | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al index ce16acf7723..2774c842c25 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMPrivacyNotice.Codeunit.al @@ -60,11 +60,11 @@ codeunit 7242 "MDM Privacy Notice" exit; ErrInfo.Message := NotApprovedErr; ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry - if MasterDataManagementSetup.Get() then begin + // The remedy is always the setup page, even before the record exists; only RecordId needs an existing record. + ErrInfo.PageNo := Page::"Master Data Management Setup"; + ErrInfo.AddNavigationAction(OpenSetupActionTxt); + if MasterDataManagementSetup.Get() then ErrInfo.RecordId := MasterDataManagementSetup.RecordId(); - ErrInfo.PageNo := Page::"Master Data Management Setup"; - ErrInfo.AddNavigationAction(OpenSetupActionTxt); - end; Error(ErrInfo); end; } From 984ed6bfb33cae6c3075873392249f20af8b1347 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 2 Sep 2026 18:27:43 +0200 Subject: [PATCH 52/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 4 ++++ .../test/src/MDMCrossEnvConsumerTests.Codeunit.al | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 190fe30a780..e5bebd7b2d7 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -88,6 +88,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" HasMore := SourceResponse.HasMore(Response); if HasMore then begin EndCursor := SourceResponse.GetNextCursor(Response); + if EndCursor = '' then begin // hasMore without a resume cursor is malformed: don't restart from the watermark and persist a bad resume point + LogParseFailure(IntegrationTableMapping."Integration Table ID", InvalidResponseReasonTok); + Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableMapping."Integration Table ID")))); + end; Selector := EndCursor; end; until (not HasMore) or ((MaxPages > 0) and (PagesFetched >= MaxPages)); diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 2dee62e6399..74a5e0f4dde 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -736,8 +736,10 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" MasterDataManagementSetup: Record "Master Data Management Setup"; InProcessTransport: Codeunit "MDM In-Process Transport"; LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + PagingConfig: Codeunit "MDM Test Paging Config"; begin InProcessTransport.Deactivate(); + PagingConfig.Deactivate(); // a paging test that failed before its CleanUp must not leak its forced page cap into later tests LibraryMasterDataMgt.ApproveCrossEnvPrivacyNotice(); // the source API is consent-gated; approve for the gated paths // SetSourceEnvironmentName validates the setup, which schedules the detector job and commits, so a mapping // created earlier in a test survives AutoRollback; clear leftovers to keep tests independent. From af28a0df18e647e6a8552c274f0451e55ff2debe Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 3 Sep 2026 14:16:05 +0200 Subject: [PATCH 53/64] [Master Data Management] Cross-environment synchonization --- .../pages/MasterDataManagementSetup.Page.al | 42 +++++++++++ .../tables/MasterDataManagementSetup.Table.al | 11 +++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 71 +++++++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index 7946ba13487..177514108a0 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -29,8 +29,23 @@ page 7230 "Master Data Management Setup" { ApplicationArea = Suite; Editable = IsEditable; + Visible = not CrossEnvConfigured; // same-environment source company; hidden once a source environment is configured ToolTip = 'Specifies the name of the source company that you synchronize data from.'; } + field("Source Environment Name"; Rec."Source Environment Name") + { + ApplicationArea = Suite; + Editable = false; + Visible = CrossEnvConfigured; + ToolTip = 'Specifies the source Business Central environment that master data is read from. Use the Cross-Environment Setup action to change it.'; + } + field("Source Company Name"; Rec."Source Company Name") + { + ApplicationArea = Suite; + Editable = false; + Visible = CrossEnvConfigured; + ToolTip = 'Specifies the company in the source environment that master data is read from. Use the Cross-Environment Setup action to change it.'; + } field("Is Enabled"; Rec."Is Enabled") { ApplicationArea = Suite; @@ -61,6 +76,30 @@ page 7230 "Master Data Management Setup" trigger OnAction() begin Page.RunModal(Page::"MDM Connection Details"); + if Rec.Get() then; // the wizard may have configured or cleared the cross-environment connection + RefreshData(); + CurrPage.Update(false); + end; + } + action(ClearCrossEnvSetup) + { + ApplicationArea = Suite; + Caption = 'Clear Cross-Environment Setup'; + Image = RemoveLine; + Enabled = IsEditable; + Visible = CrossEnvConfigured; + ToolTip = 'Remove the cross-environment connection - the source environment, company, and stored credentials - and return to same-environment synchronization.'; + + trigger OnAction() + begin + if not Confirm(ClearCrossEnvConfirmQst, false) then + exit; + Rec.Validate("Source Environment Name", ''); // clear the source env; runs the enabled guard and detector cleanup + Rec.ClearCrossEnvConnection(); + Rec.Modify(true); + if Rec.Get() then; + RefreshData(); + CurrPage.Update(false); end; } action(ResetConfiguration) @@ -273,7 +312,9 @@ page 7230 "Master Data Management Setup" SynchronizeModifiedQst: Label 'This will synchronize all modified records in all integration table mappings. \\The synchronization will run in the background so you can continue with other tasks. \\Do you want to continue?'; SyncNowScheduledMsg: Label 'Synchronization of modified records is scheduled. \\You can view details on the %1 page.', Comment = '%1 = The localized caption of page Integration Synch. Job List'; SetupSuccessfulMsg: Label 'The default setup for Business Central synchronization has completed successfully.'; + ClearCrossEnvConfirmQst: Label 'This removes the cross-environment connection and its stored credentials, and returns to same-environment synchronization. Do you want to continue?'; IsEditable: Boolean; + CrossEnvConfigured: Boolean; SynchronizationImportedMsg: label 'The synchronization setup is imported. \\To view or edit the synchronization table setup, choose action Synchronization Tables.\\To view or edit the synchronization field setup, select a synchronization table and choose action Synchronization Fields.'; NoCoupledRecordsMsg: label 'No records are currently coupled to records from the source company. \\Choose the action Start Initial Synchronization.'; @@ -285,6 +326,7 @@ page 7230 "Master Data Management Setup" local procedure UpdateEnableFlags() begin IsEditable := (not Rec."Is Enabled"); + CrossEnvConfigured := Rec.IsCrossEnvironment(); end; local procedure GetJobQueueEntriesObjectIDToRunFilter(): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 246b42835bc..8dc028f6e41 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -195,6 +195,17 @@ table 7230 "Master Data Management Setup" exit("Source Environment Name" <> ''); end; + // Reverting to same-environment: drop the source connection details and the stored secret. + internal procedure ClearCrossEnvConnection() + begin + "Source Environment URL" := ''; + "Source Company Name" := ''; + "Source OAuth Client Id" := ''; + if not IsNullGuid("Source Client Secret Key") then + if not IsolatedStorage.Delete("Source Client Secret Key", DataScope::Company) then; + Clear("Source Client Secret Key"); + end; + [NonDebuggable] internal procedure SetSourceClientSecret(ClientSecret: SecretText) begin diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 74a5e0f4dde..e73071f0d3b 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -378,6 +378,71 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + [HandlerFunctions('ConfirmHandlerYes')] + procedure ClearCrossEnvSetupRevertsToSameEnvironment() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + SetupPage: TestPage "Master Data Management Setup"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Clear Cross-Environment Setup removes the source environment, company, client id, and the stored + // secret, reverting the setup to same-environment synchronization. + Initialize(); + // [GIVEN] a configured cross-environment connection with a stored secret + MasterDataManagementSetup.Get(); + MasterDataManagementSetup.Validate("Source Environment Name", 'CONTOSO-PROD'); + MasterDataManagementSetup."Source Environment URL" := 'https://api.businesscentral.dynamics.com/v2.0/contoso-prod'; + MasterDataManagementSetup."Source Company Name" := 'CRONUS'; + MasterDataManagementSetup."Source OAuth Client Id" := '11111111-2222-3333-4444-555555555555'; + MasterDataManagementSetup."Source Client Secret Key" := CreateGuid(); // simulate a stored secret key + MasterDataManagementSetup.Modify(true); + Assert.AreNotEqual('', MasterDataManagementSetup."Source Environment Name", 'Precondition: setup should be cross-environment'); + + // [WHEN] the user runs Clear Cross-Environment Setup and confirms + SetupPage.OpenEdit(); + SetupPage.ClearCrossEnvSetup.Invoke(); + SetupPage.Close(); + + // [THEN] every cross-environment field and the stored secret are cleared + MasterDataManagementSetup.Get(); + Assert.AreEqual('', MasterDataManagementSetup."Source Environment Name", 'Source environment should be cleared'); + Assert.AreEqual('', MasterDataManagementSetup."Source Environment URL", 'Source URL should be cleared'); + Assert.AreEqual('', MasterDataManagementSetup."Source Company Name", 'Source company should be cleared'); + Assert.AreEqual('', MasterDataManagementSetup."Source OAuth Client Id", 'Source client id should be cleared'); + Assert.IsTrue(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Secret key should be cleared'); // empty Source Environment Name (asserted above) means same-environment again + + CleanUp(); + end; + + [Test] + procedure ClearCrossEnvSetupDisabledWhileSynchronizationEnabled() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + SetupPage: TestPage "Master Data Management Setup"; + begin + // [FEATURE] [AI test 0.4] + // [SCENARIO] Clear Cross-Environment Setup is disabled while synchronization is enabled, so the user must + // disable synchronization before the connection can be cleared. + Initialize(); + // [GIVEN] a cross-environment setup with synchronization enabled + MasterDataManagementSetup.Get(); + MasterDataManagementSetup.Validate("Source Environment Name", 'CONTOSO-PROD'); + MasterDataManagementSetup."Is Enabled" := true; // set directly to skip the enable side effects; the action's Enabled binding is what we assert + MasterDataManagementSetup.Modify(false); + + // [THEN] the Clear Cross-Environment Setup action is disabled on the setup page + SetupPage.OpenEdit(); + Assert.IsFalse(SetupPage.ClearCrossEnvSetup.Enabled(), 'Clear action should be disabled while synchronization is enabled'); + SetupPage.Close(); + + // reset the enabled flag so it does not leak into later tests + MasterDataManagementSetup.Get(); + MasterDataManagementSetup."Is Enabled" := false; + MasterDataManagementSetup.Modify(false); + CleanUp(); + end; + [Test] procedure CrossEnvGetByFilterReturnsRecordsPastTheWatermark() var @@ -752,6 +817,12 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" MasterDataManagementSetup.Modify(false); end; + [ConfirmHandler] + procedure ConfirmHandlerYes(Question: Text; var Reply: Boolean) + begin + Reply := true; + end; + local procedure CleanUp() var InProcessTransport: Codeunit "MDM In-Process Transport"; From 4f3c8f2b29fd148e0e36518d2dccdc919eba3723 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 3 Sep 2026 15:04:59 +0200 Subject: [PATCH 54/64] [Master Data Management] Cross-environment synchonization --- .../src/codeunits/MDMInlineMedia.Codeunit.al | 25 ++++++++++++++++--- .../codeunits/MDMSourceResponse.Codeunit.al | 12 +++++++-- .../src/pages/MDMConnectionDetails.Page.al | 2 +- .../pages/MasterDataManagementSetup.Page.al | 18 ++++++------- 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al index c8d76c8f30d..b5a3c083408 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMInlineMedia.Codeunit.al @@ -21,6 +21,7 @@ codeunit 7232 "MDM Inline Media" NameByKey: Dictionary of [Text, Text]; MimeByKey: Dictionary of [Text, Text]; ClearedByKey: Dictionary of [Text, Boolean]; + MalformedMediaErr: Label 'The source returned media content that could not be decoded.', Locked = true; procedure Reset() begin @@ -59,19 +60,37 @@ codeunit 7232 "MDM Inline Media" procedure TryGet(SystemId: Guid; FieldNo: Integer; var FileName: Text; var MimeType: Text; var TempBlob: Codeunit "Temp Blob"): Boolean var - Base64Convert: Codeunit "Base64 Convert"; MediaKey: Text; ContentBase64: Text; - ContentOutStream: OutStream; begin MediaKey := MakeKey(SystemId, FieldNo); if not ContentByKey.Get(MediaKey, ContentBase64) then exit(false); NameByKey.Get(MediaKey, FileName); MimeByKey.Get(MediaKey, MimeType); + if not TryDecodeContent(ContentBase64, TempBlob) then // undecodable media content is a broken record entry from the source + Error(MalformedMediaContent()); + exit(true); + end; + + [TryFunction] + local procedure TryDecodeContent(ContentBase64: Text; var TempBlob: Codeunit "Temp Blob") + var + Base64Convert: Codeunit "Base64 Convert"; + ContentOutStream: OutStream; + begin TempBlob.CreateOutStream(ContentOutStream); Base64Convert.FromBase64(ContentBase64, ContentOutStream); - exit(true); + end; + + local procedure MalformedMediaContent(): ErrorInfo + var + ErrInfo: ErrorInfo; + begin + ErrInfo.Message := MalformedMediaErr; + ErrInfo.DataClassification := DataClassification::SystemMetadata; // Message is emitted to telemetry + ErrInfo.ErrorType := ErrorType::Internal; + exit(ErrInfo); end; local procedure MakeKey(SystemId: Guid; FieldNo: Integer): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index ff183fcd1e4..dcbc6917dcb 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -79,6 +79,14 @@ codeunit 7248 "MDM Source Response" Value := Token.AsValue().AsBoolean(); end; + [TryFunction] + local procedure TryFromBase64(ContentBase64: Text; ContentOutStream: OutStream) + var + Base64Convert: Codeunit "Base64 Convert"; + begin + Base64Convert.FromBase64(ContentBase64, ContentOutStream); + end; + local procedure MalformedControlField(PropertyName: Text): ErrorInfo var ErrInfo: ErrorInfo; @@ -227,7 +235,6 @@ codeunit 7248 "MDM Source Response" // the destination (the same path same-env uses for mapped blobs), so no destination-side apply is needed. local procedure ApplyInlineBlob(var DestField: FieldRef; FieldNo: Integer; TableId: Integer; ValueToken: JsonToken) var - Base64Convert: Codeunit "Base64 Convert"; TempBlob: Codeunit "Temp Blob"; BlobObject: JsonObject; ContentToken: JsonToken; @@ -250,7 +257,8 @@ codeunit 7248 "MDM Source Response" if not ContentToken.IsValue() then // a non-scalar content payload is a broken record entry Error(MalformedRecordEntry()); TempBlob.CreateOutStream(ContentOutStream); - Base64Convert.FromBase64(ContentToken.AsValue().AsText(), ContentOutStream); + if not TryFromBase64(ContentToken.AsValue().AsText(), ContentOutStream) then // undecodable content is a broken record entry + Error(MalformedRecordEntry()); TempBlob.ToFieldRef(DestField); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 5c797529de4..5c3e8cd25a4 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -170,7 +170,7 @@ page 7232 "MDM Connection Details" action(TestConnection) { ApplicationArea = Suite; - Caption = 'Test Connection'; + Caption = 'Test connection'; ToolTip = 'Test that the source environment can be reached with the current URL and credentials.'; Visible = TestConnectionEnabled; Image = InteractionTemplateSetup; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index 177514108a0..83d2e886dd0 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -69,7 +69,7 @@ page 7230 "Master Data Management Setup" action(ConnectionDetails) { ApplicationArea = Suite; - Caption = 'Cross-Environment Setup'; + Caption = 'Cross-environment setup'; Image = LinkAccount; ToolTip = 'Set up the connection to a company in a different Business Central environment for cross-environment synchronization.'; @@ -84,7 +84,7 @@ page 7230 "Master Data Management Setup" action(ClearCrossEnvSetup) { ApplicationArea = Suite; - Caption = 'Clear Cross-Environment Setup'; + Caption = 'Clear cross-environment setup'; Image = RemoveLine; Enabled = IsEditable; Visible = CrossEnvConfigured; @@ -105,7 +105,7 @@ page 7230 "Master Data Management Setup" action(ResetConfiguration) { ApplicationArea = Suite; - Caption = 'Use Default Synchronization Setup'; + Caption = 'Use default synchronization setup'; Image = ResetStatus; ToolTip = 'Resets the synchronization tables, fields, and job queue entries to the default values for the connection with the source company. All current synchronization tables are deleted and recreated.'; @@ -124,7 +124,7 @@ page 7230 "Master Data Management Setup" action(ExportSetup) { ApplicationArea = Suite; - Caption = 'Export Setup'; + Caption = 'Export setup'; Image = ExportFile; ToolTip = 'Export the setup tables.'; @@ -141,7 +141,7 @@ page 7230 "Master Data Management Setup" action(ImportSetup) { ApplicationArea = Suite; - Caption = 'Import Setup'; + Caption = 'Import setup'; Image = Import; ToolTip = 'Import the setup tables.'; @@ -165,7 +165,7 @@ page 7230 "Master Data Management Setup" action(StartInitialSynchAction) { ApplicationArea = Suite; - Caption = 'Start Initial Synchronization'; + Caption = 'Start initial synchronization'; Enabled = Rec."Is Enabled"; Image = RefreshLines; ToolTip = 'Start all the default synchronization jobs for synchronizing data from the source company. Data is synchronized according to the mappings defined on the Synchronization Tables page.'; @@ -174,7 +174,7 @@ page 7230 "Master Data Management Setup" action(SynchronizeNow) { ApplicationArea = Suite; - Caption = 'Synchronize Modified Records'; + Caption = 'Synchronize modified records'; Enabled = Rec."Is Enabled"; Image = Refresh; ToolTip = 'Synchronize records that have been modified since the last time they were synchronized.'; @@ -202,7 +202,7 @@ page 7230 "Master Data Management Setup" action("Synch. Job Queue Entries") { ApplicationArea = Suite; - Caption = 'Synch. Job Queue Entries'; + Caption = 'Synch. job queue entries'; Image = JobListSetup; ToolTip = 'View the job queue entries that manage the scheduled data synchronization.'; @@ -221,7 +221,7 @@ page 7230 "Master Data Management Setup" action(IntegrationTableMappings) { ApplicationArea = Suite; - Caption = 'Synchronization Tables'; + Caption = 'Synchronization tables'; Image = MapAccounts; ToolTip = 'View the list of tables to synchronize.'; From 16d31896b41cc3cacf0d54809acf9d4aca70ef8b Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Thu, 3 Sep 2026 16:36:05 +0200 Subject: [PATCH 55/64] [Master Data Management] Cross-environment synchonization --- .../test/src/MDMCrossEnvConsumerTests.Codeunit.al | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index e73071f0d3b..819a92bf719 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -370,7 +370,9 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [THEN] the setup holds the connection and a stored client secret MasterDataManagementSetup.Get(); Assert.AreEqual('CONTOSO-PROD', MasterDataManagementSetup."Source Environment Name", 'Source environment not saved'); + Assert.AreEqual('https://api.businesscentral.dynamics.com/v2.0/contoso-prod', MasterDataManagementSetup."Source Environment URL", 'Source URL not saved'); Assert.AreEqual('CRONUS', MasterDataManagementSetup."Source Company Name", 'Source company not saved'); + Assert.AreEqual('11111111-2222-3333-4444-555555555555', MasterDataManagementSetup."Source OAuth Client Id", 'Source client id not saved'); Assert.IsFalse(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Client secret should be stored'); // Restore privacy state so this configuration test doesn't leak consent into later privacy-notice scenarios. From f80c3f979eeb3f08c0c4164d2790f6c6aacf15fe Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 4 Sep 2026 06:39:29 +0200 Subject: [PATCH 56/64] [Master Data Management] Cross-environment synchonization --- .../MDMHttpSourceTransport.Codeunit.al | 17 +++++ .../src/pages/MDMConnectionDetails.Page.al | 66 +++---------------- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 25 ++++--- 3 files changed, 42 insertions(+), 66 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 092615cabfe..72698f7d377 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -38,6 +38,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" SourceHostProdTok: Label 'api.businesscentral.dynamics.com', Locked = true; SourceHostPPETok: Label 'api.businesscentral.dynamics-tie.com', Locked = true; ActionUrlTok: Label '%1/ODataV4/%2_%3?company=%4', Locked = true, Comment = '%1 = base url, %2 = service, %3 = action, %4 = company'; + BaseUrlTok: Label 'https://%1/v2.0/%2/%3', Locked = true, Comment = '%1 = api host, %2 = Entra tenant id, %3 = source environment name'; TelemetryCategoryTok: Label 'MDM Cross-Environment', Locked = true; TokenAcquiredAuditTxt: Label 'Acquired an application access token to read master data from source environment %1.', Comment = '%1 = source environment name'; TokenFailedAuditTxt: Label 'Failed to acquire an application access token for source environment %1.', Comment = '%1 = source environment name'; @@ -294,6 +295,22 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" exit(TokenEndpointTok); end; + // Same-tenant, same-ring: the source web-service base URL is fully derivable from the source environment name, + // this environment's tenant, and the ring host, so the wizard constructs it instead of asking for a free-text URL. + internal procedure BuildSourceApiBaseUrl(EnvironmentName: Text): Text + var + AzureADTenant: Codeunit "Azure AD Tenant"; + begin + exit(StrSubstNo(BaseUrlTok, GetSourceApiHost(), AzureADTenant.GetAadTenantId(), EnvironmentName)); + end; + + local procedure GetSourceApiHost(): Text + begin + if IsPPE() then + exit(SourceHostPPETok); + exit(SourceHostProdTok); + end; + local procedure GetConfiguredSetup(var MasterDataManagementSetup: Record "Master Data Management Setup") begin if not MasterDataManagementSetup.Get() then diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 5c3e8cd25a4..6a85ef3bd74 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -25,39 +25,7 @@ page 7232 "MDM Connection Details" group(Introduction) { Caption = 'Welcome'; - InstructionalText = 'This guide helps you connect to a company in a different Business Central environment so you can synchronize master data from it. Before you continue, make sure the source environment has this extension installed and exposes its data, and that you have a Microsoft Entra application with a read-only permission set on the source.'; - } - group(TermsAndConditions) - { - Caption = 'Review the terms and conditions'; - InstructionalText = 'By enabling this, you consent to sharing data between Business Central environments that may be in different geographies: this environment reads master data from the source environment you configure, and the filters it sends there may include your data. Your privacy is important to us. To learn more, follow the link below.'; - - field(Consent; ConsentState) - { - ApplicationArea = All; - Caption = 'I accept'; - ToolTip = 'Accept the terms and conditions.'; - - trigger OnValidate() - begin - // Ticking "I accept" records the durable platform privacy-notice approval. - if ConsentState then - ConsentState := MDMPrivacyNotice.ConfirmApproval(); - SetControls(); - end; - } - field(LearnMore; LearnMoreLbl) - { - ApplicationArea = All; - Editable = false; - ShowCaption = false; - ToolTip = 'View information about privacy.'; - - trigger OnDrillDown() - begin - Hyperlink(PrivacyLinkTxt); - end; - } + InstructionalText = 'This guide helps you connect to a company in a different Business Central environment so you can synchronize master data from it. Before you continue, make sure the source environment has this extension installed and exposes its data, and that you have a Microsoft Entra application with a read-only permission set on the source. When you choose Next, you are asked to review and accept the privacy terms for sharing data between environments.'; } } group(ConnectionTab) @@ -81,19 +49,6 @@ page 7232 "MDM Connection Details" SetControls(); end; } - field(SourceEnvironmentUrl; SourceEnvironmentUrl) - { - Caption = 'Source Environment URL'; - ApplicationArea = Suite; - ExtendedDatatype = URL; - ShowMandatory = true; - ToolTip = 'Specifies the base URL of the source environment''s web services, up to but not including /ODataV4.'; - - trigger OnValidate() - begin - SetControls(); - end; - } field(SourceCompanyName; SourceCompanyName) { Caption = 'Source Company Name'; @@ -238,17 +193,14 @@ page 7232 "MDM Connection Details" MDMPrivacyNotice: Codeunit "MDM Privacy Notice"; Step: Option Welcome,Connection,TestConnection,Finish; NextEnabled, BackEnabled, FinishEnabled, TestConnectionEnabled : Boolean; - ConsentState, SecretAlreadyStored : Boolean; + SecretAlreadyStored: Boolean; SourceEnvironmentName: Text[100]; - SourceEnvironmentUrl: Text[250]; SourceCompanyName: Text[100]; OAuth2ClientId: Text[100]; [NonDebuggable] OAuth2ClientSecret: Text; - LearnMoreLbl: Label 'Privacy and Cookies'; - PrivacyLinkTxt: Label 'https://go.microsoft.com/fwlink/?linkid=521839', Locked = true; ConnectionOkMsg: Label 'Successfully connected to the source environment (contract version %1).', Comment = '%1 = wire contract version'; - ConnectionFailedErr: Label 'Could not connect to the source environment. Check the URL, company, and credentials, then try again.'; + ConnectionFailedErr: Label 'Could not connect to the source environment. Check the source environment, company, and credentials, then try again.'; local procedure LoadConfiguration() var @@ -257,7 +209,6 @@ page 7232 "MDM Connection Details" if not MasterDataManagementSetup.Get() then exit; SourceEnvironmentName := MasterDataManagementSetup."Source Environment Name"; - SourceEnvironmentUrl := MasterDataManagementSetup."Source Environment URL"; SourceCompanyName := MasterDataManagementSetup."Source Company Name"; OAuth2ClientId := MasterDataManagementSetup."Source OAuth Client Id"; SecretAlreadyStored := not IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"); @@ -267,13 +218,15 @@ page 7232 "MDM Connection Details" local procedure SaveConfiguration() var MasterDataManagementSetup: Record "Master Data Management Setup"; + HttpSourceTransport: Codeunit "MDM Http Source Transport"; begin if not MasterDataManagementSetup.Get() then begin MasterDataManagementSetup.Init(); MasterDataManagementSetup.Insert(); end; MasterDataManagementSetup.Validate("Source Environment Name", SourceEnvironmentName); - MasterDataManagementSetup."Source Environment URL" := SourceEnvironmentUrl; + // Same-tenant, same-ring: derive the source web-service URL from the environment name instead of asking for it. + MasterDataManagementSetup."Source Environment URL" := CopyStr(HttpSourceTransport.BuildSourceApiBaseUrl(SourceEnvironmentName), 1, MaxStrLen(MasterDataManagementSetup."Source Environment URL")); MasterDataManagementSetup."Source Company Name" := SourceCompanyName; MasterDataManagementSetup."Source OAuth Client Id" := OAuth2ClientId; if OAuth2ClientSecret <> '' then begin @@ -311,6 +264,9 @@ page 7232 "MDM Connection Details" local procedure NextStep() begin + // Leaving the Welcome step requires the durable platform privacy-notice approval; show it once if needed. + if (Step = Step::Welcome) and not (MDMPrivacyNotice.IsApproved() or MDMPrivacyNotice.ConfirmApproval()) then + exit; Step += 1; SetControls(); end; @@ -332,10 +288,8 @@ page 7232 "MDM Connection Details" local procedure StepIsComplete(): Boolean begin case Step of - Step::Welcome: - exit(ConsentState); Step::Connection: - exit((SourceEnvironmentName <> '') and (SourceEnvironmentUrl <> '') and (SourceCompanyName <> '') and + exit((SourceEnvironmentName <> '') and (SourceCompanyName <> '') and (OAuth2ClientId <> '') and ((OAuth2ClientSecret <> '') or SecretAlreadyStored)); else exit(true); diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 819a92bf719..2c9cfd654d3 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -144,21 +144,21 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" ConnectionWizard: TestPage "MDM Connection Details"; begin // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] [Privacy] - // [SCENARIO] Ticking consent in the wizard opens the platform privacy notice - a regression guard that the + // [SCENARIO] Choosing Next on the Welcome step opens the platform privacy notice - a regression guard that the // wizard actually calls ConfirmApproval (the handler below fires only if the notice dialog is shown). Initialize(); - // [GIVEN] the privacy notice has no recorded decision, so accepting consent must prompt it + // [GIVEN] the privacy notice has no recorded decision, so continuing past Welcome must prompt it LibraryMasterDataMgt.PrivacyNoticeResetApproval(); WizardPrivacyNoticeOpenCount := 0; - // [WHEN] the admin ticks consent in the connection wizard + // [WHEN] the admin chooses Next on the Welcome step ConnectionWizard.OpenEdit(); - ConnectionWizard.Consent.SetValue(true); + ConnectionWizard.ActionNext.Invoke(); ConnectionWizard.Close(); // [THEN] the privacy-notice dialog was shown exactly once - proving the wizard invoked ConfirmApproval - Assert.AreEqual(1, WizardPrivacyNoticeOpenCount, 'Ticking consent should open the privacy notice exactly once (call ConfirmApproval)'); + Assert.AreEqual(1, WizardPrivacyNoticeOpenCount, 'Choosing Next on the Welcome step should open the privacy notice exactly once (call ConfirmApproval)'); LibraryMasterDataMgt.PrivacyNoticeResetApproval(); CleanUp(); @@ -345,6 +345,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" MasterDataManagementSetup: Record "Master Data Management Setup"; PrivacyNotice: Codeunit "Privacy Notice"; LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + AzureADTenant: Codeunit "Azure AD Tenant"; ConnectionDetails: TestPage "MDM Connection Details"; begin // [FEATURE] [AI test 0.4] @@ -354,12 +355,10 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" PrivacyNotice.SetApprovalState(LibraryMasterDataMgt.PrivacyNoticeId(), "Privacy Notice Approval State"::Agreed); ConnectionDetails.OpenEdit(); - // Welcome step: accept the terms so Next is enabled. - ConnectionDetails.Consent.SetValue(true); + // Welcome step: consent is pre-approved above, so Next advances without prompting. ConnectionDetails.ActionNext.Invoke(); - // Connection step: provide the source environment and credentials. + // Connection step: provide the source environment and credentials (the URL is derived from the environment name). ConnectionDetails.SourceEnvironmentName.SetValue('CONTOSO-PROD'); - ConnectionDetails.SourceEnvironmentUrl.SetValue('https://api.businesscentral.dynamics.com/v2.0/contoso-prod'); ConnectionDetails.SourceCompanyName.SetValue('CRONUS'); ConnectionDetails.OAuth2ClientId.SetValue('11111111-2222-3333-4444-555555555555'); ConnectionDetails.OAuth2ClientSecret.SetValue('super-secret'); @@ -370,7 +369,13 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" // [THEN] the setup holds the connection and a stored client secret MasterDataManagementSetup.Get(); Assert.AreEqual('CONTOSO-PROD', MasterDataManagementSetup."Source Environment Name", 'Source environment not saved'); - Assert.AreEqual('https://api.businesscentral.dynamics.com/v2.0/contoso-prod', MasterDataManagementSetup."Source Environment URL", 'Source URL not saved'); + // The URL is constructed from the current tenant, ring host, and source environment name. + Assert.IsTrue( + StrPos(MasterDataManagementSetup."Source Environment URL", '/v2.0/' + AzureADTenant.GetAadTenantId() + '/CONTOSO-PROD') > 0, + 'Source URL should embed the tenant id and source environment name'); + Assert.IsTrue( + StrPos(LowerCase(MasterDataManagementSetup."Source Environment URL"), 'https://api.businesscentral.dynamics') = 1, + 'Source URL should target the Business Central API host'); Assert.AreEqual('CRONUS', MasterDataManagementSetup."Source Company Name", 'Source company not saved'); Assert.AreEqual('11111111-2222-3333-4444-555555555555', MasterDataManagementSetup."Source OAuth Client Id", 'Source client id not saved'); Assert.IsFalse(IsNullGuid(MasterDataManagementSetup."Source Client Secret Key"), 'Client secret should be stored'); From 6ddc237491055fbf843460829fef956d4a49cc23 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Fri, 4 Sep 2026 10:33:28 +0200 Subject: [PATCH 57/64] [Master Data Management] Cross-environment synchonization --- .../src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al | 6 +++--- .../app/src/codeunits/MDMSourceResponse.Codeunit.al | 2 ++ .../app/src/pages/MasterDataSynchFields.Page.al | 2 +- .../app/src/pages/MasterDataSynchTables.Page.al | 8 ++++---- .../test/src/MDMCrossEnvDetectorTests.Codeunit.al | 3 ++- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al index 048a4bc8705..2413f3b9e62 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvChangeDetector.Codeunit.al @@ -58,7 +58,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" // Still non-fatal to this recurring job: skip the poll, the next scheduled run retries. if not TryNegotiateDetectionSupport(Transport, Supported) then begin Dimensions.Add('Category', MasterDataManagement.GetTelemetryCategory()); - Session.LogMessage('0000VAZ', DetectorCapabilitiesFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); + Session.LogMessage('0000VAZ', DetectorCapabilitiesFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, Dimensions); exit; end; // Older source doesn't advertise the detection action: skip rather than error every run. @@ -72,7 +72,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" exit; end; if not SourceResponse.TryParse(ResponseText, Response) then begin - Session.LogMessage('0000VAP', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAP', DetectorParseFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; @@ -128,7 +128,7 @@ codeunit 7245 "MDM Cross-Env Change Detector" if SourceResponse.ConsentRequired(Response) then exit; // source hasn't consented to sharing; the sync job surfaces the actionable error, the detector skips if (not Response.Get('tables', TablesToken)) or (not TablesToken.IsArray()) then begin - Session.LogMessage('0000VAQ', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VAQ', DetectionContractFailedTxt, Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); exit; end; Tables := TablesToken.AsArray(); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al index dcbc6917dcb..2d9b5a9feb6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMSourceResponse.Codeunit.al @@ -115,6 +115,8 @@ codeunit 7248 "MDM Source Response" begin if not Response.Get('nextCursor', Token) then exit(''); + if not Token.IsObject() then // a non-object cursor is malformed; report "no cursor" so the caller fails the response instead of replaying from the start + exit(''); Token.WriteTo(CursorText); exit(CursorText); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al index 603c49183c6..ca63af7e94f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchFields.Page.al @@ -94,7 +94,7 @@ page 7236 "Master Data Synch. Fields" action(FieldMapping) { ApplicationArea = Suite; - Caption = 'Update Fields'; + Caption = 'Update fields'; Image = Relationship; ToolTip = 'Updates field mappings to match table schema. Use this action if you added fields to the table with an extension.'; diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al index 3b7058f9531..bf09f7885e3 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataSynchTables.Page.al @@ -231,7 +231,7 @@ page 7233 "Master Data Synch. Tables" action(ResetConfiguration) { ApplicationArea = Suite; - Caption = 'Use Default Synchronization Setup'; + Caption = 'Use default synchronization setup'; Image = ResetStatus; ToolTip = 'Resets the tables, fields and synchronization jobs to the default values for the connection with the source company. All default synchronization table definitions are deleted and recreated.'; @@ -323,7 +323,7 @@ page 7233 "Master Data Synch. Tables" action(SynchronizeNow) { ApplicationArea = Suite; - Caption = 'Synchronize Modified Records'; + Caption = 'Synchronize modified records'; Enabled = HasRecords and (Rec."Parent Name" = '') and DataSynchEnabled; Image = Refresh; ToolTip = 'Synchronize records that have been modified since the last time they were synchronized.'; @@ -352,7 +352,7 @@ page 7233 "Master Data Synch. Tables" action(SynchronizeAll) { ApplicationArea = Suite; - Caption = 'Run Full Synchronization'; + Caption = 'Run full synchronization'; Enabled = HasRecords and (Rec."Parent Name" = '') and DataSynchEnabled; Image = RefreshLines; ToolTip = 'Start a job for full synchronization from records in the chosen source company for each of the selected tables.'; @@ -409,7 +409,7 @@ page 7233 "Master Data Synch. Tables" action(RemoveCoupling) { ApplicationArea = Suite; - Caption = 'Delete Couplings'; + Caption = 'Delete couplings'; Enabled = HasRecords and (Rec."Parent Name" = '') and DataSynchEnabled; Image = UnLinkAccount; ToolTip = 'Delete couplings for the selected tables.'; diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al index 638abc14cf4..96003fb47bc 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvDetectorTests.Codeunit.al @@ -162,7 +162,8 @@ codeunit 139933 "MDM Cross-Env Detector Tests" begin InProcessTransport.Deactivate(); DetectorProbe.Deactivate(); - // RunChangeDetector goes through Codeunit.Run, which commits, so prior test data survives rollback. + // RunChangeDetector calls DetectChanges() directly (no Codeunit.Run) and the detector never commits, so it runs + // inside the test transaction; DeleteTestArtifacts() defensively clears any residue from an aborted prior run. DeleteTestArtifacts(); if not MasterDataManagementSetup.Get() then begin MasterDataManagementSetup.Init(); From af483dcb1e766823e53c8a97f64368792733feb9 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Tue, 8 Sep 2026 18:08:27 +0200 Subject: [PATCH 58/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtSubscribers.Codeunit.al | 161 +++++++++++------- .../MasterDataMgtUpgrade.Codeunit.al | 21 +-- .../src/pages/MDMConnectionDetails.Page.al | 19 ++- .../src/MDMCrossEnvConsumerTests.Codeunit.al | 7 + 4 files changed, 124 insertions(+), 84 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 2353cc5c232..07ff2320a1d 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -808,13 +808,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup.Get() then exit; - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - - if not Company.Get(SourceCompanyName) then - exit; - IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Table ID", Database::Customer); IntegrationTableMapping.SetRange("Integration Table ID", Database::Customer); @@ -823,8 +816,17 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; - if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then - exit; + // Same-environment reads the source company's relations directly; cross-environment reads the locally + // replicated relations (they sync before customers/vendors), since the source isn't a local company. + if not MasterDataManagementSetup.IsCrossEnvironment() then begin + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; + if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then + exit; + end; ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::Customer); ContactBusinessRelation.SetRange("No.", Customer."No."); @@ -853,13 +855,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup.Get() then exit; - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - - if not Company.Get(SourceCompanyName) then - exit; - IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Table ID", Database::Vendor); IntegrationTableMapping.SetRange("Integration Table ID", Database::Vendor); @@ -868,8 +863,17 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; - if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then - exit; + // Same-environment reads the source company's relations directly; cross-environment reads the locally + // replicated relations (they sync before customers/vendors), since the source isn't a local company. + if not MasterDataManagementSetup.IsCrossEnvironment() then begin + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; + if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then + exit; + end; ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::Vendor); ContactBusinessRelation.SetRange("No.", Vendor."No."); @@ -898,13 +902,6 @@ codeunit 7237 "Master Data Mgt. Subscribers" if not MasterDataManagementSetup.Get() then exit; - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - - if not Company.Get(SourceCompanyName) then - exit; - IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Table ID", Database::"Bank Account"); IntegrationTableMapping.SetRange("Integration Table ID", Database::"Bank Account"); @@ -913,8 +910,17 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; - if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then - exit; + // Same-environment reads the source company's relations directly; cross-environment reads the locally + // replicated relations (they sync before customers/vendors), since the source isn't a local company. + if not MasterDataManagementSetup.IsCrossEnvironment() then begin + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; + if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then + exit; + end; ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::"Bank Account"); ContactBusinessRelation.SetRange("No.", BankAccount."No."); @@ -1171,72 +1177,95 @@ codeunit 7237 "Master Data Mgt. Subscribers" MasterDataManagement: Codeunit "Master Data Management"; RecRef: RecordRef; RecordModifiedAfterLastSync: Boolean; + CrossEnvironment: Boolean; LinkType: Enum "Contact Business Relation Link To Table"; begin if not MasterDataManagement.IsEnabled() then exit(false); MasterDataManagementSetup.Get(); - // Cross-environment: related contact/customer resolution reads the source company directly; deferred for now. - if MasterDataManagementSetup."Source Environment Name" <> '' then - exit(false); + // Cross-environment: the source's contact business relations are replicated locally, so resolve the related + // customer/vendor from the local relation (by No.) instead of reading the source company directly. + CrossEnvironment := MasterDataManagementSetup.IsCrossEnvironment(); DestinationRecordRef.SetTable(Contact); - IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); + if not CrossEnvironment then + IntegrationContact.ChangeCompany(MasterDataManagementSetup."Company Name"); SourceRecordRef.SetTable(IntegrationContact); - IntegrationContactBusinessRelation.ChangeCompany(MasterDataManagementSetup."Company Name"); + if not CrossEnvironment then + IntegrationContactBusinessRelation.ChangeCompany(MasterDataManagementSetup."Company Name"); case IntegrationContact."Contact Business Relation" of IntegrationContact."Contact Business Relation"::Customer: begin - IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); + if not CrossEnvironment then + IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); if IntegrationContactBusinessRelation.FindByContact(LinkType::Customer, IntegrationContact."No.") then - if IntegrationCustomer.Get(IntegrationContactBusinessRelation."No.") then - if FindCustomerByIntegrationSystemId(IntegrationCustomer.SystemId, Customer) then - if Customer."Primary Contact No." = '' then - if IntegrationTableMapping.FindMapping(Database::Customer, Database::Customer) then - if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin - RecRef.GetTable(Customer); - RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); - Customer.Validate("Primary Contact No.", Contact."No."); - Customer.Modify(); - if not RecordModifiedAfterLastSync then begin - MasterDataMgtCoupling.SetRange("Local System ID", Customer.SystemId); - if MasterDataMgtCoupling.FindFirst() then begin - MasterDataMgtCoupling."Last Synch. Modified On" := Customer.SystemModifiedAt; - MasterDataMgtCoupling.Modify(); - end; + if ResolvePrimaryContactCustomer(CrossEnvironment, IntegrationContactBusinessRelation."No.", IntegrationCustomer, Customer) then + if Customer."Primary Contact No." = '' then + if IntegrationTableMapping.FindMapping(Database::Customer, Database::Customer) then + if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin + RecRef.GetTable(Customer); + RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); + Customer.Validate("Primary Contact No.", Contact."No."); + Customer.Modify(); + if not RecordModifiedAfterLastSync then begin + MasterDataMgtCoupling.SetRange("Local System ID", Customer.SystemId); + if MasterDataMgtCoupling.FindFirst() then begin + MasterDataMgtCoupling."Last Synch. Modified On" := Customer.SystemModifiedAt; + MasterDataMgtCoupling.Modify(); end; - exit(true); end; + exit(true); + end; end; IntegrationContact."Contact Business Relation"::Vendor: begin - IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); + if not CrossEnvironment then + IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); if IntegrationContactBusinessRelation.FindByContact(LinkType::Vendor, IntegrationContact."No.") then - if IntegrationVendor.Get(IntegrationContactBusinessRelation."No.") then - if FindVendorByIntegrationSystemId(IntegrationVendor.SystemId, Vendor) then - if Vendor."Primary Contact No." = '' then - if IntegrationTableMapping.FindMapping(Database::Vendor, Database::Vendor) then - if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin - RecRef.GetTable(Vendor); - RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); - Vendor.Validate("Primary Contact No.", Contact."No."); - Vendor.Modify(); - if not RecordModifiedAfterLastSync then begin - MasterDataMgtCoupling.SetRange("Local System ID", Vendor.SystemId); - if MasterDataMgtCoupling.FindFirst() then begin - MasterDataMgtCoupling."Last Synch. Modified On" := Vendor.SystemModifiedAt; - MasterDataMgtCoupling.Modify(); - end; + if ResolvePrimaryContactVendor(CrossEnvironment, IntegrationContactBusinessRelation."No.", IntegrationVendor, Vendor) then + if Vendor."Primary Contact No." = '' then + if IntegrationTableMapping.FindMapping(Database::Vendor, Database::Vendor) then + if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin + RecRef.GetTable(Vendor); + RecordModifiedAfterLastSync := IntegrationRecSynchInvoke.WasModifiedAfterLastSynch(IntegrationTableMapping, RecRef); + Vendor.Validate("Primary Contact No.", Contact."No."); + Vendor.Modify(); + if not RecordModifiedAfterLastSync then begin + MasterDataMgtCoupling.SetRange("Local System ID", Vendor.SystemId); + if MasterDataMgtCoupling.FindFirst() then begin + MasterDataMgtCoupling."Last Synch. Modified On" := Vendor.SystemModifiedAt; + MasterDataMgtCoupling.Modify(); end; - exit(true); end; + exit(true); + end; end; else exit(false) end; end; + // Same-environment maps the source customer to the destination via its source SystemId coupling; cross-environment + // resolves the destination customer directly by No. (numbers are aligned by synchronization). + local procedure ResolvePrimaryContactCustomer(CrossEnvironment: Boolean; CustomerNo: Code[20]; var IntegrationCustomer: Record Customer; var Customer: Record Customer): Boolean + begin + if CrossEnvironment then + exit(Customer.Get(CustomerNo)); + if not IntegrationCustomer.Get(CustomerNo) then + exit(false); + exit(FindCustomerByIntegrationSystemId(IntegrationCustomer.SystemId, Customer)); + end; + + local procedure ResolvePrimaryContactVendor(CrossEnvironment: Boolean; VendorNo: Code[20]; var IntegrationVendor: Record Vendor; var Vendor: Record Vendor): Boolean + begin + if CrossEnvironment then + exit(Vendor.Get(VendorNo)); + if not IntegrationVendor.Get(VendorNo) then + exit(false); + exit(FindVendorByIntegrationSystemId(IntegrationVendor.SystemId, Vendor)); + end; + local procedure FindCustomerByIntegrationSystemId(IntegrationSystemId: Guid; var Customer: Record Customer): Boolean var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al index 8f8041e4c32..dc73cfb9af6 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al @@ -28,19 +28,17 @@ codeunit 7238 "Master Data Mgt. Upgrade" end; // Guaranteed provisioning path: install codeunits are skipped when BC is pre-baked into a package and mounted per tenant. + // NOT gated by an upgrade tag: SetAllUpgradeTags on a fresh install can set the tag without the service ever being + // created, which would permanently skip registration and 404 the source API. Ensure the record exists instead - + // idempotent and self-healing on every install and upgrade. Access stays gated by the "Cross Env" permission set, not by publishing. internal procedure RegisterCrossEnvSourceWebService() var TenantWebService: Record "Tenant Web Service"; WebServiceManagement: Codeunit "Web Service Management"; - UpgradeTag: Codeunit "Upgrade Tag"; begin - if UpgradeTag.HasDatabaseUpgradeTag(GetCrossEnvWebServiceUpgradeTag()) then + if TenantWebService.Get(TenantWebService."Object Type"::Codeunit, CrossEnvSourceWebServiceName()) then exit; - - // Idempotent: creates or updates the service. Access stays gated by the dedicated "Cross Env" permission set, not by publishing. WebServiceManagement.CreateTenantWebService(TenantWebService."Object Type"::Codeunit, Codeunit::"MDM Cross-Env Source API", CrossEnvSourceWebServiceName(), true); - - UpgradeTag.SetDatabaseUpgradeTag(GetCrossEnvWebServiceUpgradeTag()); end; internal procedure CrossEnvSourceWebServiceName(): Text[240] @@ -129,21 +127,10 @@ codeunit 7238 "Master Data Mgt. Upgrade" exit('MS-543635-MDMJobQueueFrequency-20240830'); end; - local procedure GetCrossEnvWebServiceUpgradeTag(): Code[250] - begin - exit('MS-647660-MDMCrossEnvWebService-20260826'); - end; - [EventSubscriber(ObjectType::Codeunit, Codeunit::"Upgrade Tag", 'OnGetPerCompanyUpgradeTags', '', false, false)] local procedure RegisterPerCompanyTags(var PerCompanyUpgradeTags: List of [Code[250]]) begin PerCompanyUpgradeTags.Add(GetSynchTableCaptionUpgradeTag()); PerCompanyUpgradeTags.Add(GetJobQueueFrequencyUpgradeTag()); end; - - [EventSubscriber(ObjectType::Codeunit, Codeunit::"Upgrade Tag", 'OnGetPerDatabaseUpgradeTags', '', false, false)] - local procedure RegisterPerDatabaseTags(var PerDatabaseUpgradeTags: List of [Code[250]]) - begin - PerDatabaseUpgradeTags.Add(GetCrossEnvWebServiceUpgradeTag()); - end; } \ No newline at end of file diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al index 6a85ef3bd74..af669575d96 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MDMConnectionDetails.Page.al @@ -112,7 +112,7 @@ page 7232 "MDM Connection Details" group(AllDone) { Caption = 'All done'; - InstructionalText = 'You''re all set. Choose Finish to save your connection settings.'; + InstructionalText = 'You''re all set. Choose Finish to save your connection settings. You can enable data synchronization right away.'; } } } @@ -176,6 +176,7 @@ page 7232 "MDM Connection Details" trigger OnAction() begin SaveConfiguration(); + EnableSynchronizationOnFinish(); CurrPage.Close(); end; } @@ -201,6 +202,7 @@ page 7232 "MDM Connection Details" OAuth2ClientSecret: Text; ConnectionOkMsg: Label 'Successfully connected to the source environment (contract version %1).', Comment = '%1 = wire contract version'; ConnectionFailedErr: Label 'Could not connect to the source environment. Check the source environment, company, and credentials, then try again.'; + EnableNowQst: Label 'Your cross-environment connection is saved. Do you want to enable data synchronization now?'; local procedure LoadConfiguration() var @@ -238,6 +240,21 @@ page 7232 "MDM Connection Details" MasterDataManagementSetup.Modify(true); end; + // Enabling here starts synchronization straight from Finish so the admin doesn't have to flip the setup toggle afterward. + local procedure EnableSynchronizationOnFinish() + var + MasterDataManagementSetup: Record "Master Data Management Setup"; + begin + if not MasterDataManagementSetup.Get() then + exit; + if MasterDataManagementSetup."Is Enabled" then + exit; + if not Confirm(EnableNowQst) then + exit; + MasterDataManagementSetup.Validate("Is Enabled", true); + MasterDataManagementSetup.Modify(true); + end; + [NonDebuggable] local procedure TestConnectionToSource() var diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 2c9cfd654d3..48899c80441 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -340,6 +340,7 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" end; [Test] + [HandlerFunctions('ConfirmHandlerNo')] procedure ConnectionDetailsWizardSavesConfiguration() var MasterDataManagementSetup: Record "Master Data Management Setup"; @@ -830,6 +831,12 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" Reply := true; end; + [ConfirmHandler] + procedure ConfirmHandlerNo(Question: Text; var Reply: Boolean) + begin + Reply := false; + end; + local procedure CleanUp() var InProcessTransport: Codeunit "MDM In-Process Transport"; From e7e59b616e1704f7c91a263b21bdc0714da77ba4 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 9 Sep 2026 10:00:41 +0200 Subject: [PATCH 59/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataManagement.Codeunit.al | 20 +++++++++- .../src/LibraryMasterDataMgt.Codeunit.al | 9 +++++ .../src/MasterDataMgtSynchTests.Codeunit.al | 38 +++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al index 467363b396e..895c0671732 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataManagement.Codeunit.al @@ -2309,13 +2309,29 @@ codeunit 7233 "Master Data Management" IntegrationTableMapping.SetFilter("Integration Table ID", '<>0'); if IntegrationTableMapping.FindSet() then repeat - // Route through the data source: find which enabled mapping's source table holds this record. + // Route through the data source: find which enabled mapping's source table holds this record, then + // confirm it is within that mapping's integration table filter (GetBySystemId is a key lookup and ignores it). if MasterDataManagementSetup.GetDataSource().GetBySystemId(IntegrationTableMapping."Integration Table ID", MasterDataMgtCoupling."Integration System ID", IntegrationRecordRef) then - exit(true); + if MasterDataManagementSetup.IsCrossEnvironment() or IntegrationRecordMatchesMappingFilter(IntegrationTableMapping, IntegrationRecordRef, MasterDataMgtCoupling."Integration System ID") then + exit(true); until IntegrationTableMapping.Next() = 0; exit(false); end; + // GetBySystemId retrieves by key and ignores filters; same-environment must re-check the mapping's integration + // table filter here (the previous SetView-based lookup enforced it). Cross-environment applies the filter source-side. + local procedure IntegrationRecordMatchesMappingFilter(IntegrationTableMapping: Record "Integration Table Mapping"; var IntegrationRecordRef: RecordRef; IntegrationSystemId: Guid): Boolean + var + IntegrationTableFilter: Text; + begin + IntegrationTableFilter := IntegrationTableMapping.GetIntegrationTableFilter(); + if IntegrationTableFilter = '' then + exit(true); + IntegrationRecordRef.SetView(IntegrationTableFilter); + IntegrationRecordRef.Field(IntegrationTableMapping."Integration Table UID Fld. No.").SetRange(IntegrationSystemId); + exit(not IntegrationRecordRef.IsEmpty()); + end; + internal procedure CheckSetupPermissions() var MasterDataManagementSetup: Record "Master Data Management Setup"; diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 3bce0f514b1..729e651cc48 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -81,6 +81,15 @@ codeunit 139757 "Library - Master Data Mgt." MasterDataManagementSetup.Modify(false); end; + /// Finds the enabled Master Data Management mapping whose source table holds the coupling's integration record, respecting the mapping's integration table filter. + /// Returns the matched mapping. + /// The coupling whose integration record is resolved. + /// True if an enabled mapping holds the record within its integration table filter; otherwise false. + procedure FindMappingByIntegrationRecordId(var IntegrationTableMapping: Record "Integration Table Mapping"; var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"): Boolean + begin + exit(MasterDataManagement.FindMappingByIntegrationRecordId(IntegrationTableMapping, MasterDataMgtCoupling)); + end; + /// Gets the integration record reference for a coupling. /// The integration table ID to resolve. /// The coupling whose integration record is requested. diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al index 25fa554b710..b59c07b5127 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al @@ -869,6 +869,44 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" Assert.ExpectedError('The cross-environment connection to the source is not configured yet'); end; + [Test] + [HandlerFunctions('SynchronizationEnabledMessageHandler')] + procedure FindMappingByIntegrationRecordIdRespectsIntegrationTableFilter() + var + IntegrationTableMapping: Record "Integration Table Mapping"; + CustomerInFilter: Record Customer; + CustomerOutsideFilter: Record Customer; + MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + CustomerRecRef: RecordRef; + begin + // [SCENARIO] FindMappingByIntegrationRecordId matches a mapping only when the source record is within the + // mapping's integration table filter - GetBySystemId is a key lookup that ignores the filter. + Initialize(); + LibraryMasterDataMgt.SetSourceCompanyToCurrent(); + + // [GIVEN] two source customers, and the Customer mapping's integration table filter includes only the first + LibrarySales.CreateCustomer(CustomerInFilter); + LibrarySales.CreateCustomer(CustomerOutsideFilter); + GetCustomerMapping(IntegrationTableMapping); + CustomerRecRef.Open(Database::Customer); + CustomerRecRef.Field(CustomerInFilter.FieldNo("No.")).SetRange(CustomerInFilter."No."); + IntegrationTableMapping.SetIntegrationTableFilter(CustomerRecRef.GetView()); + CustomerRecRef.Close(); + IntegrationTableMapping.Modify(); + + // [WHEN] resolving the mapping for the in-filter record [THEN] it matches + Clear(MasterDataMgtCoupling); + MasterDataMgtCoupling."Integration System ID" := CustomerInFilter.SystemId; + Assert.IsTrue(LibraryMasterDataMgt.FindMappingByIntegrationRecordId(IntegrationTableMapping, MasterDataMgtCoupling), 'A source record within the integration table filter should match its mapping.'); + + // [WHEN] resolving the mapping for the out-of-filter record [THEN] it does not match (the filter is enforced) + Clear(IntegrationTableMapping); + Clear(MasterDataMgtCoupling); + MasterDataMgtCoupling."Integration System ID" := CustomerOutsideFilter.SystemId; + Assert.IsFalse(LibraryMasterDataMgt.FindMappingByIntegrationRecordId(IntegrationTableMapping, MasterDataMgtCoupling), 'A source record outside the integration table filter must not match the mapping.'); + end; + local procedure GetCustomerMapping(var IntegrationTableMapping: Record "Integration Table Mapping") begin IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); From 599ba4b4beddcb5780fd002be716d1c9589c1ed1 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 9 Sep 2026 11:33:30 +0200 Subject: [PATCH 60/64] [Master Data Management] Cross-environment synchonization --- .../MDMCrossEnvRead.PermissionSet.al | 1 + .../MDMContactRelationCache.Codeunit.al | 151 ++++++++++++++++++ .../MDMCrossEnvDataSource.Codeunit.al | 65 ++++++++ .../MasterDataMgtSubscribers.Codeunit.al | 121 +++++++++----- .../MasterDataMgtUpgrade.Codeunit.al | 18 ++- .../src/LibraryMasterDataMgt.Codeunit.al | 12 ++ .../src/MDMCrossEnvConsumerTests.Codeunit.al | 74 +++++++++ 7 files changed, 402 insertions(+), 40 deletions(-) create mode 100644 src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al index f826696425c..42921205af4 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MDMCrossEnvRead.PermissionSet.al @@ -40,6 +40,7 @@ permissionset 7242 "MDM Cross-Env Read" tabledata Vendor = R, tabledata Contact = R, tabledata "Business Relation" = R, + tabledata "Contact Business Relation" = R, tabledata "Country/Region" = R, tabledata "Post Code" = R, tabledata Currency = R, diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al new file mode 100644 index 00000000000..b41446b138c --- /dev/null +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al @@ -0,0 +1,151 @@ +namespace Microsoft.Integration.MDM; + +using Microsoft.CRM.BusinessRelation; +using Microsoft.Integration.SyncEngine; + +/// +/// Resolves source contact business relations cross-environment for the contact auto-create/primary-contact logic. +/// During a synchronization run it prefetches all of a link type's relations in a single call and answers lookups +/// from memory, turning the otherwise O(N) per-contact over-the-wire reads into O(1) per link type. Outside a run +/// (a one-off customer/vendor insert), or when the source reports the set is too large to serve in bulk, it reads +/// the single requested relation over the wire instead. Single instance so the prefetched snapshot survives across +/// the many record inserts of a run; the snapshot is reset at each run boundary. +/// +codeunit 7234 "MDM Contact Relation Cache" +{ + Access = Internal; + SingleInstance = true; + + var + ContactNoByRelation: Dictionary of [Text, Code[20]]; + RelationNoByContact: Dictionary of [Text, Code[20]]; + BulkAttempted: Dictionary of [Integer, Boolean]; + Degraded: Dictionary of [Integer, Boolean]; + InSyncRun: Boolean; + + /// Resolves the source contact number linked to a customer/vendor/bank account relation. + procedure TryGetSourceContactNo(LinkToTable: Enum "Contact Business Relation Link To Table"; RelationNo: Code[20]; var SourceContactNo: Code[20]): Boolean + begin + if UseBulk(LinkToTable) then + exit(ContactNoByRelation.Get(RelationKey(LinkToTable, RelationNo), SourceContactNo) and (SourceContactNo <> '')); + exit(ReadSingleContactNo(LinkToTable, RelationNo, SourceContactNo)); + end; + + /// Resolves the source relation number (customer/vendor No.) linked to a source contact. + procedure TryGetSourceRelationNo(LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]; var RelationNo: Code[20]): Boolean + begin + if UseBulk(LinkToTable) then + exit(RelationNoByContact.Get(ContactKey(LinkToTable, SourceContactNo), RelationNo) and (RelationNo <> '')); + exit(ReadSingleRelationNo(LinkToTable, SourceContactNo, RelationNo)); + end; + + local procedure UseBulk(LinkToTable: Enum "Contact Business Relation Link To Table"): Boolean + begin + if not InSyncRun then + exit(false); + EnsureBulkLoaded(LinkToTable); + exit(not Degraded.ContainsKey(LinkToTable.AsInteger())); + end; + + local procedure EnsureBulkLoaded(LinkToTable: Enum "Contact Business Relation Link To Table") + begin + if BulkAttempted.ContainsKey(LinkToTable.AsInteger()) then + exit; + BulkAttempted.Add(LinkToTable.AsInteger(), true); + LoadLinkType(LinkToTable); + end; + + local procedure LoadLinkType(LinkToTable: Enum "Contact Business Relation Link To Table") + var + FilterContactBusinessRelation: Record "Contact Business Relation"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + SourceRecordRef: RecordRef; + NotIndexed: Boolean; + RelationNo: Code[20]; + SourceContactNo: Code[20]; + begin + FilterContactBusinessRelation.SetRange("Link to Table", LinkToTable); + if not CrossEnvDataSource.TryBulkGetSourceRecordsByFilter(Database::"Contact Business Relation", FilterContactBusinessRelation.GetView(), SourceRecordRef, NotIndexed) then begin + // The source could not serve the whole set (too large to index, or a transient/consent issue): fall back + // to reading the one requested relation per lookup, which preserves per-record error isolation. + Degraded.Add(LinkToTable.AsInteger(), true); + exit; + end; + if not SourceRecordRef.FindSet() then + exit; + repeat + RelationNo := SourceRecordRef.Field(FilterContactBusinessRelation.FieldNo("No.")).Value(); + SourceContactNo := SourceRecordRef.Field(FilterContactBusinessRelation.FieldNo("Contact No.")).Value(); + if (RelationNo <> '') and (SourceContactNo <> '') then begin + AddUnique(ContactNoByRelation, RelationKey(LinkToTable, RelationNo), SourceContactNo); + AddUnique(RelationNoByContact, ContactKey(LinkToTable, SourceContactNo), RelationNo); + end; + until SourceRecordRef.Next() = 0; + end; + + local procedure ReadSingleContactNo(LinkToTable: Enum "Contact Business Relation Link To Table"; RelationNo: Code[20]; var SourceContactNo: Code[20]): Boolean + var + SourceContactBusinessRelation: Record "Contact Business Relation"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + SourceRecordRef: RecordRef; + begin + SourceContactBusinessRelation.SetRange("Link to Table", LinkToTable); + SourceContactBusinessRelation.SetRange("No.", RelationNo); + if not CrossEnvDataSource.GetSourceRecordsByFilter(Database::"Contact Business Relation", SourceContactBusinessRelation.GetView(), SourceRecordRef) then + exit(false); + SourceContactNo := SourceRecordRef.Field(SourceContactBusinessRelation.FieldNo("Contact No.")).Value(); + exit(SourceContactNo <> ''); + end; + + local procedure ReadSingleRelationNo(LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]; var RelationNo: Code[20]): Boolean + var + SourceContactBusinessRelation: Record "Contact Business Relation"; + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + SourceRecordRef: RecordRef; + begin + SourceContactBusinessRelation.SetRange("Link to Table", LinkToTable); + SourceContactBusinessRelation.SetRange("Contact No.", SourceContactNo); + if not CrossEnvDataSource.GetSourceRecordsByFilter(Database::"Contact Business Relation", SourceContactBusinessRelation.GetView(), SourceRecordRef) then + exit(false); + RelationNo := SourceRecordRef.Field(SourceContactBusinessRelation.FieldNo("No.")).Value(); + exit(RelationNo <> ''); + end; + + local procedure AddUnique(var Cache: Dictionary of [Text, Code[20]]; KeyText: Text; Value: Code[20]) + begin + if not Cache.ContainsKey(KeyText) then + Cache.Add(KeyText, Value); + end; + + local procedure RelationKey(LinkToTable: Enum "Contact Business Relation Link To Table"; RelationNo: Code[20]): Text + begin + exit(Format(LinkToTable.AsInteger()) + '|' + RelationNo); + end; + + local procedure ContactKey(LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]): Text + begin + exit(Format(LinkToTable.AsInteger()) + '|' + SourceContactNo); + end; + + local procedure ClearSnapshot() + begin + Clear(ContactNoByRelation); + Clear(RelationNoByContact); + Clear(BulkAttempted); + Clear(Degraded); + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Master Data Synch.", 'OnBeforeRun', '', false, false)] + local procedure MarkSyncRunStart(IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) + begin + ClearSnapshot(); + InSyncRun := true; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Master Data Synch.", 'OnAfterRun', '', false, false)] + local procedure MarkSyncRunEnd(IntegrationTableMapping: Record "Integration Table Mapping") + begin + ClearSnapshot(); + InSyncRun := false; + end; +} diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index e5bebd7b2d7..0617815b04f 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -266,6 +266,71 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceResponse.InsertRecords(Response, SourceRecordRef); end; + // Reads a RELATED source table (not one of the synchronized mappings) narrowed by a row filter, materializing + // matches into a temporary record. Used to resolve the source's contact business relations cross-environment, + // mirroring the same-env ChangeCompany read. Access is gated by the source's cross-environment read permission set. + internal procedure GetSourceRecordsByFilter(TableId: Integer; RowFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + begin + SourceRecordRef.Close(); + SourceRecordRef.Open(TableId, true); + InlineMedia.Reset(); + SourceWatermark.Reset(); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + // Full read from the start (selector '{}' = no watermark), narrowed to the related rows by the row filter. + ParseOrError(TableId, Transport.GetRecords(TableId, BuildFieldIdsForTable(TableId), '{}', PageSize(), RowFilter), Response); + SourceResponse.InsertRecords(Response, SourceRecordRef); + exit(SourceRecordRef.FindSet()); + end; + + // Bulk variant of GetSourceRecordsByFilter that never errors: it returns false (with NotIndexed set when the + // source reports the filtered set is too large to serve without an index) so the caller can fall back to + // per-record reads. Used to prefetch all of a link type's contact business relations in one call, turning the + // per-contact O(N) lookups during a sync run into O(1) calls. + internal procedure TryBulkGetSourceRecordsByFilter(TableId: Integer; RowFilter: Text; var SourceRecordRef: RecordRef; var NotIndexed: Boolean): Boolean + var + Transport: Interface "IMDM Source Transport"; + Response: JsonObject; + begin + NotIndexed := false; + SourceRecordRef.Close(); + SourceRecordRef.Open(TableId, true); + InlineMedia.Reset(); + SourceWatermark.Reset(); + Transport := GetTransport(); + SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); + if not TryParseCleanResponse(Transport.GetRecords(TableId, BuildFieldIdsForTable(TableId), '{}', PageSize(), RowFilter), Response, NotIndexed) then + exit(false); + SourceResponse.InsertRecords(Response, SourceRecordRef); + exit(true); + end; + + // Like ParseOrError but non-fatal: returns false instead of erroring, so a failed bulk prefetch degrades to + // per-record reads (which then surface any genuine consent/availability error one record at a time). + local procedure TryParseCleanResponse(ResponseText: Text; var Response: JsonObject; var NotIndexed: Boolean): Boolean + var + UnavailableFields: JsonArray; + begin + Clear(Response); + NotIndexed := false; + if not SourceResponse.TryParse(ResponseText, Response) then + exit(false); + if SourceResponse.ConsentRequired(Response) then + exit(false); + if not SourceResponse.TableAvailable(Response) then + exit(false); + if not SourceResponse.Indexed(Response) then begin + NotIndexed := true; + exit(false); + end; + if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then + exit(false); + exit(true); + end; + // A malformed wire response is an internal integration defect, not something the user can act on. local procedure InternalError(MessageText: Text): ErrorInfo var diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 07ff2320a1d..9f9c9867f67 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -800,7 +800,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" LocalContact: Record Contact; IntegrationTableMapping: Record "Integration Table Mapping"; MasterDataManagement: Codeunit "Master Data Management"; + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; SourceCompanyName: Text[30]; + SourceContactNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit; @@ -816,18 +818,27 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; - // Same-environment reads the source company's relations directly; cross-environment reads the locally - // replicated relations (they sync before customers/vendors), since the source isn't a local company. - if not MasterDataManagementSetup.IsCrossEnvironment() then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - if not Company.Get(SourceCompanyName) then - exit; - if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then - exit; + if MasterDataManagementSetup.IsCrossEnvironment() then begin + // Contact business relations are not replicated locally; resolve the source contact number cross-env + // (bulk-prefetched for the whole run, or read per-record outside a run). + if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::Customer, Customer."No.", SourceContactNo) then + if not LocalContact.Get(SourceContactNo) then begin + Contact."No." := SourceContactNo; + Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption(), Customer.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + IsHandled := true; + end; + exit; end; + // Same-environment reads the source company's relations directly via ChangeCompany. + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; + if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then + exit; + ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::Customer); ContactBusinessRelation.SetRange("No.", Customer."No."); if ContactBusinessRelation.FindFirst() then @@ -847,7 +858,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" LocalContact: Record Contact; IntegrationTableMapping: Record "Integration Table Mapping"; MasterDataManagement: Codeunit "Master Data Management"; + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; SourceCompanyName: Text[30]; + SourceContactNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit; @@ -863,18 +876,27 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; - // Same-environment reads the source company's relations directly; cross-environment reads the locally - // replicated relations (they sync before customers/vendors), since the source isn't a local company. - if not MasterDataManagementSetup.IsCrossEnvironment() then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - if not Company.Get(SourceCompanyName) then - exit; - if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then - exit; + if MasterDataManagementSetup.IsCrossEnvironment() then begin + // Contact business relations are not replicated locally; resolve the source contact number cross-env + // (bulk-prefetched for the whole run, or read per-record outside a run). + if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::Vendor, Vendor."No.", SourceContactNo) then + if not LocalContact.Get(SourceContactNo) then begin + Contact."No." := SourceContactNo; + Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption(), Vendor.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + IsHandled := true; + end; + exit; end; + // Same-environment reads the source company's relations directly via ChangeCompany. + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; + if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then + exit; + ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::Vendor); ContactBusinessRelation.SetRange("No.", Vendor."No."); if ContactBusinessRelation.FindFirst() then @@ -894,7 +916,9 @@ codeunit 7237 "Master Data Mgt. Subscribers" LocalContact: Record Contact; IntegrationTableMapping: Record "Integration Table Mapping"; MasterDataManagement: Codeunit "Master Data Management"; + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; SourceCompanyName: Text[30]; + SourceContactNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit; @@ -910,18 +934,27 @@ codeunit 7237 "Master Data Mgt. Subscribers" if IntegrationTableMapping.IsEmpty() then exit; - // Same-environment reads the source company's relations directly; cross-environment reads the locally - // replicated relations (they sync before customers/vendors), since the source isn't a local company. - if not MasterDataManagementSetup.IsCrossEnvironment() then begin - MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); - if SourceCompanyName = '' then - SourceCompanyName := MasterDataManagementSetup."Company Name"; - if not Company.Get(SourceCompanyName) then - exit; - if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then - exit; + if MasterDataManagementSetup.IsCrossEnvironment() then begin + // Contact business relations are not replicated locally; resolve the source contact number cross-env + // (bulk-prefetched for the whole run, or read per-record outside a run). + if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::"Bank Account", BankAccount."No.", SourceContactNo) then + if not LocalContact.Get(SourceContactNo) then begin + Contact."No." := SourceContactNo; + Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption(), BankAccount.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + IsHandled := true; + end; + exit; end; + // Same-environment reads the source company's relations directly via ChangeCompany. + MasterDataManagement.OnSetSourceCompanyName(SourceCompanyName, Database::Contact); + if SourceCompanyName = '' then + SourceCompanyName := MasterDataManagementSetup."Company Name"; + if not Company.Get(SourceCompanyName) then + exit; + if not ContactBusinessRelation.ChangeCompany(SourceCompanyName) then + exit; + ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::"Bank Account"); ContactBusinessRelation.SetRange("No.", BankAccount."No."); if ContactBusinessRelation.FindFirst() then @@ -1179,13 +1212,14 @@ codeunit 7237 "Master Data Mgt. Subscribers" RecordModifiedAfterLastSync: Boolean; CrossEnvironment: Boolean; LinkType: Enum "Contact Business Relation Link To Table"; + RelationNo: Code[20]; begin if not MasterDataManagement.IsEnabled() then exit(false); MasterDataManagementSetup.Get(); - // Cross-environment: the source's contact business relations are replicated locally, so resolve the related - // customer/vendor from the local relation (by No.) instead of reading the source company directly. + // Cross-environment: the source's contact business relations are NOT replicated locally, so read the matching + // source relation over the wire to resolve the related customer/vendor (numbers are aligned by synchronization). CrossEnvironment := MasterDataManagementSetup.IsCrossEnvironment(); DestinationRecordRef.SetTable(Contact); if not CrossEnvironment then @@ -1199,8 +1233,8 @@ codeunit 7237 "Master Data Mgt. Subscribers" begin if not CrossEnvironment then IntegrationCustomer.ChangeCompany(MasterDataManagementSetup."Company Name"); - if IntegrationContactBusinessRelation.FindByContact(LinkType::Customer, IntegrationContact."No.") then - if ResolvePrimaryContactCustomer(CrossEnvironment, IntegrationContactBusinessRelation."No.", IntegrationCustomer, Customer) then + if ResolveSourceRelationNo(CrossEnvironment, IntegrationContactBusinessRelation, LinkType::Customer, IntegrationContact."No.", RelationNo) then + if ResolvePrimaryContactCustomer(CrossEnvironment, RelationNo, IntegrationCustomer, Customer) then if Customer."Primary Contact No." = '' then if IntegrationTableMapping.FindMapping(Database::Customer, Database::Customer) then if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin @@ -1222,8 +1256,8 @@ codeunit 7237 "Master Data Mgt. Subscribers" begin if not CrossEnvironment then IntegrationVendor.ChangeCompany(MasterDataManagementSetup."Company Name"); - if IntegrationContactBusinessRelation.FindByContact(LinkType::Vendor, IntegrationContact."No.") then - if ResolvePrimaryContactVendor(CrossEnvironment, IntegrationContactBusinessRelation."No.", IntegrationVendor, Vendor) then + if ResolveSourceRelationNo(CrossEnvironment, IntegrationContactBusinessRelation, LinkType::Vendor, IntegrationContact."No.", RelationNo) then + if ResolvePrimaryContactVendor(CrossEnvironment, RelationNo, IntegrationVendor, Vendor) then if Vendor."Primary Contact No." = '' then if IntegrationTableMapping.FindMapping(Database::Vendor, Database::Vendor) then if IntegrationTableMapping.Direction in [IntegrationTableMapping.Direction::Bidirectional, IntegrationTableMapping.Direction::FromIntegrationTable] then begin @@ -1266,6 +1300,21 @@ codeunit 7237 "Master Data Mgt. Subscribers" exit(FindVendorByIntegrationSystemId(IntegrationVendor.SystemId, Vendor)); end; + // Resolves the source contact business relation's "No." (the related customer/vendor number) for a given source + // contact. Same-environment reads the source company's relation via ChangeCompany; cross-environment reads the + // matching relation over the wire, since contact business relations are not replicated locally. + local procedure ResolveSourceRelationNo(CrossEnvironment: Boolean; var IntegrationContactBusinessRelation: Record "Contact Business Relation"; LinkToTable: Enum "Contact Business Relation Link To Table"; SourceContactNo: Code[20]; var RelationNo: Code[20]): Boolean + var + ContactRelationCache: Codeunit "MDM Contact Relation Cache"; + begin + if CrossEnvironment then + exit(ContactRelationCache.TryGetSourceRelationNo(LinkToTable, SourceContactNo, RelationNo)); + if not IntegrationContactBusinessRelation.FindByContact(LinkToTable, SourceContactNo) then + exit(false); + RelationNo := IntegrationContactBusinessRelation."No."; + exit(true); + end; + local procedure FindCustomerByIntegrationSystemId(IntegrationSystemId: Guid; var Customer: Record Customer): Boolean var MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al index dc73cfb9af6..6ad9d221ae1 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtUpgrade.Codeunit.al @@ -29,16 +29,26 @@ codeunit 7238 "Master Data Mgt. Upgrade" // Guaranteed provisioning path: install codeunits are skipped when BC is pre-baked into a package and mounted per tenant. // NOT gated by an upgrade tag: SetAllUpgradeTags on a fresh install can set the tag without the service ever being - // created, which would permanently skip registration and 404 the source API. Ensure the record exists instead - - // idempotent and self-healing on every install and upgrade. Access stays gated by the "Cross Env" permission set, not by publishing. + // created, which would permanently skip registration and 404 the source API. Ensure the record exists AND points at the + // right object and is published instead - idempotent and self-healing on every install and upgrade. Access stays gated by + // the "Cross Env" permission set, not by publishing. internal procedure RegisterCrossEnvSourceWebService() var TenantWebService: Record "Tenant Web Service"; WebServiceManagement: Codeunit "Web Service Management"; begin - if TenantWebService.Get(TenantWebService."Object Type"::Codeunit, CrossEnvSourceWebServiceName()) then + if not TenantWebService.Get(TenantWebService."Object Type"::Codeunit, CrossEnvSourceWebServiceName()) then begin + WebServiceManagement.CreateTenantWebService(TenantWebService."Object Type"::Codeunit, Codeunit::"MDM Cross-Env Source API", CrossEnvSourceWebServiceName(), true); exit; - WebServiceManagement.CreateTenantWebService(TenantWebService."Object Type"::Codeunit, Codeunit::"MDM Cross-Env Source API", CrossEnvSourceWebServiceName(), true); + end; + + if (TenantWebService."Object ID" = Codeunit::"MDM Cross-Env Source API") and TenantWebService.Published then + exit; + + // Repair a pre-existing row that points at the wrong object or is unpublished, rather than trusting the name alone. + TenantWebService.Validate("Object ID", Codeunit::"MDM Cross-Env Source API"); + TenantWebService.Validate(Published, true); + TenantWebService.Modify(true); end; internal procedure CrossEnvSourceWebServiceName(): Text[240] diff --git a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al index 729e651cc48..506b307abb7 100644 --- a/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test library/src/LibraryMasterDataMgt.Codeunit.al @@ -199,6 +199,18 @@ codeunit 139757 "Library - Master Data Mgt." exit(CrossEnvDataSource.GetModifiedBatch(IntegrationTableMapping, TableFilter, StartCursor, MaxPages, SourceRecordRef, EndCursor, HasMore)); end; + /// Reads a related source table narrowed by a row filter from the cross-environment data source. + /// The source table ID to read. + /// The source row filter (view) restricting which records are returned. + /// Returns the record reference positioned on the materialized set. + /// True if any matching records were found; otherwise false. + procedure DataSourceGetRecordsByFilter(TableId: Integer; RowFilter: Text; var SourceRecordRef: RecordRef): Boolean + var + CrossEnvDataSource: Codeunit "MDM Cross-Env Data Source"; + begin + exit(CrossEnvDataSource.GetSourceRecordsByFilter(TableId, RowFilter, SourceRecordRef)); + end; + // Setting a Source Environment Name routes GetDataSource() to the cross-environment implementation. /// Sets the source environment name, routing the data source to the cross-environment implementation. /// The source environment name to set. diff --git a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al index 48899c80441..514a59bb359 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MDMCrossEnvConsumerTests.Codeunit.al @@ -200,6 +200,60 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" CleanUp(); end; + [Test] + procedure CrossEnvGetRecordsByFilterMaterializesContactBusinessRelation() + var + ContactBusinessRelation: Record "Contact Business Relation"; + FilterContactBusinessRelation: Record "Contact Business Relation"; + SourceContact: Record Contact; + OtherContact: Record Contact; + LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; + InProcessTransport: Codeunit "MDM In-Process Transport"; + SourceRecordRef: RecordRef; + RelationNo: Code[20]; + OtherRelationNo: Code[20]; + begin + // [FEATURE] [AI test 0.4] [Master Data Management] [Cross-Environment] + // [SCENARIO] The over-the-wire filtered read resolves a source Contact Business Relation (used to align + // auto-created contact numbers), which is not replicated locally cross-environment, and the row + // filter narrows the result to the requested link server-side. + Initialize(); + + // [GIVEN] two source contact business relations under distinct customer numbers, each linked to its own source + // contact; synthetic numbers keep a customer's own auto-created relation from matching the filter + RelationNo := UniqueCode(); + OtherRelationNo := UniqueCode(); + SourceContact := SeedContactRelation(RelationNo); + OtherContact := SeedContactRelation(OtherRelationNo); + + // [GIVEN] a subsidiary configured for cross-env with the in-process (pass-through) transport + LibraryMasterDataMgt.SetSourceEnvironmentName('PROD'); + InProcessTransport.Activate(); + + // [WHEN] the relation is read over the wire by a row filter on (Link to Table, No.) of the first relation + FilterContactBusinessRelation.SetRange("Link to Table", FilterContactBusinessRelation."Link to Table"::Customer); + FilterContactBusinessRelation.SetRange("No.", RelationNo); + + // [THEN] exactly the matching relation materializes with the source contact number preserved (the other is filtered out) + Assert.IsTrue( + LibraryMasterDataMgt.DataSourceGetRecordsByFilter(Database::"Contact Business Relation", FilterContactBusinessRelation.GetView(), SourceRecordRef), + 'The filtered read should return the source contact business relation'); + Assert.AreEqual(1, SourceRecordRef.Count(), 'The row filter should narrow the result to the one requested relation'); + Assert.IsTrue(SourceRecordRef.FindFirst(), 'The materialized relation should be positioned'); + Assert.AreEqual( + SourceContact."No.", + Format(SourceRecordRef.Field(ContactBusinessRelation.FieldNo("Contact No.")).Value()), + 'The source Contact No. should round-trip through the wire'); + + // seeded contacts/relations live outside the MDMXENV artifact set, so remove them explicitly + ContactBusinessRelation.SetRange("Link to Table", ContactBusinessRelation."Link to Table"::Customer); + ContactBusinessRelation.SetFilter("No.", '%1|%2', RelationNo, OtherRelationNo); + ContactBusinessRelation.DeleteAll(); + SourceContact.Delete(); + OtherContact.Delete(); + CleanUp(); + end; + [Test] procedure HttpTransportUnwrapsODataValueEnvelope() var @@ -804,6 +858,26 @@ codeunit 139932 "MDM Cross-Env Consumer Tests" TestTableA.Modify(); end; + local procedure SeedContactRelation(RelationNo: Code[20]) SourceContact: Record Contact + var + ContactBusinessRelation: Record "Contact Business Relation"; + begin + SourceContact.Init(); + SourceContact."No." := UniqueCode(); + SourceContact.Insert(); + ContactBusinessRelation.Init(); + ContactBusinessRelation."Contact No." := SourceContact."No."; + ContactBusinessRelation."Business Relation Code" := 'MDMTEST'; + ContactBusinessRelation."Link to Table" := ContactBusinessRelation."Link to Table"::Customer; + ContactBusinessRelation."No." := RelationNo; + ContactBusinessRelation.Insert(); + end; + + local procedure UniqueCode(): Code[20] + begin + exit(CopyStr(DelChr(Format(CreateGuid()), '=', '{}-'), 1, 20)); + end; + local procedure Initialize() var MasterDataManagementSetup: Record "Master Data Management Setup"; From aa335a6f00853b1741a993d6848cb310ba34c713 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 9 Sep 2026 12:55:51 +0200 Subject: [PATCH 61/64] [Master Data Management] Cross-environment synchonization --- .../MasterDataMgtObjects.PermissionSet.al | 1 + .../codeunits/MDMHttpSourceTransport.Codeunit.al | 6 +++--- .../codeunits/MasterDataMgtSubscribers.Codeunit.al | 14 +++++++------- .../codeunits/MasterDataMgtTableCouple.Codeunit.al | 13 +++++++------ .../src/pages/MasterDataManagementSetup.Page.al | 6 ++++-- .../src/tables/MasterDataManagementSetup.Table.al | 7 +++++-- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al index 1e5e94d5e1f..71f7d18dfac 100644 --- a/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al +++ b/src/Apps/W1/MasterDataManagement/app/permissions/MasterDataMgtObjects.PermissionSet.al @@ -27,6 +27,7 @@ permissionset 7230 "Master Data Mgt. - Objects" codeunit "MDM Source Capabilities" = X, codeunit "MDM Inline Media" = X, codeunit "MDM Source Watermark" = X, + codeunit "MDM Contact Relation Cache" = X, codeunit "MDM Privacy Notice" = X, page * = X, table * = X, diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al index 72698f7d377..1bba9a4b3af 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMHttpSourceTransport.Codeunit.al @@ -119,8 +119,8 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" // The response body can echo source-environment record content, so it is never emitted to telemetry; only // the action and HTTP status (non-content diagnostics) are logged. Dimensions.Add('Category', TelemetryCategoryTok); - Dimensions.Add('action', ActionName); - Dimensions.Add('httpStatusCode', Format(ResponseMessage.HttpStatusCode())); + Dimensions.Add('Action', ActionName); + Dimensions.Add('HttpStatusCode', Format(ResponseMessage.HttpStatusCode())); Session.LogMessage('0000VAT', StrSubstNo(RequestFailedTelemetryTxt, ActionName, ResponseMessage.HttpStatusCode()), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); // Security audit: an authorization failure crossing the environment boundary. if ResponseMessage.HttpStatusCode() in [401, 403] then @@ -134,7 +134,7 @@ codeunit 7247 "MDM Http Source Transport" implements "IMDM Source Transport" // GetLastErrorText() can contain record keys or file names, so the raw error is never emitted to telemetry; // only the action is logged. Dimensions.Add('Category', TelemetryCategoryTok); - Dimensions.Add('action', ActionName); + Dimensions.Add('Action', ActionName); Session.LogMessage('0000VAU', StrSubstNo(TransportFailedTelemetryTxt, ActionName), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, Dimensions); end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al index 9f9c9867f67..30fddaed90b 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtSubscribers.Codeunit.al @@ -37,7 +37,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" UnsupportedKeyLengthErr: label 'Table %1 has a primary key that consists of %2 fields. Off-the page, synchronization engine doesn''t support renaming with primary key length of more than 10 fields.\\Subscribe to event OnRenameDestination in codeunit "Master Data Management" to implement the rename.', Comment = '%1 - a table caption, %2 - an integer'; MappingDoesNotAllowDirectionErr: label 'The only supported direction for the data synchronization is %1.', Comment = '%1 - a text: From Integration Table'; RunningFullSynchTelemetryTxt: Label 'Running full synch job for table mapping %1', Locked = true; - SetContactNoFromSourceCompanyTxt: Label 'For %1 %2, initialized company contact No. to be equal the No. of the company contact from the source company %3.', Locked = true; + SetContactNoFromSourceCompanyTxt: Label 'Initialized the %1 company contact number to match the source company contact.', Locked = true; [EventSubscriber(ObjectType::Table, Database::"Integration Table Mapping", 'OnAfterDeleteEvent', '', false, false)] local procedure HandleOnAfterDeleteIntegrationTableMapping(var Rec: Record "Integration Table Mapping"; RunTrigger: Boolean) @@ -824,7 +824,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::Customer, Customer."No.", SourceContactNo) then if not LocalContact.Get(SourceContactNo) then begin Contact."No." := SourceContactNo; - Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption(), Customer.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; exit; @@ -844,7 +844,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactBusinessRelation.FindFirst() then if not LocalContact.Get(ContactBusinessRelation."Contact No.") then begin Contact."No." := ContactBusinessRelation."Contact No."; - Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption(), Customer.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT4', StrSubstNo(SetContactNoFromSourceCompanyTxt, Customer.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; end; @@ -882,7 +882,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::Vendor, Vendor."No.", SourceContactNo) then if not LocalContact.Get(SourceContactNo) then begin Contact."No." := SourceContactNo; - Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption(), Vendor.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; exit; @@ -902,7 +902,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactBusinessRelation.FindFirst() then if not LocalContact.Get(ContactBusinessRelation."Contact No.") then begin Contact."No." := ContactBusinessRelation."Contact No."; - Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption(), Vendor.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT5', StrSubstNo(SetContactNoFromSourceCompanyTxt, Vendor.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; end; @@ -940,7 +940,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactRelationCache.TryGetSourceContactNo("Contact Business Relation Link To Table"::"Bank Account", BankAccount."No.", SourceContactNo) then if not LocalContact.Get(SourceContactNo) then begin Contact."No." := SourceContactNo; - Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption(), BankAccount.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; exit; @@ -960,7 +960,7 @@ codeunit 7237 "Master Data Mgt. Subscribers" if ContactBusinessRelation.FindFirst() then if not LocalContact.Get(ContactBusinessRelation."Contact No.") then begin Contact."No." := ContactBusinessRelation."Contact No."; - Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption(), BankAccount.SystemId, MasterDataManagementSetup."Company Name"), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JT6', StrSubstNo(SetContactNoFromSourceCompanyTxt, BankAccount.TableCaption()), Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); IsHandled := true; end; end; diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al index cbd2f49efe8..e30179bbba5 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MasterDataMgtTableCouple.Codeunit.al @@ -26,9 +26,9 @@ codeunit 7235 "Master Data Mgt. Table Couple" SynchActionType: Option "None",Insert,Modify,ForceModify,IgnoreUnchanged,Fail,Skip,Delete,Uncouple,Couple; NoMatchingCriteriaDefinedErr: Label 'You must specify which fields on the table %1 should be used for match-based coupling.', Comment = '%1 - integration table mapping name'; NoMatchFoundErr: Label 'Failed to couple %2 record(s), because no unique uncoupled matching entity was found in %1 with the specified matching criteria.', Comment = '%1 - comma-separated list of field names, %2 - A URL, %3 - an integer, number of records'; - NoMatchFoundTelemetryErr: Label 'No matching entity was found for %1 in %3 by matching on following fields: %2.', Locked = true; - SingleMatchAlreadyCoupledTelemetryErr: Label 'Single matching entity was found for %1 in %3 by matching on following fields: %2, but it is already coupled.', Locked = true; - MultipleMatchesFoundTelemetryErr: Label 'Multiple matching entities found for %1 in %3 by matching on following fields: %2.', Locked = true; + NoMatchFoundTelemetryErr: Label 'No matching entity was found by matching on the following fields: %1.', Locked = true; + SingleMatchAlreadyCoupledTelemetryErr: Label 'A single matching entity was found by matching on the following fields: %1, but it is already coupled.', Locked = true; + MultipleMatchesFoundTelemetryErr: Label 'Multiple matching entities were found by matching on the following fields: %1.', Locked = true; NoMatchingCriteriaDefinedTelemetryErr: Label 'User is trying to schedule match based coupling for integration table mapping %1 without having specified the matchin criteria.', Locked = true; NoConflictResolutionStrategyDefinedTelemetryErr: Label 'User is trying to schedule match based coupling for integration table mapping %1 without having specified the conflict resolution strategy.', Locked = true; SkippingPostCouplingSynchTelemetryUserChoiceMsg: Label 'Skipping post-coupling synchronization for integration table mapping %1, because the user chose not to run it.', Locked = true; @@ -288,7 +288,8 @@ codeunit 7235 "Master Data Mgt. Table Couple" MatchingFieldNameList: Text; begin MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); - exit(StrSubstNo(NoMatchFoundTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); + // Record SystemId and source company name are kept out of the telemetry message; only the matching field names. + exit(StrSubstNo(NoMatchFoundTelemetryErr, MatchingFieldNameList)); end; local procedure GetMultipleMatchesFoundTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text @@ -296,7 +297,7 @@ codeunit 7235 "Master Data Mgt. Table Couple" MatchingFieldNameList: Text; begin MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); - exit(StrSubstNo(MultipleMatchesFoundTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); + exit(StrSubstNo(MultipleMatchesFoundTelemetryErr, MatchingFieldNameList)); end; local procedure GetSingleMatchAlreadyCoupledTelemetryErrorMessage(var LocalRecordRef: RecordRef; var TempMatchIntegrationFieldMapping: Record "Integration Field Mapping" temporary): Text @@ -304,7 +305,7 @@ codeunit 7235 "Master Data Mgt. Table Couple" MatchingFieldNameList: Text; begin MatchingFieldNameList := GetMatchingFieldNameList(LocalRecordRef, TempMatchIntegrationFieldMapping); - exit(StrSubstNo(SingleMatchAlreadyCoupledTelemetryErr, Format(LocalRecordRef.Field(LocalRecordRef.SystemIdNo).Value()), MatchingFieldNameList, GetIntegrationOrgCompanyName())); + exit(StrSubstNo(SingleMatchAlreadyCoupledTelemetryErr, MatchingFieldNameList)); end; local procedure GetMappingNameWithParent(var IntegrationTableMapping: Record "Integration Table Mapping"): Text diff --git a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al index 83d2e886dd0..613d9ffdd47 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al +++ b/src/Apps/W1/MasterDataManagement/app/src/pages/MasterDataManagementSetup.Page.al @@ -292,7 +292,7 @@ page 7230 "Master Data Management Setup" begin IntegrationTableMapping.SetRange(Type, IntegrationTableMapping.Type::"Master Data Management"); IntegrationTableMapping.SetRange("Delete After Synchronization", false); - Session.LogMessage('0000JIW', CompanyName(), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JIW', SetupExportedTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); Xmlport.Run(Xmlport::ExportMDMSetup, false, false, IntegrationTableMapping); end; @@ -301,11 +301,13 @@ page 7230 "Master Data Management Setup" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('0000JIX', CompanyName(), Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JIX', SetupImportedTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); Xmlport.Run(XmlPort::ImportMDMSetup, false, true); end; var + SetupExportedTelemetryTxt: Label 'Master data synchronization setup was exported.', Locked = true; + SetupImportedTelemetryTxt: Label 'Master data synchronization setup was imported.', Locked = true; ResetIntegrationTableMappingConfirmQst: Label 'This will restore the default synchronization table setup and synchronization jobs. \\All existing customizations to synchronization table setup and jobs will be overwritten.\\Do you want to continue?'; ImportIntegrationTableMappingConfirmQst: Label 'This will import the synchronization table setup from a chosen file. \\Existing synchronization tables and fields will be overwritten with the version from the file.\\Existing synchronization job queue entries will not be overwritten. Do you want to continue?'; EnableServiceQst: Label 'The %1 is not enabled. Are you sure you want to exit?', Comment = '%1 = This Page Caption (Business Central Connection Setup)'; diff --git a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al index 8dc028f6e41..6690eb05170 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al +++ b/src/Apps/W1/MasterDataManagement/app/src/tables/MasterDataManagementSetup.Table.al @@ -282,8 +282,9 @@ table 7230 "Master Data Management Setup" CurrentCompanyName := CopyStr(CompanyName(), 1, MaxStrLen(MasterDataMgtSubscriber."Company Name")); MasterDataManagement.AddSubsidiarySubscriptionToMasterCompany(Rec."Company Name", CurrentCompanyName); Message(SynchronizationEnabledMsg, Rec."Company Name"); - Session.LogMessage('0000JIM', Rec."Company Name", Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); - Session.LogMessage('0000JIN', CurrentCompanyName, Verbosity::Normal, DataClassification::OrganizationIdentifiableInformation, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + // Company names are tenant data: keep them out of the free-text telemetry message. + Session.LogMessage('0000JIM', SynchronizationEnabledSourceTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000JIN', SynchronizationEnabledSubscriberTelemetryTxt, Verbosity::Normal, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()); end; // Env name is organization-identifiable: keep it out of the free-text message and in a structured dimension. @@ -407,5 +408,7 @@ table 7230 "Master Data Management Setup" CannotChangeSourceWhileEnabledErr: label 'You cannot change the source environment while synchronization is enabled. Disable synchronization first, then change the source.'; EncryptionRequiredErr: label 'Enable data encryption before saving the source connection secret. Cross-environment credentials are never stored unencrypted.'; CrossEnvEnabledTelemetryTxt: label 'Cross-environment master data synchronization was enabled.', Locked = true; + SynchronizationEnabledSourceTelemetryTxt: label 'Master data synchronization was enabled for the source company.', Locked = true; + SynchronizationEnabledSubscriberTelemetryTxt: label 'Master data synchronization was enabled for the subscriber company.', Locked = true; ResetConfigQst: label 'There are existing synchronization table definitions in this company. Do you want to reset them to the default configuration?'; } From 2f17b74e04d43d7741503f6d454cb6ff3b2205ac Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 9 Sep 2026 13:57:03 +0200 Subject: [PATCH 62/64] [Master Data Management] Cross-environment synchonization --- .../MDMContactRelationCache.Codeunit.al | 2 ++ .../MDMCrossEnvDataSource.Codeunit.al | 20 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al index b41446b138c..fcf1542e150 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMContactRelationCache.Codeunit.al @@ -135,6 +135,8 @@ codeunit 7234 "MDM Contact Relation Cache" Clear(Degraded); end; + // The bulk path is used only within a synchronization run; the snapshot is dropped at both boundaries so it is + // fresh per run. MDM sync runs in a background job-queue session, so this state stays isolated to that session. [EventSubscriber(ObjectType::Codeunit, Codeunit::"Integration Master Data Synch.", 'OnBeforeRun', '', false, false)] local procedure MarkSyncRunStart(IntegrationTableMapping: Record "Integration Table Mapping"; var IsHandled: Boolean) begin diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 0617815b04f..424324371d2 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -27,6 +27,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" RecordsFeatureTok: Label 'records', Locked = true; LastModifiedFeatureTok: Label 'lastModifiedPerTable', Locked = true; SourceProbeTelemetryTxt: Label 'The cross-environment source-record probe failed for table %1.', Locked = true, Comment = '%1 = table id'; + SourceConsentTelemetryTxt: Label 'The source environment has not approved cross-environment master data sharing (table %1).', Locked = true, Comment = '%1 = table id'; ParseFailureTelemetryTxt: Label 'The cross-environment sync received an unusable response for table %1 (reason: %2).', Locked = true, Comment = '%1 = table id, %2 = reason'; InvalidResponseReasonTok: Label 'InvalidResponse', Locked = true; TableUnavailableReasonTok: Label 'TableUnavailable', Locked = true; @@ -131,8 +132,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; - if SourceResponse.ConsentRequired(Response) then + if SourceResponse.ConsentRequired(Response) then begin + LogConsentRequired(IntegrationTableId); Error(SourceConsentError()); + end; if (not Response.Get('tables', Token)) or (not Token.IsArray()) then begin LogProbeFailure(IntegrationTableId); Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); @@ -192,8 +195,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogProbeFailure(IntegrationTableId); Error(SourceProbeFailedErr, TableCaption(IntegrationTableId)); end; - if SourceResponse.ConsentRequired(Response) then + if SourceResponse.ConsentRequired(Response) then begin + LogConsentRequired(IntegrationTableId); Error(SourceConsentError()); + end; // An unavailable table yields no records array; treat as no matching records for the review. if not SourceResponse.TableAvailable(Response) then exit(false); @@ -389,6 +394,13 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" Session.LogMessage('0000VAS', StrSubstNo(SourceProbeTelemetryTxt, IntegrationTableId), Verbosity::Error, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; + local procedure LogConsentRequired(IntegrationTableId: Integer) + var + MasterDataManagement: Codeunit "Master Data Management"; + begin + Session.LogMessage('', StrSubstNo(SourceConsentTelemetryTxt, IntegrationTableId), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + end; + local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) var UnavailableFields: JsonArray; @@ -398,8 +410,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" LogParseFailure(IntegrationTableId, InvalidResponseReasonTok); Error(InternalError(StrSubstNo(InvalidResponseErr, TableCaption(IntegrationTableId)))); end; - if SourceResponse.ConsentRequired(Response) then + if SourceResponse.ConsentRequired(Response) then begin + LogConsentRequired(IntegrationTableId); Error(SourceConsentError()); + end; if not SourceResponse.TableAvailable(Response) then begin LogParseFailure(IntegrationTableId, TableUnavailableReasonTok); Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(TableUnavailableErr, TableCaption(IntegrationTableId)))); From 9f850c7f76b7545ac7e473fc75f4aecf43c331d6 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 9 Sep 2026 14:02:16 +0200 Subject: [PATCH 63/64] [Master Data Management] Cross-environment synchonization --- .../app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 424324371d2..0de97570371 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -398,7 +398,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" var MasterDataManagement: Codeunit "Master Data Management"; begin - Session.LogMessage('', StrSubstNo(SourceConsentTelemetryTxt, IntegrationTableId), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); + Session.LogMessage('0000VDI', StrSubstNo(SourceConsentTelemetryTxt, IntegrationTableId), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::All, 'Category', MasterDataManagement.GetTelemetryCategory()); end; local procedure ParseOrError(IntegrationTableId: Integer; ResponseText: Text; var Response: JsonObject) From df50a4a30c5b75ebba38f9498b7727e07e811a67 Mon Sep 17 00:00:00 2001 From: Djordje Cenic Date: Wed, 9 Sep 2026 16:56:58 +0200 Subject: [PATCH 64/64] [Master Data Management] Cross-environment synchonization --- .../IntegrationMasterDataSynch.Codeunit.al | 27 ++++++++++--------- .../MDMCrossEnvDataSource.Codeunit.al | 6 ++++- .../src/MasterDataMgtSynchTests.Codeunit.al | 1 - 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al index c882c95ee34..795b0dfe47c 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/IntegrationMasterDataSynch.Codeunit.al @@ -20,20 +20,21 @@ codeunit 7231 "Integration Master Data Synch." MappingName: Code[20]; begin OnBeforeRun(Rec, IsHandled); - if IsHandled then - exit; - Rec.SetOriginalJobQueueEntryOnHold(OriginalJobQueueEntry, PrevStatus); - if Rec.Direction in [Rec.Direction::ToIntegrationTable, Rec.Direction::Bidirectional] then - LatestModifiedOn[DateType::Local] := PerformScheduledSynchToIntegrationTable(Rec); - if Rec.Direction in [Rec.Direction::FromIntegrationTable, Rec.Direction::Bidirectional] then - LatestModifiedOn[DateType::Integration] := PerformScheduledSynchFromIntegrationTable(Rec); - MappingName := Rec.Name; - if not Rec.Find() then - Session.LogMessage('0000J8M', StrSubstNo(UnableToFindMappingErr, MappingName), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()) - else begin - Rec.UpdateTableMappingModifiedOn(LatestModifiedOn); - Rec.SetOriginalJobQueueEntryStatus(OriginalJobQueueEntry, PrevStatus); + // Run OnAfterRun even when a subscriber handled the run, so run-scoped subscriber state is always released. + if not IsHandled then begin + Rec.SetOriginalJobQueueEntryOnHold(OriginalJobQueueEntry, PrevStatus); + if Rec.Direction in [Rec.Direction::ToIntegrationTable, Rec.Direction::Bidirectional] then + LatestModifiedOn[DateType::Local] := PerformScheduledSynchToIntegrationTable(Rec); + if Rec.Direction in [Rec.Direction::FromIntegrationTable, Rec.Direction::Bidirectional] then + LatestModifiedOn[DateType::Integration] := PerformScheduledSynchFromIntegrationTable(Rec); + MappingName := Rec.Name; + if not Rec.Find() then + Session.LogMessage('0000J8M', StrSubstNo(UnableToFindMappingErr, MappingName), Verbosity::Warning, DataClassification::SystemMetadata, TelemetryScope::ExtensionPublisher, 'Category', MasterDataManagement.GetTelemetryCategory()) + else begin + Rec.UpdateTableMappingModifiedOn(LatestModifiedOn); + Rec.SetOriginalJobQueueEntryStatus(OriginalJobQueueEntry, PrevStatus); + end; end; OnAfterRun(Rec); diff --git a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al index 0de97570371..30cbc1a28ec 100644 --- a/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/app/src/codeunits/MDMCrossEnvDataSource.Codeunit.al @@ -309,6 +309,10 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" SourceCapabilities.EnsureSupported(Transport, RecordsFeatureTok); if not TryParseCleanResponse(Transport.GetRecords(TableId, BuildFieldIdsForTable(TableId), '{}', PageSize(), RowFilter), Response, NotIndexed) then exit(false); + // A multi-page filtered result can't be fully materialized in one bulk call; degrade to per-record reads + // rather than caching a partial (silently incomplete) snapshot. + if SourceResponse.HasMore(Response) then + exit(false); SourceResponse.InsertRecords(Response, SourceRecordRef); exit(true); end; @@ -420,7 +424,7 @@ codeunit 7249 "MDM Cross-Env Data Source" implements "IMDM Data Source" end; if not SourceResponse.Indexed(Response) then begin LogParseFailure(IntegrationTableId, NotIndexedReasonTok); - Error(NotIndexedErr, TableCaption(IntegrationTableId)); + Error(SynchTablesNavigationError(IntegrationTableId, StrSubstNo(NotIndexedErr, TableCaption(IntegrationTableId)))); end; if SourceResponse.GetUnavailableFields(Response, UnavailableFields) then begin LogParseFailure(IntegrationTableId, FieldsUnavailableReasonTok); diff --git a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al index b59c07b5127..330c50664f9 100644 --- a/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al +++ b/src/Apps/W1/MasterDataManagement/test/src/MasterDataMgtSynchTests.Codeunit.al @@ -877,7 +877,6 @@ codeunit 139758 "Master Data Mgt. Synch. Tests" CustomerInFilter: Record Customer; CustomerOutsideFilter: Record Customer; MasterDataMgtCoupling: Record "Master Data Mgt. Coupling"; - LibraryMasterDataMgt: Codeunit "Library - Master Data Mgt."; CustomerRecRef: RecordRef; begin // [SCENARIO] FindMappingByIntegrationRecordId matches a mapping only when the source record is within the