diff --git a/.gitignore b/.gitignore index 136b88c..ef15f8b 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ AGENTS.md .github/agents/ .github/instructions/ .github/prompts/ +.github/dev-trio/ .github/copilot-instructions.md .dev-trio/ diff --git a/PAX_Purview_Audit_Log_Processor_v1.11.14.ps1 b/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1 similarity index 64% rename from PAX_Purview_Audit_Log_Processor_v1.11.14.ps1 rename to PAX_Purview_Audit_Log_Processor_v1.11.15.ps1 index 7a983e0..3b6b164 100644 --- a/PAX_Purview_Audit_Log_Processor_v1.11.14.ps1 +++ b/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1 @@ -1,5 +1,4 @@ # Portable Audit eXporter (PAX) - Purview Audit Log Processor -# Version: v1.11.14 # Requirements: PowerShell 7+ for default Graph API mode; PowerShell 5.1 supported ONLY with -UseEOM (serial Exchange Online Management mode, no parallel query/explosion). # Default Activity Type: CopilotInteraction (captures ALL M365 Copilot usage including all M365 apps and Teams meetings) # DSPM for AI activity types (specified via -ActivityTypes): AIInteraction, ConnectedAIAppInteraction, AIAppInteraction @@ -96,7 +95,7 @@ -GroupNames : Filter to members of distribution/security group(s) LIVE MODE ONLY: Groups automatically expanded to individual users after authentication. - EOM mode: uses Get-DistributionGroupMember (Exchange Online RBAC) - - Graph API mode: uses Get-MgGroup + Get-MgGroupMember (requires GroupMember.Read.All) + - Graph API mode: reads /groups and /groups/{id}/transitiveMembers via Graph REST (requires GroupMember.Read.All) Accepted formats (LIVE MODE only): • Group Display Name: "Executive Leadership Team" @@ -221,7 +220,7 @@ # Rollup + Raw: CopilotInteraction-only run, keeps raw CSVs alongside rollup output pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -RollupPlusRaw -OutputPath C:\Temp\ .EXAMPLE - # Rollup with a fixed label in the unused deeper org-hierarchy levels (AI-in-One / AI Business Value) + # Rollup with a fixed label in the unused deeper org-hierarchy levels (AI-in-One / ValueLens) pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -Rollup -FillerLabel Fixed -FillerLabelText "Assistive Directs" -OutputPath C:\Temp\ .EXAMPLE # Deidentify: anonymize all identities in the output so it can be shared safely (CSV only) @@ -239,12 +238,18 @@ # Export ONLY the Microsoft Agent 365 catalog (skips audit pull; -Force auto-confirms the preflight) pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyAgent365Info -Force -OutputPathAgent365Info C:\Temp\ .EXAMPLE - # AISID dashboard — RESERVED FOR AN UPCOMING RELEASE; NOT YET AVAILABLE. Selecting -Dashboard AISID in - # this version exits immediately with a notice and does nothing. The command below shows how it will be - # used once the feature is available: a full Purview + Entra + Defender run. Enables the Defender/AISID - # pipeline and adds ThreatHunting.Read.All at sign-in (delegated) / requires it pre-consented (app-only). - # Requires -IncludeUserInfo (with its own destination); -Rollup is auto-enabled for the Purview/Entra - # output. The 6 AISID CSVs land in the dedicated -OutputPathDefenderUsage folder. + # IMPORTANT — AISID IS COMING SOON + # The AI Solutions Intelligence Dashboard (AISID) is currently under development and is not + # available for customer use in this PAX version. The -Dashboard AISID, -OutputPathDefenderUsage, + # -AppendDefenderUsage, and -DisableAISIDDeltaCache entry points are temporarily gated. The + # completed help below is retained as a preview of the capability planned for a later PAX script + # version. + # + # AISID dashboard — a full Purview + Entra + Defender run that produces the AI Solutions Intelligence + # Dashboard data set. Enables the Defender/AISID pipeline and adds ThreatHunting.Read.All at sign-in + # (delegated) / requires it pre-consented (app-only). Requires -IncludeUserInfo (with its own + # destination); -Rollup is auto-enabled for the Purview/Entra output. The AISID CSVs land in the + # dedicated -OutputPathDefenderUsage folder. pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-31 -Dashboard AISID -IncludeUserInfo -OutputPath C:\Reports\ -OutputPathUserInfo C:\Reports\ -OutputPathDefenderUsage C:\Reports\AISID\ .EXAMPLE # Combine individual users and groups @@ -337,6 +342,43 @@ ✗ Never Sent: 1 ═══════════════════════════════════════════════════════════════ + Exit Codes: + A normal run ends by returning ONE of these process codes (read $LASTEXITCODE + in automation, e.g. if ($LASTEXITCODE -ne 0) { Send-Alert }): + + 0 Clean success - run completed with no limit, failure, or gap. + 10 Record limit - a partition hit the Graph 1,000,000 / EOM 10,000 + record limit and -AutoCompleteness was NOT used. + 20 Circuit breaker - repeated block failures tripped the reliability + circuit breaker; collection stopped early. + 30 Directory fetch - Entra user-directory (org / manager) enrichment + could not be fully retrieved; audit data still written. + 40 Completed with gaps - run finished but one or more partitions failed + terminally and/or a window hit the subdivision-depth + guard; see the QUERY SUBMISSION SUMMARY for detail. + + 1 Fatal / aborted - a precondition or validation error (bad dates, + missing prerequisites, unrecoverable startup error) + stopped the run. A request for a feature that is not available + in this version also returns 1. Displaying help returns 0. + + Precedence: when a run qualifies for more than one non-zero outcome, the first + that applies wins, in this order - + 20 (circuit breaker) > 10 (record limit, unless -AutoCompleteness) > + 30 (directory fetch) > 40 (completed with gaps) > 0 (success). + Code 1 is an immediate abort taken BEFORE this end-of-run decision, so it is + not part of the ordering. + + -Resume applicability: + - 0 : nothing to resume (a clean run clears its checkpoint). + - 10 : re-run with -AutoCompleteness to resolve saturated windows; -Resume + also continues an interrupted run. + - 20 : yes - checkpointed partitions are re-processed on -Resume. + - 30 : re-run to re-attempt directory enrichment (audit data already saved). + - 40 : yes - terminally-failed and subdivision-limited windows are retried + on -Resume (the summary prints a -Resume hint for them). + - 1 : usually not resumable (abort occurs before a checkpoint exists). + Checkpoint & Resume: All authentication modes automatically create checkpoint files to preserve progress during long-running operations. This enables resumption after Ctrl+C, network failures, @@ -371,6 +413,8 @@ • -TenantId Override tenant ID (for AppRegistration) • -ClientId Override client ID (for AppRegistration) • -ClientSecret Provide client secret (for AppRegistration) + • -UserIds Override the checkpoint's user scope + • -GroupNames Override the checkpoint's group scope Usage Examples: • -Resume Auto-discover checkpoint in current directory @@ -435,7 +479,7 @@ [App-only / Delegated] Organization.Read.All (read /subscribedSkus) Group expansion (only when -GroupNames is set): - [App-only / Delegated] GroupMember.Read.All (read /groups + /groups/{id}/members) + [App-only / Delegated] GroupMember.Read.All (read /groups + /groups/{id}/transitiveMembers) AISID dashboard - Defender / AI Solutions hunting (only when -Dashboard AISID is set): [App-only / Delegated] ThreatHunting.Read.All @@ -448,8 +492,11 @@ ahead of time (a one-time customer prerequisite). Remote output - SharePoint (only when an -OutputPath* value is a SharePoint URL): - [App-only / Delegated] Sites.ReadWrite.All (resolve site/drive via /sites + /drives) - [App-only / Delegated] Files.ReadWrite.All (PUT / createUploadSession to /drives/{id}/items) + [App-only / Delegated] Sites.ReadWrite.All (resolve site/drive via /sites + /drives; + create folder via POST children; PUT / createUploadSession; download) + NOTE: Sites.ReadWrite.All alone is sufficient for every SharePoint endpoint used in + both delegated and application auth. Files.ReadWrite.All is an acceptable alternative + for the drive/file writes but is NOT requested and is not required. ALSO: the destination user / app / managed identity must have at least Member access on the target SharePoint site (granted in SharePoint, not via Graph permissions). @@ -496,9 +543,8 @@ 1. Identify (or create) the target SharePoint site and document library folder. 2. Grant the identity that runs PAX (the signed-in user for interactive auth, OR the app registration / managed identity for unattended auth) the following - Microsoft Graph application or delegated permissions, matching your -Auth choice: + Microsoft Graph application or delegated permission, matching your -Auth choice: • Sites.ReadWrite.All - • Files.ReadWrite.All 3. For app-only auth (-Auth AppRegistration / ManagedIdentity), have an admin consent the permissions in the Entra ID portal. 4. Pass the folder URL exactly as it appears in the browser address bar: @@ -595,10 +641,13 @@ SharePoint / Fabric runs. .PARAMETER OutputPathDefenderUsage - RESERVED FOR AN UPCOMING RELEASE; NOT YET AVAILABLE. This switch is tied to -Dashboard AISID, which is - not yet enabled, so supplying it has no effect in this version. When the feature is available it will - be the destination for the AISID (AI Solutions Intelligence Dashboard) output set — all 6 Defender/AISID - CSVs, including the solutions catalog, are written here. FOLDER-ONLY: supply a folder path, a + AVAILABILITY: this entry point is temporarily gated. AISID is under development and is not available + for customer use in this PAX version; supplying this switch reports that notice and stops the run. + Availability is planned for a later PAX script version. The behavior described below is retained as + a preview. + This switch is tied to -Dashboard AISID and is the destination for the AISID (AI Solutions Intelligence + Dashboard) output set — the Defender/AISID CSVs, including the solutions catalog, are written here. + FOLDER-ONLY: supply a folder path, a SharePoint folder URL, or a Fabric OneLake folder URL; a file leaf is rejected. Storage tier is inferred exactly like the other -OutputPath* switches and must match the run's other destinations. Mutually exclusive with -AppendDefenderUsage. Only meaningful with -Dashboard AISID; supplying it @@ -731,13 +780,13 @@ For targeted workload queries without M365 bundle, use -ServiceTypes with -ActivityTypes. .PARAMETER ExplodeArrays - [Deprecated] This switch is being phased out and will be removed in a future release. + [Deprecated] This switch is retained for backward compatibility and is no longer supported. .PARAMETER ExplodeDeep - [Deprecated] This switch is being phased out and will be removed in a future release. + [Deprecated] This switch is retained for backward compatibility and is no longer supported. .PARAMETER RAWInputCSV - [Deprecated] This switch is being phased out and will be removed in a future release. + [Deprecated] This switch is retained for backward compatibility and is no longer supported. .PARAMETER MaxConcurrency Maximum concurrent queries/partitions (1-10). @@ -926,7 +975,7 @@ • CopilotInteraction-only run (default activity type, or -ActivityTypes 'CopilotInteraction'): Purview_CopilotInteraction_Processor — also auto-enables -IncludeUserInfo and consumes both the Purview CSV and the Entra users CSV. - Target Analytics-Hub dashboards: AI-in-One and AI Business Value. + Target Analytics-Hub dashboards: AI-in-One and ValueLens. • -IncludeM365Usage run: Purview_M365_Usage_Bundle_Explosion_Processor — consumes the combined Purview CSV (-CombineOutput is auto-enabled by -IncludeM365Usage). Target Analytics-Hub dashboard: M365 Usage Analytics. @@ -957,7 +1006,7 @@ original run's checkpoint and cannot be added or removed on the resume command line. .PARAMETER FillerLabel - (Power BI rollup only — AI-in-One / AI Business Value dashboards.) Chooses how the + (Power BI rollup only — AI-in-One / ValueLens dashboards.) Chooses how the manager-hierarchy "level" columns are filled in below a person's own position in the org chart. The hierarchy columns themselves — each person's chain of managers up to the top of the organization, their level number, their manager, and their direct-report and total-report @@ -981,7 +1030,7 @@ Overrides custom list and default behavior. Use with -ActivityTypes to query only the specified activity types. .PARAMETER ExportWorkbook - [Deprecated] This switch is being phased out and will be removed in a future release. + [Deprecated] This switch is retained for backward compatibility and is no longer supported. .PARAMETER AppendFile Append Purview audit activity rows to an existing Purview audit CSV / XLSX instead of creating @@ -1147,6 +1196,28 @@ HasLicense are recognized when present; any other columns are preserved as passthrough. Mutually exclusive with -GroupNames. +.PARAMETER UserInfoSupplement + Path to a customer-provided CSV that ENRICHES the live Entra user directory (hybrid mode), as + opposed to -UserInfoFile which REPLACES it. The live Entra /users directory is fetched normally, + then the supplemental columns are appended to each matching Entra row. Storage tier is inferred + from the value form (drive-rooted absolute path = Local; https://*.sharepoint*/* URL = SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse/Files/.../*.csv URL = Fabric); remote inputs + are staged to local scratch before parsing. The file must be a .csv with a header row, at least one + data row, and exactly one UserPrincipalName column (the only required column; matched + case-insensitively). Rows are joined to Entra by UserPrincipalName as an Entra-left join: every + Entra user is preserved, supplemental columns are appended after the existing Entra columns in + their original order, non-matching Entra users receive blank supplemental values, and supplemental + rows that do not match any Entra user are reported and excluded (never added as directory users). + The supplemental UserPrincipalName is a join key only and is never copied into the output — the + output identity always comes from Entra. Supplemental column names/order/values are preserved + without transformation; PAX does not inspect, mask, hash, or de-identify supplemental values (with + -Deidentify, Entra-derived identity fields are still protected but supplemental values pass through + unchanged, and a prominent notice is shown). Supplemental columns flow to the normal EntraUsers CSV + and the AIO/ValueLens Rollup Users dimension, but NOT to the AISID EntraUsers.csv fixed 14-column + dashboard schema. Auto-enables -IncludeUserInfo. Mutually exclusive with -UserInfoFile, -UseEOM, + and -RAWInputCSV; compatible with -UserIds, -GroupNames, -OnlyUserInfo, AIO/ValueLens/AISID, + -AppendUserInfo, and -Deidentify. + .PARAMETER AppendAgent365Info Append the Microsoft Agent 365 catalog export into an existing file. Filename rules match -AppendFile. Auto-enables -IncludeAgent365Info. Incompatible with -OnlyAgent365Info. May be @@ -1154,9 +1225,12 @@ -OutputPathAgent365Info (overwrite/redirect), not both, for the same destination. .PARAMETER AppendDefenderUsage - RESERVED FOR AN UPCOMING RELEASE; NOT YET AVAILABLE. This switch is tied to -Dashboard AISID, which is - not yet enabled, so supplying it has no effect in this version. When the feature is available it will - append the AISID (Defender/AI Solutions) output set into an existing AISID destination folder. + AVAILABILITY: this entry point is temporarily gated. AISID is under development and is not available + for customer use in this PAX version; supplying this switch reports that notice and stops the run. + Availability is planned for a later PAX script version. The behavior described below is retained as + a preview. + This switch is tied to -Dashboard AISID and appends the AISID (Defender/AI Solutions) output set into + an existing AISID destination folder. FOLDER-ONLY (points at the folder holding the AISID CSV set); a file leaf is rejected. Mutually exclusive with -OutputPathDefenderUsage (exactly one of the pair per run). Only meaningful with -Dashboard AISID. @@ -1183,6 +1257,40 @@ 2. Conflict Resolution (ExcludeCopilotInteraction): Automatically honors -ExcludeCopilotInteraction when conflict with -ActivityTypes Use this switch for unattended/automated executions (CI/CD pipelines, scheduled tasks). +.PARAMETER ClearUncertainCreate + Operator-only SELECTIVE recovery for durable uncertain-create state on resume. An + uncertain create is a first create POST that MAY have been accepted but could not be + proven, so on resume such a partition is reconciled READ-ONLY (never re-POSTed) and, if + still not visible, is left uncertain and its worker is NOT launched. Durable uncertain + state is NEVER auto-cleared - not by elapsed time, a collection miss, a process restart, + or failure-count exhaustion. This parameter takes an EXPLICIT list of partition indexes + (e.g. -ClearUncertainCreate 3,7 clears ONLY partitions 3 and 7); it never performs an + indiscriminate global clear. A selection that is empty, contains a duplicate, or names an + index that is not currently in the durable uncertain-create set is REJECTED and nothing + is cleared. Each selected partition is first reconciled READ-ONLY: if it is still visible + by exact name it is ADOPTED (fetched, never cleared into a duplicate POST); only a + selected partition confirmed NOT visible has its durable marker cleared - selectively and + atomically, before any fresh create - and is then created fresh. Use it only after you + have confirmed no such query exists server-side for the selected indexes, since a fresh + create could otherwise duplicate an accepted-but-invisible query. + +.PARAMETER ClearUncertainContract + Operator-only EXPLICIT-CONTRACT companion to -ClearUncertainCreate for the case where a + single partition index maps to MORE THAN ONE durable uncertain-create contract (Graph + splits a run into one workload pass per service and every pass restarts partition indexes + at 1, so two different contracts can share index 1). When -ClearUncertainCreate is + ambiguous it is REJECTED before any auth or mutation with an actionable message that lists + the candidate ":" tokens (a fingerprint is a non-secret contract hash). + Pass those exact tokens to -ClearUncertainContract to clear EXACTLY the named contract(s): + each token ":" must resolve to exactly one durable uncertain-create + contract; a token that is malformed, names an unknown contract, matches more than one, or is + duplicated is REJECTED and nothing is cleared (fail-closed, no partial mutation). A valid + token clears ONLY that contract - every unselected uncertain contract is preserved exactly. + Like -ClearUncertainCreate, each selected contract is first reconciled READ-ONLY and is only + cleared for a fresh create if it is still not visible by exact name; a still-visible query is + adopted, never cleared into a duplicate POST. The simple -ClearUncertainCreate path + is unchanged for the common unambiguous case. + .PARAMETER SkipDiagnostics Skip pre-query capability diagnostics (advanced). @@ -1269,6 +1377,8 @@ • -OutputPath : Specify output directory • -OutputPathUserInfo: Per-data-type destination (overrides -OutputPath for the EntraUsers CSV) • -AppendUserInfo : Union-merge this run's EntraUsers snapshot into an existing target + • -UserIds : Scope the exported directory to these user principal names + • -GroupNames : Scope the exported directory to the members of these group(s) • -Auth : Choose authentication method (WebLogin, DeviceCode, etc.) • -CombineOutput : (Has no effect, but allowed for script compatibility) • -DisableAdaptive : (Has no effect, but allowed) @@ -1284,7 +1394,6 @@ • StreamingSchemaSample, StreamingChunkSize, ExportProgressInterval Filtering: - • UserIds, GroupNames : User filtering requires audit logs • AgentId, AgentsOnly, ExcludeAgents: Agent filtering requires audit logs • PromptFilter : Message filtering requires audit logs @@ -1377,6 +1486,7 @@ -Auth Override authentication method -TenantId, -ClientId Override auth credentials (for AppRegistration) -ClientSecret Provide client secret (for AppRegistration) + -UserIds, -GroupNames Override the checkpoint's user/group scope (fail-closed) NOT ALLOWED WITH -Resume: Any other parameter (dates, activities, explosion settings, -Deidentify, etc.) @@ -1395,10 +1505,10 @@ param( [Parameter(Mandatory = $false)] - [string]$StartDate, # Live mode: if omitted (with EndDate) auto-populated later; Replay: optional filter + [string]$StartDate, # Omit BOTH StartDate+EndDate => default [yesterday..today) UTC; omit exactly one => that side stays unbounded ('*'); Replay: optional filter. -Dashboard AISID additionally requires BOTH dates concrete. [Parameter(Mandatory = $false)] - [string]$EndDate, # Live mode: if omitted (with StartDate) auto-populated; Replay: optional filter + [string]$EndDate, # Omit BOTH StartDate+EndDate => default [yesterday..today) UTC; omit exactly one => that side stays unbounded ('*'); Replay: optional filter. -Dashboard AISID additionally requires BOTH dates concrete. [Parameter(Mandatory = $false)] [string]$OutputPath, @@ -1424,6 +1534,16 @@ param( [Parameter(Mandatory = $false)] [string]$OutputPathDefenderUsage, + # TEMPORARILY GATED: AISID is under development and is not available for customer use in this PAX + # version; supplying this switch reports that notice and stops the run. Availability is planned for + # a later PAX script version. The description below is retained as a preview. + # Turns off the incremental result cache used by the off-hours geography signal, forcing a + # full re-collection of its window on this run. Only meaningful with -Dashboard AISID. + [Parameter(Mandatory = $false)] + [switch]$DisableAISIDDeltaCache, + [Parameter(DontShow = $true)] + [switch]$VerifyPartitionStability, + [Parameter(Mandatory = $false)] [ValidateSet('WebLogin', 'DeviceCode', 'Credential', 'Silent', 'AppRegistration', 'ManagedIdentity')] [string]$Auth = 'WebLogin', @@ -1609,7 +1729,7 @@ param( [Parameter(Mandatory = $false)] [switch]$ExcludeCopilotInteraction, - # [Deprecated] Excel workbook export. Will be removed in a future release. + # [Deprecated] Excel workbook export. [Parameter(Mandatory = $false, DontShow = $true)] [switch]$ExportWorkbook, @@ -1630,6 +1750,16 @@ param( [Parameter(Mandatory = $false)] [string]$UserInfoFile, + # Hybrid directory enrichment: fetch the live Entra /users directory normally, then append the columns + # of a customer-provided supplemental CSV to each matching Entra row (Entra-left join by + # UserPrincipalName). Storage tier is inferred from the value form; remote inputs are staged to scratch + # before parsing. .csv only; requires exactly one UserPrincipalName column (join key only, never copied + # into output). Every Entra user is preserved; unmatched supplemental rows are reported and excluded. + # Auto-enables -IncludeUserInfo. Mutually exclusive with -UserInfoFile / -UseEOM / -RAWInputCSV; + # compatible with -GroupNames. + [Parameter(Mandatory = $false)] + [string]$UserInfoSupplement, + # Append Agent 365 catalog data into an existing file. Filename rules match -AppendFile. # Auto-enables -IncludeAgent365Info. Incompatible with -OnlyAgent365Info. May be used standalone or together with -AppendFile. [Parameter(Mandatory = $false)] @@ -1652,6 +1782,24 @@ param( [Parameter(Mandatory = $false)] [switch]$Force, + # Operator-only SELECTIVE recovery: an explicit list of partition indexes whose durable + # uncertain-create checkpoint state is discarded on resume so those specific partitions are + # created fresh. Durable uncertain state is never auto-cleared (see .PARAMETER + # ClearUncertainCreate); an explicit index selection is the only way to clear it, and a + # still-visible selected partition is adopted rather than cleared into a duplicate POST. + [Parameter(Mandatory = $false)] + [int[]]$ClearUncertainCreate, + + # Operator-only EXPLICIT-CONTRACT companion to -ClearUncertainCreate: an explicit list of + # ":" tokens naming durable uncertain-create contracts to clear on + # resume. It disambiguates the case where one partition index maps to more than one durable + # uncertain contract (distinct fingerprints, same index) - an ambiguous -ClearUncertainCreate + # is rejected and lists the candidate tokens to pass here. A valid token clears ONLY + # that contract; every unselected uncertain contract is preserved exactly (see .PARAMETER + # ClearUncertainContract). A still-visible selected contract is adopted, never re-POSTed. + [Parameter(Mandatory = $false)] + [string[]]$ClearUncertainContract, + # Skip pre-query capability diagnostics (advanced) [Parameter(Mandatory = $false)] [switch]$SkipDiagnostics, @@ -1715,20 +1863,22 @@ param( # Rollup target dashboard selector. Chooses which embedded Python # post-processor + output profile a rollup run produces: # AIO (default) - AI-in-One dashboard -> CopilotInteraction processor, --profile aio - # AIBV - AI Business Value dashboard -> CopilotInteraction processor, --profile aibv + # ValueLens - ValueLens dashboard -> CopilotInteraction processor, --profile aibv # M365 - M365 Usage Analytics -> M365 bundle processor (auto-enables -IncludeM365Usage) - # AISID - AI Solutions Intelligence Dashboard. RESERVED FOR AN UPCOMING RELEASE; NOT YET - # AVAILABLE — selecting it exits immediately with a notice and does nothing in this - # version. When enabled it will drive a full Purview + Entra + Defender run and the - # Defender/AISID data pipeline (the only trigger for it), require -IncludeUserInfo and a - # run not narrowed by any -Only* switch, reuse the AIO-shaped CopilotInteraction rollup - # for its Purview/Entra tables (its own Defender-sourced CSVs are exported without - # rollup processing), and add the ThreatHunting.Read.All Graph permission (delegated) / - # application permission (app-only modes) at sign-in. + # AISID - AI Solutions Intelligence Dashboard. TEMPORARILY GATED: AISID is under + # development and is not available for customer use in this PAX version; + # selecting it reports that notice and stops the run. Availability is planned + # for a later PAX script version. The description below is retained as a + # preview. Drives a full Purview + Entra + Defender run + # and the Defender/AISID data pipeline (the only trigger for it), requires + # -IncludeUserInfo and a run not narrowed by any -Only* switch, reuses the AIO-shaped + # CopilotInteraction rollup for its Purview/Entra tables (its own Defender-sourced CSVs + # are exported without rollup processing), and adds the ThreatHunting.Read.All Graph + # permission (delegated) / application permission (app-only modes) at sign-in. # Only meaningful with -Rollup / -RollupPlusRaw; if supplied without either, -Rollup is auto-enabled. # If omitted: M365 when -IncludeM365Usage is present, otherwise AIO. Case-insensitive. [Parameter(Mandatory = $false)] - [ValidateSet('AIO', 'AIBV', 'M365', 'AISID')] + [ValidateSet('AIO', 'ValueLens', 'M365', 'AISID')] [string]$Dashboard = 'AIO', # Deidentify (anonymize) all identifying values in RAW output files using a @@ -1766,6 +1916,26 @@ param( [string[]]$RemainingArgs ) +# ============================================================ +# POWERSHELL LANGUAGE MODE PREFLIGHT +# PAX requires FullLanguage mode for its .NET and runtime features. Stop before +# creating a bootstrap log, loading modules, authenticating, or writing output +# when an endpoint application-control policy places this process in another mode. +# ============================================================ +$currentLanguageMode = "$($ExecutionContext.SessionState.LanguageMode)" +if ($currentLanguageMode -ne 'FullLanguage') { + Microsoft.PowerShell.Utility\Write-Host '' + Microsoft.PowerShell.Utility\Write-Host 'PAX cannot run in the current PowerShell language mode.' -ForegroundColor Red + Microsoft.PowerShell.Utility\Write-Host ("Current mode: {0}" -f $currentLanguageMode) -ForegroundColor Yellow + Microsoft.PowerShell.Utility\Write-Host '' + Microsoft.PowerShell.Utility\Write-Host 'PAX requires PowerShell running in FullLanguage mode.' + Microsoft.PowerShell.Utility\Write-Host 'This setting is commonly controlled by your organization through WDAC or AppLocker policy.' + Microsoft.PowerShell.Utility\Write-Host 'Contact your IT or security team for an approved PowerShell environment or to have PAX allowlisted.' + Microsoft.PowerShell.Utility\Write-Host 'ExecutionPolicy Bypass does not change PowerShell language mode.' + Microsoft.PowerShell.Utility\Write-Host '' + exit 1 +} + # ============================================================ # CULTURE GUARD — DO NOT REMOVE # Pin the main thread to InvariantCulture for the entire run. @@ -1850,8 +2020,7 @@ try { # ============================================================ # DEPRECATED SWITCH GATE # Any explicit use of these switches on the command line causes an -# immediate graceful exit. These switches are slated for removal in a -# future release and are no longer supported. +# immediate graceful exit. These switches are no longer supported. # ============================================================ $script:DeprecatedSwitchesHit = @() if ($PSBoundParameters.ContainsKey('ExportWorkbook')) { $script:DeprecatedSwitchesHit += '-ExportWorkbook' } @@ -1866,19 +2035,37 @@ if ($script:DeprecatedSwitchesHit.Count -gt 0) { } # ============================================================ -# AISID DASHBOARD — RESERVED (NOT YET AVAILABLE) -# The -Dashboard AISID option (and its -OutputPathDefenderUsage / -AppendDefenderUsage -# destination switches) is reserved for an upcoming release and is not yet available for -# use. The underlying groundwork is present in this version but intentionally not enabled. -# This early gate short-circuits any -Dashboard AISID invocation here — before any parameter -# validation, permission (scope) request, or sign-in — so selecting it today has no side -# effects (no extra permission requested, no authentication attempted, no partial run). To -# enable the feature once its data pipeline is delivered, remove this single block. +# AISID AVAILABILITY GATE +# The AI Solutions Intelligence Dashboard (AISID) customer entry points are not +# available in this version. The AISID implementation stays in place; only the +# customer entry points are gated. This guard runs before any module load, +# sign-in, destination probe, output directory creation, checkpoint mutation, +# audit query creation, AISID collection, history reconciliation, upload, or +# local publication, so a gated invocation never contacts a service and never +# writes customer output. It is invoked a second time after checkpoint +# restoration so a stored AISID contract cannot reach a gated path either. # ============================================================ -if ($Dashboard -eq 'AISID') { - Microsoft.PowerShell.Utility\Write-Host "The AI Solutions Intelligence Dashboard (-Dashboard AISID) is reserved for an upcoming release and is not yet available for use in this version." -ForegroundColor Yellow - Microsoft.PowerShell.Utility\Write-Host " Use -Dashboard AIO, -Dashboard AIBV, or -Dashboard M365 (or omit -Dashboard) to run one of the currently available dashboards." -ForegroundColor Gray - exit 0 +$script:AISIDGateBlocked = $false +function script:Test-AISIDEntryPointGated { + param( + [bool]$DashboardIsAISID = $false, + [bool]$DefenderUsageOutputRequested = $false, + [bool]$DefenderUsageAppendRequested = $false, + [bool]$DeltaCacheSwitchRequested = $false + ) + if (-not ($DashboardIsAISID -or $DefenderUsageOutputRequested -or $DefenderUsageAppendRequested -or $DeltaCacheSwitchRequested)) { return $false } + if ($script:AISIDGateBlocked) { return $true } + $script:AISIDGateBlocked = $true + Microsoft.PowerShell.Utility\Write-Host "The AI Solutions Intelligence Dashboard (AISID) is currently under development and is not available for customer use in this PAX version. The AISID customer entry points will become available in a later PAX script version." -ForegroundColor Yellow + Microsoft.PowerShell.Utility\Write-Host "Gated entry points: -Dashboard AISID, -OutputPathDefenderUsage, -AppendDefenderUsage, and -DisableAISIDDeltaCache." -ForegroundColor Yellow + return $true +} +$aisidCliDashboard = ($PSBoundParameters.ContainsKey('Dashboard') -and ([string]$Dashboard).Trim().ToUpperInvariant() -eq 'AISID') +$aisidCliOutput = $PSBoundParameters.ContainsKey('OutputPathDefenderUsage') +$aisidCliAppend = $PSBoundParameters.ContainsKey('AppendDefenderUsage') +$aisidCliCache = ($PSBoundParameters.ContainsKey('DisableAISIDDeltaCache') -and [bool]$DisableAISIDDeltaCache) +if (script:Test-AISIDEntryPointGated -DashboardIsAISID $aisidCliDashboard -DefenderUsageOutputRequested $aisidCliOutput -DefenderUsageAppendRequested $aisidCliAppend -DeltaCacheSwitchRequested $aisidCliCache) { + exit 1 } # ============================================================ @@ -1902,6 +2089,59 @@ if ($RemainingArgs -and $RemainingArgs.Count -gt 0) { } } +# ============================================================ +# AISID AVAILABILITY GATE - stored resume contract (read-only pre-scan) +# When -Resume names an unambiguous checkpoint, its stored contract is inspected +# read-only here - before any module load, sign-in, destination probe, checkpoint +# mutation, audit query, collection, or publication - so a stored AISID contract is +# stopped at the same point as an explicit AISID switch. An ambiguous selection is +# left to the post-restoration guard, which still runs before sign-in, collection, +# checkpoint mutation, and publication. Nothing is created, written, or modified here. +# ============================================================ +if ($ResumeSpecified) { + $aisidResumeProbePath = $null + try { + if (-not [string]::IsNullOrWhiteSpace($Resume)) { + if (Test-Path -LiteralPath $Resume -PathType Leaf) { $aisidResumeProbePath = $Resume } + elseif (Test-Path -LiteralPath $Resume -PathType Container) { + $aisidProbeFound = @(Get-ChildItem -LiteralPath $Resume -Filter '.pax_checkpoint_*.json' -File -ErrorAction SilentlyContinue) + if ($aisidProbeFound.Count -eq 1) { $aisidResumeProbePath = $aisidProbeFound[0].FullName } + } + } else { + $aisidProbeDirs = @() + if (-not [string]::IsNullOrWhiteSpace($OutputPath)) { $aisidProbeDirs += $OutputPath } + $aisidProbeDirs += (Get-Location).Path + foreach ($aisidProbeDir in $aisidProbeDirs) { + if (-not (Test-Path -LiteralPath $aisidProbeDir -PathType Container)) { continue } + $aisidProbeFound = @(Get-ChildItem -LiteralPath $aisidProbeDir -Filter '.pax_checkpoint_*.json' -File -ErrorAction SilentlyContinue) + if ($aisidProbeFound.Count -eq 1) { $aisidResumeProbePath = $aisidProbeFound[0].FullName; break } + if ($aisidProbeFound.Count -gt 1) { break } + } + } + } catch { $aisidResumeProbePath = $null } + if ($aisidResumeProbePath) { + $aisidProbeDash = $false; $aisidProbeOut = $false; $aisidProbeApp = $false; $aisidProbeCache = $false + try { + $aisidProbeData = Get-Content -LiteralPath $aisidResumeProbePath -Raw -ErrorAction Stop | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($aisidProbeData) { + if ($aisidProbeData.Contains('aisidWindow') -and $aisidProbeData.aisidWindow) { $aisidProbeDash = $true } + if ($aisidProbeData.Contains('aisidResumeStateVersion')) { $aisidProbeDash = $true } + if ($aisidProbeData.Contains('parameters') -and $aisidProbeData.parameters) { + $aisidProbeParams = $aisidProbeData.parameters + if ($aisidProbeParams.Contains('dashboard') -and ([string]$aisidProbeParams.dashboard).Trim().ToUpperInvariant() -eq 'AISID') { $aisidProbeDash = $true } + if ($aisidProbeParams.Contains('rollupDashboard') -and ([string]$aisidProbeParams.rollupDashboard).Trim().ToUpperInvariant() -eq 'AISID') { $aisidProbeDash = $true } + $aisidProbeOut = ($aisidProbeParams.Contains('outputPathDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$aisidProbeParams.outputPathDefenderUsage)) + $aisidProbeApp = ($aisidProbeParams.Contains('appendDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$aisidProbeParams.appendDefenderUsage)) + $aisidProbeCache = ($aisidProbeParams.Contains('disableAISIDDeltaCache') -and [bool]$aisidProbeParams.disableAISIDDeltaCache) + } + } + } catch { } + if (script:Test-AISIDEntryPointGated -DashboardIsAISID $aisidProbeDash -DefenderUsageOutputRequested $aisidProbeOut -DefenderUsageAppendRequested $aisidProbeApp -DeltaCacheSwitchRequested $aisidProbeCache) { + exit 1 + } + } +} + # ============================================================ # PROMOTE AUTH PARAMETERS TO SCRIPT SCOPE # Enables access from within functions (e.g., Connect-PurviewAudit) @@ -2000,405 +2240,3925 @@ $script:ClientCertificatePassword = $ClientCertificatePassword return $finalActivityTypes } -function Send-PromptNotification { +# ============================================================ +# AI Solutions Catalog (customer-editable dimension table) +# ============================================================ +# Single source of truth for AI-app detection and the dashboard's AI_Solutions +# dimension. To extend coverage for your tenant, add a hashtable entry using the +# same keys as the existing entries; the detection-surface lists +# (CloudAppNames / SignInAppNames) may be partial — any non-empty match classifies +# an event. The catalog is also written to the AI solutions catalog CSV so the +# dashboard's AI_Solutions dimension can consume it directly. +# KEY SEMANTICS: +# AISolution - Primary key; must be unique; appears in every AISID output. +# Category - Enterprise AI | Developer AI | Unmanaged AI +# Vendor - Human-readable vendor/publisher name. +# RiskTier - Enterprise AI | Low | Medium | High (governance tier) +# DefaultDataHandling - Short narrative of the product's default data-handling posture. +# SolutionGroup - Dashboard grouping key ('Microsoft 365 Copilot', 'ChatGPT', +# 'Claude', 'Gemini', 'Perplexity', 'Grok', or 'All Other AI'). +# CloudAppNames - Values matched against Defender CloudAppEvents application names. +# SignInAppNames - Values matched against Entra and Graph sign-in application names. +# NetworkDomains - AI service hostnames matched against Defender device network event URLs +# (empty when no authoritative hostname is known for the entry). +# ============================================================ +$script:AISolutionCatalog = @( + @{ AISolution = 'Microsoft 365 Copilot'; Category = 'Enterprise AI'; Vendor = 'Microsoft'; RiskTier = 'Sanctioned'; DefaultDataHandling = 'Customer controlled, no training on tenant data'; SolutionGroup = 'Microsoft Copilot'; CloudAppNames = @('Microsoft 365 Copilot','Microsoft Copilot','Microsoft 365 Chat'); SignInAppNames = @('Microsoft 365 Copilot','Microsoft Copilot'); NetworkDomains = @('copilot.microsoft.com','copilot.cloud.microsoft') } + @{ AISolution = 'GitHub Copilot'; Category = 'Developer AI'; Vendor = 'GitHub'; RiskTier = 'Sanctioned'; DefaultDataHandling = 'Per enterprise contract'; SolutionGroup = 'Microsoft Copilot'; CloudAppNames = @('GitHub Copilot','GitHub.com'); SignInAppNames = @('GitHub Copilot','GitHub'); NetworkDomains = @() } + @{ AISolution = 'Cursor'; Category = 'Developer AI'; Vendor = 'Anysphere'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Code stored per tier'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Cursor'); SignInAppNames = @('Cursor'); NetworkDomains = @() } + @{ AISolution = 'ChatGPT'; Category = 'Unmanaged AI'; Vendor = 'OpenAI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Training by default on free tier'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('ChatGPT','ChatGPT Enterprise','OpenAI ChatGPT'); SignInAppNames = @('ChatGPT Enterprise','OpenAI'); NetworkDomains = @('chat.openai.com','chatgpt.com','api.openai.com') } + @{ AISolution = 'Claude'; Category = 'Unmanaged AI'; Vendor = 'Anthropic'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Not used for training (paid tiers)'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('Claude','Claude for Work','Anthropic Claude'); SignInAppNames = @('Claude for Work','Anthropic'); NetworkDomains = @('claude.ai','api.anthropic.com') } + @{ AISolution = 'Gemini'; Category = 'Unmanaged AI'; Vendor = 'Google'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Not used for training (Workspace tier)'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('Google Gemini','Gemini'); SignInAppNames = @('Google Gemini'); NetworkDomains = @('gemini.google.com','bard.google.com') } + @{ AISolution = 'Perplexity'; Category = 'Unmanaged AI'; Vendor = 'Perplexity'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Not disclosed in detail'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('Perplexity','Perplexity Enterprise'); SignInAppNames = @('Perplexity Enterprise','Perplexity'); NetworkDomains = @('perplexity.ai') } + @{ AISolution = 'Mistral'; Category = 'Unmanaged AI'; Vendor = 'Mistral AI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'EU-hosted'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Mistral','Mistral AI'); SignInAppNames = @('Mistral'); NetworkDomains = @() } + @{ AISolution = 'HuggingFace'; Category = 'Unmanaged AI'; Vendor = 'Hugging Face'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Model-dependent'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('HuggingFace','Hugging Face'); SignInAppNames = @('HuggingFace','Hugging Face'); NetworkDomains = @('huggingface.co') } + @{ AISolution = 'Poe'; Category = 'Unmanaged AI'; Vendor = 'Quora'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Per aggregator model'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Poe'); SignInAppNames = @('Poe'); NetworkDomains = @() } + @{ AISolution = 'Grok'; Category = 'Unmanaged AI'; Vendor = 'xAI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Training by default'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Grok','xAI'); SignInAppNames = @('Grok','xAI'); NetworkDomains = @() } + @{ AISolution = 'DeepSeek'; Category = 'Unmanaged AI'; Vendor = 'DeepSeek'; RiskTier = 'Unresolved'; DefaultDataHandling = 'PRC data residency'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('DeepSeek'); SignInAppNames = @('DeepSeek'); NetworkDomains = @() } + @{ AISolution = 'Character.AI'; Category = 'Unmanaged AI'; Vendor = 'Character.AI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Public by default'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Character.AI','Character AI'); SignInAppNames = @('Character.AI','Character AI'); NetworkDomains = @() } +) + +# ---- AI Solutions Catalog helper functions (pure lookups, no I/O) ---- + +function Get-AISIDFilteredCatalog { <# .SYNOPSIS - Plays a system beep to alert user that a prompt requires attention. - .DESCRIPTION - Useful when user is working in other windows and needs to be notified - when a prompt appears that requires input. + Returns the AI solutions catalog. #> - - try { - # Play 3 short beeps to get attention - [Console]::Beep(800, 200) # 800Hz for 200ms - Start-Sleep -Milliseconds 100 - [Console]::Beep(1000, 200) # 1000Hz for 200ms - Start-Sleep -Milliseconds 100 - [Console]::Beep(1200, 300) # 1200Hz for 300ms (slightly longer final beep) - } - catch { - # Silently fail if beep not supported (e.g., some server environments) - } + [OutputType([System.Collections.IList])] + param() + return $script:AISolutionCatalog } -# Validate -OnlyUserInfo parameter compatibility -if ($OnlyUserInfo) { - $incompatibleParams = @() - - # Date filtering parameters - if ($PSBoundParameters.ContainsKey('StartDate')) { $incompatibleParams += " - StartDate (not applicable for user-only export)" } - if ($PSBoundParameters.ContainsKey('EndDate')) { $incompatibleParams += " - EndDate (not applicable for user-only export)" } - - # Activity configuration parameters - if ($PSBoundParameters.ContainsKey('ActivityTypes')) { $incompatibleParams += " - ActivityTypes (cleared by -OnlyUserInfo)" } - if ($IncludeM365Usage) { $incompatibleParams += " - IncludeM365Usage (activity type modifier)" } - if ($ExcludeCopilotInteraction) { $incompatibleParams += " - ExcludeCopilotInteraction (activity type modifier)" } - - # Audit retrieval settings - if ($PSBoundParameters.ContainsKey('BlockHours') -and $BlockHours -ne 0.5) { $incompatibleParams += " - BlockHours (audit query partitioning)" } - if ($PSBoundParameters.ContainsKey('PartitionHours') -and $PartitionHours -ne 0) { $incompatibleParams += " - PartitionHours (audit query partitioning)" } - if ($PSBoundParameters.ContainsKey('MaxPartitions') -and $MaxPartitions -ne 160) { $incompatibleParams += " - MaxPartitions (audit query limits)" } - if ($PSBoundParameters.ContainsKey('ResultSize') -and $ResultSize -ne 10000) { $incompatibleParams += " - ResultSize (audit query page size)" } - if ($PSBoundParameters.ContainsKey('PacingMs') -and $PacingMs -ne 0) { $incompatibleParams += " - PacingMs (audit query throttling)" } - if ($AutoCompleteness) { $incompatibleParams += " - AutoCompleteness (audit log completeness checks)" } - if ($PSBoundParameters.ContainsKey('StreamingSchemaSample') -and $StreamingSchemaSample -ne 5000) { $incompatibleParams += " - StreamingSchemaSample (audit record schema sampling)" } - if ($PSBoundParameters.ContainsKey('StreamingChunkSize') -and $StreamingChunkSize -ne 5000) { $incompatibleParams += " - StreamingChunkSize (audit streaming batch size)" } - if ($PSBoundParameters.ContainsKey('ExportProgressInterval') -and $ExportProgressInterval -ne 10) { $incompatibleParams += " - ExportProgressInterval (audit export progress)" } - - # Filtering parameters - if ($PSBoundParameters.ContainsKey('AgentId')) { $incompatibleParams += " - AgentId (audit record filtering)" } - if ($AgentsOnly) { $incompatibleParams += " - AgentsOnly (audit record filtering)" } - if ($ExcludeAgents) { $incompatibleParams += " - ExcludeAgents (audit record filtering)" } - if ($PSBoundParameters.ContainsKey('PromptFilter')) { $incompatibleParams += " - PromptFilter (audit record content filtering)" } - if ($PSBoundParameters.ContainsKey('UserIds')) { $incompatibleParams += " - UserIds (audit record filtering; Entra fetch retrieves all users)" } - if ($PSBoundParameters.ContainsKey('GroupNames')) { $incompatibleParams += " - GroupNames (audit record filtering)" } - if ($PSBoundParameters.ContainsKey('RecordTypes')) { $incompatibleParams += " - RecordTypes (audit record filtering)" } - if ($PSBoundParameters.ContainsKey('ServiceTypes')) { $incompatibleParams += " - ServiceTypes (audit record filtering)" } - - # Processing mode parameters - if ($ExplodeArrays) { $incompatibleParams += " - ExplodeArrays (audit record array expansion)" } - if ($ExplodeDeep) { $incompatibleParams += " - ExplodeDeep (audit record deep expansion)" } - if ($PSBoundParameters.ContainsKey('RAWInputCSV')) { $incompatibleParams += " - RAWInputCSV (offline audit replay mode)" } - - # Parallel processing parameters - if ($EnableParallel) { $incompatibleParams += " - EnableParallel (parallel audit query execution)" } - if ($PSBoundParameters.ContainsKey('MaxConcurrency') -and $MaxConcurrency -ne 10) { $incompatibleParams += " - MaxConcurrency (concurrent query/partition limit)" } - if ($PSBoundParameters.ContainsKey('MaxParallelGroups') -and $MaxParallelGroups -ne 8) { $incompatibleParams += " - MaxParallelGroups (parallel activity group limit)" } - if ($PSBoundParameters.ContainsKey('ParallelMode') -and $ParallelMode -ne 'Auto') { $incompatibleParams += " - ParallelMode (parallel processing mode)" } - if ($DisableAdaptive) { $incompatibleParams += " - DisableAdaptive (adaptive concurrency controls)" } - if ($PSBoundParameters.ContainsKey('ProgressSmoothingAlpha') -and $ProgressSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ProgressSmoothingAlpha (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('HighLatencyMs') -and $HighLatencyMs -ne 90000) { $incompatibleParams += " - HighLatencyMs (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('MemoryPressureMB') -and $MemoryPressureMB -ne 1500) { $incompatibleParams += " - MemoryPressureMB (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('LowLatencyMs') -and $LowLatencyMs -ne 20000) { $incompatibleParams += " - LowLatencyMs (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('LowLatencyConsecutive') -and $LowLatencyConsecutive -ne 2) { $incompatibleParams += " - LowLatencyConsecutive (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('ThroughputDropPct') -and $ThroughputDropPct -ne 15) { $incompatibleParams += " - ThroughputDropPct (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('ThroughputSmoothingAlpha') -and $ThroughputSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ThroughputSmoothingAlpha (adaptive tuning)" } - if ($PSBoundParameters.ContainsKey('AdaptiveConcurrencyCeiling') -and $AdaptiveConcurrencyCeiling -ne 6) { $incompatibleParams += " - AdaptiveConcurrencyCeiling (adaptive tuning)" } - - # Reliability parameters (audit-specific) - if ($PSBoundParameters.ContainsKey('CircuitBreakerThreshold') -and $CircuitBreakerThreshold -ne 5) { $incompatibleParams += " - CircuitBreakerThreshold (block failure circuit breaker)" } - if ($PSBoundParameters.ContainsKey('CircuitBreakerCooldownSeconds') -and $CircuitBreakerCooldownSeconds -ne 120) { $incompatibleParams += " - CircuitBreakerCooldownSeconds (circuit breaker cooldown)" } - if ($PSBoundParameters.ContainsKey('BackoffBaseSeconds') -and $BackoffBaseSeconds -ne 1.0) { $incompatibleParams += " - BackoffBaseSeconds (block retry backoff)" } - if ($PSBoundParameters.ContainsKey('BackoffMaxSeconds') -and $BackoffMaxSeconds -ne 45) { $incompatibleParams += " - BackoffMaxSeconds (block retry max backoff)" } - - # Alternative modes - if ($UseEOM) { $incompatibleParams += " - UseEOM (Exchange Online Management mode incompatible with Graph Entra enrichment)" } - - # Output combination parameters - if ($CombineOutput) { $incompatibleParams += " - CombineOutput (only relevant with multiple activity types)" } - if ($AppendFile) { $incompatibleParams += " - AppendFile (appending user-only data to existing audit output not supported)" } - - if ($incompatibleParams.Count -gt 0) { - Write-Host "" - Write-Host "ERROR: The -OnlyUserInfo switch cannot be used with the following parameters:" -ForegroundColor Red - Write-Host "" - $incompatibleParams | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } - Write-Host "" - Write-Host "The -OnlyUserInfo switch exports only Entra user directory and license information (no audit logs)." -ForegroundColor Cyan - Write-Host "" - Write-Host "Compatible parameters:" -ForegroundColor Green - Write-Host " - OutputPath (where to save the file)" -ForegroundColor White - Write-Host " - Auth (authentication method: WebLogin, DeviceCode, Credential, Silent)" -ForegroundColor White - Write-Host " - Force (bypass interactive prompts)" -ForegroundColor White - Write-Host " - MaxNetworkOutageMinutes (network resilience for Graph API calls)" -ForegroundColor White - Write-Host " - EmitMetricsJson (track Entra retrieval metrics)" -ForegroundColor White - Write-Host " - MetricsPath (custom metrics output location)" -ForegroundColor White - Write-Host " - SkipDiagnostics (skip pre-query capability checks)" -ForegroundColor White - Write-Host "" - Write-Host "Please remove the incompatible parameters and try again." -ForegroundColor Cyan - Write-Host "" - exit 1 - } - - # If validation passes, configure for user-only export - Write-Host "" - Write-Host "INFO: -OnlyUserInfo mode enabled. Skipping all audit log retrieval, exporting only Entra user data." -ForegroundColor Green - Write-Host "" - $IncludeUserInfo = $true - $ActivityTypes = @() +function Get-AISIDCloudAppNames { + <# + .SYNOPSIS + Returns a de-duplicated flat array of cloud-app detection names. + #> + [OutputType([string[]])] + param() + $cat = Get-AISIDFilteredCatalog + return @($cat | ForEach-Object { $_.CloudAppNames } | Where-Object { $_ } | Select-Object -Unique) } -# PAX4A-GUARD-BEGIN -# -UserInfoFile / -GroupNames MUTUAL EXCLUSIVITY (Phase 4a) -# -UserInfoFile supplies the Entra user directory from a customer-provided CSV (bypassing the -# live /users pull). -GroupNames expands audit records against the LIVE directory. Supplying both -# is contradictory. Hard-stop here in the early parameter-validation stage so it fires on EVERY -# entry path (fresh and -Resume) BEFORE any -GroupNames consumer downstream (the RequiredScopes -# builder and the permissions banner). No partial work is done. -if ($PSBoundParameters.ContainsKey('UserInfoFile') -and $PSBoundParameters.ContainsKey('GroupNames')) { - Write-Host "" - Write-Host "ERROR: -UserInfoFile and -GroupNames cannot both be supplied." -ForegroundColor Red - Write-Host " -UserInfoFile ingests the Entra user directory from a customer-provided CSV." -ForegroundColor Yellow - Write-Host " -GroupNames expands audit records against the live Entra directory." -ForegroundColor Yellow - Write-Host " Supply exactly ONE: -UserInfoFile for a supplied directory, or -GroupNames for group filtering." -ForegroundColor Yellow - Write-Host "" - exit 1 +function Get-AISIDSignInAppNames { + <# + .SYNOPSIS + Returns a de-duplicated flat array of sign-in application detection names. + #> + [OutputType([string[]])] + param() + $cat = Get-AISIDFilteredCatalog + return @($cat | ForEach-Object { $_.SignInAppNames } | Where-Object { $_ } | Select-Object -Unique) } -# PAX4A-GUARD-END -# PAX4D-AUTOENABLE-BEGIN -# -UserInfoFile auto-enables -IncludeUserInfo (FU6): every -UserInfoFile export path (file naming, -# the multi-tab workbook, and the directory substitution seam) is gated on $IncludeUserInfo, so a -# customer need only pass -UserInfoFile. Mirrors the -OnlyUserInfo auto-enable above; fires on every -# entry path before any $IncludeUserInfo consumer. No effect when -UserInfoFile is unset. -if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile) -and -not $IncludeUserInfo) { - $IncludeUserInfo = $true -} -# PAX4D-AUTOENABLE-END +function Resolve-AISIDFromAppName { + <# + .SYNOPSIS + Resolves an application display name to its AI solutions catalog entry. + Returns $null when no match is found. + .NOTES + Matches against both CloudAppNames and SignInAppNames (case-insensitive). + #> + [OutputType([System.Collections.IDictionary])] + param([Parameter(Mandatory)][string]$AppName) -# Canonical maps for Graph filter normalization -$recordTypeCanonicalMap = @{ - 'azureactivedirectory' = 'AzureActiveDirectory' - 'azureactivedirectoryaccountlogon' = 'AzureActiveDirectoryAccountLogon' - 'azureactivedirectorystslogon' = 'AzureActiveDirectoryStsLogon' - 'exchangeadmin' = 'ExchangeAdmin' - 'exchangeitem' = 'ExchangeItem' - 'exchangemailbox' = 'ExchangeMailbox' - 'sharepointfileoperation' = 'SharePointFileOperation' - 'sharepointsharingoperation' = 'SharePointSharingOperation' - 'sharepoint' = 'SharePoint' - 'onedrive' = 'OneDrive' - 'microsoftteams' = 'MicrosoftTeams' + if ([string]::IsNullOrWhiteSpace($AppName)) { return $null } + $needle = $AppName.Trim().ToLowerInvariant() + $cat = Get-AISIDFilteredCatalog + foreach ($entry in $cat) { + $names = @() + if ($entry.CloudAppNames) { $names += $entry.CloudAppNames } + if ($entry.SignInAppNames) { $names += $entry.SignInAppNames } + foreach ($n in $names) { + if ($n -and $n.ToLowerInvariant() -eq $needle) { return $entry } + } + } + return $null } -$serviceCanonicalMap = @{ - 'azureactivedirectory' = 'AzureActiveDirectory' - 'exchange' = 'Exchange' - 'sharepoint' = 'SharePoint' - 'onedrive' = 'OneDrive' - 'teams' = 'Teams' +function Get-AISIDNetworkDomains { + <# + .SYNOPSIS + Returns a de-duplicated flat array of AI service network domains (hostnames) for host/URL + matching. Built from the catalog's NetworkDomains lists; entries with no known hostname + contribute nothing. + #> + [OutputType([string[]])] + param() + $cat = Get-AISIDFilteredCatalog + return @($cat | ForEach-Object { $_.NetworkDomains } | Where-Object { $_ } | Select-Object -Unique) } -# Normalize optional Graph filter passthrough parameters (dedupe & trim) -# Split ActivityTypes if provided as comma-separated string -if ($ActivityTypes) { - $processedActivityTypes = New-Object System.Collections.Generic.List[string] - foreach ($value in $ActivityTypes) { - if ($null -eq $value) { continue } - $raw = $value.ToString() - foreach ($piece in ($raw -split ',')) { - $token = $piece.Trim(" '""`t") - if ([string]::IsNullOrWhiteSpace($token)) { continue } - $processedActivityTypes.Add($token) +function Resolve-AISIDFromDomain { + <# + .SYNOPSIS + Resolves a host / FQDN to its AI solutions catalog entry, or $null when no match is found. + .NOTES + Matches against the catalog's NetworkDomains using endswith semantics, so a subdomain + (for example 'api.openai.com') resolves to a catalog entry that lists a parent domain. + #> + [OutputType([System.Collections.IDictionary])] + param([Parameter(Mandatory)][string]$HostName) + + if ([string]::IsNullOrWhiteSpace($HostName)) { return $null } + $needle = $HostName.Trim().ToLowerInvariant() + $cat = Get-AISIDFilteredCatalog + foreach ($entry in $cat) { + if (-not $entry.NetworkDomains) { continue } + foreach ($d in $entry.NetworkDomains) { + if (-not $d) { continue } + $lc = ([string]$d).ToLowerInvariant() + if ($needle -eq $lc -or $needle.EndsWith('.' + $lc)) { return $entry } } } - $ActivityTypes = @( - $processedActivityTypes | - ForEach-Object { $_.Trim() } | - Where-Object { $_ } | - Select-Object -Unique + return $null +} + +# Microsoft advanced hunting keeps data queryable for the last 30 days by default; reaching further +# back requires tenant-side streaming of specific tables into Microsoft Sentinel with extended +# retention, which this script cannot detect or assume. The window is fixed here as the single +# source of truth. Ref: https://learn.microsoft.com/en-us/defender-xdr/advanced-hunting-overview +$script:AISIDLookbackDaysFixed = 30 + +function Get-AISIDEffectiveLookbackDays { + <# + .SYNOPSIS + Returns the AI Solutions Intelligence Dashboard lookback window, in days. + #> + [OutputType([int])] + param() + return $script:AISIDLookbackDaysFixed +} + +function Resolve-AISIDDateWindow { + <##SYN Pure resolver for the AISID requested-vs-available date-window intersection. + Side-effect-free: no Get-Date, no logging, no prompts, no exits, no script-scope reads/writes. + All inputs are concrete UTC instants; requested and available intervals are half-open [start,end). + AvailableStartUtc = ReferenceNowUtc - RetentionDays; AvailableEndUtc = ReferenceNowUtc. + Effective = requested INTERSECT available. Status FULL = effective==requested (no clip); + PARTIAL = intervals overlap but effective differs; NONE = EffectiveStartUtc >= EffectiveEndUtc + (no overlap; an empty requested interval is NONE, never FULL). #> + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory)][datetime]$RequestedStartUtc, + [Parameter(Mandatory)][datetime]$RequestedEndUtc, + [Parameter(Mandatory)][datetime]$ReferenceNowUtc, + [int]$RetentionDays = 30 ) - if ($ActivityTypes.Count -eq 0) { - $ActivityTypes = $null + $reqStart = [datetime]::SpecifyKind($RequestedStartUtc, [System.DateTimeKind]::Utc) + $reqEnd = [datetime]::SpecifyKind($RequestedEndUtc, [System.DateTimeKind]::Utc) + $refNow = [datetime]::SpecifyKind($ReferenceNowUtc, [System.DateTimeKind]::Utc) + $availStart = $refNow.AddDays(-$RetentionDays) + $availEnd = $refNow + $effStart = if ($reqStart -gt $availStart) { $reqStart } else { $availStart } + $effEnd = if ($reqEnd -lt $availEnd) { $reqEnd } else { $availEnd } + $startClipped = ($effStart -ne $reqStart) + $endClipped = ($effEnd -ne $reqEnd) + if ($effStart -ge $effEnd) { + $status = 'NONE'; $wasClipped = $true + $clipReason = 'Requested interval does not overlap the available retention window.' + } + elseif (-not $startClipped -and -not $endClipped) { + $status = 'FULL'; $wasClipped = $false; $clipReason = '' + } + else { + $status = 'PARTIAL'; $wasClipped = $true + $parts = @() + if ($startClipped) { $parts += ("requested start {0:yyyy-MM-ddTHH:mm:ssZ} precedes the {1}-day retention floor {2:yyyy-MM-ddTHH:mm:ssZ}" -f $reqStart, $RetentionDays, $availStart) } + if ($endClipped) { $parts += ("requested end {0:yyyy-MM-ddTHH:mm:ssZ} exceeds the current retention ceiling {1:yyyy-MM-ddTHH:mm:ssZ}" -f $reqEnd, $availEnd) } + $clipReason = ($parts -join '; ') + '.' + } + return [pscustomobject][ordered]@{ + RequestedStartUtc = $reqStart + RequestedEndUtc = $reqEnd + ReferenceNowUtc = $refNow + AvailableStartUtc = $availStart + AvailableEndUtc = $availEnd + EffectiveStartUtc = $effStart + EffectiveEndUtc = $effEnd + Status = $status + WasClipped = $wasClipped + ClipReason = $clipReason + RetentionDays = [int]$RetentionDays + } +} + +function Get-AISIDCollectionBounds { + <##SYN Returns the frozen AISID collection interval [StartUtc, EndUtc) - the resolved + requested-intersect-available effective window - as a two-key hashtable. Every date-bearing + AISID collector reads its bounds from here so the whole run shares one window instant. #> + [CmdletBinding()] + param() + if ($script:AISIDWindow -and $script:AISIDWindow.EffectiveStartUtc -and $script:AISIDWindow.EffectiveEndUtc) { + return @{ + StartUtc = [datetime]::SpecifyKind([datetime]$script:AISIDWindow.EffectiveStartUtc, [System.DateTimeKind]::Utc) + EndUtc = [datetime]::SpecifyKind([datetime]$script:AISIDWindow.EffectiveEndUtc, [System.DateTimeKind]::Utc) + } } + throw "AISID collection bounds requested before the AISID date window was established (script:AISIDWindow is missing EffectiveStartUtc/EffectiveEndUtc). This is a control-flow invariant violation: every valid fresh or resumed AISID run must resolve and freeze the window before any collector runs." } -if ($RecordTypes) { - $processedRecordTypes = New-Object System.Collections.Generic.List[string] - foreach ($value in $RecordTypes) { - if ($null -eq $value) { continue } - $raw = $value.ToString() - foreach ($piece in ($raw -split ',')) { - $token = $piece.Trim(" '""`t") - if ([string]::IsNullOrWhiteSpace($token)) { continue } - $processedRecordTypes.Add($token) - } +function Write-AISIDWindowSummary { + <##SYN Prints the one-line AISID collection-window summary (terminal + log) for a completed run. + FULL/PARTIAL only; a NONE run exits before any collection and never reaches here. #> + [CmdletBinding()] + param() + if (-not $script:AISIDWindow) { return } + $w = $script:AISIDWindow + $fmt = 'yyyy-MM-ddTHH:mm:ssZ' + $reqStr = ('[{0} .. {1})' -f ([datetime]$w.RequestedStartUtc).ToString($fmt), ([datetime]$w.RequestedEndUtc).ToString($fmt)) + $effStr = ('[{0} .. {1})' -f ([datetime]$w.EffectiveStartUtc).ToString($fmt), ([datetime]$w.EffectiveEndUtc).ToString($fmt)) + if ([string]$w.Status -eq 'PARTIAL') { + Write-LogHost ("AISID collection window: PARTIAL | requested {0} | effective {1} | reason: {2}" -f $reqStr, $effStr, [string]$w.ClipReason) -ForegroundColor Yellow } - $RecordTypes = @( - $processedRecordTypes | - ForEach-Object { $_.Trim() } | - Where-Object { $_ } | - Select-Object -Unique - ) - if ($RecordTypes.Count -eq 0) { - $RecordTypes = $null - } else { - $RecordTypes = @( - foreach ($rt in $RecordTypes) { - $key = $rt.ToLowerInvariant() - if ($recordTypeCanonicalMap.ContainsKey($key)) { $recordTypeCanonicalMap[$key] } else { $rt } - } - ) | Select-Object -Unique + else { + Write-LogHost ("AISID collection window: FULL | requested {0} | effective {1}" -f $reqStr, $effStr) -ForegroundColor Green } } -if ($ServiceTypes) { - $processedServiceTypes = New-Object System.Collections.Generic.List[string] - foreach ($value in $ServiceTypes) { - if ($null -eq $value) { continue } - $raw = $value.ToString() - foreach ($piece in ($raw -split ',')) { - $token = $piece.Trim(" '""`t") - if ([string]::IsNullOrWhiteSpace($token)) { continue } - $processedServiceTypes.Add($token) - } - } - $ServiceTypes = @( - $processedServiceTypes | - ForEach-Object { $_.Trim() } | - Where-Object { $_ } | - Select-Object -Unique +# ============================================================ +# AI Solutions Intelligence Dashboard — Graph hunting rate limiter +# ============================================================ +# Microsoft's advanced hunting API guarantees at least 45 calls/min per tenant (higher for +# larger tenants) and enforces a per-tenant CPU budget that returns HTTP 429 when exhausted. +# This limiter paces proactively well under that floor and drains reactively on a rate-limit +# response. Ref: https://learn.microsoft.com/en-us/graph/api/resources/security-api-overview +$script:AISIDTokenBucket = [hashtable]::Synchronized(@{ + IntervalMs = 2000 # Proactive floor: 30 calls/min (headroom under the >=45/min guarantee) + BackoffIntervalMs = 4000 # Slowed cadence after a rate-limit response: 15 calls/min + LastEmit = [DateTime]::MinValue + BackoffUntil = [DateTime]::MinValue + PermitCount = 0 + Lock = [object]::new() +}) + +function Set-AISIDTokenBucketBackoff { + <# + .SYNOPSIS + Slows the hunting-query rate limiter to its backoff cadence for a set duration after a + rate-limit response. + #> + [CmdletBinding()] + param([int]$BackoffDurationSeconds = 60) + $script:AISIDTokenBucket.BackoffUntil = (Get-Date).AddSeconds($BackoffDurationSeconds) + Write-LogHost "[AISID] Rate-limit backoff engaged; hunting-query cadence slowed to $($script:AISIDTokenBucket.BackoffIntervalMs)ms for ${BackoffDurationSeconds}s." -ForegroundColor Yellow +} + +function Invoke-WithAISIDTokenBucket { + <# + .SYNOPSIS + Runs a scriptblock through the hunting-query rate limiter, waiting for the next permit + (base or backoff cadence) before invoking it and returning its result unchanged. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][scriptblock]$ScriptBlock, + [string]$Description = '' ) - if ($ServiceTypes.Count -eq 0) { - $ServiceTypes = $null - } else { - $ServiceTypes = @( - foreach ($svc in $ServiceTypes) { - $key = $svc.ToLowerInvariant() - if ($serviceCanonicalMap.ContainsKey($key)) { $serviceCanonicalMap[$key] } else { $svc } + + $bucket = $script:AISIDTokenBucket + $gotLock = $false + try { + [System.Threading.Monitor]::Enter($bucket.Lock, [ref]$gotLock) + $now = Get-Date + $intervalMs = if ($now -lt $bucket.BackoffUntil) { $bucket.BackoffIntervalMs } else { $bucket.IntervalMs } + if ($bucket.LastEmit -ne [DateTime]::MinValue) { + $elapsedMs = ($now - $bucket.LastEmit).TotalMilliseconds + if ($elapsedMs -lt $intervalMs) { + $waitMs = [int]($intervalMs - $elapsedMs) + [System.Threading.Monitor]::Exit($bucket.Lock); $gotLock = $false + if ($waitMs -ge 1000 -and $Description) { + Write-LogHost "[AISID] Waiting ${waitMs}ms for the next query permit ($Description)..." -ForegroundColor DarkGray + } + Start-Sleep -Milliseconds $waitMs + [System.Threading.Monitor]::Enter($bucket.Lock, [ref]$gotLock) } - ) | Select-Object -Unique + } + $bucket.LastEmit = Get-Date + $bucket.PermitCount++ + if ($gotLock) { [System.Threading.Monitor]::Exit($bucket.Lock); $gotLock = $false } + } + catch { + if ($gotLock) { [System.Threading.Monitor]::Exit($bucket.Lock); $gotLock = $false } + throw } + return (& $ScriptBlock) } -# Mapping of audit record types to supported workloads for Graph security audit queries -$recordTypeWorkloadMap = @{ - 'azureActiveDirectory' = @('AzureActiveDirectory') - 'azureActiveDirectoryAccountLogon' = @('AzureActiveDirectory') - 'azureActiveDirectoryStsLogon' = @('AzureActiveDirectory') - 'exchangeAdmin' = @('Exchange') - 'exchangeItem' = @('Exchange') - 'exchangeMailbox' = @('Exchange') - 'sharePointFileOperation' = @('SharePoint','OneDrive') - 'sharePointSharingOperation' = @('SharePoint','OneDrive') - 'sharePoint' = @('SharePoint','OneDrive') - 'onedrive' = @('OneDrive') - 'microsoftTeams' = @('Teams') - # M365 usage record types: Process in first workload pass to avoid creating additional passes - # These record types capture cross-workload activities (Office apps, Forms, Stream, Planner, PowerApps) - # Mapping to Exchange ensures they run in the first service-filtered pass - 'officeNative' = @('Exchange') - 'microsoftForms' = @('Exchange') - 'microsoftStream' = @('Exchange') - 'plannerPlan' = @('Exchange') - 'plannerTask' = @('Exchange') - 'powerAppsApp' = @('Exchange') -} +function Invoke-GraphHuntingQuery { + <# + .SYNOPSIS + Runs a Kusto advanced-hunting query through Microsoft Graph and returns a normalized + result. Single entry point for the dashboard's hunting queries. + .DESCRIPTION + Posts to the Microsoft Graph advanced-hunting endpoint over the run's existing Graph + session (Invoke-MgGraphRequest), paced by the shared rate limiter and backing off + reactively on a rate-limit response. The collection window is the fixed lookback + (Get-AISIDEffectiveLookbackDays) expressed as an ISO 8601 timespan. + Ref: https://learn.microsoft.com/en-us/graph/api/security-security-runhuntingquery + .OUTPUTS + Hashtable (never throws): Success, Results, Schema, RowCount, TruncationDetected, + ErrorMessage, HttpStatusCode, AuthQuit, Elapsed. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Query, + [string]$Description = '', + [int]$MaxTransientRetries = 3 + ) -$serviceOperationMap = @{ - 'AzureActiveDirectory' = @('UserLoggedIn','UserLoginFailed','AdminLoggedIn','ResetUserPassword','AddRegisteredUser','UpdateUser','ChangedUserSetting') - 'Exchange' = @('MailItemsAccessed','Send','SendOnBehalf','SoftDelete','HardDelete','MoveToDeletedItems','CopyToFolder','AddMailboxPermission','RemoveMailboxPermission') - 'SharePoint' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') - 'OneDrive' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') - 'Teams' = @('TeamMemberAdded','TeamMemberRemoved','ChannelAdded','ChannelDeleted','ChannelMessageSent','ChannelMessageDeleted','TeamDeleted','TeamArchived','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') - 'MicrosoftForms' = @('CreateForm','EditForm','DeleteForm','ViewForm','CreateResponse','SubmitResponse','ViewResponse','DeleteResponse') - 'MicrosoftStream' = @('StreamModified','StreamViewed','StreamDeleted','StreamDownloaded') - 'MicrosoftPlanner' = @('PlanCreated','PlanDeleted','PlanModified','TaskCreated','TaskDeleted','TaskModified','TaskAssigned','TaskCompleted') - 'PowerApps' = @('LaunchedApp','CreatedApp','EditedApp','DeletedApp','PublishedApp') -} + # Advanced hunting returns at most 100,000 rows (and 50 MB) per query; a result at or near + # the row ceiling signals possible silent truncation and is surfaced as a warning. + # Ref: https://learn.microsoft.com/en-us/graph/api/resources/security-api-overview + $rowCeiling = 100000 + $truncationWarnAt = 99000 -$copilotBaseActivityType = 'CopilotInteraction' -$m365UsageServiceBundle = @('Exchange','SharePoint','OneDrive','Teams') -$m365UsageRecordBundle = @('ExchangeAdmin','ExchangeItem','ExchangeMailbox','SharePointFileOperation','SharePointSharingOperation','SharePoint','OneDrive','MicrosoftTeams','OfficeNative','MicrosoftForms','MicrosoftStream','PlannerPlan','PlannerTask','PowerAppsApp') -# Curated, trimmed M365 usage operations targeted at the Analytics-Hub M365 Usage Analytics -# dashboard. Scope: Exchange mail access, SharePoint/OneDrive file access, Teams chat/messaging, -# Teams meeting lifecycle, and Copilot/Connected-AI interaction signals. -$m365UsageActivityBundle = @( - # === Exchange / Email === - 'MailItemsAccessed','MailboxLogin','Send', + $result = @{ + Success = $false; Results = $null; Schema = $null; RowCount = 0; TruncationDetected = $false + ErrorMessage = ''; HttpStatusCode = 0; AuthQuit = $false; Elapsed = [TimeSpan]::Zero + } - # === SharePoint / OneDrive - File access === - 'FileAccessed','FileViewed','FilePreviewed','FileModified','FileDownloaded','FileUploaded', + $lookbackDays = Get-AISIDEffectiveLookbackDays + $endpoint = 'https://graph.microsoft.com/v1.0/security/runHuntingQuery' + $body = @{ Query = $Query; Timespan = "P${lookbackDays}D" } | ConvertTo-Json -Compress -Depth 4 + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $attempt = 0 + $tokenRetryConsumed = $false - # === Teams - Chat / Messaging === - 'MessageSent','MessageRead','MessagesListed','ChatRetrieved','ChatCreated','TeamsSessionStarted', + while ($true) { + $attempt++ - # === Teams - Meeting lifecycle === - 'MeetingParticipantJoined','MeetingStarted','MeetingEnded','MeetingParticipantDetail','MeetingDetail', + # Keep the shared Graph token fresh through the existing refresh contract. + $refresh = Refresh-GraphTokenIfNeeded + if ($refresh -eq 'Quit') { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } - # === Copilot / Connected AI === - 'CopilotInteraction','ConnectedAIAppInteraction' -) | Select-Object -Unique + $httpStatus = 0; $retryAfter = 0; $errMsg = ''; $payload = $null + try { + $payload = Invoke-WithAISIDTokenBucket -Description $Description -ScriptBlock { + Invoke-MgGraphRequest -Method POST -Uri $endpoint -Body $body -ContentType 'application/json' -OutputType PSObject -ErrorAction Stop + } + $httpStatus = 200 + } + catch { + $errMsg = $_.Exception.Message + $httpStatus = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($httpStatus -eq 0 -and $errMsg -match '429|Too Many Requests|TooManyRequests') { $httpStatus = 429 } + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + } + $result.HttpStatusCode = $httpStatus -# Script version constant (must appear after param/help to keep param() valid as first executable block) -$ScriptVersion = '1.11.14' + if ($payload -and $httpStatus -ge 200 -and $httpStatus -lt 300) { + $rows = @($payload.results) + $result.Success = $true; $result.Results = $rows; $result.Schema = $payload.schema; $result.RowCount = $rows.Count + $result.TruncationDetected = ($rows.Count -ge $truncationWarnAt) + if ($result.TruncationDetected) { + Write-LogHost "[AISID] Query '$Description' returned $($rows.Count) rows, at or near the $rowCeiling-row limit; results may be truncated." -ForegroundColor Yellow + } + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } -function Invoke-PaxVersionCheck { - # Informational, non-blocking, failure-isolated version check against the public PAX repo. - # Reads versions.json from the release branch, compares the purview script version, and prints - # a single info line. Never prompts, never throws, capped at ~5s if the server is unreachable. - param([string]$CurrentVersion) - $repoUrl = 'https://github.com/microsoft/PAX' - try { - $verUrl = 'https://raw.githubusercontent.com/microsoft/PAX/release/versions.json' - $resp = Invoke-RestMethod -Uri $verUrl -TimeoutSec 5 -ErrorAction Stop - $latest = [string]$resp.products.purview.version - $relDate = [string]$resp.lastUpdated - if ($latest -and ([version]$latest) -gt ([version]$CurrentVersion)) { - $line = " Update available: PAX v$latest" - if ($relDate) { $line += " (released $relDate)" } - $line += " - you are on v$CurrentVersion. Latest: $repoUrl" - Write-LogHost $line -ForegroundColor Cyan - } else { - Write-LogHost " Version check: you are on the latest PAX version (v$CurrentVersion). $repoUrl" -ForegroundColor DarkGray + if ($httpStatus -eq 401 -and -not $tokenRetryConsumed) { + $tokenRetryConsumed = $true + $r = Refresh-GraphTokenIfNeeded + if ($r -eq 'Quit') { + $result.AuthQuit = $true; $result.ErrorMessage = 'Unauthorized and token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + continue } - } catch { - Write-LogHost " Version check skipped: the PAX GitHub repo was not reachable (offline or blocked). Latest: $repoUrl" -ForegroundColor DarkGray + + if ($httpStatus -eq 429) { + $waitSec = if ($retryAfter -gt 0) { $retryAfter } else { 60 } + Write-LogHost "[AISID] Rate limited (HTTP 429); waiting ${waitSec}s before retrying ($Description)." -ForegroundColor Yellow + Set-AISIDTokenBucketBackoff -BackoffDurationSeconds 60 + Start-Sleep -Seconds $waitSec + continue + } + + $transient = ($httpStatus -ge 500 -or $httpStatus -eq 408 -or $httpStatus -eq 0) + if ($transient -and $attempt -le $MaxTransientRetries) { + $backoffSec = [int][Math]::Min(60, [Math]::Pow(2, $attempt)) + Write-LogHost "[AISID] Transient failure (HTTP $httpStatus, attempt $attempt/$MaxTransientRetries); retrying in ${backoffSec}s. $errMsg" -ForegroundColor Yellow + Start-Sleep -Seconds $backoffSec + continue + } + + $result.ErrorMessage = "Graph hunting query failed (HTTP $httpStatus, attempt $attempt): $errMsg" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result } } -# --- Initialize/Clear persistent script variables to prevent cross-run contamination --- -# Note: Script-scoped variables persist across multiple script invocations in the same PowerShell session -$script:partitionStatus = $null -$script:processedJobIds = $null -$script:shownJobMessages = $null +function Get-AISIDHuntingRowEstimate { + <# + .SYNOPSIS + Runs a count-only hunting query and returns the estimated row count for the given KQL, or + -1 when the probe could not be evaluated. Used to decide whether a result set would exceed + the advanced-hunting response cap and therefore needs time-window subdivision. + .OUTPUTS + [long] estimated row count, or -1 on probe failure / auth quit. + #> + [OutputType([long])] + param( + [Parameter(Mandatory)][string]$ProbeQuery, + [string]$Description = 'row-estimate probe' + ) + $r = Invoke-GraphHuntingQuery -Query $ProbeQuery -Description $Description + if ($r.AuthQuit -or -not $r.Success) { return -1 } + $rows = @($r.Results) + if ($rows.Count -eq 0) { return 0 } + # The probe ends in `summarize N = count()`, so the single result row must carry a numeric field + # named exactly 'N'. OData annotation companions ('N@odata.type' = '#Int64', and any top-level + # '@odata.*') are ignored structurally. The count is read ONLY from a real 'N' property; an unrelated + # lone numeric property is NEVER accepted. Missing, non-numeric, or duplicate 'N' => unreadable (-1). + $nProps = @($rows[0].PSObject.Properties | Where-Object { $_.Name -eq 'N' }) + if ($nProps.Count -ne 1) { return -1 } + $long = [long]0 + if ($null -eq $nProps[0].Value -or -not [long]::TryParse([string]$nProps[0].Value, [ref]$long)) { return -1 } + return [long]$long +} + +function Get-AISIDTimePartitions { + <# + .SYNOPSIS + Plans a set of time sub-windows whose individual estimated row counts stay under the + advanced-hunting response cap. Probes the given window via -EstimateProvider; if the + estimate is under the cap the window is returned whole, otherwise it is split into + ceil(estimate / target) equal sub-windows, each planned recursively (re-probed, so a + non-uniform distribution is handled by splitting only the ranges that are actually hot). + Subdivision stops at -FloorMinutes; a floor window still over the cap is returned with + Truncated set and a warning, consistent with the query-level truncation warning. + .OUTPUTS + Array of ordered hashtables @{ StartUtc; EndUtc; Estimate; Truncated }. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][datetime]$StartUtc, + [Parameter(Mandatory)][datetime]$EndUtc, + [Parameter(Mandatory)][scriptblock]$EstimateProvider, + [long]$Cap = 90000, + [long]$Target = 80000, + [int]$FloorMinutes = 60 + ) -# --- Microsoft 365 Copilot license detection --- -# No hardcoded SKU GUIDs. Copilot service plans are discovered dynamically at runtime -# from /subscribedSkus by matching servicePlanName against the pattern '*COPILOT*'. -# Per-user eligibility is then computed from the user's assignedPlans where -# capabilityStatus == 'Enabled' and servicePlanId is in the discovered set. -# See Get-UserLicenseData(). + $estimate = [long](& $EstimateProvider $StartUtc $EndUtc) + $spanMinutes = ($EndUtc - $StartUtc).TotalMinutes -# --- Synchronized Timestamp & OutputPath Validation --- + # An unreadable probe (-1) must NEVER be treated as "cannot rule out over-cap" and subdivided: that + # produced an unbounded recursive fan-out down to the minute floor. Return a single window flagged + # EstimateFailed so the collector fails this table explicitly instead of silently truncating/emptying. + if ($estimate -lt 0) { + return , ([ordered]@{ StartUtc = $StartUtc; EndUtc = $EndUtc; Estimate = $estimate; Truncated = $false; EstimateFailed = $true }) + } -# Generate synchronized timestamp for all output files in this run -$global:ScriptRunTimestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + # Under the cap (and a successful probe): take the window whole. + if ($estimate -lt $Cap) { + return , ([ordered]@{ StartUtc = $StartUtc; EndUtc = $EndUtc; Estimate = $estimate; Truncated = $false; EstimateFailed = $false }) + } + # At/over the cap (or an unreadable probe, treated as "cannot rule out over-cap") but already at + # the minimum granularity: accept this window and flag possible truncation. + if ($spanMinutes -le $FloorMinutes) { + $trunc = ($estimate -ge $Cap) + if ($trunc) { + Write-LogHost ("[AISID] FileProximity: window {0:o}..{1:o} still estimates {2} rows at the {3}-minute floor; this window may be truncated at the {4}-row hunting limit." -f $StartUtc, $EndUtc, $estimate, $FloorMinutes, 100000) -ForegroundColor Yellow + } + return , ([ordered]@{ StartUtc = $StartUtc; EndUtc = $EndUtc; Estimate = $estimate; Truncated = $trunc; EstimateFailed = $false }) + } -# --- Logging Helper Functions (defined early for use throughout script) --- -# $script:LogFile was already initialized to a bootstrap temp-path file at the top -# of the script body (right after param()). DO NOT re-null it here; that would -# discard the early-startup log entries already captured. The bootstrap file is -# renamed to the final location once $OutputFile is resolved (see the relocation -# block further down). The in-memory $script:LogBuffer remains a fallback for the -# rare case where bootstrap creation failed (e.g. read-only temp). -if (-not (Get-Variable -Name LogFile -Scope Script -ErrorAction SilentlyContinue)) { - $script:LogFile = $null -} -$script:LogBuffer = New-Object System.Collections.Generic.List[string] + # Split into enough parts to bring each comfortably under target (at least 2), but never so many + # that a sub-window would fall below the floor. Each part is re-probed recursively. + $parts = 2 + if ($estimate -gt 0) { $parts = [Math]::Max(2, [int][Math]::Ceiling($estimate / [double]$Target)) } + $maxParts = [Math]::Max(2, [int][Math]::Floor($spanMinutes / [double]$FloorMinutes)) + if ($parts -gt $maxParts) { $parts = $maxParts } -function Write-Log { - param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") - $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" - $logEntry = "[$timestamp] [$Level] $Message" - Microsoft.PowerShell.Utility\Write-Host $Message - try { - if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } - else { $script:LogBuffer.Add($logEntry) | Out-Null } - } catch {} + $out = @() + $totalTicks = ($EndUtc - $StartUtc).Ticks + for ($i = 0; $i -lt $parts; $i++) { + $subStart = $StartUtc.AddTicks([long]($totalTicks * $i / $parts)) + $subEnd = if ($i -eq ($parts - 1)) { $EndUtc } else { $StartUtc.AddTicks([long]($totalTicks * ($i + 1) / $parts)) } + $out += Get-AISIDTimePartitions -StartUtc $subStart -EndUtc $subEnd -EstimateProvider $EstimateProvider -Cap $Cap -Target $Target -FloorMinutes $FloorMinutes + } + return $out } -# Log-file-only writer: never echoes to host. Used for diagnostic detail (temp paths, -# argument vectors, stack traces) that should be captured in the run log but not -# clutter the customer-facing console. -function Write-LogFile { - param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") - $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" - $logEntry = "[$timestamp] [$Level] $Message" - try { - if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } - else { $script:LogBuffer.Add($logEntry) | Out-Null } - } catch {} -} +function Test-AISIDLicensing { + <# + .SYNOPSIS + Evaluates, independently per table, which AI Solutions Intelligence Dashboard data sets + this tenant can populate. There is no combined pass/fail — each table stands on its own, + so a tenant with only Microsoft Entra access still collects the two Entra-sourced tables. + .DESCRIPTION + The four advanced-hunting tables (ActivitySessions from CloudAppEvents, OffHoursGeo from + EntraIdSignInEvents, FileProximity, and ClientChannel) are probed directly with a minimal query; a failed probe marks only + that table unavailable. The remaining three tables draw on data the full run already produces — + OAuthConsents and SSOSignIns from the run's Microsoft Entra access, and CopilotUsage from the + CopilotInteraction rollup — so they are eligible whenever the run is authenticated. + .OUTPUTS + Ordered hashtable keyed by table name, each value @{ Eligible; Source; Detail }, plus an + AuthQuit flag. No early return produces an all-or-nothing verdict. + #> + [CmdletBinding()] + param() -function Write-LogHost { - param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$ForegroundColor = "White") - Microsoft.PowerShell.Utility\Write-Host $Message -ForegroundColor $ForegroundColor - try { - $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" - $logEntry = "[$timestamp] [INFO] $Message" - if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + $verdict = [ordered]@{ + ActivitySessions = @{ Eligible = $false; Source = 'Defender advanced hunting'; Detail = '' } + OffHoursGeo = @{ Eligible = $false; Source = 'Defender advanced hunting'; Detail = '' } + FileProximity = @{ Eligible = $false; Source = 'Defender advanced hunting (MDE)'; Detail = '' } + ClientChannel = @{ Eligible = $false; Source = 'Defender advanced hunting (MDE)'; Detail = '' } + OAuthConsents = @{ Eligible = $true; Source = 'Microsoft Entra audit logs'; Detail = "Available from the run's Entra access." } + SSOSignIns = @{ Eligible = $true; Source = 'Microsoft Entra sign-in logs'; Detail = "Available from the run's Entra access." } + CopilotUsage = @{ Eligible = $true; Source = 'Microsoft Purview (CopilotInteraction rollup)'; Detail = "Available from the run's CopilotInteraction rollup." } + AuthQuit = $false + } + + $huntProbes = [ordered]@{ + ActivitySessions = 'CloudAppEvents | take 1 | project Timestamp' + OffHoursGeo = 'EntraIdSignInEvents | take 1 | project Timestamp' + FileProximity = 'DeviceNetworkEvents | take 1 | project Timestamp' + ClientChannel = 'DeviceNetworkEvents | take 1 | project Timestamp' + } + + Write-LogHost '[AISID] Checking which data sets this tenant can populate...' -ForegroundColor Cyan + foreach ($tbl in $huntProbes.Keys) { + $r = Invoke-GraphHuntingQuery -Query $huntProbes[$tbl] -Description "Eligibility probe: $tbl" + if ($r.AuthQuit) { $verdict.AuthQuit = $true } + if ($r.Success) { + $verdict[$tbl].Eligible = $true + $verdict[$tbl].Detail = 'Source data is queryable.' + } + else { + $verdict[$tbl].Detail = "Source data is not available in this tenant (status $($r.HttpStatusCode))." + } + } + + return $verdict +} + +function Write-AISIDLicensingReport { + <# + .SYNOPSIS + Prints a per-table summary of which AI Solutions Intelligence Dashboard data sets will be + collected for this tenant and which are unavailable. Reports table by table; it never + declares the dashboard as a whole unavailable. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][System.Collections.IDictionary]$Verdict) + + $tables = @('ActivitySessions', 'OffHoursGeo', 'FileProximity', 'ClientChannel', 'OAuthConsents', 'SSOSignIns', 'CopilotUsage') + $eligibleCount = @($tables | Where-Object { $Verdict[$_].Eligible }).Count + + Write-LogHost '' + Write-LogHost "[AISID] Data set availability for this tenant ($eligibleCount of $($tables.Count) available):" -ForegroundColor Cyan + foreach ($tbl in $tables) { + $v = $Verdict[$tbl] + $status = if ($v.Eligible) { 'available' } else { 'not available' } + $color = if ($v.Eligible) { 'Green' } else { 'Yellow' } + Write-LogHost ("[AISID] {0,-18} {1,-14} ({2})" -f $tbl, $status, $v.Source) -ForegroundColor $color + } + if ($eligibleCount -lt $tables.Count) { + Write-LogHost '[AISID] Data sets shown as not available depend on Microsoft Defender or Entra features this tenant does not currently expose. The available data sets are collected normally.' -ForegroundColor Gray + } + Write-LogHost '' +} + +# ============================================================ +# AI Solutions Intelligence Dashboard — table export contract + export/merge infrastructure +# ============================================================ +# The dashboard model loads each CSV by a FIXED name from one folder and applies a fixed, +# ordered TransformColumnTypes; a column-order or name mismatch surfaces as a hard Power Query +# refresh error on the customer side, so the header order below is contractual. Column names, +# order, and the merge key grain are sourced from the model schema captured in +# PBIT_GROUND_TRUTH.md (table AI_Activity) and the FileProximity stub contract — never typed +# from memory. The FileProximity entry is the permanent header-only stub (always empty). +$script:AISIDTableSchemas = @{ + ActivitySessions = @{ + FileName = 'ai_activity_sessions.csv' + Columns = @('UPN', 'AISolution', 'YearMonth', 'Sessions', 'ActiveDays', 'EstimatedPrompts', 'DistinctDevices', 'Category', 'RiskTier') + Key = @('UPN', 'AISolution', 'YearMonth') + } + FileProximity = @{ + FileName = 'ai_file_proximity.csv' + Columns = @('Timestamp', 'UPN', 'AISolution', 'YearMonth', 'FileName', 'FolderCategory', 'FolderPath', 'SecondsToAI', 'NameMatchesSensitivePattern', 'FolderMatchesSensitive') + Key = @('Timestamp', 'UPN', 'FileName') + } + OffHoursGeo = @{ + FileName = 'ai_offhours_geo.csv' + Columns = @('UPN', 'YearMonth', 'TotalSessions', 'OffHoursSessions', 'OffHoursPct', 'DistinctCountries', 'AnomalousCountryCount', 'AnomalousCountries') + Key = @('UPN', 'YearMonth') + } + OAuthConsents = @{ + FileName = 'ai_oauth_consents.csv' + Columns = @('UPN', 'AppName', 'YearMonth', 'ConsentCount', 'LastConsent', 'PermissionWeight', 'Permissions') + Key = @('UPN', 'AppName', 'YearMonth') + } + SSOSignIns = @{ + FileName = 'ai_sso_signins.csv' + Columns = @('UPN', 'Application', 'YearMonth', 'SignInCount', 'DistinctDays', 'IsGuest', 'Countries', 'HasConditionalAccess', 'LastSignIn') + Key = @('UPN', 'Application', 'YearMonth') + } + ClientChannel = @{ + FileName = 'ai_client_channel.csv' + Columns = @('AISite', 'Channel', 'YearMonth', 'EventCount') + Key = @('AISite', 'Channel', 'YearMonth') + } + CopilotUsage = @{ + FileName = 'ai_copilot_usage_graph.csv' + Columns = @('UserPrincipalName', 'YearMonth', 'TeamsPrompts', 'WordPrompts', 'ExcelPrompts', 'OutlookPrompts', 'PowerPointPrompts', 'ChatPrompts', 'TotalPrompts', 'ActiveDays', 'LastActivityDate') + Key = @('UserPrincipalName', 'YearMonth') + } + AppGovAlerts = @{ + FileName = 'ai_appgov_alerts.csv' + Columns = @('Timestamp', 'YearMonth', 'UPN', 'AppName', 'AlertType', 'Severity', 'Description') + Key = @() + } + CloudDiscovery = @{ + FileName = 'ai_cloud_discovery.csv' + Columns = @('AIDomain', 'AppCategory', 'YearMonth', 'RiskScore', 'UploadVolumeMB', 'DownloadVolumeMB', 'TransactionCount', 'DistinctUsers', 'SanctionStatus') + Key = @() + } + MDASessions = @{ + FileName = 'ai_mda_sessions.csv' + Columns = @('Timestamp', 'YearMonth', 'UPN', 'AppName', 'ActionType', 'PolicyHit', 'PolicyAction', 'IPAddress', 'CountryCode', 'EventCount') + Key = @() + } + EntraUsers = @{ + FileName = 'EntraUsers.csv' + Columns = @('userPrincipalName', 'displayName', 'department', 'jobTitle', 'city', 'country', 'companyName', 'accountEnabled', 'userType', 'manager_displayName', 'manager_userPrincipalName', 'hasLicense', 'assignedLicenses', 'createdDateTime') + Key = @() # Dimension table: whole file regenerated each run (overwrite); no replace-on-key merge. + } + SolutionsCatalog = @{ + FileName = 'ai_solutions_catalog.csv' + Columns = @('AISolution', 'Category', 'Vendor', 'RiskTier', 'DefaultDataHandling', 'SolutionGroup') + Key = @() # Dimension table: whole file regenerated each run (overwrite); no replace-on-key merge. + } +} + +function script:Get-AISIDScopeUpnSet { + # Resolved AISID collector scope: a case-insensitive set of the target UPNs, or $null when no + # user/group scope was requested (an unfiltered whole-directory run). Fail-closed: a REQUESTED + # scope that resolves to zero users already aborts the run upstream (exit 1), so a non-null empty + # set never occurs here; $null distinctly means "no scope requested" (do not filter). An empty + # scope therefore never widens, and a requested scope never leaks a non-target UPN into output. + $u = @($script:targetUsers) + if ($u.Count -eq 0) { return $null } + $set = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($x in $u) { $s = ([string]$x).Trim(); if ($s) { [void]$set.Add($s) } } + if ($set.Count -eq 0) { return $null } + # Unary comma: return the HashSet itself (with its OrdinalIgnoreCase comparer) instead of letting + # PowerShell enumerate it into a case-sensitive object[] on the way out. + return , $set +} + +# Per-table UPN column for AISID client-side scope enforcement. ClientChannel has no per-user grain +# (scoped at its source query) and SolutionsCatalog is intentionally global, so neither appears here; +# the EntraUsers dimension is scoped separately (target set + manager chain). +$script:AISIDScopeUpnColumn = @{ + ActivitySessions = 'UPN' + FileProximity = 'UPN' + OffHoursGeo = 'UPN' + OAuthConsents = 'UPN' + SSOSignIns = 'UPN' + CopilotUsage = 'UserPrincipalName' +} + +function script:Get-AISIDScopeKqlFilter { + # Server-side scope clause for a collector's hunting query: a KQL '| where in~ (dynamic([...]))' + # fragment. With -Users it covers exactly that batch (used by the batched runner); without -Users it + # covers the full resolved target set. Returns '' when no scope was requested (unfiltered run, + # unchanged behavior). The UPN literals are lower-cased and JSON-escaped. + param([Parameter(Mandatory)][string]$UpnColumn, [string[]]$Users) + $scope = + if ($PSBoundParameters.ContainsKey('Users')) { @($Users) } + else { $set = script:Get-AISIDScopeUpnSet; if ($null -eq $set) { @() } else { @($set) } } + if ($scope.Count -eq 0) { return '' } + $json = ($scope | ForEach-Object { '"' + (([string]$_).ToLowerInvariant() -replace '"', '\"') + '"' }) -join ',' + return ("| where {0} in~ (dynamic([{1}]))" -f $UpnColumn, $json) +} + +function script:Get-AISIDScopeBatches { + # Partitions the resolved target user set into deterministic batches sized by safe serialized request + # length, so a large scope never produces a single oversized KQL 'in~ (dynamic([...]))' array. EVERY + # target user appears in EXACTLY ONE batch (no cap, no first-N, no sampling, no truncation, no + # duplicates); batches are ordered and their union equals the full scope. Returns @() when no scope + # was requested (collectors then run one unfiltered query). + param([int]$MaxSerializedChars = 8000) + $set = script:Get-AISIDScopeUpnSet + if ($null -eq $set) { return @() } + # Deterministic order; force a clean [string[]] so no PSObject-wrapped element leaks into the batches. + [string[]]$users = @(@($set) | ForEach-Object { ([string]$_).ToLowerInvariant() } | Sort-Object -Unique) + $batches = New-Object System.Collections.Generic.List[object] + $start = 0 + while ($start -lt $users.Length) { + $len = 0; $count = 0 + for ($i = $start; $i -lt $users.Length; $i++) { + $add = $users[$i].Length + 3 + if ($count -gt 0 -and ($len + $add) -gt $MaxSerializedChars) { break } + $len += $add; $count++ + } + if ($count -lt 1) { $count = 1 } + [string[]]$batch = $users[$start..($start + $count - 1)] + [void]$batches.Add($batch) + $start += $count + } + return $batches.ToArray() +} + +function script:Invoke-AISIDBatchedHuntingQuery { + # Runs a hunting query once per target-user batch and returns a combined result hashtable + # (Success/AuthQuit/TruncationDetected/Results/ErrorMessage). The QueryBuilder receives the per-batch + # server-side scope clause and returns the full KQL for that batch, so every target user is queried + # exactly once across the batches. When no scope was requested a single unfiltered query runs + # (QueryBuilder called with ''), preserving the original one-query behavior. Any batch AuthQuit or + # failure short-circuits and is surfaced to the caller (fail-closed). Rows are concatenated in batch + # order; per-table dedup/aggregation is the caller's existing responsibility. + param( + [Parameter(Mandatory)][string]$UpnColumn, + [Parameter(Mandatory)][scriptblock]$QueryBuilder, + [Parameter(Mandatory)][string]$Description + ) + $acc = New-Object System.Collections.Generic.List[object] + $out = @{ Success = $true; AuthQuit = $false; TruncationDetected = $false; Results = @(); ErrorMessage = '' } + $batches = @(script:Get-AISIDScopeBatches) + $clauses = if ($batches.Count -eq 0) { @('') } else { @($batches | ForEach-Object { script:Get-AISIDScopeKqlFilter -UpnColumn $UpnColumn -Users $_ }) } + foreach ($clause in $clauses) { + $kql = & $QueryBuilder $clause + $api = Invoke-GraphHuntingQuery -Query $kql -Description $Description + if ($api.AuthQuit) { $out.AuthQuit = $true; $out.Success = $false; return $out } + if (-not $api.Success) { $out.Success = $false; $out.ErrorMessage = [string]$api.ErrorMessage; return $out } + if ($api.TruncationDetected) { $out.TruncationDetected = $true } + foreach ($r in @($api.Results)) { [void]$acc.Add($r) } + } + $out.Results = @($acc.ToArray()) + return $out +} + +function script:Select-AISIDScopedRows { + # Client-side scope enforcement (defense-in-depth behind the server-side KQL filter): drops any row + # whose UPN-column value is not in the resolved target set. Tables with no per-user grain + # (ClientChannel is scoped at its source query; SolutionsCatalog is intentionally global) and the + # EntraUsers dimension (scoped separately, incl. manager chain) are returned unchanged. When no + # scope was requested the rows are returned unchanged. + param([Parameter(Mandatory)][string]$Table, [object[]]$Rows = @()) + $set = script:Get-AISIDScopeUpnSet + if ($null -eq $set) { return @($Rows) } + if (-not $script:AISIDScopeUpnColumn.ContainsKey($Table)) { return @($Rows) } + $col = $script:AISIDScopeUpnColumn[$Table] + $kept = New-Object System.Collections.Generic.List[object] + foreach ($r in @($Rows)) { + if (-not $r) { continue } + $v = ([string]$r.$col).Trim() + if ($v -and $set.Contains($v)) { [void]$kept.Add($r) } + } + return $kept.ToArray() +} + +function script:Get-AISIDContainmentSet { + # The set of allowed UPN-column VALUES for a table's FINAL post-merge scope containment. When + # -Deidentify is on the raw target UPNs are mapped through the SAME deterministic Get-PaxDeidUpn + # helper the rows are hashed with, so already-deidentified history compares against deidentified + # scope keys (never raw-vs-hashed). Returns $null when no scope was requested or the table has no + # per-user grain (ClientChannel / SolutionsCatalog / EntraUsers). + param([Parameter(Mandatory)][string]$Table) + $set = script:Get-AISIDScopeUpnSet + if ($null -eq $set) { return $null } + if (-not $script:AISIDScopeUpnColumn.ContainsKey($Table)) { return $null } + $out = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($u in @($set)) { + $val = if ($script:PaxDeidEnabled) { Get-PaxDeidUpn -Value ([string]$u) } else { [string]$u } + $v = ([string]$val).Trim() + if ($v) { [void]$out.Add($v) } + } + return , $out +} + +function script:Select-AISIDContainedRows { + # Splits merged rows into in-scope and out-of-scope by the table's UPN column, using the deid-aware + # containment set. Returns a hashtable @{ InScope; OutOfScope; Applies }. Applies=$false for + # unscoped runs or non-user-grain tables (rows all treated as in-scope). + param([Parameter(Mandatory)][string]$Table, [object[]]$Rows = @()) + $cs = script:Get-AISIDContainmentSet -Table $Table + if ($null -eq $cs) { return @{ Applies = $false; InScope = @($Rows); OutOfScope = @() } } + $col = $script:AISIDScopeUpnColumn[$Table] + $in = New-Object System.Collections.Generic.List[object] + $offList = New-Object System.Collections.Generic.List[object] + foreach ($r in @($Rows)) { + if (-not $r) { continue } + $v = ([string]$r.$col).Trim() + if ($v -and $cs.Contains($v)) { [void]$in.Add($r) } else { [void]$offList.Add($r) } + } + return @{ Applies = $true; InScope = $in.ToArray(); OutOfScope = $offList.ToArray() } +} + +function script:Test-PaxZeroRecordHeaderOnlyPath { + # Decides whether the zero in-memory-record path enters the header-only-Purview-CSV emit block. TRUE + # when there are no records AND it is neither -OnlyUserInfo (Entra-only) nor a streaming-merge export + # (records stream from JSONL). -Dashboard AISID IS included here (it still needs the header-only + # Purview CSV), but the AISID case CONTINUES past the block (see + # Test-PaxZeroRecordContinuesAfterHeaderOnly) instead of returning, so a zero-record AISID run flows + # through the SAME downstream finalization (Rollup empty Fact + Users -> Entra -> AISID export -> + # completion/roster -> truthful exit) as a populated run. Zero Purview records are a valid empty + # result, never a failure. + param([int]$AllLogsCount, [bool]$OnlyUserInfo, [bool]$UseStreamingMergeForExport) + if ($AllLogsCount -ne 0) { return $false } + if ($OnlyUserInfo) { return $false } + if ($UseStreamingMergeForExport) { return $false } + return $true +} + +function script:Test-PaxZeroRecordContinuesAfterHeaderOnly { + # After the zero-record header-only Purview CSV is emitted, decides whether the run CONTINUES to the + # common downstream finalization (Rollup -> Entra -> AISID -> completion/roster) instead of taking the + # early return. TRUE only for -Dashboard AISID: a zero-record AISID run must still produce the empty + # Rollup Fact, the Rollup Users dimension, the normal Entra output, all twelve AISID CSVs (incl. + # EntraUsers.csv), the completion summary and the output roster. FALSE for every other mode, whose + # zero-record behavior (emit header-only and return) is unchanged. + param([string]$Dashboard) + return ($Dashboard -eq 'AISID') +} + +function Export-AISIDTable { + <# + .SYNOPSIS + Writes one dashboard CSV in the exact contractual column order, with the header row always + present. A table with zero rows still produces a valid header-only file (never omitted, + never an error) so the dashboard model loads it cleanly as empty. + .DESCRIPTION + Folder/destination resolution is the caller's responsibility: the caller passes the already + resolved effective directory (e.g. from Resolve-DataTypePaths for the dashboard destination), + so this writer performs no tier inference or collision handling of its own. Only the columns + named in OrderedColumns are emitted, so incidental provenance fields on the input rows never + leak into the model. + .OUTPUTS + The full path of the file written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][string]$FileName, + [Parameter(Mandatory)][string[]]$OrderedColumns, + [object[]]$Rows = @() + ) + + if (-not (Test-Path -LiteralPath $DestinationDir)) { + New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null + } + $target = Join-Path $DestinationDir $FileName + + $sb = [System.Text.StringBuilder]::new() + # Header row is written unconditionally — this is the safe-empty-table fallback. + [void]$sb.AppendLine(($OrderedColumns -join ',')) + foreach ($row in @($Rows)) { + if (-not $row) { continue } + $fields = foreach ($col in $OrderedColumns) { + $raw = if ($row -is [System.Collections.IDictionary]) { $row[$col] } else { $row.$col } + $sVal = if ($null -eq $raw) { '' } else { [string]$raw } + if ($sVal -match '[",\r\n]') { '"' + ($sVal -replace '"', '""') + '"' } else { $sVal } + } + [void]$sb.AppendLine(($fields -join ',')) + } + [System.IO.File]::WriteAllText($target, $sb.ToString(), (New-Object System.Text.UTF8Encoding($false))) + return $target +} + +function Merge-AISIDTableOnKey { + <# + .SYNOPSIS + Merges new rows into existing rows using replace-on-key semantics: a new row with the same + key overwrites the existing one; counts are never summed. Table-agnostic — the key grain is + supplied by the caller via KeyColumns, not hardcoded to any table. + .OUTPUTS + The merged row set (existing rows whose keys were not re-supplied, plus every new row). + #> + [CmdletBinding()] + param( + [object[]]$ExistingRows = @(), + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + + $index = [ordered]@{} + foreach ($r in @($ExistingRows)) { if ($r) { $index[(& $keyOf $r)] = $r } } + foreach ($r in @($NewRows)) { if ($r) { $index[(& $keyOf $r)] = $r } } + return @($index.Values) +} + +function Merge-AISIDTableAppendDedup { + <# + .SYNOPSIS + Merges new event rows into existing event rows using append-and-deduplicate semantics: every + distinct key is kept exactly once, existing history is preserved, and a re-observed event (a new + row whose key already exists) is dropped rather than duplicated or overwritten. Used for + event-grain tables where each row is an immutable observation and history accumulates across runs. + Table-agnostic — the key grain is supplied by the caller via KeyColumns, not hardcoded. + .OUTPUTS + The merged row set: all existing rows, plus each new row whose key was not already present. + #> + [CmdletBinding()] + param( + [object[]]$ExistingRows = @(), + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + + # Existing history is added first; a re-observed event (a NewRows entry whose key already exists) is + # neither re-added nor overwritten, so event history only ever grows and never double-counts. + $index = [ordered]@{} + foreach ($r in @($ExistingRows)) { if ($r) { $k = & $keyOf $r; if (-not $index.Contains($k)) { $index[$k] = $r } } } + foreach ($r in @($NewRows)) { if ($r) { $k = & $keyOf $r; if (-not $index.Contains($k)) { $index[$k] = $r } } } + return @($index.Values) +} + +# ───────────────────────────────────────────────────────────────────────────── +# AISID bounded staged history reconciliation (Blocker 2) +# The array-based merges above receive the ENTIRE prior history as an in-memory +# array (Get-AISIDExistingRows loads the whole CSV) held simultaneously with the +# keyed index — multiple copies of accumulated history. The streamed variants +# below reconcile against the prior history WITHOUT materializing it as an array: +# the prior CSV is read one row at a time through the pipeline and only one row +# per key (the output grain) is retained. Semantics are byte-for-byte identical to +# the array merges (replace-on-key = last wins; append-dedup = first wins; +# first-appearance output order), so callers see the same result set. A read that +# fails (locked / permission / malformed — as opposed to a confirmed-absent file) +# propagates so the caller preserves the prior file rather than overwriting it. +# ───────────────────────────────────────────────────────────────────────────── +function Test-AISIDCsvHasDataRow { + <# + .SYNOPSIS + Returns $true when a local CSV exists and has at least one data row (header + >=1), reading at + most the first two lines. Bounded — never loads the file. A confirmed-absent file returns $false; + any other read failure propagates so the caller can preserve prior state. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$Path) + if (-not (Test-Path -LiteralPath $Path)) { return $false } + $reader = $null + try { + $reader = [System.IO.StreamReader]::new($Path, [System.Text.UTF8Encoding]::new($false)) + $null = $reader.ReadLine() # header + $second = $reader.ReadLine() # first data row (if any) + return ($null -ne $second) + } + finally { + if ($reader) { $reader.Dispose() } + } +} + +function Merge-AISIDTableOnKeyStreamed { + <# + .SYNOPSIS + Bounded replace-on-key merge: streams the prior history CSV row-by-row into a keyed index (one + row per key), then applies the new rows (a new row overwrites the existing one on the same key). + Identical result to Merge-AISIDTableOnKey, but the full prior-history array is never held. When + the prior file is absent or header-only (no data rows), the new rows are returned unchanged — + matching the caller's "merge only when prior history exists" guard exactly. + #> + [CmdletBinding()] + param( + [string]$ExistingCsvPath, + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + $index = [ordered]@{} + $existingCount = 0 + if ($ExistingCsvPath -and (Test-Path -LiteralPath $ExistingCsvPath)) { + Import-Csv -LiteralPath $ExistingCsvPath | ForEach-Object { $index[(& $keyOf $_)] = $_; $existingCount++ } + } + if ($existingCount -eq 0) { return @($NewRows) } + foreach ($r in @($NewRows)) { if ($r) { $index[(& $keyOf $r)] = $r } } + try { Write-PaxMemoryObservation -Stage 'AISIDHistoryReconcile' -RowsProcessed ([long]$index.Count) -Force } catch {} + return @($index.Values) +} + +function Merge-AISIDTableAppendDedupStreamed { + <# + .SYNOPSIS + Bounded append-and-dedup merge: streams the prior history CSV row-by-row into a keyed index + (first occurrence wins), then applies each new row only if its key is not already present. + Identical result to Merge-AISIDTableAppendDedup, but the full prior-history array is never held. + Whether to merge-vs-stub remains the caller's decision (prior history present OR a collected run). + #> + [CmdletBinding()] + param( + [string]$ExistingCsvPath, + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + $index = [ordered]@{} + if ($ExistingCsvPath -and (Test-Path -LiteralPath $ExistingCsvPath)) { + Import-Csv -LiteralPath $ExistingCsvPath | ForEach-Object { $k = & $keyOf $_; if (-not $index.Contains($k)) { $index[$k] = $_ } } + } + foreach ($r in @($NewRows)) { if ($r) { $k = & $keyOf $r; if (-not $index.Contains($k)) { $index[$k] = $r } } } + try { Write-PaxMemoryObservation -Stage 'AISIDHistoryReconcile' -RowsProcessed ([long]$index.Count) -Force } catch {} + return @($index.Values) +} + +function Get-AISIDExistingRows { + <# + .SYNOPSIS + Reads the rows already present in a dashboard CSV so a replace-on-key export can merge new rows + into the accumulated history instead of overwriting it. Resolves the destination tier (local, + SharePoint, or Fabric/OneLake) and returns the parsed rows. + .DESCRIPTION + Critically distinguishes a genuinely absent file (a normal first-run condition -> empty set) from a + read that FAILED for another reason on a file that may well exist (locked/permission-denied/partial + or malformed CSV locally; auth/throttle/network or corrupt download remotely). Only a confirmed + absence returns empty; any other failure is thrown so the caller can preserve the existing file + rather than overwrite it with an empty-merge result (which would reproduce the original data-loss + bug from a different cause). Remote not-found is detected as an HTTP 404 via the same + $_.Exception.Response.StatusCode.value__ inspection used elsewhere for remote 404 handling. + + The reader is merge-mode-agnostic: it only returns the existing rows. The caller decides how to + combine them, so a future grain (for example event-level append-and-dedup) can reuse it unchanged. + .OUTPUTS + An array of existing rows (one object per CSV row), or an empty array when the file is confirmed + absent. Throws on any non-not-found read failure. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][string]$FileName, + [string]$ParentOverride + ) + + $mode = $script:RemoteOutputMode + + if (-not $mode -or $mode -eq 'None') { + # Local tier. A genuinely absent file is a normal first-run condition -> empty. Any OTHER failure + # (locked file, permission denied, malformed/partial CSV) must NOT masquerade as "no history" -- + # Import-Csv is left uncaught so it propagates to the caller, which preserves the existing file. + $path = Join-Path $DestinationDir $FileName + if (-not (Test-Path -LiteralPath $path)) { return @() } + return @(Import-Csv -LiteralPath $path) + } + + # Remote tier (SharePoint / Fabric-OneLake). Download to a temp path via the established remote-read + # functions, then parse. Only a confirmed HTTP 404 (file not yet present) is treated as an empty + # first-run set; every other failure (401/403/429/5xx/network, or an unparseable status) propagates so + # the caller preserves prior state rather than merging as if empty. + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("aisid_read_" + [System.Guid]::NewGuid().ToString('N') + ".csv") + try { + try { + switch ($mode) { + 'SharePoint' { Get-RemoteFile-SharePoint -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + 'Fabric' { Get-RemoteFile-OneLake -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { + Write-Verbose ("Get-AISIDExistingRows: remote '{0}' not found (404); treating as first run." -f $FileName) + return @() + } + # Not a confirmed absence -- surface the real failure; the caller preserves prior state. + throw + } + if (-not (Test-Path -LiteralPath $tmp)) { return @() } + return @(Import-Csv -LiteralPath $tmp) + } + finally { + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + } +} + +function Get-AISIDExistingCsvToScratch { + <# + .SYNOPSIS + Downloads the remote (SharePoint / Fabric-OneLake) prior-history CSV to a local scratch file and + returns the scratch path so it can be streamed through the bounded merge — the downloaded file is + never parsed into an in-memory array. Returns $null when the remote file is confirmed absent + (HTTP 404 -> first run). Any other failure (401/403/429/5xx/network/unparseable status) throws so + the caller preserves prior remote state. The CALLER owns cleanup of the returned scratch path + (removed on success, preserved for recovery on failure) — this mirrors the memory-safe + download-to-local behavior of Get-AISIDExistingRows without the whole-file Import-Csv. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$FileName, + [string]$ParentOverride + ) + $mode = $script:RemoteOutputMode + if (-not $mode -or $mode -eq 'None') { return $null } + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("aisid_hist_" + [System.Guid]::NewGuid().ToString('N') + ".csv") + try { + switch ($mode) { + 'SharePoint' { Get-RemoteFile-SharePoint -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + 'Fabric' { Get-RemoteFile-OneLake -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + if ($status -eq 404) { + Write-Verbose ("Get-AISIDExistingCsvToScratch: remote '{0}' not found (404); treating as first run." -f $FileName) + return $null + } + throw + } + if (-not (Test-Path -LiteralPath $tmp)) { return $null } + try { Write-PaxMemoryObservation -Stage 'AISIDRemoteHistoryDownloaded' -Force } catch {} + return $tmp +} + +function Export-AISIDCompatibilityTable { + <# + .SYNOPSIS + Publishes a required dashboard compatibility table without collecting new rows. A valid prior + file is copied byte-for-byte into the new run bundle; only a confirmed-absent prior file becomes + a header-only table. Read or header failures throw so prior data is never replaced by a false empty. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Table, + [Parameter(Mandatory)][string]$DestinationDir, + [hashtable]$ExistingRows = @{}, + [string]$HistoryDir = '', + [string]$ParentOverride = '' + ) + $schema = $script:AISIDTableSchemas[$Table] + if (-not $schema) { throw "Unknown AISID compatibility table: $Table" } + if ($ExistingRows.ContainsKey($Table) -and @($ExistingRows[$Table]).Count -gt 0) { + return Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows @($ExistingRows[$Table]) + } + + $source = $null + $remoteScratch = $false + if ($script:RemoteOutputMode -and $script:RemoteOutputMode -ne 'None' -and $ParentOverride) { + $source = Get-AISIDExistingCsvToScratch -FileName $schema.FileName -ParentOverride $ParentOverride + $remoteScratch = [bool]$source + } + elseif ($HistoryDir) { + $candidate = Join-Path $HistoryDir $schema.FileName + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $source = $candidate } + } + + if (-not $source) { + return Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows @() + } + + $expectedHeader = $schema.Columns -join ',' + $actualHeader = Get-Content -LiteralPath $source -TotalCount 1 -ErrorAction Stop + if ($actualHeader -cne $expectedHeader) { + throw ("{0}: prior compatibility file has an invalid header; prior file left unchanged" -f $Table) + } + if (-not (Test-Path -LiteralPath $DestinationDir)) { New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null } + $target = Join-Path $DestinationDir $schema.FileName + [System.IO.File]::Copy($source, $target, $true) + if ($remoteScratch -and (Test-Path -LiteralPath $source)) { Remove-Item -LiteralPath $source -Force -ErrorAction SilentlyContinue } + return $target +} + +function ConvertTo-AISIDInvariantUtcTimestamp { + <# + .SYNOPSIS + Serializes a timestamp value to an invariant, culture-independent ISO-8601 UTC string ending in + 'Z' (round-trip 7-digit fractional form, e.g. 2026-07-01T08:47:59.0000000Z). Format-only: the + instant is preserved, only its textual representation is normalized. Unparseable input is + returned as its original string so no value is silently dropped. + .OUTPUTS + [string] + #> + [CmdletBinding()] + param([Parameter()] $Value) + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $fmt = 'yyyy-MM-ddTHH:mm:ss.fffffffZ' + if ($null -eq $Value) { return '' } + if ($Value -is [datetime]) { return ([datetime]$Value).ToUniversalTime().ToString($fmt, $inv) } + if ($Value -is [System.DateTimeOffset]) { return ([System.DateTimeOffset]$Value).UtcDateTime.ToString($fmt, $inv) } + $s = [string]$Value + if ([string]::IsNullOrWhiteSpace($s)) { return '' } + $parsed = [datetime]::MinValue + $styles = [System.Globalization.DateTimeStyles]::AdjustToUniversal -bor [System.Globalization.DateTimeStyles]::AssumeUniversal + if ([datetime]::TryParse($s, $inv, $styles, [ref]$parsed)) { return $parsed.ToString($fmt, $inv) } + return $s +} + +function Get-AISIDFileProximity { + <# + .SYNOPSIS + Collects the File Proximity data set: files created or changed within five minutes of an AI + website visit on the same device — a temporal-proximity signal (temporal correlation only; not + proof of exfiltration). Sourced from Defender + DeviceNetworkEvents (AI site visits) joined to DeviceFileEvents (file activity). Event-grain; the + collection window is subdivided automatically so no single hunting query approaches the + advanced-hunting response cap. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $domains = @(Get-AISIDNetworkDomains) + if ($domains.Count -eq 0) { + Write-LogHost '[AISID] FileProximity: no AI network domains in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $domainJson = ($domains | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + # Collection window: the frozen requested-intersect-available effective interval (half-open). + $fpBounds = Get-AISIDCollectionBounds + $startUtc = $fpBounds.StartUtc + $endUtc = $fpBounds.EndUtc + + # Builds the DeviceNetworkEvents x DeviceFileEvents 5-minute-proximity join for one time window. + # The API Timespan already caps the source at the fixed lookback; the window bounds here subdivide + # that lookback so no single response approaches the row cap. -CountOnly returns the post-join row + # count for the estimate probe; the full form emits the contractual columns (with the visited URL, + # which is resolved to an AI solution client-side below). + # Target-user scope pushed server-side onto the joined events (before the tail projects/aggregates). + $fpScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'InitiatingProcessAccountUpn' + $buildQuery = { + param([datetime]$WinStart, [datetime]$WinEnd, [switch]$CountOnly, [string]$ScopeClause = $fpScope) + $ws = $WinStart.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $we = $WinEnd.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $tail = if ($CountOnly) { + '| summarize N = count()' + } + else { +@' +| extend SecondsToAI = datetime_diff('second', FileTimestamp, AITimestamp) +| extend UPN = tolower(InitiatingProcessAccountUpn) +| extend YearMonth = format_datetime(AITimestamp, 'yyyy-MM') +| extend FolderCategory = case( + FolderPath has 'Desktop', 'Desktop', + FolderPath has 'Downloads', 'Downloads', + FolderPath has 'Documents', 'Documents', + FolderPath has 'OneDrive', 'OneDrive', + FolderPath has 'SharePoint', 'SharePoint', + 'Other') +| extend NameMatchesSensitivePattern = iff(FileName has_any (SensitiveNamePatterns), 1, 0) +| extend FolderMatchesSensitive = iff(FolderPath has_any (SensitiveFolderPatterns), 1, 0) +| project Timestamp = AITimestamp, RemoteUrl, UPN, YearMonth, FileName, FolderCategory, FolderPath, + SecondsToAI, NameMatchesSensitivePattern, FolderMatchesSensitive +'@ + } +@" +let AIDomains = dynamic([$domainJson]); +let SensitiveNamePatterns = dynamic(["confidential","secret","password","credential","private","restricted","internal","draft","salary","ssn","pii","financial","budget","forecast","strategy","merger","acquisition","termination","layoff","patent"]); +let SensitiveFolderPatterns = dynamic(["confidential","restricted","hr","legal","finance","executive","board","compliance","audit","security"]); +let WinStart = datetime($ws); +let WinEnd = datetime($we); +let AIVisits = + DeviceNetworkEvents + | where Timestamp between (WinStart .. WinEnd) + | where ActionType == 'ConnectionSuccess' + | where RemoteUrl has_any (AIDomains) + | project AITimestamp = Timestamp, DeviceId, RemoteUrl; +let FileAccess = + DeviceFileEvents + | where Timestamp between (WinStart .. (WinEnd + 5m)) + | where ActionType in ('FileCreated', 'FileModified', 'FileRenamed', 'FileCopied') + | where FileName !endswith '.tmp' and FileName !endswith '.log' + | project FileTimestamp = Timestamp, DeviceId, FileName, FolderPath, InitiatingProcessAccountUpn; +AIVisits +| join kind=inner FileAccess on DeviceId +| where FileTimestamp between (AITimestamp .. (AITimestamp + 5m)) +$ScopeClause +$tail +"@ + } + + # Plan sub-windows so no single query approaches the response cap; each range is re-probed, so a + # non-uniform distribution only subdivides the ranges that are actually hot. + $estimateProvider = { + param($a, $b) + $probe = & $buildQuery -WinStart $a -WinEnd $b -CountOnly + Get-AISIDHuntingRowEstimate -ProbeQuery $probe -Description 'FileProximity row-estimate' + } + $partitions = @(Get-AISIDTimePartitions -StartUtc $startUtc -EndUtc $endUtc -EstimateProvider $estimateProvider) + + # A partition flagged EstimateFailed means the row-estimate probe was unreadable; fail the table + # explicitly rather than silently emitting an empty/truncated file (and never retry unbounded). + if (@($partitions | Where-Object { $_.EstimateFailed }).Count -gt 0) { + $result.ErrorMessage = 'FileProximity row-estimate probe returned an unreadable result; collection aborted to avoid an unbounded estimate retry and a false-empty output.' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + Write-LogHost ("[AISID] FileProximity: planned {0} time partition(s) for collection." -f $partitions.Count) -ForegroundColor DarkGray + + $rawRows = New-Object System.Collections.Generic.List[object] + foreach ($p in $partitions) { + if ($p.Truncated) { $result.TruncationDetected = $true } + # Batched per partition: for a large scope the source events are filtered per user-batch so every + # target user's events are queried exactly once; event-grain rows are concatenated (each event's + # InitiatingProcessAccountUpn belongs to exactly one batch, so there is no cross-batch duplication). + $__pStart = $p.StartUtc; $__pEnd = $p.EndUtc + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'InitiatingProcessAccountUpn' -Description 'FileProximity' -QueryBuilder ({ param($__scope) & $buildQuery -WinStart $__pStart -WinEnd $__pEnd -ScopeClause $__scope }.GetNewClosure()) + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "FileProximity query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if ($api.TruncationDetected) { $result.TruncationDetected = $true } + foreach ($row in @($api.Results)) { if ($row) { $rawRows.Add($row) } } + } + $result.RawRowCount = $rawRows.Count + + # Client-side: resolve each event's AI solution from the visited host (catalog-driven, mirroring how + # ActivitySessions resolves its application name), then emit the contractual event-grain columns. + $rows = New-Object System.Collections.Generic.List[object] + foreach ($r in $rawRows) { + $hostName = $null + $ru = [string]$r.RemoteUrl + if (-not [string]::IsNullOrWhiteSpace($ru)) { + $parsed = $null + if ([Uri]::TryCreate($ru, [UriKind]::Absolute, [ref]$parsed)) { $hostName = $parsed.Host } + else { $hostName = ((($ru -replace '^[A-Za-z][A-Za-z0-9+.\-]*://', '') -replace '/.*$', '') -replace ':\d+$', '') } + } + $entry = if ($hostName) { Resolve-AISIDFromDomain -HostName $hostName } else { $null } + $solution = if ($entry) { [string]$entry.AISolution } else { 'Other AI' } + $rows.Add([pscustomobject][ordered]@{ + Timestamp = (ConvertTo-AISIDInvariantUtcTimestamp -Value $r.Timestamp) + UPN = [string]$r.UPN + AISolution = $solution + YearMonth = [string]$r.YearMonth + FileName = [string]$r.FileName + FolderCategory = [string]$r.FolderCategory + FolderPath = [string]$r.FolderPath + SecondsToAI = [int]$r.SecondsToAI + NameMatchesSensitivePattern = [int]$r.NameMatchesSensitivePattern + FolderMatchesSensitive = [int]$r.FolderMatchesSensitive + }) + } + + $result.Rows = $rows.ToArray() + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function Get-AISIDClientChannel { + <# + .SYNOPSIS + Collects the Client Channel data set: AI website activity split by access channel — browser vs. + scripted/API client vs. desktop app — per AI site per month. Sourced from Defender + DeviceNetworkEvents; the AI sites are filtered by the catalog's network domains, and each event's + site and channel are labelled server-side (the channel is inferred from initiating-process + metadata, not a definitive client attribution). Aggregate-grain (one row per site/channel/month), so a + single hunting query stays well under the advanced-hunting response cap. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $domains = @(Get-AISIDNetworkDomains) + if ($domains.Count -eq 0) { + Write-LogHost '[AISID] ClientChannel: no AI network domains in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $domainJson = ($domains | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + + # The catalog's network domains drive the AI-site filter (mirroring FileProximity). The site and + # channel labels use the authoritative source mapping — visited host -> canonical site, initiating + # process -> Browser/API/Desktop — applied server-side. The shared hunting-query timespan bounds the + # collection window; the aggregate collapses to one row per site/channel/month, so no subdivision is + # needed. (The site case() carries the full source mapping; only catalog-filtered hosts reach it.) + $ccBounds = Get-AISIDCollectionBounds + $ccStart = $ccBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ccEnd = $ccBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + # Target-user scope pushed server-side onto the source events BEFORE the site/channel aggregate + # collapses UPN away (ClientChannel has no per-user output grain, so this is its only scope point). + $ccScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'InitiatingProcessAccountUpn' + # Batched: a large scope is split into size-bounded batches; the source events are filtered per batch + # (every target user's events queried exactly once) and the per-batch aggregates are RE-AGGREGATED + # below by (AISite, Channel, YearMonth) so counts sum deterministically without duplicate keys. + $null = $ccScope + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'InitiatingProcessAccountUpn' -Description 'ClientChannel' -QueryBuilder ({ + param($__scope) +@" +let AIDomains = dynamic([$domainJson]); +let WinStart = datetime($ccStart); +let WinEnd = datetime($ccEnd); +DeviceNetworkEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where ActionType == 'ConnectionSuccess' +| where RemoteUrl has_any (AIDomains) +$__scope +| extend AISite = case( + RemoteUrl has 'copilot.microsoft.com' or RemoteUrl has 'copilot.cloud.microsoft', 'copilot.microsoft.com', + RemoteUrl has 'chat.openai.com' or RemoteUrl has 'chatgpt.com', 'chatgpt.com', + RemoteUrl has 'api.openai.com', 'api.openai.com', + RemoteUrl has 'claude.ai', 'claude.ai', + RemoteUrl has 'api.anthropic.com', 'api.anthropic.com', + RemoteUrl has 'gemini.google.com' or RemoteUrl has 'bard.google.com', 'gemini.google.com', + RemoteUrl has 'perplexity.ai', 'perplexity.ai', + RemoteUrl has 'midjourney.com', 'midjourney.com', + RemoteUrl has 'huggingface.co', 'huggingface.co', + RemoteUrl has 'stability.ai', 'stability.ai', + RemoteUrl has 'jasper.ai', 'jasper.ai', + RemoteUrl has 'grammarly.com', 'grammarly.com', + RemoteUrl has 'notion.so', 'notion.so', + RemoteUrl has 'firefly.adobe.com', 'firefly.adobe.com', + RemoteUrl has 'runwayml.com', 'runwayml.com', + RemoteUrl has 'canva.com', 'canva.com', + RemoteUrl has 'app.synthesia.io', 'app.synthesia.io', + RemoteUrl) +| extend Channel = case( + InitiatingProcessFileName has_any ('chrome.exe', 'msedge.exe', 'firefox.exe', 'brave.exe', 'safari', 'opera.exe', 'iexplore.exe', 'Chrome', 'Safari', 'Firefox'), 'Browser', + InitiatingProcessFileName has_any ('python', 'node', 'java', 'curl', 'powershell', 'pwsh', 'cmd.exe', 'bash', 'dotnet', 'go'), 'API', + 'Desktop') +| extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') +| summarize EventCount = count() by AISite, Channel, YearMonth +"@ + }.GetNewClosure()) + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "ClientChannel query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $rawRows = @($api.Results) + $result.RawRowCount = $rawRows.Count + $result.TruncationDetected = [bool]$api.TruncationDetected + + # Re-aggregate across batches: the same (AISite, Channel, YearMonth) can appear in multiple batches + # (disjoint user subsets), so EventCount is SUMMED per key — deterministic, no duplicate aggregation. + $ccAgg = [ordered]@{} + foreach ($r in $rawRows) { + if (-not $r) { continue } + $k = '{0}|{1}|{2}' -f [string]$r.AISite, [string]$r.Channel, [string]$r.YearMonth + if (-not $ccAgg.Contains($k)) { + $ccAgg[$k] = [pscustomobject][ordered]@{ AISite = [string]$r.AISite; Channel = [string]$r.Channel; YearMonth = [string]$r.YearMonth; EventCount = 0 } + } + $ccAgg[$k].EventCount = [int]$ccAgg[$k].EventCount + [int]$r.EventCount + } + + $result.Rows = @($ccAgg.Values) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function ConvertTo-AISIDUtcInstant { + <# + .SYNOPSIS + Converts an audit event timestamp to a Kind=Utc [datetime] WITHOUT any machine-timezone + dependency, for AISID window filtering and per-instant (YearMonth / ActiveDays / LastActivityDate) + grouping. + .DESCRIPTION + An explicit Z or numeric offset is authoritative. A [datetime]'s Kind is honored: Utc is returned + as-is; Local is converted with .ToUniversalTime(); Unspecified (offset-free) is treated as UTC under + one explicit, documented policy -- NEVER silently as host-local time. A [datetimeoffset] uses its own + offset (.UtcDateTime). Strings are parsed with the INVARIANT culture and AssumeUniversal|AdjustToUniversal + styles so a trailing Z or offset governs and an offset-free string is taken as UTC. A [datetime] is + NEVER stringified and reparsed through the current culture (the prior defect: that produced a + host-timezone-dependent shift). Malformed/empty input fails clearly. + .OUTPUTS + [bool] -- $true and sets [ref]$Utc to a Kind=Utc [datetime] on success; $false on null/malformed input. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][AllowNull()]$Value, [Parameter(Mandatory)][ref]$Utc) + + if ($null -eq $Value) { return $false } + if ($Value -is [datetimeoffset]) { $Utc.Value = $Value.UtcDateTime; return $true } + if ($Value -is [datetime]) { + switch ([string]$Value.Kind) { + 'Utc' { $Utc.Value = $Value; return $true } + 'Local' { $Utc.Value = $Value.ToUniversalTime(); return $true } + default { $Utc.Value = [datetime]::SpecifyKind($Value, [System.DateTimeKind]::Utc); return $true } + } + } + $s = [string]$Value + if ([string]::IsNullOrWhiteSpace($s)) { return $false } + $dto = [datetimeoffset]::MinValue + $styles = [System.Globalization.DateTimeStyles]::AssumeUniversal -bor [System.Globalization.DateTimeStyles]::AdjustToUniversal + if ([datetimeoffset]::TryParse($s, [System.Globalization.CultureInfo]::InvariantCulture, $styles, [ref]$dto)) { + $Utc.Value = $dto.UtcDateTime + return $true + } + return $false +} + +function Get-AISIDCopilotUsage { + <# + .SYNOPSIS + Builds the Copilot Usage data set (per-user, per-month prompt counts across the six MODELED + surfaces: Teams / Word / Excel / Outlook / PowerPoint / Chat) from the CURRENT run's RAW Purview + CopilotInteraction CSV. No second audit search and no Graph call: it streams the raw CSV, parses + each AuditData JSON, and pivots AuditData.CopilotEventData.AppHost against a fixed, case-insensitive + allowlist. This is deliberately the six modeled surfaces ONLY - it is NOT comprehensive Microsoft 365 + Copilot usage; any AppHost that cannot map to one of the six columns (Security Copilot, Copilot Studio, + M365 Admin Center, and other unmodeled hosts) is excluded from every surface AND from TotalPrompts, so + TotalPrompts always reconciles to the six-surface sum. + .NOTES + Source path is passed explicitly (-RawPurviewCsv = the current run's raw Purview CSV, still present and + identified at the post-rollup AISID call). AuditData.CreationTime is the CopilotUsage event timestamp. + Fails gracefully (Success=$false) only for CopilotUsage when the source is absent/unreadable or every + record is unusable, so the export falls that one table back to a header-only file. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$RawPurviewCsv) + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + if ([string]::IsNullOrWhiteSpace($RawPurviewCsv) -or -not (Test-Path -LiteralPath $RawPurviewCsv -PathType Leaf)) { + $result.ErrorMessage = 'CopilotUsage: raw Purview CopilotInteraction CSV not available; cannot build the Copilot Usage data set.' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + # AISID effective collection window (frozen). Records outside [start, end) are excluded before grouping. + $cuBounds = Get-AISIDCollectionBounds + $cuStartUtc = $cuBounds.StartUtc + $cuEndUtc = $cuBounds.EndUtc + + # AppHost -> surface (six modeled surfaces ONLY). Case-insensitive exact match after trim. Any AppHost + # absent here is EXCLUDED from every surface AND from TotalPrompts (so Total reconciles to the six-surface + # sum). No wildcards; no unknown-to-Chat. Keys are lowercase for case-insensitive lookup. + $surfaceOf = @{ + 'teams' = 'TeamsPrompts' + 'word' = 'WordPrompts'; 'wordoncanvas' = 'WordPrompts' + 'excel' = 'ExcelPrompts' + 'outlook' = 'OutlookPrompts'; 'outlookoncanvas' = 'OutlookPrompts'; 'outlooksidepane' = 'OutlookPrompts' + 'powerpoint' = 'PowerPointPrompts'; 'powerpointoncanvas' = 'PowerPointPrompts' + 'bizchat' = 'ChatPrompts'; 'bing' = 'ChatPrompts'; 'office' = 'ChatPrompts'; 'edge' = 'ChatPrompts' + 'm365app' = 'ChatPrompts'; 'm365chat' = 'ChatPrompts'; 'officecopilotnotebook' = 'ChatPrompts' + 'officecopilotsearchanswer' = 'ChatPrompts'; 'onenotecopilotnotebook' = 'ChatPrompts' + } + + $agg = [ordered]@{} + $rawCount = 0; $skipped = 0; $included = 0; $outOfWindow = 0; $responseOnly = 0 + $exSec = 0; $exStudio = 0; $exAdmin = 0; $exOther = 0 + try { + Import-Csv -LiteralPath $RawPurviewCsv | ForEach-Object { + $rawCount++ + $d = $null + try { $d = $_.AuditData | ConvertFrom-Json -ErrorAction Stop } catch { $d = $null } + if (-not $d) { $skipped++; return } + $upnRaw = [string]$d.UserId + if ([string]::IsNullOrWhiteSpace($upnRaw)) { $skipped++; return } + $dtU = [datetime]::MinValue + # Convert the audit event timestamp to a UTC instant with NO machine-timezone dependency (an explicit + # Z/offset is authoritative; a DateTime Kind is honored; offset-free => UTC by documented policy). + # Never stringify-and-reparse through the current culture (the prior host-timezone-dependent defect). + if (-not (ConvertTo-AISIDUtcInstant -Value $d.CreationTime -Utc ([ref]$dtU))) { $skipped++; return } + # AISID window filter: keep only records whose UTC instant is in the frozen half-open interval. + if ($dtU -lt $cuStartUtc -or $dtU -ge $cuEndUtc) { $outOfWindow++; return } + $appHostRaw = ([string]$d.CopilotEventData.AppHost).Trim() + $key = $appHostRaw.ToLowerInvariant() + if (-not $surfaceOf.ContainsKey($key)) { + if ($appHostRaw -like 'SecurityCopilot-*') { $exSec++ } + elseif ($key -eq 'copilot studio') { $exStudio++ } + elseif ($key -eq 'm365admincenter') { $exAdmin++ } + else { $exOther++ } + return + } + $surface = $surfaceOf[$key] + $included++ + # Count actual user-prompt messages in this record (CopilotEventData.Messages[].isPrompt === $true). A + # response-only record (zero prompt messages) contributes NOTHING to any surface, TotalPrompts, ActiveDays, + # or LastActivityDate; a record with N prompt messages contributes N (no one-prompt-per-record assumption). + $promptCount = 0 + $msgs = $d.CopilotEventData.Messages + if ($null -ne $msgs) { foreach ($m in @($msgs)) { if ($m -and $m.isPrompt -eq $true) { $promptCount++ } } } + if ($promptCount -le 0) { $responseOnly++; return } + $upn = $upnRaw.ToLowerInvariant() + $ym = $dtU.ToString('yyyy-MM') + $day = $dtU.ToString('yyyy-MM-dd') + $aggKey = '{0}|{1}' -f $upn, $ym + if (-not $agg.Contains($aggKey)) { + $agg[$aggKey] = @{ + UserPrincipalName = $upn; YearMonth = $ym + TeamsPrompts = 0; WordPrompts = 0; ExcelPrompts = 0; OutlookPrompts = 0; PowerPointPrompts = 0; ChatPrompts = 0 + TotalPrompts = 0; Days = (New-Object 'System.Collections.Generic.HashSet[string]'); LastDay = '' + } + } + $b = $agg[$aggKey] + $b[$surface] = [int]$b[$surface] + $promptCount + $b.TotalPrompts = [int]$b.TotalPrompts + $promptCount + [void]$b.Days.Add($day) + if ($day -gt [string]$b.LastDay) { $b.LastDay = $day } + } + } + catch { + $result.ErrorMessage = "CopilotUsage: failed to stream raw Purview CSV: $($_.Exception.Message)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $result.RawRowCount = $rawCount + + if ($rawCount -gt 0 -and ($rawCount - $skipped) -eq 0) { + $result.ErrorMessage = "CopilotUsage: all $rawCount raw record(s) were malformed/unusable; falling back to header-only for this table." + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $rows = New-Object System.Collections.Generic.List[object] + foreach ($b in $agg.Values) { + $rows.Add([pscustomobject][ordered]@{ + UserPrincipalName = [string]$b.UserPrincipalName + YearMonth = [string]$b.YearMonth + TeamsPrompts = [int]$b.TeamsPrompts + WordPrompts = [int]$b.WordPrompts + ExcelPrompts = [int]$b.ExcelPrompts + OutlookPrompts = [int]$b.OutlookPrompts + PowerPointPrompts = [int]$b.PowerPointPrompts + ChatPrompts = [int]$b.ChatPrompts + TotalPrompts = [int]$b.TotalPrompts + ActiveDays = [int]$b.Days.Count + LastActivityDate = [string]$b.LastDay + }) + } + + if ($skipped -gt 0) { + Write-LogHost ("[AISID] CopilotUsage: skipped {0} malformed/incomplete record(s)." -f $skipped) -ForegroundColor DarkYellow + } + $exTotal = $exSec + $exStudio + $exAdmin + $exOther + Write-LogHost ("[AISID] CopilotUsage: raw={0} in-scope={1} response-only={9} out-of-window={8} excluded={2} (SecurityCopilot-={3}, Copilot Studio={4}, M365AdminCenter={5}, Other unmodeled={6}); user-month rows={7}." -f $rawCount, $included, $exTotal, $exSec, $exStudio, $exAdmin, $exOther, $rows.Count, $outOfWindow, $responseOnly) -ForegroundColor DarkGray + + $result.Rows = $rows.ToArray() + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function New-AISIDFileProximityStub { + <# + .SYNOPSIS + Emits the permanent header-only file-proximity CSV. This table is always empty by design: + the dashboard model loads the file on every refresh, but no query, licensing check, or Graph + call is involved — the header alone satisfies the model's fixed schema. + .OUTPUTS + The full path of the stub file written. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$DestinationDir) + + $schema = $script:AISIDTableSchemas['FileProximity'] + return Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows @() +} + +function Get-AISIDActivitySessions { + <# + .SYNOPSIS + Collects the Activity Sessions data set from cloud-app activity and returns enriched rows in + the contractual column shape. Server-side aggregation keeps the result small; catalog-driven + enrichment (application name -> AI solution / category / risk tier) is applied client-side and + collapsed to one row per (UPN, AISolution, YearMonth). + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $cloudAppNames = @(Get-AISIDCloudAppNames) + if ($cloudAppNames.Count -eq 0) { + Write-LogHost '[AISID] ActivitySessions: no cloud-app names in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + # The collection window is bounded by the shared hunting-query timespan; the query filters to the + # catalog's cloud-app names, resolves each event's canonical user principal name from the directory + # identity data (falling back to the raw account object id when the directory has no match for that + # identity), and aggregates per user/app/month server-side. + $appFilterJson = ($cloudAppNames | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + $asBounds = Get-AISIDCollectionBounds + $asStart = $asBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $asEnd = $asBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $asScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'UPN' + # Batched: a large target scope is split into deterministic size-bounded batches; the query runs once + # per batch (every target user queried exactly once) and the rows are concatenated. $asScope is + # retained for structural parity but the per-batch clause ($__scope) is what the runner injects. + $null = $asScope + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'UPN' -Description 'ActivitySessions' -QueryBuilder ({ + param($__scope) +@" +let AppFilter = dynamic([$appFilterJson]); +let UserUpns = IdentityInfo + | summarize take_any(AccountUpn) by AccountObjectId; +let WinStart = datetime($asStart); +let WinEnd = datetime($asEnd); +CloudAppEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where Application in~ (AppFilter) +| lookup kind=leftouter UserUpns on AccountObjectId +| extend UPN = tolower(coalesce(AccountUpn, AccountObjectId)) +| where isnotempty(UPN) +$__scope +| summarize + Sessions = count(), + ActiveDays = dcount(format_datetime(Timestamp, 'yyyy-MM-dd')), + EstimatedPrompts = countif(ActionType in ("MessageSent", "SearchPerformed", "AppAccessedViaAPI")), + DistinctDevices = dcount(DeviceType) + by UPN, AppNameMatched = Application, YearMonth = format_datetime(Timestamp, 'yyyy-MM') +"@ + }.GetNewClosure()) + + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "ActivitySessions query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $rawRows = @($api.Results) + $result.RawRowCount = $rawRows.Count + $result.TruncationDetected = [bool]$api.TruncationDetected + + # Enrich each raw row and collapse catalog aliases that resolve to the same solution. Distinct-count + # metrics (ActiveDays, DistinctDevices) cannot be re-derived exactly once aliases collapse, so take + # the max as a safe upper bound; additive count metrics (Sessions, EstimatedPrompts) sum, since each + # raw app alias contributes a disjoint set of events. + $agg = [ordered]@{} + foreach ($r in $rawRows) { + if (-not $r) { continue } + $appName = [string]$r.AppNameMatched + $resolved = Resolve-AISIDFromAppName -AppName $appName + $solution = if ($resolved) { $resolved.AISolution } else { $appName } + $category = if ($resolved) { $resolved.Category } else { 'Unknown' } + $riskTier = if ($resolved) { $resolved.RiskTier } else { 'Unknown' } + $key = '{0}|{1}|{2}' -f ([string]$r.UPN).ToLowerInvariant(), $solution, [string]$r.YearMonth + + if (-not $agg.Contains($key)) { + $agg[$key] = [ordered]@{ + UPN = [string]$r.UPN; AISolution = $solution; YearMonth = [string]$r.YearMonth + Sessions = 0; ActiveDays = 0; EstimatedPrompts = 0; DistinctDevices = 0 + Category = $category; RiskTier = $riskTier + } + } + $b = $agg[$key] + $b.Sessions = [int]$b.Sessions + [int]$r.Sessions + $b.ActiveDays = [Math]::Max([int]$b.ActiveDays, [int]$r.ActiveDays) + $b.DistinctDevices = [Math]::Max([int]$b.DistinctDevices, [int]$r.DistinctDevices) + $b.EstimatedPrompts = [int]$b.EstimatedPrompts + [int]$r.EstimatedPrompts + } + + $result.Rows = @($agg.Values | ForEach-Object { [pscustomobject]$_ }) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function script:Get-AISIDScratchRoot { + # ONE run-specific internal AISID scratch root under OS temp. Unique per run — its leaf carries the + # global ScriptRunTimestamp, the process id, a per-run random nonce, AND a hash of the resolved user + # scope — so no two runs, and no two user scopes, ever share a scratch root or an off-hours/geography + # cache. Created on first use; deleted wholesale after a successful publication + # (Remove-AISIDScratchRoot). It is NEVER placed under the customer output folder, so a successful run + # leaves no cache/scratch behind. + if (-not $script:AISIDScratchNonce) { $script:AISIDScratchNonce = [System.Guid]::NewGuid().ToString('N').Substring(0, 12) } + $scopeSrc = (@($script:targetUsers) | ForEach-Object { ([string]$_).Trim().ToLowerInvariant() } | Where-Object { $_ } | Sort-Object -Unique) -join '|' + $scopeHash = ([BitConverter]::ToString([System.Security.Cryptography.SHA256]::Create().ComputeHash([System.Text.Encoding]::UTF8.GetBytes($scopeSrc)))).Replace('-', '').Substring(0, 8) + $leaf = 'PAX_AISID_{0}_{1}_{2}_{3}' -f $global:ScriptRunTimestamp, $PID, $script:AISIDScratchNonce, $scopeHash + return (Join-Path ([System.IO.Path]::GetTempPath()) $leaf) +} + +function script:Get-AISIDCacheDir { + # The off-hours/geography delta cache lives beneath the run-specific scratch root + # (Get-AISIDScratchRoot) in OS temp, NEVER in the customer output folder. Because the scratch root is + # unique per run, the cache is intra-run only — it is never reused between independent runs, and one + # scope can never read another scope's cache. The '.aisid_cache' leaf is created under the returned + # directory by Save-AISIDOffHoursGeoCache. The DestinationDir parameter is retained for call-site + # compatibility but intentionally ignored (the cache location is no longer destination-keyed). + param([Parameter()][AllowEmptyString()][string]$DestinationDir = '') + return (script:Get-AISIDScratchRoot) +} + +function script:Remove-AISIDScratchRoot { + # Recursively delete the run-specific AISID scratch root after a successful publication and twelve-CSV + # verification. Returns $true only when the root no longer exists on disk. A residual scratch root is a + # completed-with-gaps condition (the caller sets AISIDHadGaps -> exit 40 and logs the residual path) — + # a successful AISID run leaves no cache or scratch directory anywhere. + param([Parameter()][AllowEmptyString()][string]$Root = '') + if ([string]::IsNullOrWhiteSpace($Root)) { return $true } + if (-not (Test-Path -LiteralPath $Root)) { return $true } + try { [System.GC]::Collect(); [System.GC]::WaitForPendingFinalizers() } catch {} + try { Remove-Item -LiteralPath $Root -Recurse -Force -ErrorAction Stop } catch {} + return (-not (Test-Path -LiteralPath $Root)) +} + +function script:Get-AISIDFabricRunFolderUrl { + # Defect-3: build the CANONICAL Fabric AISID run-folder URL from Resolve-FabricTarget's structured + # result so the twelve fixed-name CSV bundle ALWAYS lives under Files/ (never Tables/) and the produced + # URL round-trips through Resolve-FabricTarget. Root, GUID-root, /Files[/], and /Tables[/] + # parents all resolve to '//Files/[/]'. Tables inputs route + # the operational CSV bundle under Files/ (TablesPath is intentionally ignored), consistent with the + # established operational-artifact routing policy (Send-FileToOneLake). Never appends the leaf to the + # raw parent URL. Returns '' when the parent cannot be parsed (caller treats '' as no remote target). + param([Parameter(Mandatory)][string]$ParentUrl, [Parameter(Mandatory)][string]$Leaf) + if ([string]::IsNullOrWhiteSpace($ParentUrl)) { return '' } + try { + $r = Resolve-FabricTarget -Url $ParentUrl + $filesBase = "$($r.FilesystemBase)/$($r.ItemFull)/Files" + $rel = if ($r.FilesPath) { "$($r.FilesPath)/$Leaf" } else { [string]$Leaf } + return "$filesBase/$rel" + } + catch { return '' } +} + +function script:Resolve-AISIDRunFolder { + # Canonical AISID run-folder resolver: every AISID run publishes the twelve fixed-name CSVs into a + # per-run 'AISID_' child folder beneath the DefenderUsage destination, on ALL + # tiers (Local / SharePoint / Fabric). One resolver drives writes, existing-history reads, completion + # checks, upload destination construction, upload registration, roster, metrics/snapshots, checkpoint/ + # resume, recovery, and AppendDefenderUsage, so every consumer agrees on one location. The twelve + # filenames inside are NEVER renamed. Optional test-injection of the resolved DefenderUsage dir and + # remote parent keeps the resolver pure for focused tests. + param( + [string]$ParentOverride = '', + [string]$EffectiveDirOverride = $null, + [string]$RemoteParentOverride = $null + ) + $leaf = 'AISID_' + $global:ScriptRunTimestamp + $mode = if ($script:RemoteOutputMode) { [string]$script:RemoteOutputMode } else { 'None' } + $effDir = + if (-not [string]::IsNullOrEmpty($EffectiveDirOverride)) { [string]$EffectiveDirOverride } + else { $du = script:Resolve-DataTypePaths -DataType 'DefenderUsage'; if ($du -and $du.EffectiveDir) { [string]$du.EffectiveDir } else { '' } } + $localFolder = if ($effDir) { Join-Path $effDir $leaf } else { $leaf } + $remoteParent = + if ($ParentOverride) { [string]$ParentOverride } + elseif (-not [string]::IsNullOrEmpty($RemoteParentOverride)) { [string]$RemoteParentOverride } + elseif ($script:DestParentUrl -and $script:DestParentUrl['DefenderUsage']) { [string]$script:DestParentUrl['DefenderUsage'] } + else { '' } + # Defect-3: on Fabric the AISID bundle URL is built from Resolve-FabricTarget's structured result so + # it ALWAYS lands under Files/ and round-trips through Resolve-FabricTarget (a root/Tables parent + # appended raw would be an invalid third segment). SharePoint / other tiers keep the folder-URL join. + $remoteUrl = + if ($mode -eq 'None' -or -not $remoteParent) { '' } + elseif ($mode -eq 'Fabric') { script:Get-AISIDFabricRunFolderUrl -ParentUrl $remoteParent -Leaf $leaf } + else { (($remoteParent -replace '/+$', '') + '/' + $leaf) } + $staging = Join-Path (script:Get-AISIDScratchRoot) $leaf + $writeDir = if ($mode -ne 'None') { $staging } else { $localFolder } + $display = if ($mode -ne 'None') { $remoteUrl } else { $localFolder } + return [pscustomobject][ordered]@{ + RunLeaf = $leaf + Mode = $mode + LocalRunFolder = $localFolder + RemoteRunFolderUrl = $remoteUrl + RemoteScratchStaging = $staging + WriteDir = $writeDir + DisplayPath = $display + } +} + +function script:Get-AISIDUploadPlan { + # Blocker-1: builds the AUTHORITATIVE remote AISID upload set from the run-specific staging directory + # ($script:AISIDRunFolder.RemoteScratchStaging) — never from the general run scratch dir. For each + # expected fixed-name leaf it REQUIRES the staged file to exist under $StagingDir and takes its full + # path from there, so an unrelated same-named file elsewhere can never impersonate an AISID artifact. + # Returns the resolved staged file objects (exactly one per present leaf, deduped case-insensitively), + # the set of missing leaves, and Complete=$true only when every expected leaf was found in staging. + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$StagingDir, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$ExpectedLeafs + ) + $files = New-Object System.Collections.Generic.List[object] + $missing = New-Object System.Collections.Generic.List[string] + $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($leaf in @($ExpectedLeafs)) { + if (-not $leaf) { continue } + if (-not $seen.Add($leaf)) { continue } + $full = if ($StagingDir) { Join-Path $StagingDir $leaf } else { '' } + if ($full -and (Test-Path -LiteralPath $full -PathType Leaf)) { $files.Add((Get-Item -LiteralPath $full)) } + else { [void]$missing.Add($leaf) } + } + return [pscustomobject][ordered]@{ + Files = @($files.ToArray()) + MissingLeafs = @($missing.ToArray()) + Complete = ($missing.Count -eq 0 -and $files.Count -gt 0) + } +} + +function script:Resolve-AISIDUploadParentUrl { + # Blocker-1: the per-run AISID child URL ($script:AISIDRemoteRunFolderUrl) is the upload parent for + # every AISID fixed-name leaf on every remote tier (SharePoint / Fabric); all other artifacts keep + # their default per-data-type parent. Isolated so the upload sweep's AISID routing is behaviorally + # testable without reconstructing the sweep. + param([Parameter(Mandatory)][string]$Leaf, [Parameter()][AllowEmptyString()][string]$DefaultParentUrl = '') + if ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Contains($Leaf) -and $script:AISIDRemoteRunFolderUrl) { + return [string]$script:AISIDRemoteRunFolderUrl + } + return [string]$DefaultParentUrl +} + +function script:New-AISIDRunFolderOwnership { + # Blocker-5: acquire EXCLUSIVE ownership of a local AISID run folder atomically. A one-second run + # timestamp cannot distinguish two concurrent processes, so Test-Path+CreateDirectory is not a valid + # ownership decision (both can win the race). This uses an atomic FileMode.CreateNew marker in the + # PARENT folder — the OS guarantees exactly one creator — as the ownership gate, THEN creates the run + # folder. Returns { Acquired=[bool]; Reason=[string]; MarkerPath=[string] }. An already-existing run + # folder (a prior non-resume run) or an existing marker (a concurrent/failed run) is a collision and + # fails closed — this never deletes or overwrites another run's folder or marker. + param([Parameter(Mandatory)][string]$RunFolder) + $parent = Split-Path -Parent $RunFolder + $leaf = Split-Path -Leaf $RunFolder + if ([string]::IsNullOrWhiteSpace($parent)) { $parent = (Get-Location).Path } + try { [void][System.IO.Directory]::CreateDirectory($parent) } + catch { return [pscustomobject]@{ Acquired = $false; Reason = ('parent-uncreatable: ' + $_.Exception.Message); MarkerPath = '' } } + # An existing run folder is a collision (no resume-identity verification is available here). + if (Test-Path -LiteralPath $RunFolder) { return [pscustomobject]@{ Acquired = $false; Reason = 'run-folder-exists'; MarkerPath = '' } } + $marker = Join-Path $parent ('.aisid_own_' + $leaf) + $fs = $null + try { + $fs = [System.IO.File]::Open($marker, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + $bytes = [System.Text.Encoding]::UTF8.GetBytes(('{0}|{1}|{2}' -f $PID, $global:ScriptRunTimestamp, ((Get-Date).ToUniversalTime().ToString('o')))) + $fs.Write($bytes, 0, $bytes.Length) + } + catch [System.IO.IOException] { + if ($fs) { $fs.Dispose() } + return [pscustomobject]@{ Acquired = $false; Reason = 'ownership-marker-exists'; MarkerPath = $marker } + } + catch { + if ($fs) { $fs.Dispose() } + return [pscustomobject]@{ Acquired = $false; Reason = ('marker-uncreatable: ' + $_.Exception.Message); MarkerPath = $marker } + } + finally { if ($fs) { $fs.Dispose() } } + # Marker acquired exclusively — create the run folder now. + try { [void][System.IO.Directory]::CreateDirectory($RunFolder) } + catch { + # Defect-3 fail-closed: the marker was acquired but the run folder could not be created. Remove the + # marker immediately and verify its absence so NO unregistered/orphan marker is left behind. If the + # marker itself cannot be removed, surface a structured cleanup failure that still names it. + $__rfErr = $_.Exception.Message + $__removed = $false + try { Remove-Item -LiteralPath $marker -Force -ErrorAction Stop; $__removed = -not (Test-Path -LiteralPath $marker) } catch { $__removed = $false } + if ($__removed) { return [pscustomobject]@{ Acquired = $false; Reason = ('run-folder-uncreatable: ' + $__rfErr); MarkerPath = ''; CleanupFailed = $false } } + return [pscustomobject]@{ Acquired = $false; Reason = ('run-folder-uncreatable; marker cleanup FAILED: ' + $__rfErr); MarkerPath = $marker; CleanupFailed = $true } + } + return [pscustomobject]@{ Acquired = $true; Reason = 'acquired'; MarkerPath = $marker; CleanupFailed = $false } +} + +function script:Get-AISIDRemoteFolderProbe { + # Blocker-4: read-only existence probe for a remote AISID run-folder URL. Returns + # { Accessible=[bool]; Exists=[bool] }. A confirmed HTTP 404 is Accessible=$true/Exists=$false (normal + # first run); a 2xx is Exists=$true; any auth/throttle/network/5xx/unparseable failure is + # Accessible=$false so the caller fails closed. NEVER creates or modifies the folder — a metadata GET + # (SharePoint driveItem) / directory HEAD (Fabric DFS) only. Uses the established target resolution + # and transport. Isolated so the collision decision is behaviorally testable via injection. + param( + [Parameter(Mandatory)][string]$Url, + [Parameter(Mandatory)][ValidateSet('SharePoint', 'Fabric')][string]$Mode + ) + try { + if ($Mode -eq 'SharePoint') { + $resolved = Resolve-SharePointTarget -Url $Url + $folderPath = if ($resolved.FolderPath) { $resolved.FolderPath } else { '' } + $metaUri = if ($folderPath) { + $esc = ($folderPath -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${esc}" + } + else { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root" } + $graphToken = Get-GraphAccessToken + if (-not $graphToken) { return [pscustomobject]@{ Accessible = $false; Exists = $false } } + try { + $null = Invoke-WebRequest -Uri $metaUri -Method GET -Headers @{ Authorization = "Bearer $graphToken" } -UseBasicParsing -ErrorAction Stop + return [pscustomobject]@{ Accessible = $true; Exists = $true } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return [pscustomobject]@{ Accessible = $true; Exists = $false } } + return [pscustomobject]@{ Accessible = $false; Exists = $false } + } + } + else { + $resolved = Resolve-FabricTarget -Url $Url + $dirPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$($resolved.FilesPath)" + try { + $null = Invoke-FabricWebRequest -Uri $dirPath -Method HEAD + return [pscustomobject]@{ Accessible = $true; Exists = $true } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return [pscustomobject]@{ Accessible = $true; Exists = $false } } + return [pscustomobject]@{ Accessible = $false; Exists = $false } + } + } + } + catch { return [pscustomobject]@{ Accessible = $false; Exists = $false } } +} + +function script:Test-AISIDRemoteRunFolderState { + # Blocker-4: classify a remote AISID run-folder URL as 'Absent' | 'Exists' | 'Inaccessible' from the + # read-only probe, distinguishing a confirmed absence (safe to create/upload) from a confirmed + # existing folder (collision -> fail closed, never touched) and from a lookup/transport/authorization + # failure (fail closed instead of assuming absence). The caller blocks on 'Exists' and 'Inaccessible'. + param( + [Parameter(Mandatory)][string]$Url, + [Parameter(Mandatory)][ValidateSet('SharePoint', 'Fabric')][string]$Mode + ) + try { + $probe = script:Get-AISIDRemoteFolderProbe -Url $Url -Mode $Mode + if ($null -eq $probe -or $probe.Accessible -ne $true) { return 'Inaccessible' } + if ($probe.Exists -eq $true) { return 'Exists' } + return 'Absent' + } + catch { return 'Inaccessible' } +} + +function script:New-AISIDRemoteRunFolderSharePoint { + # Defect-1/2: EXCLUSIVELY create the exact timestamped SharePoint AISID child folder via a single POST + # to the parent's /children with '@microsoft.graph.conflictBehavior'='fail'. The POST is the + # AUTHORITATIVE ownership decision (2xx = exclusive create; 409 / nameAlreadyExists = collision), so + # two same-second runs cannot both proceed. NEVER uses 'replace'/'rename'; NEVER treats an existing + # folder as success; NEVER modifies or deletes an existing folder. Not routed through + # Test-RemoteDestination (which targets the parent destination, not this timestamped child). Returns + # { State='Acquired'|'Collision'|'Inaccessible'; Reason }. + param([Parameter(Mandatory)][string]$Url) + $resolved = Resolve-SharePointTarget -Url $Url + $folderPath = if ($resolved.FolderPath) { [string]$resolved.FolderPath } else { '' } + if (-not $folderPath) { return [pscustomobject]@{ State = 'Inaccessible'; Reason = 'resolved child URL carries no folder path' } } + # Defect-2: Resolve-SharePointTarget derives FolderPath from Uri.AbsolutePath, whose segments are + # ALREADY percent-encoded (e.g. 'PAX%20Exports/Sub%20Folder'). Re-escaping them would double-encode + # ('%2520'). Canonicalize each segment EXACTLY ONCE: decode to its display value, then encode that + # once. The Graph colon-path is built from the single-encoded parent segments; the POST body child + # 'name' is the DECODED display value (Graph stores the literal name). This round-trip preserves + # spaces, '#', '%', apostrophes, Unicode, '+', and nested paths without ever creating a literal '%20'. + $segs = @($folderPath -split '/' | Where-Object { $_ -ne '' }) + if ($segs.Count -eq 0) { return [pscustomobject]@{ State = 'Inaccessible'; Reason = 'resolved child URL carries no folder path' } } + $leafDisplay = [System.Uri]::UnescapeDataString($segs[-1]) + $parentSegs = if ($segs.Count -gt 1) { $segs[0..($segs.Count - 2)] } else { @() } + $childrenUri = if ($parentSegs.Count -gt 0) { + $parentEnc = ($parentSegs | ForEach-Object { [System.Uri]::EscapeDataString([System.Uri]::UnescapeDataString($_)) }) -join '/' + "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${parentEnc}:/children" + } + else { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root/children" } + $body = @{ name = $leafDisplay; folder = @{}; '@microsoft.graph.conflictBehavior' = 'fail' } + try { + $null = Invoke-MgGraphRequest -Method POST -Uri $childrenUri -Body $body -ErrorAction Stop + return [pscustomobject]@{ State = 'Acquired'; Reason = 'exclusive create (conflictBehavior=fail)' } + } + catch { + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($status -eq 0) { try { if ($_.Exception.Response.StatusCode -is [System.Net.HttpStatusCode]) { $status = [int]$_.Exception.Response.StatusCode } } catch {} } + $detail = '' + try { if ($_.ErrorDetails -and $_.ErrorDetails.Message) { $detail = [string]$_.ErrorDetails.Message } } catch {} + if ([string]::IsNullOrEmpty($detail)) { $detail = [string]$_.Exception.Message } + if ($status -eq 409 -or $detail -match 'nameAlreadyExists') { return [pscustomobject]@{ State = 'Collision'; Reason = ('folder already exists (HTTP {0})' -f $status) } } + return [pscustomobject]@{ State = 'Inaccessible'; Reason = ('create failed (HTTP {0}): {1}' -f $status, $detail) } + } +} + +function script:New-AISIDRemoteRunFolderFabric { + # Defect-2: EXCLUSIVELY create the exact timestamped Fabric AISID child directory via ADLS Gen2 + # "Create Path" (PUT ?resource=directory) with a conditional 'If-None-Match: *' header, so an + # already-present directory is a CONFLICT (fail closed) rather than an idempotent success. The PUT is + # the authoritative ownership decision — a prior read-only HEAD is never relied upon. NEVER overwrites + # or reuses an existing directory. Returns { State='Acquired'|'Collision'|'Inaccessible'; Reason }. + param([Parameter(Mandatory)][string]$Url) + $resolved = Resolve-FabricTarget -Url $Url + $dirUri = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$($resolved.FilesPath)?resource=directory" + try { + $null = Invoke-FabricWebRequest -Uri $dirUri -Method PUT -Headers @{ 'If-None-Match' = '*' } + return [pscustomobject]@{ State = 'Acquired'; Reason = 'exclusive create (If-None-Match:*)' } + } + catch { + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + $detail = [string]$_.Exception.Message + if ($status -eq 409 -or $status -eq 412 -or $detail -match 'PathAlreadyExists') { return [pscustomobject]@{ State = 'Collision'; Reason = ('directory already exists (HTTP {0})' -f $status) } } + return [pscustomobject]@{ State = 'Inaccessible'; Reason = ('create failed (HTTP {0}): {1}' -f $status, $detail) } + } +} + +function script:Acquire-AISIDRemoteRunFolderOwnership { + # Defect-3: single dispatcher that acquires EXCLUSIVE ownership of the remote AISID run folder BEFORE + # any upload. The read-only probe (Test-AISIDRemoteRunFolderState) runs first as an early diagnostic + # only; the tier-specific exclusive CREATE is authoritative — if the probe reports Absent but the + # create reports Collision, the result is Collision (fail closed). Returns + # { Acquired=[bool]; State='Acquired'|'Collision'|'Inaccessible'; Mode; Url; Reason; ProbeState }. + # The caller proceeds (stages/uploads) only when Acquired is $true. + param( + [Parameter(Mandatory)][string]$Url, + [Parameter(Mandatory)][ValidateSet('SharePoint', 'Fabric')][string]$Mode + ) + $probeState = try { script:Test-AISIDRemoteRunFolderState -Url $Url -Mode $Mode } catch { 'Inaccessible' } + $create = if ($Mode -eq 'SharePoint') { script:New-AISIDRemoteRunFolderSharePoint -Url $Url } else { script:New-AISIDRemoteRunFolderFabric -Url $Url } + return [pscustomobject]@{ + Acquired = ($create.State -eq 'Acquired') + State = $create.State + Mode = $Mode + Url = $Url + Reason = $create.Reason + ProbeState = $probeState + } +} + +function script:Resolve-AISIDHistorySource { + # Blocker-3: the AISID history-read source, SEPARATE from the new output run folder. Prior history is + # read ONLY from an explicitly supplied -AppendDefenderUsage prior bundle; a plain + # -OutputPathDefenderUsage run (or no destination) reads NO prior history (never the fresh output + # folder). Returns { LocalDir=[string]; RemoteParentUrl=[string] } with at most one non-empty per tier + # (Local -> LocalDir; SharePoint/Fabric -> RemoteParentUrl). The supplied prior bundle is only READ; + # the reconciled result is written into the new AISID_ child. Overrides keep it testable. + param( + [Nullable[bool]]$AppendBoundOverride = $null, + [string]$EffectiveDirOverride = $null, + [string]$RemoteParentOverride = $null + ) + $mode = if ($script:RemoteOutputMode) { [string]$script:RemoteOutputMode } else { 'None' } + $bound = + if ($null -ne $AppendBoundOverride) { [bool]$AppendBoundOverride } + elseif ($script:AppendIsBound -and $script:AppendIsBound.ContainsKey('DefenderUsage')) { [bool]$script:AppendIsBound['DefenderUsage'] } + else { $false } + if (-not $bound) { return [pscustomobject]@{ LocalDir = ''; RemoteParentUrl = '' } } + if ($mode -eq 'None') { + $eff = + if (-not [string]::IsNullOrEmpty($EffectiveDirOverride)) { [string]$EffectiveDirOverride } + else { $du = try { script:Resolve-DataTypePaths -DataType 'DefenderUsage' } catch { $null }; if ($du -and $du.EffectiveDir) { [string]$du.EffectiveDir } else { '' } } + return [pscustomobject]@{ LocalDir = $eff; RemoteParentUrl = '' } + } + $rp = + if (-not [string]::IsNullOrEmpty($RemoteParentOverride)) { [string]$RemoteParentOverride } + elseif ($script:DestParentUrl -and $script:DestParentUrl['DefenderUsage']) { [string]$script:DestParentUrl['DefenderUsage'] } + else { '' } + return [pscustomobject]@{ LocalDir = ''; RemoteParentUrl = $rp } +} + +function Get-AISIDOffHoursGeoCache { + <# + .SYNOPSIS + Returns cached off-hours/geography raw rows when a same-day cache for the same collection + window exists; otherwise returns $null. This delta cache is scoped to the off-hours/geography + signal alone — no other data set reads or writes it. + .OUTPUTS + The cached raw row set, or $null on any miss / unreadable cache. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][int]$LookbackDays, + [string]$WindowStartUtc = '', + [string]$WindowEndUtc = '' + ) + + $cacheFile = Join-Path (Join-Path $DestinationDir '.aisid_cache') 'offhoursgeo.json' + if (-not (Test-Path -LiteralPath $cacheFile)) { return $null } + try { $payload = Get-Content -LiteralPath $cacheFile -Raw | ConvertFrom-Json } catch { return $null } + if (-not $payload) { return $null } + if ([int]$payload.LookbackDays -ne $LookbackDays) { return $null } + # Cache identity includes the exact effective window bounds: same day + different bounds = miss, + # and a legacy LookbackDays-only cache (no WindowStartUtc/WindowEndUtc) is always a miss. Both sides + # are normalized to the invariant UTC ISO-8601 form first, because ConvertFrom-Json auto-parses an + # ISO-8601 'Z' string back into a [datetime] whose culture-formatted [string] would never string-equal + # the ISO literal (a same-run cache would otherwise always false-miss). + if ((ConvertTo-AISIDInvariantUtcTimestamp -Value $payload.WindowStartUtc) -ne (ConvertTo-AISIDInvariantUtcTimestamp -Value $WindowStartUtc)) { return $null } + if ((ConvertTo-AISIDInvariantUtcTimestamp -Value $payload.WindowEndUtc) -ne (ConvertTo-AISIDInvariantUtcTimestamp -Value $WindowEndUtc)) { return $null } + $cachedUtc = $null + try { $cachedUtc = [datetime]$payload.CachedUtc } catch { return $null } + # Same-window reuse is valid only within the same UTC calendar day. + if ($cachedUtc.ToUniversalTime().Date -ne (Get-Date).ToUniversalTime().Date) { return $null } + return @($payload.Rows) +} + +function Save-AISIDOffHoursGeoCache { + <# + .SYNOPSIS + Persists off-hours/geography raw rows to the off-hours/geography-scoped delta cache. Failures + are non-fatal (a run must never break because the cache could not be written). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][int]$LookbackDays, + [string]$WindowStartUtc = '', + [string]$WindowEndUtc = '', + [object[]]$Rows = @() + ) + + $cacheDir = Join-Path $DestinationDir '.aisid_cache' + try { + if (-not (Test-Path -LiteralPath $cacheDir)) { New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null } + $payload = [ordered]@{ + LookbackDays = $LookbackDays + WindowStartUtc = $WindowStartUtc + WindowEndUtc = $WindowEndUtc + CachedUtc = (Get-Date).ToUniversalTime().ToString('o') + Rows = @($Rows) + } + $json = $payload | ConvertTo-Json -Depth 6 + [System.IO.File]::WriteAllText((Join-Path $cacheDir 'offhoursgeo.json'), $json, (New-Object System.Text.UTF8Encoding($false))) + } + catch { + Write-LogHost "[AISID] OffHoursGeo cache write failed (non-fatal): $($_.Exception.Message)" -ForegroundColor Yellow + } +} + +function Get-AISIDCountryCountMap { + <# + .SYNOPSIS + Converts an advanced-hunting CountryCounts bag into a hashtable of country-key -> integer + session count. OData annotation properties emitted by the Graph security hunting API + ('@odata.type' = '#microsoft.graph.security.dynamicColumnValue', and any per-field + '@odata.type' companion) are structurally ignored, never cast; any entry whose + value is not a whole number is skipped rather than guessed. Metadata therefore can never + enter the map and can never raise an Int32 conversion error. + .OUTPUTS + [hashtable] of [string] country -> [int] count (possibly empty). + #> + [CmdletBinding()] + param([Parameter()][AllowNull()]$CountryCounts) + + $map = @{} + if ($null -eq $CountryCounts) { return $map } + + if ($CountryCounts -is [System.Collections.IDictionary]) { + $pairs = @($CountryCounts.Keys | ForEach-Object { [pscustomobject]@{ Name = [string]$_; Value = $CountryCounts[$_] } }) + } + else { + $pairs = @($CountryCounts.PSObject.Properties | ForEach-Object { [pscustomobject]@{ Name = $_.Name; Value = $_.Value } }) + } + + foreach ($p in $pairs) { + $key = [string]$p.Name + if ([string]::IsNullOrWhiteSpace($key)) { continue } + if ($key -like '*@odata*') { continue } + $parsed = [long]0 + if ([long]::TryParse([string]$p.Value, [ref]$parsed)) { $map[$key] = [int]$parsed } + } + return $map +} + +function Get-AISIDOffHoursGeo { + <# + .SYNOPSIS + Collects the off-hours / geography data set from Entra sign-in hunting events and returns rows + in the contractual column order. Uses the off-hours/geography-scoped delta cache unless + DisableDeltaCache is set, in which case a full fresh query always runs and the cache is neither + read nor written. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, CacheHit, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param( + [switch]$DisableDeltaCache, + [string]$CacheDir + ) + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; CacheHit = $false + TruncationDetected = $false; AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $signInAppNames = @(Get-AISIDSignInAppNames) + if ($signInAppNames.Count -eq 0) { + Write-LogHost '[AISID] OffHoursGeo: no sign-in app names in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $lookbackDays = Get-AISIDEffectiveLookbackDays + $appFilterJson = ($signInAppNames | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + # The collection window is bounded by the shared hunting-query timespan; the query filters to the + # catalog's sign-in app names and aggregates per user/month, with a per-country breakdown so the + # "primary country vs. anomalous" split can be computed client-side without a second round-trip. + $ohgBounds = Get-AISIDCollectionBounds + $ohgStart = $ohgBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ohgEnd = $ohgBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ohgScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'UPN' + # Batched: a large scope is split into size-bounded batches; both the outer and the joined subquery + # receive the SAME per-batch scope clause so every target user is queried exactly once and the + # per-user rows are concatenated across batches. $ohgScope retained for structural parity. + $null = $ohgScope + $ohgQueryBuilder = ({ + param($__scope) +@" +let AppFilter = dynamic([$appFilterJson]); +let WinStart = datetime($ohgStart); +let WinEnd = datetime($ohgEnd); +EntraIdSignInEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where Application in~ (AppFilter) +| extend UPN = tolower(AccountUpn) +| where isnotempty(UPN) +$__scope +| extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') +| extend CountryKey = iff(isempty(Country), 'Unknown', Country) +| summarize CountryCount = count() by UPN, YearMonth, CountryKey +| summarize + TotalSessions = sum(CountryCount), + DistinctCountries = dcount(CountryKey), + CountryCounts = make_bag(pack(CountryKey, CountryCount), 200) + by UPN, YearMonth +| join kind=inner ( + EntraIdSignInEvents + | where Timestamp >= WinStart and Timestamp < WinEnd + | where Application in~ (AppFilter) + | extend UPN = tolower(AccountUpn) + | where isnotempty(UPN) +$__scope + | extend Hour = datetime_part('hour', Timestamp) + | extend IsOffHours = iff(Hour < 7 or Hour > 19, 1, 0) + | extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') + | summarize OffHoursSessions = sum(IsOffHours) by UPN, YearMonth + ) on UPN, YearMonth +| project UPN, YearMonth, TotalSessions, OffHoursSessions, DistinctCountries, CountryCounts +"@ + }.GetNewClosure()) + + + $rawRows = $null + # A resumed AISID run whose stored cache-window identity cannot be trusted forces a guaranteed cache miss on READ (the original + # run's destination/cache identity cannot be trusted to match), so the window is re-queried fresh. + # The switch semantics of -DisableAISIDDeltaCache are unchanged; the cache WRITE below still runs. + if (-not $DisableDeltaCache -and -not $script:AISIDResumeForceCacheMiss -and $CacheDir) { + $cached = Get-AISIDOffHoursGeoCache -DestinationDir $CacheDir -LookbackDays $lookbackDays -WindowStartUtc $ohgStart -WindowEndUtc $ohgEnd + if ($null -ne $cached) { + $rawRows = @($cached); $result.CacheHit = $true + Write-LogHost "[AISID] OffHoursGeo: delta cache hit ($($rawRows.Count) rows)." -ForegroundColor DarkGray + } + } + + if ($null -eq $rawRows) { + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'UPN' -Description 'OffHoursGeo' -QueryBuilder $ohgQueryBuilder + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "OffHoursGeo query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $rawRows = @($api.Results) + $result.TruncationDetected = [bool]$api.TruncationDetected + if (-not $DisableDeltaCache -and $CacheDir) { + Save-AISIDOffHoursGeoCache -DestinationDir $CacheDir -LookbackDays $lookbackDays -WindowStartUtc $ohgStart -WindowEndUtc $ohgEnd -Rows $rawRows + } + } + + $result.RawRowCount = @($rawRows).Count + + # Client-side enrichment: off-hours percentage, and anomalous countries = every non-primary country. + $rows = New-Object System.Collections.Generic.List[object] + foreach ($r in $rawRows) { + if (-not $r) { continue } + $total = [int]$r.TotalSessions + $off = [int]$r.OffHoursSessions + $pct = if ($total -gt 0) { [Math]::Round(($off / [double]$total), 4) } else { 0 } + + $countryMap = Get-AISIDCountryCountMap -CountryCounts $r.CountryCounts + $anomalous = @() + if ($countryMap.Count -gt 1) { + # Deterministic primary: highest session count; ties broken by alphabetically-first country key. + $primary = ($countryMap.GetEnumerator() | Sort-Object @{ Expression = { $_.Value }; Descending = $true }, @{ Expression = { $_.Key }; Descending = $false } | Select-Object -First 1).Key + $anomalous = @($countryMap.Keys | Where-Object { $_ -ne $primary } | Sort-Object) + } + + $rows.Add([pscustomobject][ordered]@{ + UPN = [string]$r.UPN + YearMonth = [string]$r.YearMonth + TotalSessions = $total + OffHoursSessions = $off + OffHoursPct = $pct + DistinctCountries = [int]$r.DistinctCountries + AnomalousCountryCount = $anomalous.Count + AnomalousCountries = ($anomalous -join ';') + }) + } + $result.Rows = $rows.ToArray() + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +# Severity-tier reference for OAuth permission scopes. The tiering signal is anchored on Microsoft's own +# admin-consent-requirement model: application permissions always require admin consent, many high-privilege +# delegated permissions require admin consent, and delegated permissions that do not require admin consent are +# lower-privilege by design. See: +# https://learn.microsoft.com/en-us/entra/identity-platform/permissions-consent-overview +# https://learn.microsoft.com/en-us/graph/permissions-overview +# https://learn.microsoft.com/en-us/graph/permissions-reference +# The admin-consent facts are Microsoft-documented; the specific High/Medium/Low assignment per scope is +# informed practitioner judgment layered on top of that signal, not a Microsoft-official severity ranking. +# Any scope not listed here resolves to 'Unclassified' (never a numeric guess). +$script:AISIDPermissionRiskCatalog = @{ + # High: broad directory/role/app control, tenant-wide write, or mail send/read-write (all admin-consent). + 'Directory.ReadWrite.All' = 'High' + 'RoleManagement.ReadWrite.Directory' = 'High' + 'Application.ReadWrite.All' = 'High' + 'AppRoleAssignment.ReadWrite.All' = 'High' + 'User.ReadWrite.All' = 'High' + 'Group.ReadWrite.All' = 'High' + 'Sites.FullControl.All' = 'High' + 'Files.ReadWrite.All' = 'High' + 'Mail.ReadWrite' = 'High' + 'Mail.Send' = 'High' + 'Directory.AccessAsUser.All' = 'High' + 'full_access_as_app' = 'High' + # Medium: tenant-wide read of sensitive data, or scoped write of user content (typically admin-consent). + 'Directory.Read.All' = 'Medium' + 'User.Read.All' = 'Medium' + 'Group.Read.All' = 'Medium' + 'Mail.Read' = 'Medium' + 'Files.Read.All' = 'Medium' + 'Sites.Read.All' = 'Medium' + 'Calendars.ReadWrite' = 'Medium' + 'Contacts.ReadWrite' = 'Medium' + 'People.Read.All' = 'Medium' + 'Chat.Read.All' = 'Medium' + # Low: self-scoped or low-privilege scopes that do not require admin consent by design. + 'User.Read' = 'Low' + 'User.ReadBasic.All' = 'Low' + 'openid' = 'Low' + 'profile' = 'Low' + 'email' = 'Low' + 'offline_access' = 'Low' + 'Calendars.Read' = 'Low' + 'Contacts.Read' = 'Low' + 'Files.Read' = 'Low' + 'Mail.ReadBasic' = 'Low' +} + +function Resolve-AISIDPermissionTier { + <# + .SYNOPSIS + Maps a single Graph permission scope string to a severity tier via the reference catalog. Any scope + not present in the catalog resolves to 'Unclassified' — this is the only fallback, and it is never a + numeric guess. + .OUTPUTS + One of: 'High', 'Medium', 'Low', 'Unclassified'. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][AllowEmptyString()][string]$Scope) + + $key = $Scope.Trim() + if ($key -and $script:AISIDPermissionRiskCatalog.ContainsKey($key)) { + return $script:AISIDPermissionRiskCatalog[$key] + } + return 'Unclassified' +} + +function Get-AISIDPermissionWeight { + <# + .SYNOPSIS + Computes the integer PermissionWeight for one consent grant's set of permission scopes. Each scope is + mapped to a severity tier and scored (High=3, Medium=2, Low=1, Unclassified=0); the row's weight is the + HIGHEST score across all granted scopes, because one severe scope defines the risk regardless of how + many trivial scopes accompany it. A grant with no scopes returns 0 without ever entering the tier + lookup; the companion Permissions column distinguishes 'no scopes' (empty) from 'only unrecognized + scopes' (populated) at the same weight of 0. + .OUTPUTS + [int] weight (0-3). + #> + [CmdletBinding()] + param([string[]]$Permissions = @()) + + $tierScore = @{ High = 3; Medium = 2; Low = 1; Unclassified = 0 } + + # A single source value may carry one or several scope tokens; split on common delimiters and strip any + # surrounding quote/bracket characters so an exact catalog-key match is not defeated by formatting. + $tokens = New-Object System.Collections.Generic.List[string] + foreach ($p in @($Permissions)) { + if (-not $p) { continue } + foreach ($t in ($p -split '[,;\s]+')) { + $t = $t.Trim('"', "'", '[', ']') + if ($t) { $tokens.Add($t) } + } + } + if ($tokens.Count -eq 0) { return 0 } + + $max = 0 + foreach ($t in $tokens) { + $score = $tierScore[(Resolve-AISIDPermissionTier -Scope $t)] + if ($score -gt $max) { $max = $score } + } + return $max +} + +function Get-AISIDOAuthConsents { + <# + .SYNOPSIS + Collects OAuth consent activity from the Entra audit log (directoryAudits) on the existing + Graph session, correlates it to catalog AI apps, and aggregates to one row per + (UPN, AppName, YearMonth). This table is cache-free by design (the delta cache is + off-hours/geography-only) and issues no hunting query. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0 + TruncationDetected = $false; AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $aiAppNames = @(Get-AISIDSignInAppNames) + $aiAppLower = @{} + foreach ($n in $aiAppNames) { $aiAppLower[$n.ToLowerInvariant()] = $true } + + $ocBounds = Get-AISIDCollectionBounds + $startIso = $ocBounds.StartUtc.ToString('o') + $endIso = $ocBounds.EndUtc.ToString('o') + # Consent-related operations recorded in the Entra audit log. + $ops = @('Consent to application', 'Add delegated permission grant', 'Add app role assignment to service principal') + $opClause = ($ops | ForEach-Object { "activityDisplayName eq '$_'" }) -join ' or ' + $filter = "($opClause) and activityDateTime ge $startIso and activityDateTime lt $endIso" + $encoded = [System.Uri]::EscapeDataString($filter) + $uri = "https://graph.microsoft.com/v1.0/auditLogs/directoryAudits?`$filter=$encoded&`$top=200" + + $collected = New-Object System.Collections.Generic.List[object] + try { + while ($uri) { + # Refresh on the existing session before each page to avoid mid-pagination 401s. + $refresh = Refresh-GraphTokenIfNeeded + if ($refresh -eq 'Quit') { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $resp = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + if ($resp.value) { foreach ($rec in $resp.value) { $collected.Add($rec) } } + $uri = $resp.'@odata.nextLink' + } + } + catch { + $result.ErrorMessage = "OAuthConsents Graph query failed: $($_.Exception.Message)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $rawRecords = @($collected.ToArray()) + $result.RawRowCount = $rawRecords.Count + + # Client-side catalog match + aggregate by (UPN, AppName, YearMonth). + $agg = [ordered]@{} + foreach ($rec in $rawRecords) { + if (-not $rec) { continue } + $target = if ($rec.targetResources) { @($rec.targetResources)[0] } else { $null } + $appName = if ($target -and $target.displayName) { [string]$target.displayName } else { $null } + if (-not $appName) { continue } + + $resolved = Resolve-AISIDFromAppName -AppName $appName + if (-not ($resolved -or $aiAppLower.ContainsKey($appName.ToLowerInvariant()))) { continue } + $solution = if ($resolved) { $resolved.AISolution } else { $appName } + + $upn = $null + if ($rec.initiatedBy -and $rec.initiatedBy.user -and $rec.initiatedBy.user.userPrincipalName) { + $upn = [string]$rec.initiatedBy.user.userPrincipalName + } + if (-not $upn) { continue } + $upn = $upn.ToLowerInvariant() + + $ts = $null; try { $ts = [datetime]$rec.activityDateTime } catch {} + if (-not $ts) { continue } + $tsUtc = $ts.ToUniversalTime() + $yearMonth = $tsUtc.ToString('yyyy-MM') + + $perms = @() + # Granted scopes come from the consent event's DelegatedPermissionGrant.Scope modified property + # (TargetResources[].ModifiedProperties[].NewValue) — the field-tested source for the actual granted + # scope strings. The value can arrive wrapped in quotes/brackets, so surrounding quote/bracket + # characters are trimmed before it is stored (the per-token tokenizer strips them again defensively). + if ($rec.targetResources) { + foreach ($tr in @($rec.targetResources)) { + if (-not $tr.modifiedProperties) { continue } + foreach ($mp in @($tr.modifiedProperties)) { + $dn = if ($mp.displayName) { [string]$mp.displayName } elseif ($mp.DisplayName) { [string]$mp.DisplayName } else { '' } + if ($dn -ne 'DelegatedPermissionGrant.Scope') { continue } + $nv = if ($mp.newValue) { [string]$mp.newValue } elseif ($mp.NewValue) { [string]$mp.NewValue } else { '' } + $nv = $nv.Trim().Trim('"', "'", '[', ']').Trim() + if ($nv) { $perms += $nv } + } + } + } + + $key = '{0}|{1}|{2}' -f $upn, $solution, $yearMonth + if (-not $agg.Contains($key)) { + $agg[$key] = [ordered]@{ + UPN = $upn; AppName = $solution; YearMonth = $yearMonth + ConsentCount = 0; LastConsent = $tsUtc + PermSet = (New-Object System.Collections.Generic.HashSet[string]) + } + } + $b = $agg[$key] + $b.ConsentCount = [int]$b.ConsentCount + 1 + if ($tsUtc -gt [datetime]$b.LastConsent) { $b.LastConsent = $tsUtc } + foreach ($p in $perms) { [void]$b.PermSet.Add($p) } + } + + $rows = foreach ($b in $agg.Values) { + $permArr = @($b.PermSet) + [pscustomobject][ordered]@{ + UPN = $b.UPN + AppName = $b.AppName + YearMonth = $b.YearMonth + ConsentCount = [int]$b.ConsentCount + LastConsent = ([datetime]$b.LastConsent).ToString('o') + PermissionWeight = Get-AISIDPermissionWeight -Permissions $permArr + Permissions = ($permArr -join ';') + } + } + $result.Rows = @($rows) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function Get-AISIDSSOSignIns { + <# + .SYNOPSIS + Collects SSO sign-in activity from Entra sign-in hunting events, aggregated per + (UPN, Application, YearMonth). Cache-free by design (the delta cache is off-hours/geography-only). + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0 + TruncationDetected = $false; AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $signInAppNames = @(Get-AISIDSignInAppNames) + if ($signInAppNames.Count -eq 0) { + Write-LogHost '[AISID] SSOSignIns: no sign-in app names in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $appFilterJson = ($signInAppNames | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + # Guest is inferred from the external-identity marker in the UPN; conditional-access applicability is + # inferred from a non-empty conditional-access evaluation status. Both are advisory client-side reads. + $ssoBounds = Get-AISIDCollectionBounds + $ssoStart = $ssoBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ssoEnd = $ssoBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ssoScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'UPN' + # Batched: a large scope is split into size-bounded batches; the query runs once per batch (every + # target user queried exactly once) and the per-user rows are concatenated. $ssoScope retained for + # structural parity; the per-batch clause ($__scope) is what the runner injects. + $null = $ssoScope + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'UPN' -Description 'SSOSignIns' -QueryBuilder ({ + param($__scope) +@" +let AppFilter = dynamic([$appFilterJson]); +let WinStart = datetime($ssoStart); +let WinEnd = datetime($ssoEnd); +EntraIdSignInEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where Application in~ (AppFilter) +| extend UPN = tolower(AccountUpn) +| where isnotempty(UPN) +$__scope +| extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') +| extend DayKey = format_datetime(Timestamp, 'yyyy-MM-dd') +| extend IsGuestFlag = iff(UPN has '#ext#', 'True', 'False') +| extend CAApplied = iff(ConditionalAccessStatus == 'success' or ConditionalAccessStatus == 'failure', 'Yes', 'No') +| extend CountryKey = iff(isempty(Country), 'Unknown', Country) +| summarize + SignInCount = count(), + DistinctDays = dcount(DayKey), + IsGuest = max(IsGuestFlag), + HasConditionalAccess = max(CAApplied), + Countries = make_set(CountryKey, 100), + LastSignIn = max(Timestamp) + by UPN, Application, YearMonth +"@ + }.GetNewClosure()) + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "SSOSignIns query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $rawRows = @($api.Results) + $result.RawRowCount = $rawRows.Count + $result.TruncationDetected = [bool]$api.TruncationDetected + + $rows = foreach ($r in $rawRows) { + if (-not $r) { continue } + $countries = @() + if ($r.Countries) { + if ($r.Countries -is [System.Array]) { $countries = @($r.Countries | ForEach-Object { [string]$_ }) } + else { $countries = @([string]$r.Countries) } + } + $last = '' + if ($r.LastSignIn) { try { $last = ([datetime]$r.LastSignIn).ToUniversalTime().ToString('o') } catch { $last = [string]$r.LastSignIn } } + [pscustomobject][ordered]@{ + UPN = [string]$r.UPN + Application = [string]$r.Application + YearMonth = [string]$r.YearMonth + SignInCount = [int]$r.SignInCount + DistinctDays = [int]$r.DistinctDays + IsGuest = [string]$r.IsGuest + Countries = ($countries -join ';') + HasConditionalAccess = [string]$r.HasConditionalAccess + LastSignIn = $last + } + } + $result.Rows = @($rows) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function Get-AISIDSolutionsCatalogExport { + <# + .SYNOPSIS + Projects the in-memory AI solutions reference catalog to the dashboard's dimension CSV columns. + This is dimension data — the whole file is regenerated each run (overwrite); there is no + time-series grain and no per-row key merge. + .OUTPUTS + An array of rows in the contractual column order. + #> + [CmdletBinding()] + param() + + $rows = foreach ($entry in $script:AISolutionCatalog) { + if (-not $entry) { continue } + [pscustomobject][ordered]@{ + AISolution = [string]$entry.AISolution + Category = [string]$entry.Category + Vendor = [string]$entry.Vendor + RiskTier = [string]$entry.RiskTier + DefaultDataHandling = [string]$entry.DefaultDataHandling + SolutionGroup = [string]$entry.SolutionGroup + } + } + return @($rows) +} + +function Get-AISIDEntraUsersProjection { + <# + .SYNOPSIS + Projects the already-collected general EntraUsers export data down to the dashboard's fixed + column set, in contractual order. Reads in-memory data only — it performs no directory query of + its own and leaves the general export untouched. Dimension data: the whole file is regenerated + each run (overwrite); no per-row key merge. + .OUTPUTS + An array of rows in the contractual column order. + #> + [CmdletBinding()] + param([object[]]$SourceRows = @($script:EntraUsersData)) + + $SourceRows = @($SourceRows) + + # Target-user scope for the EntraUsers dimension: when a user/group scope was requested, restrict + # the projection to the target users PLUS their full manager chain (managers of managers, up to the + # top). No non-target UPN reaches the dimension except those manager-chain rows, which the dashboard + # hierarchy needs to resolve. Unscoped runs project the source rows unchanged. + $scope = script:Get-AISIDScopeUpnSet + if ($null -ne $scope) { + $byUpn = @{} + foreach ($u in $SourceRows) { + if (-not $u) { continue } + $g = if ($u -is [System.Collections.IDictionary]) { [string]$u['userPrincipalName'] } else { [string]$u.userPrincipalName } + $g = ([string]$g).Trim() + if ($g) { $byUpn[$g.ToLowerInvariant()] = $u } + } + $keep = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($t in @($scope)) { + $cur = ([string]$t).Trim() + $guard = 0 + while ($cur -and $keep.Add($cur) -and $guard -lt 10000) { + $guard++ + $row = $byUpn[$cur.ToLowerInvariant()] + if (-not $row) { break } + $mgr = if ($row -is [System.Collections.IDictionary]) { [string]$row['manager_userPrincipalName'] } else { [string]$row.manager_userPrincipalName } + $cur = ([string]$mgr).Trim() + } + } + $SourceRows = @($SourceRows | Where-Object { + $g = if ($_ -is [System.Collections.IDictionary]) { [string]$_['userPrincipalName'] } else { [string]$_.userPrincipalName } + $g = ([string]$g).Trim() + $g -and $keep.Contains($g) + }) + } + + $rows = foreach ($u in @($SourceRows)) { + if (-not $u) { continue } + $get = { + param($n) + if ($u -is [System.Collections.IDictionary]) { $u[$n] } else { $u.$n } + } + [pscustomobject][ordered]@{ + userPrincipalName = [string](& $get 'userPrincipalName') + displayName = [string](& $get 'displayName') + department = [string](& $get 'department') + jobTitle = [string](& $get 'jobTitle') + city = [string](& $get 'city') + country = [string](& $get 'country') + companyName = [string](& $get 'companyName') + accountEnabled = [string](& $get 'accountEnabled') + userType = [string](& $get 'userType') + manager_displayName = [string](& $get 'manager_displayName') + manager_userPrincipalName = [string](& $get 'manager_userPrincipalName') + hasLicense = [string](& $get 'hasLicense') + assignedLicenses = [string](& $get 'assignedLicenses') + createdDateTime = [string](& $get 'createdDateTime') + } + } + return @($rows) +} + +function Export-AISIDPilotTables { + <# + .SYNOPSIS + Processes the five query-backed data sets (Activity Sessions, Off-Hours/Geography, File + Proximity, OAuth Consents, SSO Sign-Ins) through the shared export/merge infrastructure, + producing the full dashboard output set. Each query-backed table is collected only when its + per-table licensing verdict is eligible; an ineligible or empty table still yields a valid + header-only CSV. File Proximity is event-grain (append-and-dedup merge); when it is ineligible + or its query fails and no prior history exists, the header-only stub is emitted as its fallback. + The off-hours/geography delta cache is used unless DisableDeltaCache is set. + .OUTPUTS + Ordered hashtable of written paths per table, plus AuthQuit. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][System.Collections.IDictionary]$Verdict, + [switch]$DisableDeltaCache, + [hashtable]$ExistingRows = @{}, + [string]$ParentOverride, + [string]$HistoryDir = '', + [string]$RawPurviewCsv + ) + + $out = [ordered]@{ + ActivitySessions = $null; OffHoursGeo = $null; OAuthConsents = $null; SSOSignIns = $null + FileProximity = $null; ClientChannel = $null; CopilotUsage = $null; AppGovAlerts = $null; CloudDiscovery = $null; MDASessions = $null + EntraUsers = $null; SolutionsCatalog = $null; AuthQuit = $false + ReadFailures = (New-Object System.Collections.Generic.List[string]) + Failed = (New-Object System.Collections.Generic.List[string]) + TableStatus = [ordered]@{} + } + # A1 state contract: every dashboard table gets an EXPLICIT terminal state before completeness is + # evaluated. Seed each as Incomplete ("collector not reached") so a table skipped by an early exit is + # never silently treated as clean; per-table processing below overwrites this with the decided state. + foreach ($t in @('ActivitySessions', 'OffHoursGeo', 'FileProximity', 'OAuthConsents', 'SSOSignIns', 'ClientChannel', 'CopilotUsage', 'AppGovAlerts', 'CloudDiscovery', 'MDASessions', 'SolutionsCatalog', 'EntraUsers')) { + $out.TableStatus[$t] = [ordered]@{ State = 'Incomplete'; Reason = 'collector not reached' } + } + + # Table -> collector. Each collector returns the standard result hashtable (Success/Rows/AuthQuit). + # Off-Hours/Geography is the only cache-aware collector; every other table is cache-free. + $collect = { + param($table) + switch ($table) { + 'ActivitySessions' { Get-AISIDActivitySessions } + 'OffHoursGeo' { Get-AISIDOffHoursGeo -DisableDeltaCache:$DisableDeltaCache -CacheDir (script:Get-AISIDCacheDir -DestinationDir $DestinationDir) } + 'OAuthConsents' { Get-AISIDOAuthConsents } + 'SSOSignIns' { Get-AISIDSSOSignIns } + 'ClientChannel' { Get-AISIDClientChannel } + 'CopilotUsage' { Get-AISIDCopilotUsage -RawPurviewCsv $RawPurviewCsv } + } + } + + foreach ($table in @('ActivitySessions', 'OffHoursGeo', 'OAuthConsents', 'SSOSignIns', 'ClientChannel', 'CopilotUsage')) { + $schema = $script:AISIDTableSchemas[$table] + $rows = @() + $state = 'Unavailable'; $reason = 'licensing ineligible' + if ($Verdict[$table] -and $Verdict[$table].Eligible) { + try { + $q = & $collect $table + if ($q.AuthQuit) { $out.AuthQuit = $true } + if ($q.Success) { $rows = @($q.Rows); if (@($rows).Count -gt 0) { $state = 'Succeeded'; $reason = 'collected rows' } else { $state = 'Empty'; $reason = 'query succeeded with zero rows' } } + else { $state = 'Failed'; $reason = 'collector returned Success=false'; [void]$out.Failed.Add($table) } + } + catch { + # Per-table isolation: an UNEXPECTED throw (as opposed to a Success=$false return) is + # caught here so the remaining tables in the loop still run. $rows stays empty, so the + # read-before-merge + export below fall this table back to its safe-empty / history- + # preserving output exactly as an ineligible or Success=$false table does. + Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); falling back to safe-empty output for this table." -f $table, $_.Exception.Message) -ForegroundColor Yellow + $state = 'Failed'; $reason = ("collector threw: {0}" -f $_.Exception.Message) + [void]$out.Failed.Add($table) + } + } + # Ineligible or empty: $rows stays empty; Export-AISIDTable still writes a header-only CSV. + + # Client-side target-user scope enforcement (defense-in-depth behind the server-side KQL + # filter): when a user/group scope was requested, drop any newly collected row whose UPN is + # not in the resolved target set so no non-target user reaches this run's output. No-op for + # unscoped runs and for non-user-grain / global tables (ClientChannel, SolutionsCatalog) and + # the separately-scoped EntraUsers dimension. + $rows = script:Select-AISIDScopedRows -Table $table -Rows $rows + + # Deidentify this table's NEW rows BEFORE the read-before-merge below, so on-disk (already + # deidentified) rows and these newly-collected rows compare on matching keys during merge. + # No-op when -Deidentify is off. CopilotUsage is now deidentified here too: its source is the + # IDENTIFIED raw Purview CSV (read before any raw scrub), so its UserPrincipalName must pass once + # through the shared Get-PaxDeidUpn path to match the deidentified EntraUsers key. + $rows = ConvertTo-AISIDDeidentifiedRows -Table $table -Rows $rows + + # ClientChannel scope provenance: ClientChannel aggregates away UPN, so a prior ClientChannel + # bundle cannot be proven to share this run's user scope. For a SCOPED run, refuse to merge + # ClientChannel history whose scope provenance cannot be established — fail closed (don't merge, + # don't publish, preserve prior, AISID gap) rather than silently mix tenant-wide and scoped + # aggregates. + if ($table -eq 'ClientChannel' -and $null -ne (script:Get-AISIDScopeUpnSet)) { + $__ccHasPrior = + ($ExistingRows.ContainsKey($table) -and @($ExistingRows[$table]).Count -gt 0) -or + ((-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') -and $HistoryDir -and (Test-AISIDCsvHasDataRow -Path (Join-Path $HistoryDir $schema.FileName))) -or + ($script:RemoteOutputMode -and $script:RemoteOutputMode -ne 'None' -and $ParentOverride) + if ($__ccHasPrior) { + Write-LogHost "[AISID] ClientChannel: a scoped run cannot prove the prior ClientChannel bundle shares this user scope (aggregates carry no UPN); refusing to merge and leaving the prior file unchanged (AISID gap)." -ForegroundColor Yellow + [void]$out.Failed.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Failed'; Reason = 'scoped ClientChannel append: prior scope provenance unprovable; prior file left unchanged' } + $script:AISIDHadGaps = $true + continue + } + } + + # Read-before-merge for replace-on-key tables: reconcile against the accumulated history already + # on disk so prior periods are preserved instead of being overwritten with only this run's rows. + # Overwrite/dimension tables (empty Key) never enter this path. An explicitly injected ExistingRows + # entry takes precedence over the on-disk read. On the LOCAL tier the prior history is streamed + # row-by-row through the bounded merge (never materialized as an array); the remote tier still + # downloads-and-reads (inherent) then array-merges. + if ($schema.Key -and $schema.Key.Count -gt 0) { + if ($ExistingRows.ContainsKey($table)) { + $existing = @($ExistingRows[$table]) + if (@($existing).Count -gt 0) { + $rows = Merge-AISIDTableOnKey -ExistingRows $existing -NewRows $rows -KeyColumns $schema.Key + } + } + elseif (-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') { + # Blocker-3: prior history comes from the SEPARATE supplied prior-bundle folder ($HistoryDir), + # never from the fresh output run folder ($DestinationDir). No $HistoryDir + # (-OutputPathDefenderUsage without -AppendDefenderUsage) => no prior history read; this run's + # rows stand alone. A read failure (locked/malformed, not a confirmed absence) propagates so the + # prior file is left untouched this run. + $__histPath = if ($HistoryDir) { Join-Path $HistoryDir $schema.FileName } else { '' } + if ($__histPath -and (Test-Path -LiteralPath $__histPath)) { + try { + $rows = Merge-AISIDTableOnKeyStreamed -ExistingCsvPath $__histPath -NewRows $rows -KeyColumns $schema.Key + } + catch { + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f $table, $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Incomplete'; Reason = 'existing file unreadable; prior file left untouched' } + continue + } + } + } + else { + # Remote tier: prior history from the supplied prior-bundle URL ($ParentOverride); none when + # empty (no -AppendDefenderUsage). Download prior history to a local scratch file (memory-safe), + # then stream the scratch through the SAME bounded merge. Scratch is removed on success and + # preserved for recovery on failure. + if ($ParentOverride) { + $scratchPath = $null + try { + $scratchPath = Get-AISIDExistingCsvToScratch -FileName $schema.FileName -ParentOverride $ParentOverride + $rows = Merge-AISIDTableOnKeyStreamed -ExistingCsvPath $scratchPath -NewRows $rows -KeyColumns $schema.Key + } + catch { + if ($scratchPath -and (Test-Path -LiteralPath $scratchPath)) { Write-LogHost ("[AISID] {0}: remote history scratch preserved for recovery: {1}" -f $table, $scratchPath) -ForegroundColor DarkYellow } + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f $table, $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Incomplete'; Reason = 'existing file unreadable; prior file left untouched' } + continue + } + if ($scratchPath -and (Test-Path -LiteralPath $scratchPath)) { Remove-Item -LiteralPath $scratchPath -Force -ErrorAction SilentlyContinue } + } + } + } + + # Final post-merge scope containment (defense-in-depth): after the read-before-merge, verify NO + # row in the merged result falls outside the effective target scope. History (injected, streamed + # local, or downloaded remote) that reintroduced a non-target user is caught here; when + # -Deidentify is on the comparison uses deidentified scope keys. If any out-of-scope row survives, + # the merged file is NOT published, the prior file is left unchanged, the table is marked + # Failed/Incomplete, and an AISID gap (exit 40) is flagged — a scoped run never emits a non-target + # user and never overwrites prior history with a scope it cannot prove. + $__cont = script:Select-AISIDContainedRows -Table $table -Rows $rows + if ($__cont.Applies -and @($__cont.OutOfScope).Count -gt 0) { + Write-LogHost ("[AISID] {0}: {1} row(s) outside the requested user scope survived the history merge; refusing to publish and leaving the prior file unchanged (AISID gap)." -f $table, @($__cont.OutOfScope).Count) -ForegroundColor Yellow + [void]$out.Failed.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Failed'; Reason = 'post-merge scope containment failed; prior file left unchanged' } + $script:AISIDHadGaps = $true + continue + } + + $out[$table] = Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows $rows + $out.TableStatus[$table] = [ordered]@{ State = $state; Reason = $reason } + } + + # File Proximity: event-grain, append-and-dedup. Same licensing-gated pattern as the query-backed + # tables above (eligibility -> collect -> read-before-merge -> merge -> export), but with the + # append+dedup merge so accumulated event history only ever grows. The header-only stub is the + # fallback only when the table is ineligible or its query fails AND no prior history exists. + $fpSchema = $script:AISIDTableSchemas['FileProximity'] + $fpRows = @() + $fpCollected = $false + $fpState = 'Unavailable'; $fpReason = 'licensing ineligible' + if ($Verdict['FileProximity'] -and $Verdict['FileProximity'].Eligible) { + try { + $fpQ = Get-AISIDFileProximity + if ($fpQ.AuthQuit) { $out.AuthQuit = $true } + if ($fpQ.Success) { $fpRows = @($fpQ.Rows); $fpCollected = $true; if (@($fpRows).Count -gt 0) { $fpState = 'Succeeded'; $fpReason = 'collected rows' } else { $fpState = 'Empty'; $fpReason = 'query succeeded with zero rows' } } else { $fpState = 'Failed'; $fpReason = 'collector returned Success=false'; [void]$out.Failed.Add('FileProximity') } + } + catch { + # Per-table isolation: an unexpected throw is caught so the dimension tables still run. + # $fpCollected stays $false and $fpRows empty -> the append-dedup / stub path below falls + # back to this table's safe output (stub when no prior history, else history preserved). + Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); falling back to safe-empty output for this table." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + $fpState = 'Failed'; $fpReason = ("collector threw: {0}" -f $_.Exception.Message) + [void]$out.Failed.Add('FileProximity') + } + } + # Blocker-2: client-side target-user scope enforcement on FileProximity's NEW rows (same order as the + # generic loop: scope BEFORE deidentify) so no non-target user reaches this run's output. No-op when + # unscoped. Then deidentify the NEW rows before the append-dedup read/merge (no-op when off). + $fpRows = script:Select-AISIDScopedRows -Table 'FileProximity' -Rows $fpRows + $fpRows = ConvertTo-AISIDDeidentifiedRows -Table 'FileProximity' -Rows $fpRows + $fpExisting = @() + $fpReadFailed = $false + $fpHasHistory = $false + $fpUseStreamed = $false + $fpStreamedPath = $null + $fpScratchPath = $null + if ($ExistingRows.ContainsKey('FileProximity')) { + $fpExisting = @($ExistingRows['FileProximity']) + $fpHasHistory = (@($fpExisting).Count -gt 0) + } + elseif (-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') { + # Blocker-3: FileProximity prior event history comes from the SEPARATE supplied prior-bundle folder + # ($HistoryDir), never from the fresh output run folder. No $HistoryDir => no prior history. + try { + $fpStreamedPath = if ($HistoryDir) { Join-Path $HistoryDir $fpSchema.FileName } else { '' } + if ($fpStreamedPath) { + $fpHasHistory = Test-AISIDCsvHasDataRow -Path $fpStreamedPath + $fpUseStreamed = $true + } + } + catch { + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add('FileProximity') + $fpState = 'Incomplete'; $fpReason = 'existing file unreadable; prior file left untouched' + $fpReadFailed = $true + } + } + else { + # Remote tier: prior event history from the supplied prior-bundle URL ($ParentOverride); none when + # empty (no -AppendDefenderUsage). Download prior history to a local scratch file (memory-safe), + # then stream the scratch through the bounded append-dedup merge — never parsed into an array. + if ($ParentOverride) { + try { + $fpScratchPath = Get-AISIDExistingCsvToScratch -FileName $fpSchema.FileName -ParentOverride $ParentOverride + $fpStreamedPath = $fpScratchPath + $fpHasHistory = ($fpScratchPath -and (Test-AISIDCsvHasDataRow -Path $fpScratchPath)) + $fpUseStreamed = $true + } + catch { + # Not a confirmed absence: leave the prior file untouched this run rather than risk an + # empty-merge overwrite of accumulated event history. + if ($fpScratchPath -and (Test-Path -LiteralPath $fpScratchPath)) { Write-LogHost ("[AISID] {0}: remote history scratch preserved for recovery: {1}" -f 'FileProximity', $fpScratchPath) -ForegroundColor DarkYellow } + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add('FileProximity') + $fpState = 'Incomplete'; $fpReason = 'existing file unreadable; prior file left untouched' + $fpReadFailed = $true + } + } + } + if (-not $fpReadFailed) { + if ($fpHasHistory -or $fpCollected) { + if ($fpUseStreamed) { + try { + $fpMerged = Merge-AISIDTableAppendDedupStreamed -ExistingCsvPath $fpStreamedPath -NewRows $fpRows -KeyColumns $fpSchema.Key + } + catch { + if ($fpScratchPath -and (Test-Path -LiteralPath $fpScratchPath)) { Write-LogHost ("[AISID] {0}: remote history scratch preserved for recovery: {1}" -f 'FileProximity', $fpScratchPath) -ForegroundColor DarkYellow } + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add('FileProximity') + $fpState = 'Incomplete'; $fpReason = 'existing file unreadable; prior file left untouched' + $fpReadFailed = $true + } + } + else { + $fpMerged = Merge-AISIDTableAppendDedup -ExistingRows $fpExisting -NewRows $fpRows -KeyColumns $fpSchema.Key + } + if (-not $fpReadFailed) { + # Blocker-2: final post-merge scope containment (deidentify-aware) — history (injected, streamed + # local, or downloaded remote) that reintroduced a non-target user is caught here. If any + # out-of-scope row survives, DO NOT publish; mark Failed; preserve the prior file; flag an AISID + # gap. A scoped run never emits a non-target FileProximity row. + $__fpCont = script:Select-AISIDContainedRows -Table 'FileProximity' -Rows $fpMerged + if ($__fpCont.Applies -and @($__fpCont.OutOfScope).Count -gt 0) { + Write-LogHost ("[AISID] FileProximity: {0} row(s) outside the requested user scope survived the history merge; refusing to publish and leaving the prior file unchanged (AISID gap)." -f @($__fpCont.OutOfScope).Count) -ForegroundColor Yellow + [void]$out.Failed.Add('FileProximity') + $fpState = 'Failed'; $fpReason = 'post-merge scope containment failed; prior file left unchanged' + $script:AISIDHadGaps = $true + } + else { + $out.FileProximity = Export-AISIDTable -DestinationDir $DestinationDir -FileName $fpSchema.FileName -OrderedColumns $fpSchema.Columns -Rows $fpMerged + } + } + } + else { + # Ineligible or query-failed with no prior history: emit the header-only stub fallback. + $out.FileProximity = New-AISIDFileProximityStub -DestinationDir $DestinationDir + } + } + # Remove the downloaded remote history scratch on success; on a read/merge failure it is preserved + # above for recovery. + if (-not $fpReadFailed -and $fpScratchPath -and (Test-Path -LiteralPath $fpScratchPath)) { + Remove-Item -LiteralPath $fpScratchPath -Force -ErrorAction SilentlyContinue + } + $out.TableStatus['FileProximity'] = [ordered]@{ State = $fpState; Reason = $fpReason } + + # Dimension tables: always regenerated in full (overwrite), independent of any licensing verdict and + # with no per-row key merge. Both project already-collected in-memory data. Each collector call is + # isolated so an unexpected throw falls back to a header-only file and never skips the other table. + $catSchema = $script:AISIDTableSchemas['SolutionsCatalog'] + $catRows = @() + $catState = 'Succeeded'; $catReason = 'dimension regenerated' + try { $catRows = @(Get-AISIDSolutionsCatalogExport) } + catch { Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); the dimension FAILED (recorded as a gap)." -f 'SolutionsCatalog', $_.Exception.Message) -ForegroundColor Red; $catState = 'Failed'; $catReason = ("collector threw: {0}" -f $_.Exception.Message); [void]$out.Failed.Add('SolutionsCatalog') } + if ($catState -eq 'Succeeded' -and @($catRows).Count -eq 0) { $catState = 'Empty'; $catReason = 'dimension produced zero rows' } + $catRows = ConvertTo-AISIDDeidentifiedRows -Table 'SolutionsCatalog' -Rows $catRows + $out.SolutionsCatalog = Export-AISIDTable -DestinationDir $DestinationDir -FileName $catSchema.FileName -OrderedColumns $catSchema.Columns -Rows $catRows + $out.TableStatus['SolutionsCatalog'] = [ordered]@{ State = $catState; Reason = $catReason } + + $euSchema = $script:AISIDTableSchemas['EntraUsers'] + $euRows = @() + $euState = 'Succeeded'; $euReason = 'dimension regenerated' + try { $euRows = @(Get-AISIDEntraUsersProjection) } + catch { Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); the dimension FAILED (recorded as a gap)." -f 'EntraUsers', $_.Exception.Message) -ForegroundColor Red; $euState = 'Failed'; $euReason = ("collector threw: {0}" -f $_.Exception.Message); [void]$out.Failed.Add('EntraUsers') } + if ($euState -eq 'Succeeded' -and @($euRows).Count -eq 0) { $euState = 'Empty'; $euReason = 'dimension produced zero rows' } + # F3 backstop (defense-in-depth): for a SCOPED AISID run the requested target(s) are REQUIRED in the + # EntraUsers dimension. The early user-scope gate already fails closed when an explicit target is absent + # from the directory; this second line of defense catches a resolved target that is nevertheless missing + # from the projected rows and records it as a real gap (Failed) — NOT a factual Empty. Only the explicit + # target set is required (manager-chain rows merely accompany it); an unscoped run keeps ordinary Empty. + if ($euState -ne 'Failed') { + $euScope = script:Get-AISIDScopeUpnSet + if ($null -ne $euScope) { + $euProjected = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($er in @($euRows)) { + if (-not $er) { continue } + $eg = if ($er -is [System.Collections.IDictionary]) { [string]$er['userPrincipalName'] } else { [string]$er.userPrincipalName } + $eg = ([string]$eg).Trim() + if ($eg) { [void]$euProjected.Add($eg) } + } + $euMissingTargets = @(@($euScope) | Where-Object { $t = ([string]$_).Trim(); $t -and -not $euProjected.Contains($t) }) + if ($euMissingTargets.Count -gt 0) { + Write-LogHost ("[AISID] EntraUsers: required scoped target(s) absent from the directory projection ({0}); the dimension FAILED (recorded as a gap), not a factual empty." -f (@($euMissingTargets) -join ', ')) -ForegroundColor Red + $euState = 'Failed'; $euReason = ("required scoped target(s) not present in projection: {0}" -f (@($euMissingTargets) -join ', ')) + [void]$out.Failed.Add('EntraUsers') + } + } + } + $euRows = ConvertTo-AISIDDeidentifiedRows -Table 'EntraUsers' -Rows $euRows + $out.EntraUsers = Export-AISIDTable -DestinationDir $DestinationDir -FileName $euSchema.FileName -OrderedColumns $euSchema.Columns -Rows $euRows + $out.TableStatus['EntraUsers'] = [ordered]@{ State = $euState; Reason = $euReason } + + # The dashboard loads these MDA-tier tables unconditionally. Until their collectors are available, + # preserve valid prior history byte-for-byte; when none exists, publish exact header-only compatibility + # files so tenants without those data sources refresh cleanly. + foreach ($stubTable in @('AppGovAlerts', 'CloudDiscovery', 'MDASessions')) { + try { + $out[$stubTable] = Export-AISIDCompatibilityTable -Table $stubTable -DestinationDir $DestinationDir -ExistingRows $ExistingRows -HistoryDir $HistoryDir -ParentOverride $ParentOverride + $out.TableStatus[$stubTable] = [ordered]@{ State = 'Unavailable'; Reason = 'data source not implemented; prior history preserved or header-only compatibility file emitted' } + } + catch { + Write-LogHost ("[AISID] {0}: existing compatibility data could not be preserved ({1}); leaving the prior file unchanged." -f $stubTable, $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add($stubTable) + $out.TableStatus[$stubTable] = [ordered]@{ State = 'Incomplete'; Reason = 'existing compatibility file unreadable or invalid; prior file left unchanged' } + } + } + return $out +} + +function ConvertTo-AISIDDeidentifiedRows { + <# + .SYNOPSIS + Deidentifies the PII columns of an AISID table's newly-collected rows, reusing the shared + deterministic Get-PaxDeid* helpers (same salt/algorithm as the main pipeline and the Python + rollup, so a given real identity maps to the identical token everywhere and cross-table joins + survive). No new hashing logic is introduced. + .DESCRIPTION + No-op when -Deidentify is off ($script:PaxDeidEnabled = $false): the input rows are returned + unchanged (same reference). Only the columns named in the per-table map are transformed; every + other column is copied verbatim and column order is preserved (the source row is never mutated). + Tables with NO map entry are returned unchanged: ClientChannel and SolutionsCatalog carry no PII. + CopilotUsage IS deidentified in this path now: its source is the run's IDENTIFIED raw Purview CSV, + so its UserPrincipalName is hashed once through the shared Get-PaxDeid* path to the identical token + used everywhere. Applied to a table's NEW rows BEFORE the read-before-merge/merge so on-disk (already + deidentified) rows and these rows compare on matching keys. Org/geo attributes (department, + jobTitle, city, country, companyName, ...) are intentionally retained, matching the main + pipeline's established Invoke-PaxRawDeidentify policy. + .OUTPUTS + The rows with mapped PII columns deidentified, or the input unchanged (off / no map entry). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Table, + [object[]]$Rows = @() + ) + if (-not $script:PaxDeidEnabled) { return $Rows } + + # Per-table PII column -> Get-PaxDeid* helper kind. Direct identifiers only. ClientChannel and + # SolutionsCatalog carry no PII and are deliberately absent; CopilotUsage IS mapped here now. + $deidMap = @{ + ActivitySessions = [ordered]@{ UPN = 'Upn' } + OffHoursGeo = [ordered]@{ UPN = 'Upn' } + OAuthConsents = [ordered]@{ UPN = 'Upn' } + SSOSignIns = [ordered]@{ UPN = 'Upn' } + FileProximity = [ordered]@{ UPN = 'Upn'; FileName = 'File'; FolderPath = 'Resource' } + CopilotUsage = [ordered]@{ UserPrincipalName = 'Upn' } + EntraUsers = [ordered]@{ userPrincipalName = 'Upn'; displayName = 'Name'; manager_userPrincipalName = 'Upn'; manager_displayName = 'Name' } + } + $colMap = $deidMap[$Table] + if (-not $colMap) { return $Rows } + + $out = foreach ($row in $Rows) { + if (-not $row) { continue } + # Ordered clone: preserves column order and never mutates the source row object. + $clone = [ordered]@{} + foreach ($p in $row.PSObject.Properties) { $clone[$p.Name] = $p.Value } + foreach ($col in $colMap.Keys) { + if ($clone.Contains($col) -and -not [string]::IsNullOrEmpty([string]$clone[$col])) { + $sv = [string]$clone[$col] + $clone[$col] = switch ($colMap[$col]) { + 'Upn' { Get-PaxDeidUpn -Value $sv } + 'Name' { Get-PaxDeidName -Value $sv } + 'File' { Get-PaxDeidFile -Value $sv } + 'Resource' { Get-PaxDeidResource -Value $sv } + } + } + } + [pscustomobject]$clone + } + return @($out) +} + +function Invoke-AISIDPilotExport { + <# + .SYNOPSIS + End-of-run orchestrator for the AI Solutions Intelligence Dashboard (AISID) export: evaluates + per-table licensing, reports it, and writes all twelve dashboard CSVs through the shared + export/merge infrastructure. Intended to be called exactly once, only under -Dashboard AISID, + from the point in the rollup post-processing where the CopilotInteraction Fact CSV has just + been written and BEFORE any -AppendFile merge/delete can consume it (so Get-AISIDCopilotUsage's + source is guaranteed present). + .DESCRIPTION + Whole-call failure isolation: any unexpected error is caught, logged loudly, and swallowed so + the AISID export can never abort the run or disturb the Purview/rollup output already on disk. + Per-table handling (each collector's licensing-ineligible/empty result and each read failure -> + safe header-only fallback / ReadFailures) is owned by Export-AISIDPilotTables; this outer guard + covers Test-AISIDLicensing and any orchestration-level error. + .OUTPUTS + The Export-AISIDPilotTables result hashtable (written paths per table + AuthQuit), or $null on + an isolated failure. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [switch]$DisableDeltaCache, + [string]$RawPurviewCsv, + [string]$ParentOverride, + [string]$HistoryDir = '' + ) + try { + $verdict = Test-AISIDLicensing + Write-AISIDLicensingReport -Verdict $verdict + return Export-AISIDPilotTables -DestinationDir $DestinationDir -Verdict $verdict -DisableDeltaCache:$DisableDeltaCache -RawPurviewCsv $RawPurviewCsv -ParentOverride $ParentOverride -HistoryDir $HistoryDir + } + catch { + Write-LogHost ("[AISID] Dashboard export failed (non-fatal; the run's Purview/rollup output is unaffected): {0}" -f $_.Exception.Message) -ForegroundColor Red + return $null + } +} + +function Get-AISIDCompletionStatus { + <# + .SYNOPSIS + Evaluates whether the AISID dashboard export completed truthfully, using the per-table state + contract (TableStatus: Succeeded / Empty / Unavailable / Failed / Incomplete) produced by + Export-AISIDPilotTables cross-checked against the twelve contractual dashboard CSVs (each must + exist with its exact header). A collector Failed/Incomplete state, a missing/invalid required + file, an auth-quit interruption, or an orchestration-level null result is a GAP and must never + be reported as clean. Header-only output alone never decides WHY a table is empty — the + collector's recorded state does. + .OUTPUTS + [pscustomobject] with Status, HadGaps, AllTablesTerminal, CleanCompletion, Missing, Failed, + Incomplete, Unavailable, Empty, Succeeded, AuthQuit, ExportErrored, PerTable. + #> + [CmdletBinding()] + param([object]$Result, [Parameter(Mandatory)][string]$DestinationDir) + + $schemas = $script:AISIDTableSchemas + $order = @('ActivitySessions', 'OffHoursGeo', 'FileProximity', 'OAuthConsents', 'SSOSignIns', 'ClientChannel', 'CopilotUsage', 'AppGovAlerts', 'CloudDiscovery', 'MDASessions', 'SolutionsCatalog', 'EntraUsers') + $exportErrored = ($null -eq $Result) + $ts = if ($Result -and $Result.TableStatus) { $Result.TableStatus } else { $null } + + $missing = @(); $failed = @(); $incomplete = @(); $unavailable = @(); $empty = @(); $succeeded = @() + $perTable = [ordered]@{} + $allTerminal = (-not $exportErrored) + + foreach ($t in $order) { + $sch = $schemas[$t] + # Collector-recorded state (authoritative for WHY). Absent => the export never decided this table. + $state = 'Incomplete'; $reason = 'export did not return a state for this table' + if ($ts -and $ts.Contains($t)) { $state = [string]$ts[$t].State; $reason = [string]$ts[$t].Reason } + else { $allTerminal = $false } + + # File presence + header validity (the dashboard model binds each CSV by a fixed header). + $fileOk = $false; $fileNote = 'missing' + if ($sch) { + $path = Join-Path $DestinationDir $sch.FileName + if (Test-Path -LiteralPath $path) { + $header = $null + try { $header = (Get-Content -LiteralPath $path -TotalCount 1 -ErrorAction Stop) } catch { $header = $null } + if (-not [string]::IsNullOrEmpty($header) -and ($header.Trim() -eq ($sch.Columns -join ','))) { $fileOk = $true; $fileNote = 'present' } + else { $fileNote = 'invalid header' } + } + } + if (-not $fileOk) { $missing += $sch.FileName } + + # A Succeeded/Empty/Unavailable state that lost its file is a contradiction -> treat as a gap. + $effective = $state + if (-not $fileOk -and $state -in @('Succeeded', 'Empty', 'Unavailable')) { $effective = 'Failed'; $reason = ("recorded {0} but output file is {1}" -f $state, $fileNote) } + + switch ($effective) { + 'Succeeded' { $succeeded += $t } + 'Empty' { $empty += $t } + 'Unavailable' { $unavailable += $t } + 'Failed' { $failed += $t } + default { $incomplete += $t } + } + $perTable[$t] = [ordered]@{ State = $effective; Reason = $reason; File = $fileNote } + } + + $authQuit = [bool]($Result -and $Result.AuthQuit) + $hadGaps = $exportErrored -or ($failed.Count -gt 0) -or ($incomplete.Count -gt 0) -or ($missing.Count -gt 0) -or $authQuit + $cleanCompletion = (-not $hadGaps) -and $allTerminal -and (($succeeded.Count + $empty.Count + $unavailable.Count) -eq $order.Count) + $status = if ($authQuit) { 'interrupted' } elseif ($hadGaps) { 'completed-with-gaps' } elseif ($unavailable.Count -gt 0) { 'complete-with-unavailable' } elseif ($empty.Count -gt 0) { 'complete-with-empty' } else { 'complete' } + + Write-LogHost '[AISID] ===== Dashboard completion summary =====' -ForegroundColor Cyan + Write-LogHost ("[AISID] Required dashboard files present: {0}/{1}" -f ($order.Count - @($missing | Sort-Object -Unique).Count), $order.Count) -ForegroundColor Gray + if ($succeeded.Count -gt 0) { Write-LogHost ("[AISID] Succeeded: {0}" -f ($succeeded -join ', ')) -ForegroundColor Green } + if ($empty.Count -gt 0) { Write-LogHost ("[AISID] Empty (query succeeded, zero rows): {0}" -f ($empty -join ', ')) -ForegroundColor Gray } + if ($unavailable.Count -gt 0) { Write-LogHost ("[AISID] Unavailable (licensing ineligible): {0}" -f ($unavailable -join ', ')) -ForegroundColor Gray } + if ($failed.Count -gt 0) { Write-LogHost ("[AISID] FAILED (real data gap, NOT a factual empty): {0}" -f (@($failed | Sort-Object -Unique) -join ', ')) -ForegroundColor Red } + if ($incomplete.Count -gt 0) { Write-LogHost ("[AISID] Incomplete (collector not reached / existing file unreadable): {0}" -f (@($incomplete | Sort-Object -Unique) -join ', ')) -ForegroundColor Yellow } + if (@($missing | Sort-Object -Unique).Count -gt 0) { Write-LogHost ("[AISID] Missing/invalid required files: {0}" -f (@($missing | Sort-Object -Unique) -join ', ')) -ForegroundColor Red } + if ($authQuit) { Write-LogHost '[AISID] Collection was interrupted (authentication quit) before all data sets finished.' -ForegroundColor Red } + if ($exportErrored) { Write-LogHost '[AISID] Dashboard export failed at the orchestration level (isolated error); output is incomplete.' -ForegroundColor Red } + Write-LogHost ("[AISID] Overall AISID status: {0}" -f $status) -ForegroundColor Cyan + + return [pscustomobject]@{ + Status = $status; HadGaps = $hadGaps; AllTablesTerminal = $allTerminal; CleanCompletion = $cleanCompletion + Missing = @($missing | Sort-Object -Unique); Failed = @($failed | Sort-Object -Unique); Incomplete = @($incomplete | Sort-Object -Unique) + Unavailable = @($unavailable); Empty = @($empty); Succeeded = @($succeeded); AuthQuit = $authQuit; ExportErrored = $exportErrored; PerTable = $perTable + } +} + +function Send-PromptNotification { + <# + .SYNOPSIS + Plays a system beep to alert user that a prompt requires attention. + .DESCRIPTION + Useful when user is working in other windows and needs to be notified + when a prompt appears that requires input. + #> + + try { + # Play 3 short beeps to get attention + [Console]::Beep(800, 200) # 800Hz for 200ms + Start-Sleep -Milliseconds 100 + [Console]::Beep(1000, 200) # 1000Hz for 200ms + Start-Sleep -Milliseconds 100 + [Console]::Beep(1200, 300) # 1200Hz for 300ms (slightly longer final beep) + } + catch { + # Silently fail if beep not supported (e.g., some server environments) + } +} + +# Validate -OnlyUserInfo parameter compatibility +if ($OnlyUserInfo) { + $incompatibleParams = @() + + # Date filtering parameters + if ($PSBoundParameters.ContainsKey('StartDate')) { $incompatibleParams += " - StartDate (not applicable for user-only export)" } + if ($PSBoundParameters.ContainsKey('EndDate')) { $incompatibleParams += " - EndDate (not applicable for user-only export)" } + + # Activity configuration parameters + if ($PSBoundParameters.ContainsKey('ActivityTypes')) { $incompatibleParams += " - ActivityTypes (cleared by -OnlyUserInfo)" } + if ($IncludeM365Usage) { $incompatibleParams += " - IncludeM365Usage (activity type modifier)" } + if ($ExcludeCopilotInteraction) { $incompatibleParams += " - ExcludeCopilotInteraction (activity type modifier)" } + + # Audit retrieval settings + if ($PSBoundParameters.ContainsKey('BlockHours') -and $BlockHours -ne 0.5) { $incompatibleParams += " - BlockHours (audit query partitioning)" } + if ($PSBoundParameters.ContainsKey('PartitionHours') -and $PartitionHours -ne 0) { $incompatibleParams += " - PartitionHours (audit query partitioning)" } + if ($PSBoundParameters.ContainsKey('MaxPartitions') -and $MaxPartitions -ne 160) { $incompatibleParams += " - MaxPartitions (audit query limits)" } + if ($PSBoundParameters.ContainsKey('ResultSize') -and $ResultSize -ne 10000) { $incompatibleParams += " - ResultSize (audit query page size)" } + if ($PSBoundParameters.ContainsKey('PacingMs') -and $PacingMs -ne 0) { $incompatibleParams += " - PacingMs (audit query throttling)" } + if ($AutoCompleteness) { $incompatibleParams += " - AutoCompleteness (audit log completeness checks)" } + if ($PSBoundParameters.ContainsKey('StreamingSchemaSample') -and $StreamingSchemaSample -ne 5000) { $incompatibleParams += " - StreamingSchemaSample (audit record schema sampling)" } + if ($PSBoundParameters.ContainsKey('StreamingChunkSize') -and $StreamingChunkSize -ne 5000) { $incompatibleParams += " - StreamingChunkSize (audit streaming batch size)" } + if ($PSBoundParameters.ContainsKey('ExportProgressInterval') -and $ExportProgressInterval -ne 10) { $incompatibleParams += " - ExportProgressInterval (audit export progress)" } + + # Filtering parameters + if ($PSBoundParameters.ContainsKey('AgentId')) { $incompatibleParams += " - AgentId (audit record filtering)" } + if ($AgentsOnly) { $incompatibleParams += " - AgentsOnly (audit record filtering)" } + if ($ExcludeAgents) { $incompatibleParams += " - ExcludeAgents (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('PromptFilter')) { $incompatibleParams += " - PromptFilter (audit record content filtering)" } + if ($PSBoundParameters.ContainsKey('RecordTypes')) { $incompatibleParams += " - RecordTypes (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('ServiceTypes')) { $incompatibleParams += " - ServiceTypes (audit record filtering)" } + + # Processing mode parameters + if ($ExplodeArrays) { $incompatibleParams += " - ExplodeArrays (audit record array expansion)" } + if ($ExplodeDeep) { $incompatibleParams += " - ExplodeDeep (audit record deep expansion)" } + if ($PSBoundParameters.ContainsKey('RAWInputCSV')) { $incompatibleParams += " - RAWInputCSV (offline audit replay mode)" } + + # Parallel processing parameters + if ($EnableParallel) { $incompatibleParams += " - EnableParallel (parallel audit query execution)" } + if ($PSBoundParameters.ContainsKey('MaxConcurrency') -and $MaxConcurrency -ne 10) { $incompatibleParams += " - MaxConcurrency (concurrent query/partition limit)" } + if ($PSBoundParameters.ContainsKey('MaxParallelGroups') -and $MaxParallelGroups -ne 8) { $incompatibleParams += " - MaxParallelGroups (parallel activity group limit)" } + if ($PSBoundParameters.ContainsKey('ParallelMode') -and $ParallelMode -ne 'Auto') { $incompatibleParams += " - ParallelMode (parallel processing mode)" } + if ($DisableAdaptive) { $incompatibleParams += " - DisableAdaptive (adaptive concurrency controls)" } + if ($PSBoundParameters.ContainsKey('ProgressSmoothingAlpha') -and $ProgressSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ProgressSmoothingAlpha (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('HighLatencyMs') -and $HighLatencyMs -ne 90000) { $incompatibleParams += " - HighLatencyMs (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('MemoryPressureMB') -and $MemoryPressureMB -ne 1500) { $incompatibleParams += " - MemoryPressureMB (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('LowLatencyMs') -and $LowLatencyMs -ne 20000) { $incompatibleParams += " - LowLatencyMs (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('LowLatencyConsecutive') -and $LowLatencyConsecutive -ne 2) { $incompatibleParams += " - LowLatencyConsecutive (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('ThroughputDropPct') -and $ThroughputDropPct -ne 15) { $incompatibleParams += " - ThroughputDropPct (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('ThroughputSmoothingAlpha') -and $ThroughputSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ThroughputSmoothingAlpha (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('AdaptiveConcurrencyCeiling') -and $AdaptiveConcurrencyCeiling -ne 6) { $incompatibleParams += " - AdaptiveConcurrencyCeiling (adaptive tuning)" } + + # Reliability parameters (audit-specific) + if ($PSBoundParameters.ContainsKey('CircuitBreakerThreshold') -and $CircuitBreakerThreshold -ne 5) { $incompatibleParams += " - CircuitBreakerThreshold (block failure circuit breaker)" } + if ($PSBoundParameters.ContainsKey('CircuitBreakerCooldownSeconds') -and $CircuitBreakerCooldownSeconds -ne 120) { $incompatibleParams += " - CircuitBreakerCooldownSeconds (circuit breaker cooldown)" } + if ($PSBoundParameters.ContainsKey('BackoffBaseSeconds') -and $BackoffBaseSeconds -ne 1.0) { $incompatibleParams += " - BackoffBaseSeconds (block retry backoff)" } + if ($PSBoundParameters.ContainsKey('BackoffMaxSeconds') -and $BackoffMaxSeconds -ne 45) { $incompatibleParams += " - BackoffMaxSeconds (block retry max backoff)" } + + # Alternative modes + if ($UseEOM) { $incompatibleParams += " - UseEOM (Exchange Online Management mode incompatible with Graph Entra enrichment)" } + + # Output combination parameters + if ($CombineOutput) { $incompatibleParams += " - CombineOutput (only relevant with multiple activity types)" } + if ($AppendFile) { $incompatibleParams += " - AppendFile (appending user-only data to existing audit output not supported)" } + + if ($incompatibleParams.Count -gt 0) { + Write-Host "" + Write-Host "ERROR: The -OnlyUserInfo switch cannot be used with the following parameters:" -ForegroundColor Red + Write-Host "" + $incompatibleParams | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } + Write-Host "" + Write-Host "The -OnlyUserInfo switch exports only Entra user directory and license information (no audit logs)." -ForegroundColor Cyan + Write-Host "" + Write-Host "Compatible parameters:" -ForegroundColor Green + Write-Host " - OutputPath (where to save the file)" -ForegroundColor White + Write-Host " - UserIds / GroupNames (scope the exported directory to these users / group members)" -ForegroundColor White + Write-Host " - Auth (authentication method: WebLogin, DeviceCode, Credential, Silent)" -ForegroundColor White + Write-Host " - Force (bypass interactive prompts)" -ForegroundColor White + Write-Host " - MaxNetworkOutageMinutes (network resilience for Graph API calls)" -ForegroundColor White + Write-Host " - EmitMetricsJson (track Entra retrieval metrics)" -ForegroundColor White + Write-Host " - MetricsPath (custom metrics output location)" -ForegroundColor White + Write-Host " - SkipDiagnostics (skip pre-query capability checks)" -ForegroundColor White + Write-Host "" + Write-Host "Please remove the incompatible parameters and try again." -ForegroundColor Cyan + Write-Host "" + exit 1 + } + + # If validation passes, configure for user-only export + Write-Host "" + Write-Host "INFO: -OnlyUserInfo mode enabled. Skipping all audit log retrieval, exporting only Entra user data." -ForegroundColor Green + Write-Host "" + $IncludeUserInfo = $true + $ActivityTypes = @() +} + +# PAX4A-GUARD-BEGIN +# -UserInfoFile / -GroupNames MUTUAL EXCLUSIVITY +# -UserInfoFile supplies the Entra user directory from a customer-provided CSV (bypassing the +# live /users pull). -GroupNames expands audit records against the LIVE directory. Supplying both +# is contradictory. Hard-stop here in the early parameter-validation stage so it fires on EVERY +# entry path (fresh and -Resume) BEFORE any -GroupNames consumer downstream (the RequiredScopes +# builder and the permissions banner). No partial work is done. +if ($PSBoundParameters.ContainsKey('UserInfoFile') -and $PSBoundParameters.ContainsKey('GroupNames')) { + Write-Host "" + Write-Host "ERROR: -UserInfoFile and -GroupNames cannot both be supplied." -ForegroundColor Red + Write-Host " -UserInfoFile ingests the Entra user directory from a customer-provided CSV." -ForegroundColor Yellow + Write-Host " -GroupNames expands audit records against the live Entra directory." -ForegroundColor Yellow + Write-Host " Supply exactly ONE: -UserInfoFile for a supplied directory, or -GroupNames for group filtering." -ForegroundColor Yellow + Write-Host "" + exit 1 +} +# PAX4A-GUARD-END + +# PAX4D-AUTOENABLE-BEGIN +# -UserInfoFile auto-enables -IncludeUserInfo (FU6): every -UserInfoFile export path (file naming, +# the multi-tab workbook, and the directory substitution seam) is gated on $IncludeUserInfo, so a +# customer need only pass -UserInfoFile. Mirrors the -OnlyUserInfo auto-enable above; fires on every +# entry path before any $IncludeUserInfo consumer. No effect when -UserInfoFile is unset. +if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile) -and -not $IncludeUserInfo) { + $IncludeUserInfo = $true +} +# PAX4D-AUTOENABLE-END + +# PAX-UIS-GUARD-BEGIN +# -UserInfoSupplement is the HYBRID directory-enrichment mode: the live Entra directory is fetched +# normally and the supplemental CSV columns are appended by an Entra-left join. It is distinct from +# -UserInfoFile (which REPLACES the directory), so the two are mutually exclusive. It also requires the +# live /users pull, so it is incompatible with -UseEOM (no Graph directory) and -RAWInputCSV (offline +# replay). Unlike -UserInfoFile, it is COMPATIBLE with -GroupNames. These hard-stops fire on every entry +# path (fresh and -Resume) before any consumer. It auto-enables -IncludeUserInfo. +$script:UserInfoSupplementActive = ($PSBoundParameters.ContainsKey('UserInfoSupplement') -and -not [string]::IsNullOrWhiteSpace($UserInfoSupplement)) +if ($script:UserInfoSupplementActive) { + if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile)) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement and -UserInfoFile cannot both be supplied." -ForegroundColor Red + Write-Host " -UserInfoSupplement ENRICHES the live Entra directory (hybrid); -UserInfoFile REPLACES it." -ForegroundColor Yellow + Write-Host " Supply exactly ONE." -ForegroundColor Yellow + Write-Host "" + exit 1 + } + if ($UseEOM) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement requires the Graph /users directory and is not supported with -UseEOM." -ForegroundColor Red + Write-Host "" + exit 1 + } + if ($PSBoundParameters.ContainsKey('RAWInputCSV') -and -not [string]::IsNullOrWhiteSpace($RAWInputCSV)) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement enriches a live directory fetch and cannot be combined with -RAWInputCSV (offline replay)." -ForegroundColor Red + Write-Host "" + exit 1 + } + if (-not $IncludeUserInfo) { $IncludeUserInfo = $true } +} +# PAX-UIS-GUARD-END + +# Canonical maps for Graph filter normalization +$recordTypeCanonicalMap = @{ + 'azureactivedirectory' = 'AzureActiveDirectory' + 'azureactivedirectoryaccountlogon' = 'AzureActiveDirectoryAccountLogon' + 'azureactivedirectorystslogon' = 'AzureActiveDirectoryStsLogon' + 'exchangeadmin' = 'ExchangeAdmin' + 'exchangeitem' = 'ExchangeItem' + 'exchangemailbox' = 'ExchangeMailbox' + 'sharepointfileoperation' = 'SharePointFileOperation' + 'sharepointsharingoperation' = 'SharePointSharingOperation' + 'sharepoint' = 'SharePoint' + 'onedrive' = 'OneDrive' + 'microsoftteams' = 'MicrosoftTeams' +} + +$serviceCanonicalMap = @{ + 'azureactivedirectory' = 'AzureActiveDirectory' + 'exchange' = 'Exchange' + 'sharepoint' = 'SharePoint' + 'onedrive' = 'OneDrive' + 'teams' = 'Teams' +} + +# Normalize optional Graph filter passthrough parameters (dedupe & trim) +# Split ActivityTypes if provided as comma-separated string +if ($ActivityTypes) { + $processedActivityTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $ActivityTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedActivityTypes.Add($token) + } + } + $ActivityTypes = @( + $processedActivityTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($ActivityTypes.Count -eq 0) { + $ActivityTypes = $null + } +} + +if ($RecordTypes) { + $processedRecordTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $RecordTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedRecordTypes.Add($token) + } + } + $RecordTypes = @( + $processedRecordTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($RecordTypes.Count -eq 0) { + $RecordTypes = $null + } else { + $RecordTypes = @( + foreach ($rt in $RecordTypes) { + $key = $rt.ToLowerInvariant() + if ($recordTypeCanonicalMap.ContainsKey($key)) { $recordTypeCanonicalMap[$key] } else { $rt } + } + ) | Select-Object -Unique + } +} + +if ($ServiceTypes) { + $processedServiceTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $ServiceTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedServiceTypes.Add($token) + } + } + $ServiceTypes = @( + $processedServiceTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($ServiceTypes.Count -eq 0) { + $ServiceTypes = $null + } else { + $ServiceTypes = @( + foreach ($svc in $ServiceTypes) { + $key = $svc.ToLowerInvariant() + if ($serviceCanonicalMap.ContainsKey($key)) { $serviceCanonicalMap[$key] } else { $svc } + } + ) | Select-Object -Unique + } +} + +# Mapping of audit record types to supported workloads for Graph security audit queries +$recordTypeWorkloadMap = @{ + 'azureActiveDirectory' = @('AzureActiveDirectory') + 'azureActiveDirectoryAccountLogon' = @('AzureActiveDirectory') + 'azureActiveDirectoryStsLogon' = @('AzureActiveDirectory') + 'exchangeAdmin' = @('Exchange') + 'exchangeItem' = @('Exchange') + 'exchangeMailbox' = @('Exchange') + 'sharePointFileOperation' = @('SharePoint','OneDrive') + 'sharePointSharingOperation' = @('SharePoint','OneDrive') + 'sharePoint' = @('SharePoint','OneDrive') + 'onedrive' = @('OneDrive') + 'microsoftTeams' = @('Teams') + # M365 usage record types: Process in first workload pass to avoid creating additional passes + # These record types capture cross-workload activities (Office apps, Forms, Stream, Planner, PowerApps) + # Mapping to Exchange ensures they run in the first service-filtered pass + 'officeNative' = @('Exchange') + 'microsoftForms' = @('Exchange') + 'microsoftStream' = @('Exchange') + 'plannerPlan' = @('Exchange') + 'plannerTask' = @('Exchange') + 'powerAppsApp' = @('Exchange') +} + +$serviceOperationMap = @{ + 'AzureActiveDirectory' = @('UserLoggedIn','UserLoginFailed','AdminLoggedIn','ResetUserPassword','AddRegisteredUser','UpdateUser','ChangedUserSetting') + 'Exchange' = @('MailItemsAccessed','Send','SendOnBehalf','SoftDelete','HardDelete','MoveToDeletedItems','CopyToFolder','AddMailboxPermission','RemoveMailboxPermission') + 'SharePoint' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'OneDrive' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'Teams' = @('TeamMemberAdded','TeamMemberRemoved','ChannelAdded','ChannelDeleted','ChannelMessageSent','ChannelMessageDeleted','TeamDeleted','TeamArchived','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'MicrosoftForms' = @('CreateForm','EditForm','DeleteForm','ViewForm','CreateResponse','SubmitResponse','ViewResponse','DeleteResponse') + 'MicrosoftStream' = @('StreamModified','StreamViewed','StreamDeleted','StreamDownloaded') + 'MicrosoftPlanner' = @('PlanCreated','PlanDeleted','PlanModified','TaskCreated','TaskDeleted','TaskModified','TaskAssigned','TaskCompleted') + 'PowerApps' = @('LaunchedApp','CreatedApp','EditedApp','DeletedApp','PublishedApp') +} + +$copilotBaseActivityType = 'CopilotInteraction' +$m365UsageServiceBundle = @('Exchange','SharePoint','OneDrive','Teams') +$m365UsageRecordBundle = @('ExchangeAdmin','ExchangeItem','ExchangeMailbox','SharePointFileOperation','SharePointSharingOperation','SharePoint','OneDrive','MicrosoftTeams','OfficeNative','MicrosoftForms','MicrosoftStream','PlannerPlan','PlannerTask','PowerAppsApp') +# Curated, trimmed M365 usage operations targeted at the Analytics-Hub M365 Usage Analytics +# dashboard. Scope: Exchange mail access, SharePoint/OneDrive file access, Teams chat/messaging, +# Teams meeting lifecycle, and Copilot/Connected-AI interaction signals. +$m365UsageActivityBundle = @( + # === Exchange / Email === + 'MailItemsAccessed','MailboxLogin','Send', + + # === SharePoint / OneDrive - File access === + 'FileAccessed','FileViewed','FilePreviewed','FileModified','FileDownloaded','FileUploaded', + + # === Teams - Chat / Messaging === + 'MessageSent','MessageRead','MessagesListed','ChatRetrieved','ChatCreated','TeamsSessionStarted', + + # === Teams - Meeting lifecycle === + 'MeetingParticipantJoined','MeetingStarted','MeetingEnded','MeetingParticipantDetail','MeetingDetail', + + # === Copilot / Connected AI === + 'CopilotInteraction','ConnectedAIAppInteraction' +) | Select-Object -Unique + +# Script version constant (must appear after param/help to keep param() valid as first executable block) +$ScriptVersion = '1.11.15' + +function Invoke-PaxVersionCheck { + # Informational, non-blocking, failure-isolated version check against the public PAX repo. + # Reads versions.json from the release branch, compares the purview script version, and prints + # a single info line. Never prompts, never throws, capped at ~5s if the server is unreachable. + param([string]$CurrentVersion) + $repoUrl = 'https://github.com/microsoft/PAX' + try { + $verUrl = 'https://raw.githubusercontent.com/microsoft/PAX/release/versions.json' + $resp = Invoke-RestMethod -Uri $verUrl -TimeoutSec 5 -ErrorAction Stop + $latest = [string]$resp.products.purview.version + $relDate = [string]$resp.lastUpdated + if ($latest -and ([version]$latest) -gt ([version]$CurrentVersion)) { + $line = " Update available: PAX v$latest" + if ($relDate) { $line += " (released $relDate)" } + $line += " - you are on v$CurrentVersion. Latest: $repoUrl" + Write-LogHost $line -ForegroundColor Cyan + } else { + Write-LogHost " Version check: you are on the latest PAX version (v$CurrentVersion). $repoUrl" -ForegroundColor DarkGray + } + } catch { + Write-LogHost " Version check skipped: the PAX GitHub repo was not reachable (offline or blocked). Latest: $repoUrl" -ForegroundColor DarkGray + } +} + +# --- Initialize/Clear persistent script variables to prevent cross-run contamination --- +# Note: Script-scoped variables persist across multiple script invocations in the same PowerShell session +$script:partitionStatus = $null +$script:processedJobIds = $null +$script:shownJobMessages = $null + +# --- Microsoft 365 Copilot license detection --- +# No hardcoded SKU GUIDs. Copilot service plans are discovered dynamically at runtime +# from /subscribedSkus by matching servicePlanName against the pattern '*COPILOT*'. +# Per-user eligibility is then computed from the user's assignedPlans where +# capabilityStatus == 'Enabled' and servicePlanId is in the discovered set. +# See Get-UserLicenseData(). + +# --- Synchronized Timestamp & OutputPath Validation --- + +# Generate synchronized timestamp for all output files in this run +$global:ScriptRunTimestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + +# --- Logging Helper Functions (defined early for use throughout script) --- +# $script:LogFile was already initialized to a bootstrap temp-path file at the top +# of the script body (right after param()). DO NOT re-null it here; that would +# discard the early-startup log entries already captured. The bootstrap file is +# renamed to the final location once $OutputFile is resolved (see the relocation +# block further down). The in-memory $script:LogBuffer remains a fallback for the +# rare case where bootstrap creation failed (e.g. read-only temp). +if (-not (Get-Variable -Name LogFile -Scope Script -ErrorAction SilentlyContinue)) { + $script:LogFile = $null +} +$script:LogBuffer = New-Object System.Collections.Generic.List[string] + +function Write-Log { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [$Level] $Message" + Microsoft.PowerShell.Utility\Write-Host $Message + try { + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# Log-file-only writer: never echoes to host. Used for diagnostic detail (temp paths, +# argument vectors, stack traces) that should be captured in the run log but not +# clutter the customer-facing console. +function Write-LogFile { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [$Level] $Message" + try { + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# --- Shared memory / stage observability --------------------------------------- +# Lightweight, additive heartbeat: samples the parent process working set and +# managed heap, an optional Python child process working set, rows/pages +# processed, and temporary storage bytes, then appends a single diagnostic line +# to the existing run log (Write-LogFile). No new required output file is +# created, and it NEVER surfaces identifying row values. Emission is throttled to +# the progress interval (default 30s) unless -Force is given. Every metric read +# is guarded so a missing child process or an inaccessible counter can never +# crash the run. This is observation only; it changes no success/failure result. +function Write-PaxMemoryObservation { + [CmdletBinding()] + param( + [string] $Stage = '', + [AllowNull()] [System.Nullable[int]] $ChildPid = $null, + [AllowNull()] [System.Nullable[long]] $RowsProcessed = $null, + [AllowNull()] [System.Nullable[long]] $PagesProcessed = $null, + [string] $TempDir = '', + [switch] $Force + ) + try { + $intervalSec = 30 + try { if ($script:ProgressIntervalSeconds -and [int]$script:ProgressIntervalSeconds -gt 0) { $intervalSec = [int]$script:ProgressIntervalSeconds } } catch {} + $nowUtc = [DateTime]::UtcNow + if (-not $Force) { + $last = $script:PaxLastMemObsUtc + if ($last -and (($nowUtc - $last).TotalSeconds -lt $intervalSec)) { return } + } + $script:PaxLastMemObsUtc = $nowUtc + + $proc = [System.Diagnostics.Process]::GetCurrentProcess() + $parentWs = [long]$proc.WorkingSet64 + $heap = [long][System.GC]::GetTotalMemory($false) + $childWs = $null + if ($null -ne $ChildPid) { + try { $cp = [System.Diagnostics.Process]::GetProcessById([int]$ChildPid); $childWs = [long]$cp.WorkingSet64 } catch { $childWs = $null } + } + $tempBytes = $null + if ($TempDir -and (Test-Path -LiteralPath $TempDir)) { + try { $tempBytes = [long](@(Get-ChildItem -LiteralPath $TempDir -Recurse -File -ErrorAction SilentlyContinue | Measure-Object -Property Length -Sum).Sum) } catch { $tempBytes = $null } + } + $lastProg = $null + try { $lastProg = $script:PaxLastProgressUtc } catch {} + + $parts = New-Object System.Collections.Generic.List[string] + $parts.Add(("ts={0}" -f $nowUtc.ToString('o'))) + if ($Stage) { $parts.Add("stage=$Stage") } + $parts.Add("pid=$($proc.Id)") + $parts.Add(("parentWS_MB={0:N1}" -f ($parentWs / 1MB))) + $parts.Add(("heap_MB={0:N1}" -f ($heap / 1MB))) + if ($null -ne $ChildPid) { $parts.Add("childPID=$ChildPid") } + if ($null -ne $childWs) { $parts.Add(("childWS_MB={0:N1}" -f ($childWs / 1MB))) } + if ($null -ne $RowsProcessed) { $parts.Add("rows=$RowsProcessed") } + if ($null -ne $PagesProcessed) { $parts.Add("pages=$PagesProcessed") } + if ($null -ne $tempBytes) { $parts.Add("tempBytes=$tempBytes") } + if ($lastProg) { try { $parts.Add("lastProgress=$($lastProg.ToString('o'))") } catch {} } + try { Write-LogFile ("MEMOBS " + ($parts -join ' ')) } catch {} + + # Optional in-memory metrics sink (append-only) when the run maintains one. + try { + if ($null -ne $script:PaxMemObsList) { + [void]$script:PaxMemObsList.Add([ordered]@{ ts = $nowUtc.ToString('o'); stage = $Stage; pid = $proc.Id; parentWSBytes = $parentWs; heapBytes = $heap; childPID = $ChildPid; childWSBytes = $childWs; rows = $RowsProcessed; pages = $PagesProcessed; tempBytes = $tempBytes }) + } + } catch {} + } + catch {} +} + +function Write-LogHost { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$ForegroundColor = "White") + Microsoft.PowerShell.Utility\Write-Host $Message -ForegroundColor $ForegroundColor + try { + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [INFO] $Message" + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } else { $script:LogBuffer.Add($logEntry) | Out-Null } } catch {} } @@ -2851,41 +6611,78 @@ function Invoke-PaxRawDeidentify { # NOT flat-hashed — their PII lives in nested keys the column map cannot reach. $jsonCols = @('AuditData', 'CopilotEventData') - $rows = @(Import-Csv -LiteralPath $Path) - if ($rows.Count -eq 0) { return } - $columns = $rows[0].PSObject.Properties.Name - $active = @{} - foreach ($c in $columns) { if ($map.ContainsKey($c)) { $active[$c] = $map[$c] } } - $activeJson = @($jsonCols | Where-Object { $columns -contains $_ }) - if ($active.Count -eq 0 -and $activeJson.Count -eq 0) { return } - - foreach ($row in $rows) { - foreach ($col in @($active.Keys)) { - $val = $row.$col - if ([string]::IsNullOrEmpty($val)) { continue } - $row.$col = switch ($active[$col]) { - 'Upn' { Get-PaxDeidUpn -Value $val } - 'Name' { Get-PaxDeidName -Value $val } - 'Guid' { Get-PaxDeidGuid -Value $val } - 'Sid' { Get-PaxDeidSid -Value $val } - 'Token' { Get-PaxDeidToken -Value $val } - 'Resource' { Get-PaxDeidResource -Value $val } - 'File' { Get-PaxDeidFile -Value $val } - 'Proxy' { Get-PaxDeidProxy -Value $val } - 'Ip' { Get-PaxDeidIp -Value $val } - } + # Stream the RAW file record-by-record (RFC-4180) and write each deidentified row + # through the existing streaming CSV writer in bounded batches, so the complete + # file is never materialized. Token generation is stateless/deterministic, so no + # per-identity map is retained. Column order is preserved from the header; output + # quoting is byte-identical to the prior in-place rewrite (same writer). + $reader = [System.IO.StreamReader]::new($Path, $true) + $readerDone = $false + try { + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $header) { return } + $columns = @($header | ForEach-Object { [string]$_ }) + $active = @{} + foreach ($c in $columns) { if ($map.ContainsKey($c)) { $active[$c] = $map[$c] } } + $activeJson = @($jsonCols | Where-Object { $columns -contains $_ }) + if ($active.Count -eq 0 -and $activeJson.Count -eq 0) { return } + $colCount = $columns.Count + $activeKeys = @($active.Keys) + $tmp = "$Path.deid.tmp" + $rowCount = 0 + $batch = New-Object System.Collections.Generic.List[object] + try { + try { + Open-CsvWriter -Path $tmp -Columns $columns + while ($true) { + $fields = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $fields) { break } + $row = [ordered]@{} + for ($i = 0; $i -lt $colCount; $i++) { $row[$columns[$i]] = if ($i -lt $fields.Length) { [string]$fields[$i] } else { '' } } + foreach ($col in $activeKeys) { + $val = $row[$col] + if ([string]::IsNullOrEmpty($val)) { continue } + $row[$col] = switch ($active[$col]) { + 'Upn' { Get-PaxDeidUpn -Value $val } + 'Name' { Get-PaxDeidName -Value $val } + 'Guid' { Get-PaxDeidGuid -Value $val } + 'Sid' { Get-PaxDeidSid -Value $val } + 'Token' { Get-PaxDeidToken -Value $val } + 'Resource' { Get-PaxDeidResource -Value $val } + 'File' { Get-PaxDeidFile -Value $val } + 'Proxy' { Get-PaxDeidProxy -Value $val } + 'Ip' { Get-PaxDeidIp -Value $val } + } + } + foreach ($jc in $activeJson) { + if (-not [string]::IsNullOrEmpty($row[$jc])) { $row[$jc] = Get-PaxDeidJson -Value $row[$jc] } + } + $batch.Add([pscustomobject]$row) + $rowCount++ + if ($batch.Count -ge 5000) { Write-CsvRows -Rows $batch -Columns $columns; $batch.Clear() } + } + if ($batch.Count -gt 0) { Write-CsvRows -Rows $batch -Columns $columns } + } + finally { Close-CsvWriter } + # Release the input handle BEFORE the in-place replace so Move-Item can + # overwrite $Path. + $reader.Dispose(); $readerDone = $true + if ($rowCount -eq 0) { + # Header-only source: the prior pass was a no-op (Import-Csv => 0 rows => return). + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + return + } + # Atomic in-place publish (temp + replace), preserving original column order. + Move-Item -LiteralPath $tmp -Destination $Path -Force -ErrorAction Stop } - foreach ($jc in $activeJson) { - if (-not [string]::IsNullOrEmpty($row.$jc)) { $row.$jc = Get-PaxDeidJson -Value $row.$jc } + catch { + # Fail closed: never leave a partial temp; the original file is untouched + # because the replace happens only on the last line above. + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + throw } } - - # Atomic in-place rewrite (temp + replace), preserving original column order. - $tmp = "$Path.deid.tmp" - Open-CsvWriter -Path $tmp -Columns $columns - Write-CsvRows -Rows $rows -Columns $columns - Close-CsvWriter - Move-Item -LiteralPath $tmp -Destination $Path -Force + finally { if (-not $readerDone) { $reader.Dispose() } } } function Test-PaxFileDeidentified { @@ -2987,7 +6784,7 @@ function Test-PAYGBillingEnabled { # Note: The most reliable way to detect PAYG is to attempt a query for AIAppInteraction # and check for specific error responses. However, this requires actual data/timeframe. - # For now, we return $null to indicate "unknown" and let the post-query detection handle it. + # Returns $null to indicate "unknown" and lets the post-query detection handle it. Write-LogHost "PAYG billing detection: Deferred to post-query validation" -ForegroundColor DarkGray return $null @@ -3306,7 +7103,7 @@ $script:NormalizeFolderUrl = { if ($script:RemoteOutputUrl) { $script:RemoteOutputUrl = & $script:NormalizeFolderUrl $script:RemoteOutputUrl } -foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { +foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if (-not $script:DestRaw.ContainsKey($k)) { continue } if (-not $script:DestTier.ContainsKey($k)) { continue } if ($script:DestTier[$k] -eq 'Local') { continue } @@ -3524,6 +7321,41 @@ if ($env:IDENTITY_ENDPOINT -and $Auth -ne 'ManagedIdentity') { # Data-type keys: 'Purview', 'UserInfo', 'Agent365Info', 'DefenderUsage', 'Log'. # When a non-Purview data type was not bound on the CLI, it inherits the Purview # tier/destination so legacy single-destination behavior is preserved. +function script:Get-AISIDResolvedOutputDir { + # Blocker-6: single source of truth for the resolved AISID output BUNDLE folder shared by the + # parameter snapshot, the checkpoint parameter snapshot, the metrics JSON, recovery messages, the + # output roster, and resume validation. Reports the ACTUAL per-run bundle location — the + # 'AISID_' child folder — via the canonical run-folder resolver, NOT the flat + # DefenderUsage parent, and NEVER the internal OS-temp scratch staging path. Local returns + # \AISID_; SharePoint/Fabric returns /AISID_. Returns '' for non-AISID runs. + if ($Dashboard -ne 'AISID') { return '' } + try { + $_run = script:Resolve-AISIDRunFolder + if ($_run -and $_run.DisplayPath) { return [string]$_run.DisplayPath } + } catch {} + return '' +} + +function script:Get-AISIDSnapshotFields { + # Builds the AISID field group embedded in the metrics JSON. Always returns all three keys so + # the metrics schema stays stable across runs; AISID-only values are null/false on a non-AISID + # run, matching the existing metrics-field convention. Pure (all inputs passed in) so the + # parameter-snapshot coverage tests can exercise it directly without running the main script. + param( + [string]$Dashboard, + [string]$ResolvedOutputDir, + [string]$AppendDefenderUsage, + [bool]$DisableAISIDDeltaCache, + [bool]$ExportCompleted = $true + ) + $isAisid = ($Dashboard -eq 'AISID') + return [ordered]@{ + resolvedAISIDOutputDir = if ($isAisid -and $ResolvedOutputDir -and $ExportCompleted) { [string]$ResolvedOutputDir } else { $null } + appendDefenderUsage = if ($isAisid -and $AppendDefenderUsage) { [string]$AppendDefenderUsage } else { $null } + disableAISIDDeltaCache = if ($isAisid) { [bool]$DisableAISIDDeltaCache } else { $false } + } +} + function script:Resolve-DataTypePaths { param( [Parameter(Mandatory)] [ValidateSet('Purview','UserInfo','Agent365Info','DefenderUsage','Log')] [string]$DataType, @@ -3804,13 +7636,13 @@ if ($Dashboard -eq 'AISID') { } } -# Folder-only guard: the AISID destination addresses the whole 6-CSV set, so a file leaf is +# Folder-only guard: the AISID destination addresses the whole twelve-CSV set, so a file leaf is # rejected on either switch — supply a folder path / SharePoint folder URL / Fabric OneLake folder URL. foreach ($duPair in @(@{ N = 'OutputPathDefenderUsage'; V = $OutputPathDefenderUsage }, @{ N = 'AppendDefenderUsage'; V = $AppendDefenderUsage })) { if ($PSBoundParameters.ContainsKey($duPair.N) -and -not [string]::IsNullOrWhiteSpace($duPair.V)) { $duNorm = $duPair.V.Trim().TrimEnd('/', '\') if ($duNorm -match '\.[a-z0-9]{2,5}$') { - Write-Host ("ERROR: -{0} must be a FOLDER, not a file leaf. The AISID output is a set of 6 CSVs." -f $duPair.N) -ForegroundColor Red + Write-Host ("ERROR: -{0} must be a FOLDER, not a file leaf. The AISID output is a set of 9 CSVs." -f $duPair.N) -ForegroundColor Red Write-Host (" Supplied: {0}" -f $duPair.V) -ForegroundColor Yellow Write-Host " Provide a folder path, a SharePoint folder URL, or a Fabric OneLake folder URL." -ForegroundColor Yellow exit 1 @@ -4247,7 +8079,7 @@ $script:RollupProcessorMode = 'None' $script:RollupDashboard = 'None' $script:RollupDashboardProfile = $null -# Resolves -Dashboard into auto-enabled switches so the existing +# Resolves -Dashboard into auto-enabled switches so the existing # rollup gate + mode decision below operate on a consistent state. Acts only when # the user explicitly passed -Dashboard; the default 'AIO' is inert so a bare # -Rollup / -RollupPlusRaw keeps today's behavior exactly. @@ -4255,11 +8087,11 @@ $dashboardExplicit = $PSBoundParameters.ContainsKey('Dashboard') $dashboardUC = $Dashboard.ToUpperInvariant() $dashboardImpliedRollup = $false if ($dashboardExplicit) { - # C1: AIO/AIBV (CopilotInteraction) and the M365 usage bundle are different data + # C1: AIO/ValueLens (CopilotInteraction) and the M365 usage bundle are different data # pulls AND different processors — refuse to guess; make the user pick one. - if (($dashboardUC -eq 'AIO' -or $dashboardUC -eq 'AIBV') -and $IncludeM365Usage) { + if (($dashboardUC -eq 'AIO' -or $dashboardUC -eq 'VALUELENS') -and $IncludeM365Usage) { Write-Host "ERROR: -Dashboard $Dashboard and -IncludeM365Usage are incompatible (different data source AND different processor). Pick one." -ForegroundColor Red - Write-Host " -Dashboard AIO|AIBV : CopilotInteraction rollup (AI-in-One / AI Business Value dashboards)." -ForegroundColor Yellow + Write-Host " -Dashboard AIO|ValueLens : CopilotInteraction rollup (AI-in-One / ValueLens dashboards)." -ForegroundColor Yellow Write-Host " -Dashboard M365 : M365 Usage Analytics rollup (or simply use -IncludeM365Usage)." -ForegroundColor Yellow exit 1 } @@ -4351,7 +8183,7 @@ if ($Rollup -or $RollupPlusRaw) { # tables, so it maps to the AIO profile here — no AISID-specific rollup mode/profile is # introduced, and 'AISID' never leaks into the rollup/checkpoint machinery. AISID's own # Defender CSVs are produced by the (later-phase) Defender pipeline, not this processor. - $script:RollupDashboard = if ($dashboardExplicit) { if ($dashboardUC -eq 'AISID') { 'AIO' } else { $dashboardUC } } else { 'AIO' } + $script:RollupDashboard = if ($dashboardExplicit) { if ($dashboardUC -eq 'AISID') { 'AIO' } elseif ($dashboardUC -eq 'VALUELENS') { 'AIBV' } else { $dashboardUC } } else { 'AIO' } $script:RollupDashboardProfile = $script:RollupDashboard.ToLowerInvariant() } else { @@ -4394,12 +8226,12 @@ if ($fillerTextBound -and -not $fillerLabelBound) { if ($fillerLabelBound) { if (-not ($Rollup -or $RollupPlusRaw)) { Write-Host "ERROR: -FillerLabel requires -Rollup or -RollupPlusRaw." -ForegroundColor Red - Write-Host " -FillerLabel only affects the rolled-up AI-in-One / AI Business Value Users output." -ForegroundColor Yellow + Write-Host " -FillerLabel only affects the rolled-up AI-in-One / ValueLens Users output." -ForegroundColor Yellow exit 1 } if ($script:RollupProcessorMode -eq 'M365Bundle') { Write-Host "ERROR: -FillerLabel is not valid with the M365 dashboard (-IncludeM365Usage / -Dashboard M365)." -ForegroundColor Red - Write-Host " The org / manager hierarchy is produced only for the AI-in-One and AI Business Value dashboards." -ForegroundColor Yellow + Write-Host " The org / manager hierarchy is produced only for the AI-in-One and ValueLens dashboards." -ForegroundColor Yellow exit 1 } $fillMode = ([string]$FillerLabel).Trim() @@ -4437,14 +8269,14 @@ if ($fillerLabelBound) { # resolved Python interpreter, and deleted in finally. Single-quoted here- # strings prevent any PowerShell variable expansion of the Python source. # ============================================================================ -$Script:EMBEDDED_PROCESSOR_COPILOT_VERSION = '4.2.0' -$Script:EMBEDDED_PROCESSOR_M365_VERSION = '2.6.1' +$Script:EMBEDDED_PROCESSOR_COPILOT_VERSION = '4.2.1' +$Script:EMBEDDED_PROCESSOR_M365_VERSION = '2.6.2' # >>> BEGIN-EMBEDDED-COPILOT-PROCESSOR $Script:EMBEDDED_PROCESSOR_COPILOT = @' #!/usr/bin/env python3 """ -Purview CopilotInteraction Processor v4.1.0 +Purview CopilotInteraction Processor v4.2.1 ------------------------------------------- Two-input / two-output preprocessor for the AI Business Value Dashboard and AI-in-One Rollup PBIPs. @@ -4457,11 +8289,14 @@ Output profiles (--profile): Behavior_Plausible, Workflow_Action, Delegation_Event_Key, Is_Agent_Activity/Web_Grounded_Signal promoted into the grain for sliceability) + Audit_UserId passthrough. - aio : AI-in-One Dashboard. 36-column fact — 5-value Environment - {Autonomous Agent, Cowork, Agents, Licensed M365 Copilot, - Unlicensed Chat}. Reproduces the v3.1.0 AIO output - BYTE-IDENTICALLY (validated), so the AIO dashboard is - unaffected. ~41% smaller than the aibv fact. + aio : AI-in-One Dashboard. 36-column fact + 2 trailing append- + reconciliation columns (Message_Id_Raw, ThreadId_Raw) — + 5-value Environment {Autonomous Agent, Cowork, + Agents, Licensed M365 Copilot, Unlicensed Chat}. Reproduces + the AIO output byte-identically in the original 36 + columns (validated); the two raw keys are appended last so + the AIO dashboard is unaffected. ~41% smaller than the aibv + fact. Inputs: --purview (required) @@ -4524,7 +8359,9 @@ import hashlib import hmac import os import re +import sqlite3 import sys +import tempfile import time from datetime import datetime, timedelta, timezone from pathlib import Path @@ -4550,22 +8387,22 @@ except ImportError: _JSON_ENGINE = "json (stdlib)" -SCRIPT_VERSION = "4.2.0" +SCRIPT_VERSION = "4.2.1" # --------------------------------------------------------------------------- # Output schemas — TWO PROFILES # -# --profile aio : reproduces the v3.1.0 AIO-faithful output EXACTLY except +# --profile aio : reproduces the AIO-faithful output EXACTLY except # for two provenance columns (Message_Id_Raw, ThreadId_Raw) -# APPENDED LAST in v4.2.0 for cross-run append reconciliation. +# APPENDED LAST for cross-run append reconciliation. # The original 36 columns are unchanged in name, order, and # value (5-value Environment vocabulary), so the pre-existing -# AIO output stays byte-identical to v3.1.0; only the two +# AIO output is unchanged; only the two # trailing raw keys are new. This is the contract the # AI-in-One dashboard already consumes. # --profile aibv : the AIBV-faithful superset (50-col fact, 3-value # Environment, all offloaded calc cols + grain-promoted -# sliceable flags) built in this v4.0.0 effort. +# sliceable flags). # # Both share one classification CODEBASE; the per-profile vocabulary is # selected by the `profile` argument threaded through the classifiers. @@ -4591,7 +8428,7 @@ _GRAIN_KEYS_COMMON: tuple[str, ...] = ( "ThreadId", ) -# AIO grain = the common 16 (matches v3.1.0 exactly). +# AIO grain = the common 16. GRAIN_KEYS_AIO: tuple[str, ...] = _GRAIN_KEYS_COMMON # AIBV grain = common 16 + 3 promoted per-resource flags (sliceability fix). @@ -4604,7 +8441,7 @@ GRAIN_KEYS_AIBV: tuple[str, ...] = _GRAIN_KEYS_COMMON + ( "Workflow_Action", ) -# Cross-run append reconciliation keys (v4.2.0): the stable raw GUIDs behind the +# Cross-run append reconciliation keys: the stable raw GUIDs behind the # INT surrogates Message_Id (message) and ThreadId (thread). Appended as the FINAL # two columns of EVERY profile so all pre-existing column positions are unchanged. # The PAX append layer (ConvertTo-FactSeedMaps / Merge-FactCsv) dedups cross-run on @@ -4615,7 +8452,7 @@ _RAW_ID_ATTRS: tuple[str, ...] = ( "ThreadId_Raw", ) -# AIO non-grain carried attrs = exactly the v3.1.0 set (ends at ActivityDate); +# AIO non-grain carried attrs end at ActivityDate; # the trailing _RAW_ID_ATTRS are appended below to form _NONGRAIN_ATTRS_AIO. _NONGRAIN_ATTRS_AIO_BASE: tuple[str, ...] = ( "CreationDate", @@ -4634,12 +8471,12 @@ _NONGRAIN_ATTRS_AIO_BASE: tuple[str, ...] = ( "ModelTransparencyDetails_ModelName", "Agent_TitleID", "Message_isPrompt", - # Calc cols ported from DAX (present in AIO since v3.1.0) + # Calc cols ported from DAX "Behavior_Source", "Value_Outcome", "ActivityDate", ) -# AIO carried attrs = the v3.1.0 base set + a stable user-identity column + the +# AIO carried attrs = the base set + a stable user-identity column + the # trailing raw reconciliation keys. _NONGRAIN_ATTRS_AIO: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + ( # Stable, deid-consistent user identity for AIO. Mirrors the @@ -4691,6 +8528,17 @@ def schema_for(profile: str) -> tuple[tuple[str, ...], tuple[str, ...], list[str # same renaming so the dim CSV is drop-in compatible with all downstream DAX. UPN_VARIANTS_NORMALIZED = {"userprincipalname", "upn", "personid"} DEPARTMENT_VARIANT_NORMALIZED = "department" +# Organization source precedence (most meaningful first). `department` carries the +# human-readable org name in a directory export; a column literally named +# Organization/Organisation is accepted only when no readable department exists, +# because in real exports it is frequently a numeric department identifier. +_DEPARTMENT_SOURCE_PREFERENCE: tuple[str, ...] = ( + DEPARTMENT_VARIANT_NORMALIZED, + "organisation", + "organization", +) +# Name used to retain a displaced Organization-named identifier column. +_DISPLACED_ORG_COLUMN = "Organization_Id" JOBTITLE_RAW_NAME = "jobTitle" # exact-match rename to "JobTitle" HAS_LICENSE_VARIANTS = ( "Has license", @@ -5091,7 +8939,7 @@ def compute_license_status(has_license_raw: str) -> str: def compute_environment(profile: str, has_license_raw: str, agent_name: str, agent_id: str, app_host: str) -> str: license_val = (has_license_raw or "").strip().upper() if profile == "aio": - # v3.1.0 AIO vocabulary (5-value, keyed off app_host + agent presence). + # AIO vocabulary (5-value, keyed off app_host + agent presence). host = (app_host or "").lower() has_agent = bool((agent_name or "").strip()) or bool((agent_id or "").strip()) if host in {"autonomous", "logic app"}: @@ -5164,7 +9012,7 @@ def _resource_behavior( if res_type in {"teamsmessage", "teamschat", "teamschannel"}: return "Teams Messaging" if profile == "aio": - # v3.1.0: any flow/connector/http resource -> "Workflow Execution". + # Any flow/connector/http resource -> "Workflow Execution". if res_type in {"flow", "connector", "http"}: return "Workflow Execution" else: @@ -5270,7 +9118,7 @@ def _context_behavior(profile: str, app_host: str, ctx_type: str, is_active: boo if app_host == "copilot studio": return "Domain-Specific Agent" if profile == "aio": - # v3.1.0: autonomous OR logic app -> "Workflow Execution". + # Autonomous OR logic app -> "Workflow Execution". if app_host in {"autonomous", "logic app"}: return "Workflow Execution" else: @@ -5343,7 +9191,7 @@ def compute_behavior_enriched(profile: str, behavior_category: str, agent_name: # Autonomy_Pattern — profile-aware. -# AIO (v3.1.0): keyed off the 5-value Environment. +# AIO: keyed off the 5-value Environment. # AIBV: SWITCH(Cowork->3, Is_Agent_Activity->2, Licensed->1, else BLANK). @functools.lru_cache(maxsize=None) def compute_autonomy_pattern(profile: str, environment: str, is_agent_activity_str: str) -> str: @@ -5804,7 +9652,7 @@ def compute_delegation_event_key( # NOTE: DAX FORMAT "mm" resolves to MONTH here (not minute) because it does # not follow h/hh — so interaction_date_str ("%Y-%m-%d") matches exactly. # COALESCE: we treat empty/whitespace as blank (skip it). The blank-vs-empty - # nuance + rollup-vs-fanout grain interaction are flagged for Phase 4 live + # nuance + rollup-vs-fanout grain interaction affect the # parity (measure: [Delegation Events] = DISTINCTCOUNT, filtered Usage_Mode # = "5 - Delegating"). tail = "unknown-workflow" @@ -5835,7 +9683,7 @@ def compute_agent_publish_status(agent_id: str, agent_name: str) -> str: # Verbatim port of AIBV calc col `Is_Agent_Activity` (emitted as TRUE/FALSE text, # mirroring the Is_Sensitive / Message_isPrompt convention; the PBIT types it -# logical in Phase 3). res_type is the per-resource AccessedResource_Type. +# logical). res_type is the per-resource AccessedResource_Type. @functools.lru_cache(maxsize=None) def compute_is_agent_activity(agent_name: str, agent_id: str, app_host: str, res_type: str) -> str: has_agent = bool((agent_name or "").strip()) @@ -5885,12 +9733,35 @@ def detect_upn_column(headers: list[str]) -> str | None: def detect_department_column(headers: list[str]) -> str | None: - for h in headers: - if _normalize_col_name(h) in {DEPARTMENT_VARIANT_NORMALIZED, "organization", "organisation"}: - return h + # Selection is by meaning, not by source-column position: the readable department + # name owns `Organization` whenever one is present, because the dashboards bind + # their Organization slicers to that value. + for wanted in _DEPARTMENT_SOURCE_PREFERENCE: + for h in headers: + if _normalize_col_name(h) == wanted: + return h return None +def detect_displaced_org_columns(headers: list[str], chosen: str | None) -> list[str]: + """Organization-named source columns displaced by the chosen department column. + + When a readable `department` column is promoted to `Organization`, any + pre-existing column already named Organization/Organisation would collide. + Those are preserved under a separate truthful name rather than dropped, + because a numeric department identifier is legitimate data in its own right. + """ + if not chosen: + return [] + displaced: list[str] = [] + for h in headers: + if h == chosen: + continue + if _normalize_col_name(h) in {"organization", "organisation"}: + displaced.append(h) + return displaced + + def detect_jobtitle_column(headers: list[str]) -> str | None: for h in headers: if _normalize_col_name(h) == "jobtitle": @@ -6047,12 +9918,132 @@ def _build_org_hierarchy(uk_by_id, uk_by_upn, mgr_ptr, name_by_uk) -> dict: return result +class _SqliteRowView: + """Re-iterable, disk-backed drop-in replacement for ``list(csv.DictReader)``. + + Stages every source row into a temporary SQLite table (source sequence + + JSON payload) in bounded batches during construction, then serves ordered + re-iteration by reopening a cursor on each pass. The complete row set is + never held in memory. It behaves like a list for the only two operations the + callers use — ``len()`` and (repeatable) iteration — and each yielded item is + the original ``csv.DictReader`` dict reconstructed identically, so downstream + calculations, ordering, and output are unchanged. + """ + + def __init__(self, reader, batch_size: int = 2000): + import json + fd, self._path = tempfile.mkstemp(prefix="pax_rowview_", suffix=".sqlite") + os.close(fd) + self._conn = sqlite3.connect(self._path) + self._conn.execute("PRAGMA journal_mode=OFF") + self._conn.execute("PRAGMA synchronous=OFF") + self._conn.execute("CREATE TABLE rows (seq INTEGER PRIMARY KEY, payload TEXT NOT NULL)") + seq = 0 + batch: list = [] + try: + self._conn.execute("BEGIN") + for row in reader: + batch.append((seq, json.dumps(row, ensure_ascii=False))) + seq += 1 + if len(batch) >= batch_size: + self._conn.executemany("INSERT INTO rows (seq, payload) VALUES (?, ?)", batch) + batch.clear() + if batch: + self._conn.executemany("INSERT INTO rows (seq, payload) VALUES (?, ?)", batch) + self._conn.execute("COMMIT") + except Exception: + try: + self._conn.execute("ROLLBACK") + except Exception: + pass + self.close() + raise + self._count = seq + + def __len__(self) -> int: + return self._count + + def __iter__(self): + import json + cur = self._conn.cursor() + try: + cur.execute("SELECT payload FROM rows ORDER BY seq") + for (payload,) in cur: + yield json.loads(payload) + finally: + cur.close() + + def close(self) -> None: + conn = getattr(self, "_conn", None) + if conn is not None: + try: + conn.close() + except Exception: + pass + self._conn = None + path = getattr(self, "_path", None) + if path: + try: + if os.path.exists(path): + os.remove(path) + except Exception: + pass + + def __del__(self): + self.close() + + +# Exact-case Users columns the AIO semantic model exposes as source columns. Power Query M is +# case-sensitive, so these names must match exactly. PowerShell CSV readers, by contrast, treat +# headers case-INSENSITIVELY and reject a file carrying both `displayName` and `DisplayName`, so +# these two are RENAMED to their canonical form rather than duplicated. `mail` and `Email` differ +# by more than case and may safely coexist, so `mail` is retained and `Email` added alongside it. +# ValueLens consumes the directory-native names and is deliberately unaffected. +_AIO_CANONICAL_RENAMES: tuple[tuple[str, str], ...] = ( + ("displayName", "DisplayName"), + ("country", "Country"), +) +_AIO_EMAIL_SOURCE = "mail" +_AIO_EMAIL_CANONICAL = "Email" + + +# UserKey allocation. A new key is always taken from a counter positioned STRICTLY +# ABOVE every key already reserved by a prior append target, so a key that is no +# longer present in the current directory export can never be handed to a different +# identity. With no reserved keys the counter starts at 1 and allocates 1, 2, 3, ... +# in encounter order. +_NEXT_USER_KEY = 1 + + +def reset_user_key_allocator(user_key_map: dict[str, int]) -> None: + """Position the allocator above every already-reserved key.""" + global _NEXT_USER_KEY + highest = 0 + for reserved in user_key_map.values(): + if reserved > highest: + highest = reserved + _NEXT_USER_KEY = highest + 1 + + +def mint_user_key(user_key_map: dict[str, int], normalized_identity: str) -> int: + """Return the key reserved for an identity, allocating the next free key once.""" + global _NEXT_USER_KEY + reserved = user_key_map.get(normalized_identity) + if reserved is not None: + return reserved + assigned = _NEXT_USER_KEY + _NEXT_USER_KEY = assigned + 1 + user_key_map[normalized_identity] = assigned + return assigned + + def load_entra_and_write_users( entra_csv: str, users_out_csv: str, user_key_map: dict[str, int], licensing_csv: str | None = None, quiet: bool = False, + profile: str = "aibv", ) -> dict[str, dict[str, str]]: """ Read the Entra CSV, write the Users dim CSV (with PBIP-compatible renames + @@ -6103,27 +10094,62 @@ def load_entra_and_write_users( rename_map[upn_col] = "PersonId" if dept_col and dept_col != "Organization": rename_map[dept_col] = "Organization" + # Retain any other Organization-named source column (typically a numeric + # department identifier) under a separate name so promoting the readable + # department neither collides with it nor discards it. + for displaced in detect_displaced_org_columns(original_headers, dept_col): + alt = _DISPLACED_ORG_COLUMN + suffix = 2 + while alt in original_headers or alt in rename_map.values(): + alt = f"{_DISPLACED_ORG_COLUMN}_{suffix}" + suffix += 1 + rename_map[displaced] = alt if jobtitle_col and jobtitle_col != "JobTitle": rename_map[jobtitle_col] = "JobTitle" if has_license_col and has_license_col != "Has license": rename_map[has_license_col] = "Has license" + # AIO canonical identity headers. Renamed (not duplicated) so the emitted CSV never + # carries two headers differing only by case, which PowerShell CSV readers reject. + # When the export already supplies the exact canonical header, the case-only variant is + # dropped from the OUTPUT but its source name is remembered: a blank canonical value + # still falls back to the variant's value rather than emitting a blank field. + aio_fallback_source_by_canonical: dict[str, str] = {} + if profile == "aio": + for _src, _canon in _AIO_CANONICAL_RENAMES: + _srcs = [h for h in original_headers if h == _src] + _canons = [h for h in original_headers if h == _canon] + if _canons: + for h in _srcs: + rename_map[h] = None + aio_fallback_source_by_canonical[_canon] = h + elif _srcs: + rename_map[_srcs[0]] = _canon + # Final header list for users CSV — preserve original order, apply renames, # then append injected columns. UserKey is the INT surrogate that joins # to the fact table. renamed_headers = [rename_map.get(h, h) for h in original_headers] + # A None target means the source column is intentionally dropped (AIO case-only variant + # superseded by an already canonical header). + renamed_headers = [h for h in renamed_headers if h is not None] injected = ["UserKey", "PersonId_Normalized", "License Status", "TotalEmployees"] if "Has license" not in renamed_headers: renamed_headers.append("Has license") for inj in injected: if inj not in renamed_headers: renamed_headers.append(inj) + # AIO exact-case alias headers (aio profile only, additive). ValueLens output shape + # is deliberately unchanged. + if profile == "aio": + if _AIO_EMAIL_CANONICAL not in renamed_headers: + renamed_headers.append(_AIO_EMAIL_CANONICAL) # Org/manager hierarchy columns (always appended; AIO/AIBV Users dim). for hc in _HIER_COLUMNS: if hc not in renamed_headers: renamed_headers.append(hc) - rows = list(reader) + rows = _SqliteRowView(reader) total_rows = len(rows) user_lookup: dict[str, dict[str, str]] = {} @@ -6144,10 +10170,7 @@ def load_entra_and_write_users( pid_norm = pid.strip().lower() if not pid_norm: continue - uk = user_key_map.get(pid_norm) - if uk is None: - uk = len(user_key_map) + 1 - user_key_map[pid_norm] = uk + uk = mint_user_key(user_key_map, pid_norm) uk_by_upn[pid_norm] = uk rid = src_row.get("id", "") rid = "" if rid is None else str(rid) @@ -6193,6 +10216,16 @@ def load_entra_and_write_users( if tgt_h in out_row: out_row[tgt_h] = "" if value is None else str(value) + # AIO canonical selection: prefer a nonblank exact canonical value, otherwise fall + # back to the dropped case-only variant's source value. Done BEFORE the + # de-identification block so the selected value is the one that gets transformed. + # Lookups are explicit and case-sensitive - no case-insensitive dictionary access. + if profile == "aio": + for _canon, _srcname in aio_fallback_source_by_canonical.items(): + if not out_row.get(_canon, ""): + _fb = src_row.get(_srcname, "") + out_row[_canon] = "" if _fb is None else str(_fb) + # Deidentification (no-op unless --deidentify): transform identity columns # IN PLACE before PersonId_Normalized / UserKey derive from PersonId, so the # fact<->Users join and manager links stay consistent across the hash. @@ -6201,6 +10234,12 @@ def load_entra_and_write_users( out_row["PersonId"] = deid_upn(out_row["PersonId"]) if "displayName" in out_row: out_row["displayName"] = deid_name(out_row["displayName"]) + # AIO emits the canonical header instead of the case-only variant, so the same + # transform must be applied to it (dict keys are case-sensitive in Python). + if "DisplayName" in out_row: + out_row["DisplayName"] = deid_name(out_row["DisplayName"]) + if "Email" in out_row: + out_row["Email"] = deid_upn(out_row["Email"]) if "mail" in out_row: out_row["mail"] = deid_upn(out_row["mail"]) if "givenName" in out_row: @@ -6237,10 +10276,7 @@ def load_entra_and_write_users( # UserKey (INT surrogate; assigned in Entra-file order) if person_id_norm: - user_key = user_key_map.get(person_id_norm) - if user_key is None: - user_key = len(user_key_map) + 1 - user_key_map[person_id_norm] = user_key + user_key = mint_user_key(user_key_map, person_id_norm) out_row["UserKey"] = str(user_key) else: out_row["UserKey"] = "" @@ -6287,6 +10323,14 @@ def load_entra_and_write_users( for hc in _HIER_COLUMNS: out_row[hc] = hrec.get(hc, "") + # AIO exact-case aliases. Copied AFTER the de-identification transforms above so a + # deidentified run carries the transformed value and never the original. An alias + # already supplied by the directory export is preserved rather than overwritten + # with a blank source value. + if profile == "aio": + if not out_row.get(_AIO_EMAIL_CANONICAL, ""): + out_row[_AIO_EMAIL_CANONICAL] = out_row.get(_AIO_EMAIL_SOURCE, "") + writer.writerow(out_row) # Build fact-lookup dict (dedupe on normalized key, last-wins matches @@ -6315,6 +10359,7 @@ def load_entra_and_write_users( print(f" License col detected: NO RECOGNIZED LICENSE COLUMN FOUND IN ENTRA CSV") print(f" Fallback: every user will be tagged 'Unlicensed' until a recognized column is present.") + rows.close() return user_lookup @@ -6368,10 +10413,7 @@ def explode_record( # INT and stash so subsequent rows for the same user reuse it. The # caller tracks unmatched-vs-Entra via the lookup membership check. if audit_user_id_norm: - user_key = user_key_map.get(audit_user_id_norm) - if user_key is None: - user_key = len(user_key_map) + 1 - user_key_map[audit_user_id_norm] = user_key + user_key = mint_user_key(user_key_map, audit_user_id_norm) else: user_key = "" # ThreadId INT surrogate. deid_guid is a no-op unless --deidentify; under @@ -6419,7 +10461,7 @@ def explode_record( # the per-resource fields). Built once per record; copied per emitted row # and updated with the resource-varying keys. Common keys first; AIBV-only # keys appended only for the aibv profile so the AIO output stays the exact - # v3.1.0 column set. + # Column set. base_nongrain: dict[str, Any] = { "CreationDate": creation_date_str, "WeekStart": week_start_str, @@ -6569,7 +10611,7 @@ def explode_record( nongrain["Behavior_Plausible"] = behavior_plausible nongrain["Delegation_Event_Key"] = delegation_event_key else: - # AIO (v3.1.0): autonomy keyed purely off Environment; 16-key grain. + # AIO: autonomy keyed purely off Environment; 16-key grain. autonomy_pattern = compute_autonomy_pattern(profile, environment, "") grain_tuple = common_grain + ( autonomy_pattern, @@ -6823,7 +10865,7 @@ def compute_and_write_aggregates( ) if not quiet: - print(" Pre-aggregated tables (AIBV):") + print(" Pre-aggregated tables (ValueLens):") print(f" ActiveDaysSummary: {counts['active_days']:,} rows") print(f" UserMonthMetrics: {counts['user_month_metrics']:,} rows") print(f" Licensed User Rankings: {counts['licensed_rankings']:,} rows") @@ -6855,9 +10897,11 @@ def run_processor( "unmatched_users": 0, } + profile_label = "ValueLens" if profile == "aibv" else "AI-in-One" + if not quiet: print(f"Purview CopilotInteraction Processor v{SCRIPT_VERSION}") - print(f" Profile: {profile}") + print(f" Profile: {profile_label}") print(f" JSON engine: {_JSON_ENGINE}") print(f" Purview input: {purview_csv}") print(f" Entra input: {entra_csv}") @@ -6877,7 +10921,7 @@ def run_processor( thread_key_map: dict[str, int] = {} mid_to_int: dict[str, int] = {} - # PAX append support (not in the standalone v4.0.0 reference): pre-seed the + # PAX append support: pre-seed the # three INT-surrogate maps from JSON snapshots of the append-target's existing # surrogates so retained rows keep stable Message_Id / ThreadKey / UserKey # values across runs. No-op when the seed paths are None (the standalone @@ -6899,9 +10943,10 @@ def run_processor( _load_int_seed(seed_thread_map_path, thread_key_map) if seed_mid_map_path: _load_int_seed(seed_mid_map_path, mid_to_int) + reset_user_key_allocator(user_key_map) user_lookup = load_entra_and_write_users( - entra_csv, users_out_csv, user_key_map, licensing_csv=licensing_csv, quiet=quiet + entra_csv, users_out_csv, user_key_map, licensing_csv=licensing_csv, quiet=quiet, profile=profile ) if not quiet: @@ -6969,7 +11014,7 @@ def run_processor( print() print("Writing rolled-up fact CSV...") - # Profile-specific output schema (AIO = v3.1.0 36-col; AIBV = 50-col superset). + # Profile-specific output schema (AIO = 36-col; AIBV = 50-col superset). grain_keys, nongrain_attrs_sel, fact_header = schema_for(profile) with open(fact_out_csv, "w", encoding="utf-8", newline="") as fout: writer = csv.writer(fout, lineterminator="\n") @@ -7007,7 +11052,9 @@ def run_processor( print(f" Rollup rows: {len(rollup):,} ({reduction_pct:.1f}% reduction)") print(f" Distinct Message_Ids: {len(mid_to_int):,}") print(f" Distinct ThreadIds: {len(thread_key_map):,}") - print(f" Distinct UserKeys: {len(user_key_map):,}") + fact_user_keys = {k[0][0] for k in rollup.keys()} + print(f" Users in dimension: {len(user_key_map):,}") + print(f" Users represented in fact: {len(fact_user_keys):,}") print(f" Unmatched users: {stats['unmatched_users']:,}") print(f" Elapsed: {elapsed:.2f}s") @@ -7025,7 +11072,7 @@ def main() -> None: f"Purview CopilotInteraction Processor v{SCRIPT_VERSION} - " "Two/three-input, two-output preprocessor that produces a rolled-up " "Interactions fact CSV (~85% row reduction via PromptCount grain) " - "and a Users dim CSV for the AI Business Value Dashboard PBIP." + "and a Users dim CSV for the ValueLens Dashboard PBIP." ) ) parser.add_argument( @@ -7052,7 +11099,7 @@ def main() -> None: "--entra is treated as a users-only file and license status is merged " "in from this file (the 3-file workflow). Omit to use a single " "combined Entra file (license column auto-detected). Applies to both " - "--profile aibv and --profile aio." + "ValueLens and AI-in-One profiles." ), ) parser.add_argument( @@ -7078,10 +11125,10 @@ def main() -> None: choices=("aibv", "aio"), default="aibv", help=( - "Output profile. 'aibv' (default) = AI Business Value Dashboard " + "Output profile. ValueLens Dashboard (default; internally selected by PAX) " "superset (50-col fact, 3-value Environment). 'aio' = AI-in-One " "Dashboard (36-col fact, 5-value Environment) — reproduces the " - "v3.1.0 AIO output exactly." + "AIO output exactly." ), ) parser.add_argument( @@ -7096,9 +11143,9 @@ def main() -> None: action="store_true", default=False, help=( - "Also write the AIBV pre-aggregated tables (ActiveDaysSummary, " + "Also write the ValueLens pre-aggregated tables (ActiveDaysSummary, " "UserMonthMetrics, Licensed/Unlicensed user rankings, Licensed user " - "summary). OFF by default — the AIBV template only needs the two core " + "summary). OFF by default — the ValueLens template only needs the two core " "rollup files (Interactions + Users). No effect for --profile aio." ), ) @@ -7122,7 +11169,7 @@ def main() -> None: ), ) parser.add_argument( - # PAX append support (not in the standalone v4.0.0 reference). + # PAX append support. "--seed-mid-map", default=None, help=( @@ -7155,7 +11202,7 @@ def main() -> None: default="none", help=( "Filler for org-hierarchy level slots deeper than a user's own level " - "(Users dim, AIO/AIBV). 'none' (default) leaves them blank; 'self' " + "(Users dim, AIO/ValueLens). 'none' (default) leaves them blank; 'self' " "repeats the user; 'manager' repeats the user's manager; 'fixed' uses " "--hierarchy-fill-label. The hierarchy columns are always emitted." ), @@ -7219,7 +11266,7 @@ def main() -> None: purview_stem = Path(purview_path).stem entra_stem = Path(entra_path).stem run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") - # PAX embed (differs from the standalone v4.0.0, which timestamps these two): + # PAX embed (the standalone timestamps these two): # the core rollup outputs use NON-timestamped names so PAX's post-run # Merge-FactCsv / Merge-UsersCsv resolve them by exact name. The run timestamp # is already baked into the input filenames (Purview_Audit_*_.csv, @@ -7266,7 +11313,7 @@ if __name__ == "__main__": $Script:EMBEDDED_PROCESSOR_M365 = @' #!/usr/bin/env python3 """ -Purview M365 Usage Bundle Explosion Processor v2.6.1 +Purview M365 Usage Bundle Explosion Processor v2.6.2 ===================================================== Two-mode processor for Purview audit log CSV exports: @@ -7277,24 +11324,6 @@ Two-mode processor for Purview audit log CSV exports: Targets 80%+ row reduction for Power BI ingestion. Streaming — no exploded rows held in memory. - v2.3.0 CHANGES (validated against MS Learn audit-log schema + DAX TMDL fingerprint): - • Canonical Operation names enforced. Three legacy/wrong names auto-renamed at - intake (OP_RENAME), preserving historical data while emitting canonical values: - FileViewed → FileAccessed - MeetingParticipantJoined → MeetingParticipantDetail - ConnectedAIAppInteraction → AIAppInteraction - • TEAMS_OPS / FILE_OPS / COPILOT_OPS updated to the 14 DAX-required ops only. - • Rollup CSV header extended with 4 agent telemetry columns (AgentId, AgentName, - ContextType, IsAgentInteraction) so M365Usage.tmdl fingerprint check passes - without Power Query post-processing. - • Multi-file input supported: pass --input/-i multiple paths (or repeat the flag) - to combine N Purview exports into one rollup + UserStats + SessionCohort bundle. - Validated 4-pull strategy: Teams + Outlook + Files + Copilot in a single run. - • is_copilot() now also recognises AIAppInteraction (agent/connected-app events). - • All performance characteristics preserved: streaming intake, orjson, no buffered - explosion, per-record cap. Goal is to move processing OUT of Power BI INTO Python - so the .pbix loads/refreshes faster without losing fidelity. - After the rollup CSV is written, a second pass streams through it to produce two additional analytics files (unless --no-userstats is specified): - UserStats: One row per user with 66 columns of pre-computed metrics @@ -7307,7 +11336,7 @@ Two-mode processor for Purview audit log CSV exports: recalculating expensive DAX/M expressions, cutting dashboard load times. EVENT-LEVEL MODE (--mode event-level): v1-compatible 153-column explosion output. - Identical behavior to v1.0.0 for debugging and reconciliation. + For debugging and reconciliation. UserStats and SessionCohort files are NOT generated in this mode. Requirements: @@ -7316,10 +11345,10 @@ Requirements: Usage: # (A) Single PAX / PowerShell export: - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py --pax + python Purview_M365_Usage_Bundle_Explosion_Processor.py --pax # (B) Manual 4-pull export from Purview Audit: - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py \ + python Purview_M365_Usage_Bundle_Explosion_Processor.py \ --teams --outlook --files --copilot Common optional flags: @@ -7340,7 +11369,7 @@ Output files (rollup mode — all share the same timestamp): ResponseCount, AgentSessionCount) matches AI in One DISTINCTCOUNT(ThreadId); AgentPromptCount = prompts on agent-flagged - threads (v2.6.1) + threads ( = input file's stem for single input, or '_Combined' for multi-input. Rename the output file or use --output-dir if you want a tenant-specific name.) @@ -7359,24 +11388,24 @@ Arguments: Examples: # Default rollup (13-column output + UserStats + SessionCohort) - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv # Combine the validated 4-pull bundle (Teams + Outlook + Files + Copilot) in one run - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py \ + python Purview_M365_Usage_Bundle_Explosion_Processor.py \ -i Teams_Export.csv Outlook_Export.csv Files_Export.csv Copilot_Export.csv \ --combined-stem ZavaCorp_2025_11 # Rollup with output in a different directory - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --output-dir ./output + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --output-dir ./output # Rollup only — skip UserStats and SessionCohort generation - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --no-userstats + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --no-userstats # v1-compatible event-level explosion (153-column output) - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --mode event-level + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --mode event-level # Rollup with sample-based reconciliation check - python Purview_M365_Usage_Bundle_Explosion_Processor_v2.6.1.py -i Purview_Export.csv --reconcile + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --reconcile Validated 4-pull strategy (Purview Audit → Activities filter, type+click each chip): Teams (7d): MessageSent, MessageRead, ChatCreated, TeamsSessionStarted, @@ -7386,7 +11415,6 @@ Validated 4-pull strategy (Purview Audit → Activities filter, type+click each Copilot (30d): CopilotInteraction, AIAppInteraction (filter by record type) Author: Microsoft Copilot Growth ROI Advisory Team (copilot-roi-advisory-team-gh@microsoft.com) -Version: 2.6.1 """ from __future__ import annotations @@ -7402,7 +11430,7 @@ import re import sys import time from collections import defaultdict -from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED from datetime import datetime, timezone, date, timedelta from pathlib import Path from typing import Any @@ -7437,7 +11465,7 @@ except ImportError: # CONSTANTS # ═════════════════════════════════════════════════════════════════════════════ -SCRIPT_VERSION = "2.6.1" +SCRIPT_VERSION = "2.6.2" EXPLOSION_PER_RECORD_ROW_CAP = 1000 STREAMING_CHUNK_SIZE = 5000 @@ -7504,7 +11532,7 @@ ROLLUP_HEADER: list[str] = [ # Reconciliation sample size RECONCILE_SAMPLE_SIZE = 10_000 -# ── Operation canonicalization (v2.3.0) ────────────────────────────────────── +# ── Operation canonicalization ────────────────────────────────────── # Legacy/wrong names that have appeared in older exports or older DAX models. # Renamed at intake so historical data merges cleanly with current canonical pulls. OP_RENAME: dict[str, str] = { @@ -7538,7 +11566,7 @@ COPILOT_OPS: set[str] = {"CopilotInteraction", "AIAppInteraction"} # AIAppInter # AppHost values that indicate an agent / connected-app interaction. AGENT_APPHOSTS: set[str] = {"agent", "copilotstudio", "declarativeagent", "customengineagent"} -# ── DAX-aligned op/ext sets (for the CE/LP precomputed columns added in v2.4.0) ── +# ── DAX-aligned op/ext sets (for the CE/LP precomputed columns) ── # These mirror the exact filters in the PBIT measures Word/Excel/PowerPoint/Outlook/ # Teams Activity *V2, Copilot All Apps Total, and CE Copilot Percentile. # Important: ops are matched AFTER OP_RENAME canonicalization, so the legacy names @@ -7569,7 +11597,7 @@ USERSTATS_HEADER: list[str] = [ "TeamsActivitySegment", "OutlookActivitySegment", "WordActivitySegment", "ExcelActivitySegment", "PowerPointActivitySegment", "OfficeFilesActivitySegment", "OverallM365ActivitySegment", - # ── v2.5.0: precomputed raw activity counts + CE percentile ranks per window. + # ── Precomputed raw activity counts + CE percentile ranks per window. # Windows: _L30 = trailing 30 days ending at max(CreationDate); _L60 = trailing 60; # _Full = entire data range. Filters match the corresponding DAX measures exactly # (post-canonicalization). CE ranks are integer 0-100; blank when raw is 0. @@ -7588,16 +11616,16 @@ USERSTATS_HEADER: list[str] = [ "CECopilotPercentile_L30", "CECopilotPercentile_L60", "CECopilotPercentile_Full", ] -# v2.5.0: percentile window codes used in column names. Order matters for writer. +# Percentile window codes used in column names. Order matters for writer. RANK_WINDOWS: tuple[str, ...] = ("L30", "L60", "Full") SESSIONCOHORT_HEADER: list[str] = ["UserId", "AppColumn", "SessionCohort"] -# v2.6.0: SessionStats — AI in One parity. Per (UserId, CreationDate, AppHost) we count +# SessionStats — AI in One parity. Per (UserId, CreationDate, AppHost) we count # DISTINCT ThreadIds (matches Microsoft AI in One `Sessions` measure), plus prompt / # response counts and an agent-only thread count. License filtering happens downstream # in DAX via the EntraUsers relationship; this CSV stays license-agnostic. -# v2.6.1: Added AgentPromptCount — exact chat vs agent split at message-tally time +# AgentPromptCount — exact chat vs agent split at message-tally time # (uses the same is_agent flag the rollup already determines per record). SESSIONSTATS_HEADER: list[str] = [ "UserId", "CreationDate", "AppHost", @@ -7617,7 +11645,7 @@ _CREATION_DATE_FORMATS: tuple[str, ...] = ( "%m/%d/%Y", ) -# GroupKey type (v2.3.0): adds agent_id, agent_name, context_type so multi-agent users +# GroupKey type: adds agent_id, agent_name, context_type so multi-agent users # don't collapse rows together. IsAgentInteraction is derived on write from AgentId. # (user_id_lower, creation_date_normalized, operation, workload, sfe_lower, app_host, # agent_id, agent_name, context_type) @@ -7655,7 +11683,7 @@ SessionKey = tuple[str, str, str] class SessionAccum: - """Per-(user, date, app_host) Copilot session accumulator (v2.6.0). + """Per-(user, date, app_host) Copilot session accumulator. Mirrors the AI in One `Sessions` measure: DISTINCTCOUNT(ThreadId) where at least one message in the thread is a user prompt (isPrompt=True). Threads with only @@ -7671,7 +11699,7 @@ class SessionAccum: self.thread_ids: set[str] = set() self.agent_thread_ids: set[str] = set() self.prompt_count: int = 0 - self.agent_prompt_count: int = 0 # v2.6.1: prompts on agent-flagged threads + self.agent_prompt_count: int = 0 # prompts on agent-flagged threads self.response_count: int = 0 self.original_user_id = original_uid @@ -8898,70 +12926,86 @@ def run_explosion( if not quiet: print("Phase 2: Processing records...") - # Accumulate all exploded rows (we need dynamic columns before writing header) - all_rows: list[dict] = [] + # Streamed explode-and-write: the output header is fixed (final_header), + # so exploded rows never need to be pooled for column discovery. Input is + # read in bounded chunks and each chunk's exploded rows are written to the + # output CSV as soon as the chunk completes, then released. Neither the full + # input nor the full exploded row set is held in memory; peak usage stays a + # small window of in-flight chunks regardless of input size or explosion ratio. + if not quiet: + print("Phase 3: Writing output CSV...") - # Read CSV in chunks - chunks: list[list[dict]] = [] - current_chunk: list[dict] = [] + os.makedirs(os.path.dirname(os.path.abspath(output_csv)), exist_ok=True) + output_rows = 0 + total_input = 0 - with open(input_csv, "r", encoding="utf-8-sig", newline="") as f: - reader = csv.DictReader(f) + def _read_chunks(reader): + current: list[dict] = [] for row in reader: - current_chunk.append(row) - if len(current_chunk) >= chunk_size: - chunks.append(current_chunk) - current_chunk = [] - if current_chunk: - chunks.append(current_chunk) - - total_input = sum(len(c) for c in chunks) - stats["input_records"] = total_input - - if not quiet: - print(f" Loaded {total_input:,} input records in {len(chunks)} chunk(s)") - - # Determine whether to use multiprocessing - use_parallel = workers > 1 and len(chunks) > 1 + current.append(row) + if len(current) >= chunk_size: + yield current + current = [] + if current: + yield current + + with open(input_csv, "r", encoding="utf-8-sig", newline="") as f_in, \ + open(output_csv, "w", encoding="utf-8", newline="") as f_out: + reader = csv.DictReader(f_in) + writer = csv.DictWriter(f_out, fieldnames=final_header, extrasaction="ignore", lineterminator="\n") + writer.writeheader() - if use_parallel: - chunk_args = [(chunk, prompt_filter) for chunk in chunks] - with ProcessPoolExecutor(max_workers=workers, initializer=_deid_init_worker, initargs=(_DEIDENTIFY,)) as executor: - futures = {executor.submit(_process_chunk, arg): idx for idx, arg in enumerate(chunk_args)} - for future in as_completed(futures): - try: - exploded, _in_count, err_count = future.result() - all_rows.extend(exploded) - stats["errors"] += err_count - stats["chunks_processed"] += 1 - if not quiet and stats["chunks_processed"] % 5 == 0: - print(f" Chunks completed: {stats['chunks_processed']}/{len(chunks)}") - except Exception as exc: - stats["errors"] += 1 - if not quiet: - print(f" Chunk failed: {exc}", file=sys.stderr) - else: - for chunk in chunks: - exploded, _in_count, err_count = _process_chunk((chunk, prompt_filter)) - all_rows.extend(exploded) - stats["errors"] += err_count - stats["chunks_processed"] += 1 - if not quiet and stats["chunks_processed"] % 5 == 0: - print(f" Chunks completed: {stats['chunks_processed']}/{len(chunks)}") + use_parallel = workers > 1 + + if use_parallel: + with ProcessPoolExecutor(max_workers=workers, initializer=_deid_init_worker, initargs=(_DEIDENTIFY,)) as executor: + inflight: set = set() + max_inflight = max(1, workers * 2) + chunk_iter = _read_chunks(reader) + exhausted = False + while not exhausted or inflight: + while not exhausted and len(inflight) < max_inflight: + try: + chunk = next(chunk_iter) + except StopIteration: + exhausted = True + break + total_input += len(chunk) + inflight.add(executor.submit(_process_chunk, (chunk, prompt_filter))) + if not inflight: + break + done, inflight = wait(inflight, return_when=FIRST_COMPLETED) + for future in done: + try: + exploded, _in_count, err_count = future.result() + if exploded: + writer.writerows(exploded) + output_rows += len(exploded) + stats["errors"] += err_count + stats["chunks_processed"] += 1 + if not quiet and stats["chunks_processed"] % 5 == 0: + print(f" Chunks completed: {stats['chunks_processed']}") + except Exception as exc: + stats["errors"] += 1 + if not quiet: + print(f" Chunk failed: {exc}", file=sys.stderr) + else: + for chunk in _read_chunks(reader): + total_input += len(chunk) + exploded, _in_count, err_count = _process_chunk((chunk, prompt_filter)) + if exploded: + writer.writerows(exploded) + output_rows += len(exploded) + stats["errors"] += err_count + stats["chunks_processed"] += 1 + if not quiet and stats["chunks_processed"] % 5 == 0: + print(f" Chunks completed: {stats['chunks_processed']}") - stats["output_rows"] = len(all_rows) + stats["input_records"] = total_input + stats["output_rows"] = output_rows - # ── Phase 3: Write output CSV with fixed header ────────────────────── if not quiet: - print("Phase 3: Writing output CSV...") - - # Write output using fixed M code-ordered header - os.makedirs(os.path.dirname(os.path.abspath(output_csv)), exist_ok=True) - with open(output_csv, "w", encoding="utf-8", newline="") as f: - writer = csv.DictWriter(f, fieldnames=final_header, extrasaction="ignore", lineterminator="\n") - writer.writeheader() - for row in all_rows: - writer.writerow(row) + print(f" Loaded {total_input:,} input records") t_elapsed = time.perf_counter() - t_start @@ -9120,7 +13164,7 @@ def run_rollup( is_agent=is_agent, ) - # ── SessionStats accumulation (v2.6.0 — AI in One parity) ─── + # ── SessionStats accumulation (AI in One parity) ─── # Only records with a CopilotEventData payload contribute. Threads # without at least one user prompt are excluded (matches AI in One # `Message_isPrompt = TRUE` filter). @@ -9145,7 +13189,7 @@ def run_rollup( sacc.prompt_count += prompts_here sacc.response_count += responses_here if is_agent: - sacc.agent_prompt_count += prompts_here # v2.6.1: exact chat/agent split + sacc.agent_prompt_count += prompts_here # exact chat/agent split if thread_id and prompts_here > 0: sacc.thread_ids.add(thread_id) if is_agent: @@ -9181,7 +13225,7 @@ def run_rollup( "TRUE" if acc.is_agent_interaction else "FALSE", ]) - # ── SessionStats CSV (v2.6.0 — AI in One parity) ───────────────────── + # ── SessionStats CSV (AI in One parity) ───────────────────── if track_sessions: os.makedirs(os.path.dirname(os.path.abspath(session_stats_csv)), exist_ok=True) with open(session_stats_csv, "w", encoding="utf-8", newline="") as f: @@ -9197,7 +13241,7 @@ def run_rollup( ah, session_count, sacc.prompt_count, - sacc.agent_prompt_count, # v2.6.1 + sacc.agent_prompt_count, sacc.response_count, len(sacc.agent_thread_ids), ]) @@ -9414,7 +13458,7 @@ def run_reconcile( _ru_count(Operation="CopilotInteraction", AppHost="Teams"), _ev_count(Operation="CopilotInteraction", AppHost="Teams")) - # Check 4: Excel FileAccessed (was FileViewed in pre-v2.3 — renamed via OP_RENAME) + # Check 4: Excel FileAccessed _check("Excel FileAccessed (Operation=FileAccessed, SourceFileExtension in xlsx/xls/xlsm/csv)", _ru_count(Operation="FileAccessed", SourceFileExtension={"xlsx", "xls", "xlsm", "csv"}), _ev_count(Operation="FileAccessed", SourceFileExtension={"xlsx", "xls", "xlsm", "csv"})) @@ -9458,7 +13502,7 @@ def write_userstats_files( *_UserStats.csv — one row per unique UserId with pre-computed metrics *_SessionCohort.csv — one row per (UserId, AppColumn) with session cohort label - When `session_stats_csv_path` is provided (v2.6.0+), the CECopilotPercentile_* + When `session_stats_csv_path` is provided, the CECopilotPercentile_* columns are computed from per-user PromptCount (human interactions) instead of raw audit-event counts. This matches the AI in One semantics and prevents service-principal / plugin-chain inflation from skewing the CE Quadrant. @@ -9494,7 +13538,7 @@ def write_userstats_files( o_ec: dict[str, int] = defaultdict(int) off_ec: dict[str, int] = defaultdict(int) - # ── v2.5.0: DAX-aligned per-user raw activity counts, computed per window. + # ── DAX-aligned per-user raw activity counts, computed per window. # Three windows (L30, L60, Full) feed both the LP Weighted measures and the # CE percentile ranks. Each is a dict keyed by window code → {uid: count}. def _wbuckets() -> dict[str, dict[str, int]]: @@ -9514,7 +13558,7 @@ def write_userstats_files( # normalize the engagement segmentation to active-days-per-week. all_dates: set[str] = set() - # ── v2.5.0: Pass 1 — determine the trailing-window cutoffs ────────── + # ── Pass 1 — determine the trailing-window cutoffs ────────── # Scan CreationDate only to find the most-recent date in the rollup. Cutoffs # are inclusive lower bounds; a row qualifies for window W iff date_key >= cutoff[W]. # The "Full" window has no cutoff and always qualifies. @@ -9603,7 +13647,7 @@ def write_userstats_files( if ext in OFFICE_EXTS and op in FILE_OPS: off_ec[uid_lower] += event_count - # ── v2.5.0: DAX-aligned raw counts, accumulated per window. + # ── DAX-aligned raw counts, accumulated per window. # Helper closure: write to Full always; to L60/L30 only if the row's # date_key satisfies the trailing-window cutoff. def _bump(buckets: dict[str, dict[str, int]], n: int) -> None: @@ -9693,7 +13737,7 @@ def write_userstats_files( cop_rank = compute_ranks({u: cop_ec.get(u, 0) for u in copilot_uids}) m365_rank = compute_ranks({u: m365_ec.get(u, 0) for u in all_uids}) - # ── v2.5.0: CE percentile ranks per window (integer 0–100, match DAX exactly) ── + # ── CE percentile ranks per window (integer 0–100, match DAX exactly) ── # DAX formula: ROUND( COUNTROWS(users with score <= mine) / COUNTROWS(users with score > 0) * 100 , 0) # Users with score 0 / no activity → BLANK (we emit empty string). def _ce_rank_pct(scores: dict[str, int]) -> dict[str, str]: @@ -9728,7 +13772,7 @@ def write_userstats_files( ce_rank_ppt = {w: _ce_rank_pct({u: ppt_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} ce_rank_all = {w: _ce_rank_pct(m365_all_apps_raw[w]) for w in RANK_WINDOWS} - # ── v2.6.0: CE Copilot Percentile based on PROMPT COUNT (human interactions) ── + # ── CE Copilot Percentile based on PROMPT COUNT (human interactions) ── # Read the SessionStats CSV (if produced by run_rollup) and tally PromptCount per # user per window. This is the AI in One semantics: one count per `isPrompt=TRUE` # message — resistant to AI-response fanout, plugin chains, retries, and most @@ -9830,7 +13874,7 @@ def write_userstats_files( t_act, o_act, off_act, t_seg, o_seg, w_seg, x_seg, p_seg, off_seg, overall_seg, - # v2.5.0: precomputed raw + CE rank columns per window (order must + # Precomputed raw + CE rank columns per window (order must # match USERSTATS_HEADER: 6 raws × 3 windows, then 7 ranks × 3 windows) teams_raw["L30"].get(uid, 0), teams_raw["L60"].get(uid, 0), teams_raw["Full"].get(uid, 0), outlook_raw["L30"].get(uid, 0), outlook_raw["L60"].get(uid, 0), outlook_raw["Full"].get(uid, 0), @@ -9925,7 +13969,7 @@ EXAMPLES --outlook Outlook_Export.csv ^ --files Files_Export.csv ^ --copilot Copilot_Export.csv ^ - --output-dir .\output + --output-dir ./output OUTPUT (rollup mode, both layouts produce the same three files): @@ -10317,133 +14361,1837 @@ function Resolve-PythonExe { } } - if (-not $installed) { - throw "Rollup: failed to auto-install Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+. Install Python manually from https://www.python.org/downloads/ and re-run." - } + if (-not $installed) { + throw "Rollup: failed to auto-install Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+. Install Python manually from https://www.python.org/downloads/ and re-run." + } + + # Refresh PATH from process + machine + user scopes so new install is visible without restart. + $paths = @( + [Environment]::GetEnvironmentVariable('Path', 'Machine'), + [Environment]::GetEnvironmentVariable('Path', 'User'), + $env:Path + ) | Where-Object { $_ } + $env:Path = ($paths -join ';') + + # Recurse once with auto-install disabled to prevent loops. + return Resolve-PythonExe -AllowAutoInstall:$false +} + +function Write-PaxRollupStage { + <# + .SYNOPSIS + Emit one sanitized rollup stage record (entry, heartbeat, completion, skip, or failure). + + .DESCRIPTION + Rollup preparation runs several phases that can each take a long time on a large + append target, and without stage records a run looks idle while it is working. Each + record names the stage, its status, and the seconds elapsed in that stage, followed + by caller-supplied AGGREGATE counters only. + + Nothing identifying is accepted or written. Counter values are emitted only when + they are numeric or boolean; any other value is replaced by a placeholder so a key, + row, field, package identifier, path, URL, user principal name, tenant or account + identifier, or credential can never reach the record. Emission is routed through the + host log helper when it exists and is inert when it does not, so the function is safe + in isolated unit extraction. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Stage, + [ValidateSet('entry', 'heartbeat', 'complete', 'skipped', 'failed')][string]$Status = 'entry', + [hashtable]$Counters + ) + if ($null -eq $script:PaxRollupStageStarted) { $script:PaxRollupStageStarted = @{} } + $nowUtc = [System.DateTime]::UtcNow + if ($Status -eq 'entry' -or -not $script:PaxRollupStageStarted.ContainsKey($Stage)) { + $script:PaxRollupStageStarted[$Stage] = $nowUtc + } + $elapsed = [double](($nowUtc - [datetime]$script:PaxRollupStageStarted[$Stage]).TotalSeconds) + $parts = [System.Collections.Generic.List[string]]::new() + if ($Counters) { + foreach ($name in @($Counters.Keys | Sort-Object)) { + $v = $Counters[$name] + if ($v -is [bool]) { [void]$parts.Add(('{0}={1}' -f $name, $v)); continue } + if ($v -is [int] -or $v -is [int64] -or $v -is [int16] -or $v -is [uint32] -or $v -is [uint64] -or $v -is [byte]) { + [void]$parts.Add(('{0}={1:N0}' -f $name, [int64]$v)); continue + } + if ($v -is [double] -or $v -is [single] -or $v -is [decimal]) { + [void]$parts.Add(('{0}={1:N2}' -f $name, [double]$v)); continue + } + [void]$parts.Add(('{0}=' -f $name)) + } + } + $detail = if ($parts.Count -gt 0) { ' ' + ($parts.ToArray() -join ' ') } else { '' } + $color = if ($Status -eq 'failed') { 'Yellow' } else { 'DarkGray' } + $msg = ('Rollup stage [{0}] {1}: elapsed={2:N1}s{3}' -f $Stage, $Status, $elapsed, $detail) + $lh = Microsoft.PowerShell.Core\Get-Command -Name 'Write-LogHost' -CommandType Function -ErrorAction SilentlyContinue + if ($lh) { try { & $lh -Message $msg -ForegroundColor $color } catch { } } +} + +function Write-PaxRollupReaderMode { + <# + .SYNOPSIS + Record exactly once which CSV record reader backend serves the bounded rollup pipelines. + + .DESCRIPTION + The record reader chooses its backend once per process. This helper reports that single + decision one time so a run states plainly whether the compiled reader or the built-in + parser is active, without repeating the fact on every stage. Only the mode is recorded. + #> + [CmdletBinding()] + param() + if ($script:PaxRollupReaderModeLogged) { return } + $script:PaxRollupReaderModeLogged = $true + $compiled = $false + try { + if ($script:PaxCsvBackendOverride -ne 'Fallback') { + $compiled = [bool](script:Initialize-PaxCsvCompiledReader) + } + } + catch { $compiled = $false } + Write-PaxRollupStage -Stage 'csv-reader' -Status 'complete' -Counters @{ compiledReader = $compiled } +} + +function Invoke-PaxBoundedProcess { + <# + .SYNOPSIS + Run a child process under a wall-clock bound and never wait on it indefinitely. + + .DESCRIPTION + Optional package handling contacts an external tool that can stall without ever + exiting. Waiting on such a child with no bound would hold the whole run. This helper + starts the child with redirected output, waits at most the supplied number of + seconds, and on expiry terminates ONLY the process it started (and that process's + own descendants), then reports the outcome. + + The returned record carries the outcome, the exit code when the child exited on its + own, and the captured output. Nothing is thrown for a non-zero exit or a timeout, so + callers decide whether the outcome matters. Only inbox types are used. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $FilePath, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $ArgumentList, + [int] $TimeoutSeconds = 120 + ) + + $result = [pscustomobject]@{ + Outcome = 'Failed' + ExitCode = $null + StdOut = '' + StdErr = '' + Message = '' + Seconds = [double]0 + } + + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $FilePath + foreach ($a in @($ArgumentList)) { [void]$psi.ArgumentList.Add([string]$a) } + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + + $proc = $null + $sw = [System.Diagnostics.Stopwatch]::StartNew() + try { + $proc = [System.Diagnostics.Process]::Start($psi) + if ($null -eq $proc) { + $result.Message = 'process could not be started' + return $result + } + # Read both streams asynchronously so a full pipe buffer can never deadlock the wait. + $outTask = $proc.StandardOutput.ReadToEndAsync() + $errTask = $proc.StandardError.ReadToEndAsync() + $bound = [int]([math]::Max(1, $TimeoutSeconds)) * 1000 + if ($proc.WaitForExit($bound)) { + [void]$outTask.Wait(5000) + [void]$errTask.Wait(5000) + try { $result.StdOut = [string]$outTask.Result } catch { $result.StdOut = '' } + try { $result.StdErr = [string]$errTask.Result } catch { $result.StdErr = '' } + $result.ExitCode = [int]$proc.ExitCode + $result.Outcome = if ($result.ExitCode -eq 0) { 'Succeeded' } else { 'ExitedNonZero' } + } + else { + $result.Outcome = 'TimedOut' + $result.Message = ("no exit within {0}s" -f $TimeoutSeconds) + # Terminate ONLY this cycle-started child and its own descendants. + try { $proc.Kill($true) } catch { try { $proc.Kill() } catch { } } + try { [void]$proc.WaitForExit(5000) } catch { } + } + } + catch { + $result.Outcome = 'Failed' + $result.Message = $_.Exception.Message + } + finally { + $sw.Stop() + $result.Seconds = [double]$sw.Elapsed.TotalSeconds + if ($proc) { try { $proc.Dispose() } catch { } } + } + return $result +} + +function Install-OrjsonIfMissing { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs, + [int] $ProbeTimeoutSeconds = 60, + [int] $InstallTimeoutSeconds = 180 + ) + + # Best-effort. Both embedded processors fall back to stdlib json on import failure, so + # every outcome here is reported and the run continues. Both the probe and the install + # are bounded so neither can hold the run open, and a child that outlives its bound is + # terminated before the run moves on. + Write-PaxRollupStage -Stage 'json-package' -Status 'entry' + + $showArgs = @($LauncherArgs) + @('-m', 'pip', '--disable-pip-version-check', 'show', 'orjson') + $probe = Invoke-PaxBoundedProcess -FilePath $PythonExe -ArgumentList $showArgs -TimeoutSeconds $ProbeTimeoutSeconds + if ($probe.Outcome -eq 'Succeeded') { + Write-PaxRollupStage -Stage 'json-package' -Status 'complete' -Counters @{ alreadyPresent = $true; installAttempted = $false } + return $true + } + if ($probe.Outcome -eq 'TimedOut') { + Write-LogHost "Rollup: optional JSON package check did not finish within its time limit and was stopped. Continuing with the built-in JSON reader." -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ probeTimedOut = $true; installAttempted = $false } + return $false + } + if ($probe.Outcome -eq 'Failed') { + Write-LogHost "Rollup: optional JSON package check could not run. Continuing with the built-in JSON reader." -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ probeFailed = $true; installAttempted = $false } + return $false + } + + Write-LogHost "Rollup: 'orjson' not installed; installing for faster JSON parsing (~5-10x). Falls back to stdlib json on failure." -ForegroundColor Cyan + $installArgs = @($LauncherArgs) + @('-m', 'pip', '--disable-pip-version-check', 'install', '--quiet', '--user', 'orjson') + $install = Invoke-PaxBoundedProcess -FilePath $PythonExe -ArgumentList $installArgs -TimeoutSeconds $InstallTimeoutSeconds + if ($install.StdOut) { foreach ($ln in ($install.StdOut -split "`r?`n")) { if ($ln.Trim()) { Write-Log $ln } } } + if ($install.StdErr) { foreach ($ln in ($install.StdErr -split "`r?`n")) { if ($ln.Trim()) { Write-Log $ln } } } + + if ($install.Outcome -eq 'Succeeded') { + Write-PaxRollupStage -Stage 'json-package' -Status 'complete' -Counters @{ alreadyPresent = $false; installAttempted = $true; installed = $true } + return $true + } + if ($install.Outcome -eq 'TimedOut') { + Write-LogHost "Rollup: 'orjson' install did not finish within its time limit and was stopped. Continuing with stdlib json." -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ installTimedOut = $true; installAttempted = $true; installed = $false } + return $false + } + if ($install.Outcome -eq 'ExitedNonZero') { + Write-LogHost "Rollup: 'orjson' install returned exit code $($install.ExitCode). Continuing with stdlib json." -ForegroundColor Yellow + } + else { + Write-LogHost "Rollup: 'orjson' install could not run: $($install.Message). Continuing with stdlib json." -ForegroundColor Yellow + } + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ installAttempted = $true; installed = $false } + return $false +} + +function Install-DeltalakeIfMissing { + <# + .SYNOPSIS + Ensures the Python 'deltalake' package is installed for Fabric Lakehouse Delta-table writes. + + .DESCRIPTION + Invoked only when the run will write to a Fabric Lakehouse target (Tables/ namespace). + Performs a quiet per-user install on first use. On failure, returns $false with a clear + actionable message — the caller must abort the Delta write path rather than continue. + + Unlike Install-OrjsonIfMissing, the deltalake package is NOT optional: the Fabric Tables + write path cannot fall back to a pure-Python implementation. Callers must check the + return value and abort the run cleanly if the install fails. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + + $showArgs = @($LauncherArgs) + @('-m', 'pip', 'show', 'deltalake') + & $PythonExe @showArgs 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { return $true } + + Write-LogHost "Fabric: 'deltalake' Python package not installed; installing (required for Lakehouse Tables/ writes)." -ForegroundColor Cyan + $installArgs = @($LauncherArgs) + @('-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', '--user', 'deltalake>=0.15') + try { + & $PythonExe @installArgs 2>&1 | ForEach-Object { Write-Log $_ } + if ($LASTEXITCODE -eq 0) { + Write-LogHost "Fabric: 'deltalake' installed successfully." -ForegroundColor Green + return $true + } + Write-LogHost "Fabric: 'deltalake' install returned exit code $LASTEXITCODE. Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red + return $false + } + catch { + Write-LogHost "Fabric: 'deltalake' install threw: $($_.Exception.Message). Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red + return $false + } +} + +function ConvertTo-UsersSeedMap { + <# + .SYNOPSIS + Extract {PersonId_Normalized -> UserKey (INT)} from a target Users CSV and write it + as a compact JSON file the embedded CopilotInteraction processor consumes via + `--seed-userkey-map`. + + .DESCRIPTION + Pre-flight helper for -AppendUserInfo. Read the existing target Users CSV produced by a + prior run, build a dict of PersonId_Normalized -> UserKey, and write JSON. Skips rows + with empty PersonId_Normalized or non-integer UserKey. First occurrence of any + PersonId_Normalized wins (target is the source of truth). + + .OUTPUTS + Int count of entries written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUsersCsv, + [Parameter(Mandatory)] [string] $OutJsonPath, + # Fixed in-memory buffer budget for the bounded sort (bytes). Governs only + # WHEN buffered pairs spill; never caps how many rows are processed. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default. It is forwarded verbatim to + # the bounded sorts, which pass the callback an AGGREGATE snapshot only and swallow + # any exception it raises. Supplying it cannot change the seed JSON bytes, the key + # set, the ordering, or the success of the build. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 + ) + if (-not (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf)) { + throw "ConvertTo-UsersSeedMap: target Users CSV not found: '$TargetUsersCsv'" + } + + # Bounded seed builder (shared shape with ConvertTo-FactSeedMaps): stream the + # target once, spill (rawKey, sequence, int-value) pairs, fold to distinct + # first-occurrence keys via the frozen external-sort machinery, and stream the + # compact JSON one single-property object at a time so escaping / integer + # formatting match the prior ConvertTo-Json output byte-for-byte. The complete + # target is never materialized. + $buildSeedJson = { + param([string]$PairsCsvPath, [string]$OutTempJsonPath, [string]$WorkDir, [string]$Tag, [int64]$Budget, [scriptblock]$Progress, [double]$ProgressInterval) + $u8 = [System.Text.UTF8Encoding]::new($false) + $byKeyPath = Join-Path $WorkDir ($Tag + '_bykey.csv') + $firstPath = Join-Path $WorkDir ($Tag + '_first.csv') + $bySeqPath = Join-Path $WorkDir ($Tag + '_byseq.csv') + [void](script:Invoke-PaxExternalSort -InputPath $PairsCsvPath -OutputPath $byKeyPath -KeyColumn @('RawKey', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + $cmp = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $r1 = [System.IO.StreamReader]::new($byKeyPath, $true) + $w1 = [System.IO.StreamWriter]::new($firstPath, $false, $u8) + $distinct = [int64]0 + try { + $null = script:Read-PaxCsvRecord -Reader $r1 + script:Write-PaxCsvRecord -Writer $w1 -Fields @('RawKey', 'SeqPad', 'Val') + $haveLast = $false; $lastKey = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r1 + if ($null -eq $row) { break } + $k = if ($row.Length -ge 1) { [string]$row[0] } else { '' } + if ((-not $haveLast) -or ($cmp.Compare($k, $lastKey) -ne 0)) { + script:Write-PaxCsvRecord -Writer $w1 -Fields $row + $distinct++ + $lastKey = $k; $haveLast = $true + } + } + } + finally { $w1.Flush(); $w1.Dispose(); $r1.Dispose() } + [void](script:Invoke-PaxExternalSort -InputPath $firstPath -OutputPath $bySeqPath -KeyColumn @('SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + $r2 = [System.IO.StreamReader]::new($bySeqPath, $true) + $w2 = [System.IO.StreamWriter]::new($OutTempJsonPath, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $r2 + $w2.Write('{') + $wrote = $false + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r2 + if ($null -eq $row) { break } + $rawKey = [string]$row[0] + $valStr = if ($row.Length -ge 3) { [string]$row[2] } else { '0' } + $obj = [ordered]@{} + $obj[$rawKey] = [int]::Parse($valStr, [System.Globalization.CultureInfo]::InvariantCulture) + $oneLine = $obj | ConvertTo-Json -Compress -Depth 2 + $inner = $oneLine.Substring(1, $oneLine.Length - 2) + if ($wrote) { $w2.Write(',') } + $w2.Write($inner) + $wrote = $true + } + $w2.Write('}') + } + finally { $w2.Flush(); $w2.Dispose(); $r2.Dispose() } + return $distinct + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_usersseed_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $outTemp = $OutJsonPath + '.seedtmp' + $count = 0 + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $pairs = Join-Path $workDir 'pairs.csv' + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + $reader = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + $pw = [System.IO.StreamWriter]::new($pairs, $false, $utf8NoBom) + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('RawKey', 'SeqPad', 'Val') + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -ne $header) { + # Match the prior Import-Csv fail-closed behavior on a duplicated member. + $seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($h in $header) { if (-not $seen.Add([string]$h)) { throw "ConvertTo-UsersSeedMap: duplicate header column '$h' in target Users CSV." } } + $pidIdx = -1; $ukIdx = -1 + for ($i = 0; $i -lt $header.Length; $i++) { + $name = [string]$header[$i] + if ([string]::Equals($name, 'PersonId_Normalized', [System.StringComparison]::OrdinalIgnoreCase)) { $pidIdx = $i } + elseif ([string]::Equals($name, 'UserKey', [System.StringComparison]::OrdinalIgnoreCase)) { $ukIdx = $i } + } + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $seq = [int64]0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $row) { break } + $pidRaw = if ($pidIdx -ge 0 -and $pidIdx -lt $row.Length) { [string]$row[$pidIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($pidRaw)) { + $ukStr = if ($ukIdx -ge 0 -and $ukIdx -lt $row.Length) { [string]$row[$ukIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($ukStr)) { + $ukInt = 0 + if ([int]::TryParse($ukStr, [ref]$ukInt)) { + script:Write-PaxCsvRecord -Writer $pw -Fields @($pidRaw, $seq.ToString('D19', $inv), $ukInt.ToString($inv)) + } + } + } + $seq++ + } + } + } + finally { $pw.Flush(); $pw.Dispose(); $reader.Dispose() } + $count = [int](& $buildSeedJson $pairs $outTemp $workDir 'seed' $MemoryBudgetBytes $ProgressCallback $ProgressIntervalSeconds) + try { $null = (Get-Content -LiteralPath $outTemp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "ConvertTo-UsersSeedMap: seed JSON validation failed for '$outTemp': $($_.Exception.Message)" } + # Aggregate-only stage record. Routed through a host lookup so the function stays + # inert when it is extracted and exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + if ($__paxStage) { + $__seedBytes = try { [int64](Get-Item -LiteralPath $outTemp).Length } catch { [int64]0 } + try { & $__paxStage -Stage 'users-seed-validate' -Status 'complete' -Counters @{ distinctKeys = [int64]$count; seedBytes = $__seedBytes } } catch { } + } + $outDir = Split-Path -Parent $OutJsonPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + Move-Item -LiteralPath $outTemp -Destination $OutJsonPath -Force -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $outTemp) { try { Remove-Item -LiteralPath $outTemp -Force -ErrorAction SilentlyContinue } catch {} } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + return $count +} + +function ConvertTo-PaxUserKeyContinuity { + <# + .SYNOPSIS + Build one combined {identity -> UserKey} continuity map from whichever existing + append targets are present and emit the paired continuity seed and audit-only + identity list. + + .DESCRIPTION + A Users target on its own describes only the identities that were present in a + directory export. An identity that appears solely in audit activity lives only in + the Fact target, so a map built from the Users target alone reserves none of those + keys and a later run can hand one of them to a different person. + + Either target may be absent. Continuity is then derived from the one that is + present, so a run that appends only the activity data still keeps every key it + already handed out. + + Both targets are streamed once through the bounded record reader and the bounded + external-sort machinery; neither file is held in memory. Identity comparison is + case-insensitive and culture-independent, matching the rest of the append path. + + The prior mapping is validated BEFORE anything is published. The build fails closed + when one identity carries more than one key, when one key is claimed by more than + one identity, when a key is absent or is not a positive whole number where + continuity would be assumed, when a required identity or key column is missing, + when either file cannot be read as well-formed CSV, or when the paired outputs + cannot both be completed. Only aggregate counts are reported; no identity and no + key value ever reaches a message. + + .OUTPUTS + Hashtable: SeededKeys, UsersIdentities, AuditOnlyIdentities, MaxReservedUserKey. + #> + [CmdletBinding()] + param( + [Parameter()] [string] $TargetUsersCsv, + [Parameter()] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $OutJsonPath, + [Parameter(Mandatory)] [string] $OutAuditOnlyCsvPath, + # Fixed in-memory buffer budget for the bounded sorts (bytes). Governs only + # WHEN buffered pairs spill; never caps how many rows are processed. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default and forwarded verbatim to the + # bounded sorts, which pass an AGGREGATE snapshot and swallow any exception it + # raises. Supplying it cannot change any output byte. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 + ) + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + $cmpId = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + + # First matching column position for any of the supplied names (BOM strip / trim, + # case-insensitive), or -1. + $findCol = { + param($header, [string[]]$names) + for ($i = 0; $i -lt $header.Length; $i++) { + $h = [string]$header[$i] + if ($i -eq 0) { $h = $h -replace '^\uFEFF', '' } + $h = $h.Trim() + foreach ($n in $names) { + if ([string]::Equals($h, $n, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } + } + } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_ukcont_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $seedTemp = $OutJsonPath + '.conttmp' + $auditTemp = $OutAuditOnlyCsvPath + '.conttmp' + $seededKeys = 0; $usersIdentities = 0; $auditOnlyIdentities = 0; $maxReserved = 0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $pairs = Join-Path $workDir 'pairs.csv' + $pw = [System.IO.StreamWriter]::new($pairs, $false, $u8) + $seq = [int64]0 + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('RawKey', 'KeyPad', 'SeqPad', 'Val', 'Side') + + # --- Side 0: the existing Users target, in file order. --- + if ($TargetUsersCsv -and (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf)) { + $ur = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + try { + $uh = script:Read-PaxCsvRecord -Reader $ur + if ($null -ne $uh) { + $uPid = & $findCol $uh @('PersonId_Normalized') + $uKey = & $findCol $uh @('UserKey') + if ($uPid -lt 0 -or $uKey -lt 0) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Users target does not carry both the normalized identity column and the UserKey column; refusing to assume key continuity.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ur + if ($null -eq $row) { break } + $idRaw = if ($uPid -lt $row.Length) { ([string]$row[$uPid]).Trim() } else { '' } + if (-not [string]::IsNullOrWhiteSpace($idRaw)) { + $kStr = if ($uKey -lt $row.Length) { ([string]$row[$uKey]).Trim() } else { '' } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Users target holds an identity whose UserKey is absent or is not a positive whole number; refusing to assume key continuity.' + } + if ($kInt -gt $maxReserved) { $maxReserved = $kInt } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), $seq.ToString('D19', $inv), $kInt.ToString($inv), '0') + $seq++ + } + } + } + } + finally { $ur.Dispose() } + } + + # --- Side 1: the existing Fact target, in file order. --- + if ($TargetFactCsv -and (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { + $fr = [System.IO.StreamReader]::new($TargetFactCsv, $true) + try { + $fh = script:Read-PaxCsvRecord -Reader $fr + if ($null -ne $fh) { + $fId = & $findCol $fh @('User_Id_Normalized', 'Audit_UserId_Normalized') + $fKey = & $findCol $fh @('UserKey') + if ($fId -lt 0 -or $fKey -lt 0) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Fact target does not carry both the normalized identity column and the UserKey column; refusing to assume key continuity.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $row) { break } + $idRaw = if ($fId -lt $row.Length) { ([string]$row[$fId]).Trim() } else { '' } + if (-not [string]::IsNullOrWhiteSpace($idRaw)) { + $kStr = if ($fKey -lt $row.Length) { ([string]$row[$fKey]).Trim() } else { '' } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Fact target holds an identity whose UserKey is absent or is not a positive whole number; refusing to assume key continuity.' + } + if ($kInt -gt $maxReserved) { $maxReserved = $kInt } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), $seq.ToString('D19', $inv), $kInt.ToString($inv), '1') + $seq++ + } + } + } + } + finally { $fr.Dispose() } + } + } + finally { $pw.Flush(); $pw.Dispose() } + + # --- Identity view: one group per identity, first appearance first. --- + $byIdentity = Join-Path $workDir 'by_identity.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byIdentity -KeyColumn @('RawKey', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $identities = Join-Path $workDir 'identities.csv' + $splitIdentities = 0 + $ir = [System.IO.StreamReader]::new($byIdentity, $true) + $iw = [System.IO.StreamWriter]::new($identities, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $ir + script:Write-PaxCsvRecord -Writer $iw -Fields @('RawKey', 'SeqPad', 'Val', 'HasUsers') + $haveGroup = $false; $gRaw = ''; $gSeq = ''; $gVal = ''; $gHasUsers = '0'; $gSplit = $false + $rRaw = ''; $rSeq = ''; $rVal = ''; $rSide = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ir + $isEnd = ($null -eq $row) + if (-not $isEnd) { + $rRaw = [string]$row[0] + $rSeq = [string]$row[2] + $rVal = [string]$row[3] + $rSide = [string]$row[4] + } + if ($haveGroup -and ($isEnd -or ($cmpId.Compare($rRaw, $gRaw) -ne 0))) { + if ($gSplit) { $splitIdentities++ } + script:Write-PaxCsvRecord -Writer $iw -Fields @($gRaw, $gSeq, $gVal, $gHasUsers) + if ($gHasUsers -eq '1') { $usersIdentities++ } else { $auditOnlyIdentities++ } + $haveGroup = $false + } + if ($isEnd) { break } + if (-not $haveGroup) { + $haveGroup = $true; $gRaw = $rRaw; $gSeq = $rSeq; $gVal = $rVal; $gHasUsers = '0'; $gSplit = $false + } + if ($rVal -ne $gVal) { $gSplit = $true } + if ($rSide -eq '0') { $gHasUsers = '1' } + } + } + finally { $iw.Flush(); $iw.Dispose(); $ir.Dispose() } + + # --- Key view: one group per reserved key. --- + $byKey = Join-Path $workDir 'by_key.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byKey -KeyColumn @('KeyPad', 'SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $sharedKeys = 0 + $kr = [System.IO.StreamReader]::new($byKey, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $kr + $haveK = $false; $kPad = ''; $kFirstId = ''; $kShared = $false + $rPad = ''; $rId = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $kr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId = [string]$row[0]; $rPad = [string]$row[1] } + if ($haveK -and ($isEnd -or ($rPad -ne $kPad))) { + if ($kShared) { $sharedKeys++ } + $haveK = $false + } + if ($isEnd) { break } + if (-not $haveK) { $haveK = $true; $kPad = $rPad; $kFirstId = $rId; $kShared = $false } + if ($cmpId.Compare($rId, $kFirstId) -ne 0) { $kShared = $true } + } + } + finally { $kr.Dispose() } + + if ($splitIdentities -gt 0 -or $sharedKeys -gt 0) { + Write-PaxRollupStage -Stage 'users-continuity' -Status 'failed' -Counters @{ splitIdentities = [int64]$splitIdentities; sharedUserKeys = [int64]$sharedKeys } + throw ('ConvertTo-PaxUserKeyContinuity: the existing append targets disagree about user keys ({0} identity group(s) carry more than one key; {1} key(s) are claimed by more than one identity). Refusing to build a continuity map that would corrupt the append.' -f $splitIdentities, $sharedKeys) + } + + # --- Emit the paired outputs in first-appearance order. --- + $ordered = Join-Path $workDir 'ordered.csv' + [void](script:Invoke-PaxExternalSort -InputPath $identities -OutputPath $ordered -KeyColumn @('SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $or = [System.IO.StreamReader]::new($ordered, $true) + $jw = [System.IO.StreamWriter]::new($seedTemp, $false, $u8) + $aw = [System.IO.StreamWriter]::new($auditTemp, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $or + script:Write-PaxCsvRecord -Writer $aw -Fields @('PersonId_Normalized', 'UserKey') + $jw.Write('{') + $wrote = $false + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $or + if ($null -eq $row) { break } + $rawKey = [string]$row[0] + $valStr = [string]$row[2] + $hasUsers = [string]$row[3] + $obj = [ordered]@{} + $obj[$rawKey] = [int]::Parse($valStr, $inv) + $oneLine = $obj | ConvertTo-Json -Compress -Depth 2 + $innerJson = $oneLine.Substring(1, $oneLine.Length - 2) + if ($wrote) { $jw.Write(',') } + $jw.Write($innerJson) + $wrote = $true + $seededKeys++ + if ($hasUsers -ne '1') { script:Write-PaxCsvRecord -Writer $aw -Fields @($rawKey, $valStr) } + } + $jw.Write('}') + } + finally { $jw.Flush(); $jw.Dispose(); $aw.Flush(); $aw.Dispose(); $or.Dispose() } + + try { $null = (Get-Content -LiteralPath $seedTemp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "ConvertTo-PaxUserKeyContinuity: continuity seed validation failed: $($_.Exception.Message)" } + if (-not (Test-Path -LiteralPath $auditTemp -PathType Leaf)) { + throw 'ConvertTo-PaxUserKeyContinuity: the audit-only identity list was not produced; refusing to publish an incomplete pair.' + } + foreach ($outPath in @($OutJsonPath, $OutAuditOnlyCsvPath)) { + $outDir = Split-Path -Parent $outPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + } + Move-Item -LiteralPath $seedTemp -Destination $OutJsonPath -Force -ErrorAction Stop + try { Move-Item -LiteralPath $auditTemp -Destination $OutAuditOnlyCsvPath -Force -ErrorAction Stop } + catch { + if (Test-Path -LiteralPath $OutJsonPath) { try { Remove-Item -LiteralPath $OutJsonPath -Force -ErrorAction SilentlyContinue } catch { } } + throw + } + Write-PaxRollupStage -Stage 'users-continuity' -Status 'complete' -Counters @{ reservedKeys = [int64]$seededKeys; directoryIdentities = [int64]$usersIdentities; auditOnlyIdentities = [int64]$auditOnlyIdentities; highestReservedKey = [int64]$maxReserved } + } + catch { + foreach ($tempPath in @($seedTemp, $auditTemp)) { + if (Test-Path -LiteralPath $tempPath) { try { Remove-Item -LiteralPath $tempPath -Force -ErrorAction SilentlyContinue } catch { } } + } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch { } } + } + + return @{ + SeededKeys = [int]$seededKeys + UsersIdentities = [int]$usersIdentities + AuditOnlyIdentities = [int]$auditOnlyIdentities + MaxReservedUserKey = [int]$maxReserved + } +} + +function Add-PaxAuditOnlyUserRows { + <# + .SYNOPSIS + Write a copy of the current-run Users CSV that also carries one row for every + audit-only identity, so every Fact user resolves to a Users row. + + .DESCRIPTION + An identity that only ever appears in audit activity has no directory row, so the + current run emits no Users row for it and the Fact rows that carry its key have + nothing to join to. This helper adds one row per such identity carrying ONLY the + normalized identity and the key already reserved for it. Every descriptive, + licensing, organizational, manager, hierarchy and profile value is left blank; + nothing is invented. + + The missing identities are taken from two sources: the prior audit-only list, and + the candidate activity data for this run. The second source covers a person seen + for the very first time in this run's activity, who has no prior row anywhere and + would otherwise leave the key their Fact rows carry unresolved. + + A real directory row always wins: an audit-only identity that IS present in the + current run is skipped entirely, and a later run that obtains a real row folds it + in through the union merge while keeping the same key. + + Both inputs are streamed through the bounded machinery. Publication is refused when + the audit-only list repeats an identity or repeats a key, when one identity carries + more than one key in the candidate activity data, or when a reserved key disagrees + with the key that identity carries in the candidate activity data. + + .OUTPUTS + Int count of rows added. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $CurrentUsersCsv, + [Parameter(Mandatory)] [string] $AuditOnlyCsv, + [Parameter(Mandatory)] [string] $OutputPath, + [Parameter()] [string] $CandidateFactCsv, + [int64] $MemoryBudgetBytes = 33554432 + ) + if (-not (Test-Path -LiteralPath $CurrentUsersCsv -PathType Leaf)) { + throw "Add-PaxAuditOnlyUserRows: current Users CSV not found: '$CurrentUsersCsv'" + } + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + $cmpId = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $findCol = { + param($header, [string]$name) + for ($i = 0; $i -lt $header.Length; $i++) { + $h = [string]$header[$i] + if ($i -eq 0) { $h = $h -replace '^\uFEFF', '' } + if ([string]::Equals($h.Trim(), $name, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } + } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_ukplace_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $tmpOut = $OutputPath + '.placing' + $added = 0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + + # --- Header of the current-run Users CSV (bounded: header record only). --- + $hdr = $null + $hr = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { $hdr = script:Read-PaxCsvRecord -Reader $hr } finally { $hr.Dispose() } + if ($null -eq $hdr) { throw 'Add-PaxAuditOnlyUserRows: the current-run Users CSV has no header record.' } + $colPidNorm = & $findCol $hdr 'PersonId_Normalized' + $colUserKey = & $findCol $hdr 'UserKey' + $colPersonId = & $findCol $hdr 'PersonId' + if ($colPidNorm -lt 0 -or $colUserKey -lt 0) { + throw 'Add-PaxAuditOnlyUserRows: the current-run Users CSV does not carry both the normalized identity column and the UserKey column; refusing to add rows.' + } + + # --- Pair both identity sources into one bounded work file. --- + $pairs = Join-Path $workDir 'pairs.csv' + $pw = [System.IO.StreamWriter]::new($pairs, $false, $u8) + $seq = [int64]0 + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('Id', 'Side', 'SeqPad', 'Val') + $cr = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $cr + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr + if ($null -eq $row) { break } + $idRaw = if ($colPidNorm -lt $row.Length) { ([string]$row[$colPidNorm]).Trim() } else { '' } + if (-not [string]::IsNullOrWhiteSpace($idRaw)) { + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, '0', $seq.ToString('D19', $inv), '') + $seq++ + } + } + } + finally { $cr.Dispose() } + + if (Test-Path -LiteralPath $AuditOnlyCsv -PathType Leaf) { + $ar = [System.IO.StreamReader]::new($AuditOnlyCsv, $true) + try { + $ah = script:Read-PaxCsvRecord -Reader $ar + if ($null -ne $ah) { + $aId = & $findCol $ah 'PersonId_Normalized' + $aKey = & $findCol $ah 'UserKey' + if ($aId -lt 0 -or $aKey -lt 0) { + throw 'Add-PaxAuditOnlyUserRows: the audit-only identity list is missing a required column; refusing to add rows.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ar + if ($null -eq $row) { break } + $idRaw = if ($aId -lt $row.Length) { ([string]$row[$aId]).Trim() } else { '' } + $kStr = if ($aKey -lt $row.Length) { ([string]$row[$aKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($idRaw)) { continue } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'Add-PaxAuditOnlyUserRows: the audit-only identity list carries a key that is not a positive whole number; refusing to add rows.' + } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, '1', $seq.ToString('D19', $inv), $kInt.ToString($inv)) + $seq++ + } + } + } + finally { $ar.Dispose() } + } + + # --- Side 2: the candidate activity data, in file order. An identity first --- + # --- seen in THIS run has no prior audit-only row, so its key is taken here. --- + if ($CandidateFactCsv -and (Test-Path -LiteralPath $CandidateFactCsv -PathType Leaf)) { + $fr = [System.IO.StreamReader]::new($CandidateFactCsv, $true) + try { + $fh = script:Read-PaxCsvRecord -Reader $fr + if ($null -ne $fh) { + $fId = & $findCol $fh 'User_Id_Normalized' + if ($fId -lt 0) { $fId = & $findCol $fh 'Audit_UserId_Normalized' } + $fKey = & $findCol $fh 'UserKey' + if ($fId -lt 0 -or $fKey -lt 0) { + throw 'Add-PaxAuditOnlyUserRows: the candidate Fact CSV does not carry both the normalized identity column and the UserKey column; refusing to add rows.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $row) { break } + $idRaw = if ($fId -lt $row.Length) { ([string]$row[$fId]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($idRaw)) { continue } + $kStr = if ($fKey -lt $row.Length) { ([string]$row[$fKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($kStr)) { continue } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'Add-PaxAuditOnlyUserRows: the candidate Fact CSV carries a key that is not a positive whole number; refusing to add rows.' + } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, '2', $seq.ToString('D19', $inv), $kInt.ToString($inv)) + $seq++ + } + } + } + finally { $fr.Dispose() } + } + } + finally { $pw.Flush(); $pw.Dispose() } - # Refresh PATH from process + machine + user scopes so new install is visible without restart. - $paths = @( - [Environment]::GetEnvironmentVariable('Path', 'Machine'), - [Environment]::GetEnvironmentVariable('Path', 'User'), - $env:Path - ) | Where-Object { $_ } - $env:Path = ($paths -join ';') + # --- Group per identity; keep only identities with no current-run row. --- + $sorted = Join-Path $workDir 'sorted.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $sorted -KeyColumn @('Id', 'Side', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) - # Recurse once with auto-install disabled to prevent loops. - return Resolve-PythonExe -AllowAutoInstall:$false -} + $additions = Join-Path $workDir 'additions.csv' + $sr = [System.IO.StreamReader]::new($sorted, $true) + $aw = [System.IO.StreamWriter]::new($additions, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $sr + script:Write-PaxCsvRecord -Writer $aw -Fields @('Id', 'KeyPad', 'Val') + $haveGroup = $false; $gId = ''; $gHasCurrent = $false; $gVal = ''; $gAuditRows = 0; $gFactRows = 0; $gFactVal = ''; $gFactSplit = $false + $rId = ''; $rSide = ''; $rVal = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $sr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId = [string]$row[0]; $rSide = [string]$row[1]; $rVal = [string]$row[3] } + if ($haveGroup -and ($isEnd -or ($cmpId.Compare($rId, $gId) -ne 0))) { + if ($gAuditRows -gt 1) { + throw 'Add-PaxAuditOnlyUserRows: the audit-only identity list repeats an identity; refusing to add rows.' + } + if ($gFactSplit) { + throw 'Add-PaxAuditOnlyUserRows: one identity carries more than one key in the candidate activity data; refusing to add rows.' + } + if ($gAuditRows -eq 1 -and $gFactRows -gt 0 -and $gFactVal -ne $gVal) { + throw 'Add-PaxAuditOnlyUserRows: an identity''s reserved key disagrees with the key it carries in the candidate activity data; refusing to add rows.' + } + if ((-not $gHasCurrent) -and $gAuditRows -eq 1) { + $kv = [int]::Parse($gVal, $inv) + script:Write-PaxCsvRecord -Writer $aw -Fields @($gId, $kv.ToString('D19', $inv), $gVal) + $added++ + } + elseif ((-not $gHasCurrent) -and $gAuditRows -eq 0 -and $gFactRows -gt 0) { + $kv = [int]::Parse($gFactVal, $inv) + script:Write-PaxCsvRecord -Writer $aw -Fields @($gId, $kv.ToString('D19', $inv), $gFactVal) + $added++ + } + $haveGroup = $false + } + if ($isEnd) { break } + if (-not $haveGroup) { $haveGroup = $true; $gId = $rId; $gHasCurrent = $false; $gVal = ''; $gAuditRows = 0; $gFactRows = 0; $gFactVal = ''; $gFactSplit = $false } + if ($rSide -eq '0') { $gHasCurrent = $true } + elseif ($rSide -eq '1') { $gAuditRows++; $gVal = $rVal } + else { + $gFactRows++ + if ($gFactRows -eq 1) { $gFactVal = $rVal } + elseif ($rVal -ne $gFactVal) { $gFactSplit = $true } + } + } + } + finally { $aw.Flush(); $aw.Dispose(); $sr.Dispose() } -function Install-OrjsonIfMissing { - [CmdletBinding()] - param( - [Parameter(Mandatory)] [string] $PythonExe, - [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs - ) + # --- Refuse a repeated key across the rows about to be added. --- + if ($added -gt 0) { + $byKey = Join-Path $workDir 'additions_bykey.csv' + [void](script:Invoke-PaxExternalSort -InputPath $additions -OutputPath $byKey -KeyColumn @('KeyPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + $dr = [System.IO.StreamReader]::new($byKey, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $dr + $prevPad = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $dr + if ($null -eq $row) { break } + $pad = [string]$row[1] + if ($null -ne $prevPad -and $pad -eq $prevPad) { + throw 'Add-PaxAuditOnlyUserRows: two audit-only identities claim the same key; refusing to add rows.' + } + $prevPad = $pad + } + } + finally { $dr.Dispose() } + } - # Best-effort install. Both embedded processors fall back to stdlib json on import failure, - # so we warn-and-continue on any error here rather than throwing. - $showArgs = @($LauncherArgs) + @('-m', 'pip', 'show', 'orjson') - & $PythonExe @showArgs 2>$null | Out-Null - if ($LASTEXITCODE -eq 0) { return $true } + # --- Publish: the current rows verbatim, then the added rows. --- + $width = $hdr.Length + $ow = [System.IO.StreamWriter]::new($tmpOut, $false, $u8) + try { + $cr2 = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr2 + if ($null -eq $row) { break } + script:Write-PaxCsvRecord -Writer $ow -Fields $row + } + } + finally { $cr2.Dispose() } + if ($added -gt 0) { + $ar2 = [System.IO.StreamReader]::new($additions, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $ar2 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ar2 + if ($null -eq $row) { break } + $fields = New-Object 'string[]' $width + for ($j = 0; $j -lt $width; $j++) { $fields[$j] = '' } + $fields[$colPidNorm] = [string]$row[0] + $fields[$colUserKey] = [string]$row[2] + if ($colPersonId -ge 0) { $fields[$colPersonId] = [string]$row[0] } + script:Write-PaxCsvRecord -Writer $ow -Fields $fields + } + } + finally { $ar2.Dispose() } + } + } + finally { $ow.Flush(); $ow.Dispose() } - Write-LogHost "Rollup: 'orjson' not installed; installing for faster JSON parsing (~5-10x). Falls back to stdlib json on failure." -ForegroundColor Cyan - $installArgs = @($LauncherArgs) + @('-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', '--user', 'orjson') - try { - & $PythonExe @installArgs 2>&1 | ForEach-Object { Write-Log $_ } - if ($LASTEXITCODE -eq 0) { return $true } - Write-LogHost "Rollup: 'orjson' install returned exit code $LASTEXITCODE. Continuing with stdlib json." -ForegroundColor Yellow - return $false + Move-Item -LiteralPath $tmpOut -Destination $OutputPath -Force -ErrorAction Stop } catch { - Write-LogHost "Rollup: 'orjson' install threw: $($_.Exception.Message). Continuing with stdlib json." -ForegroundColor Yellow - return $false + if (Test-Path -LiteralPath $tmpOut) { try { Remove-Item -LiteralPath $tmpOut -Force -ErrorAction SilentlyContinue } catch { } } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch { } } } + + return [int]$added } -function Install-DeltalakeIfMissing { +function Test-PaxUserKeyReferentialIntegrity { <# .SYNOPSIS - Ensures the Python 'deltalake' package is installed for Fabric Lakehouse Delta-table writes. + Validate a candidate Fact file and a candidate Users file TOGETHER, before either + one is allowed to replace an existing append target. .DESCRIPTION - Invoked only when the run will write to a Fabric Lakehouse target (Tables/ namespace). - Performs a quiet per-user install on first use. On failure, returns $false with a clear - actionable message — the caller must abort the Delta write path rather than continue. + Runs the joins the dashboard will run. Every check streams through the bounded + machinery, so file size does not govern memory. Only aggregate counts are + reported; no identity and no key value ever reaches a message. - Unlike Install-OrjsonIfMissing, the deltalake package is NOT optional: the Fabric Tables - write path cannot fall back to a pure-Python implementation. Callers must check the - return value and abort the run cleanly if the install fails. + Checks: no key is claimed by more than one identity; no identity carries more than + one key; every non-blank Fact key resolves to exactly one Users row; the resolved + Users identity matches the Fact identity on that row. + + The Users candidate is optional. With no Users candidate there is nothing to + resolve against, so the Fact candidate is validated for identity-to-key uniqueness + and for keys shared between two identities rather than for orphan resolution. + + .OUTPUTS + Hashtable: Passed, ConflictingUserKeys, SplitIdentities, OrphanFactKeys, + MismatchedFactRows, UsersRows, FactRows. #> [CmdletBinding()] param( - [Parameter(Mandatory)] [string] $PythonExe, - [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + [Parameter()] [string] $CandidateUsersCsv, + [Parameter()] [string] $CandidateFactCsv, + [int64] $MemoryBudgetBytes = 33554432 ) + $hasUsersCandidate = -not [string]::IsNullOrWhiteSpace($CandidateUsersCsv) + if ($hasUsersCandidate -and -not (Test-Path -LiteralPath $CandidateUsersCsv -PathType Leaf)) { + throw "Test-PaxUserKeyReferentialIntegrity: candidate Users CSV not found: '$CandidateUsersCsv'" + } + if (-not $hasUsersCandidate -and -not ($CandidateFactCsv -and (Test-Path -LiteralPath $CandidateFactCsv -PathType Leaf))) { + throw 'Test-PaxUserKeyReferentialIntegrity: neither candidate file was supplied; there is nothing to validate.' + } - $showArgs = @($LauncherArgs) + @('-m', 'pip', 'show', 'deltalake') - & $PythonExe @showArgs 2>$null | Out-Null - if ($LASTEXITCODE -eq 0) { return $true } + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + $cmpId = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $findCol = { + param($header, [string[]]$names) + for ($i = 0; $i -lt $header.Length; $i++) { + $h = [string]$header[$i] + if ($i -eq 0) { $h = $h -replace '^\uFEFF', '' } + $h = $h.Trim() + foreach ($n in $names) { + if ([string]::Equals($h, $n, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } + } + } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_ukgate_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $conflictingKeys = 0; $splitIdentities = 0; $orphanFactKeys = 0; $mismatchedFactRows = 0 + $usersRows = [int64]0; $factRows = [int64]0 - Write-LogHost "Fabric: 'deltalake' Python package not installed; installing (required for Lakehouse Tables/ writes)." -ForegroundColor Cyan - $installArgs = @($LauncherArgs) + @('-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', '--user', 'deltalake>=0.15') try { - & $PythonExe @installArgs 2>&1 | ForEach-Object { Write-Log $_ } - if ($LASTEXITCODE -eq 0) { - Write-LogHost "Fabric: 'deltalake' installed successfully." -ForegroundColor Green - return $true + [void][System.IO.Directory]::CreateDirectory($workDir) + $pairs = Join-Path $workDir 'pairs.csv' + $pw = [System.IO.StreamWriter]::new($pairs, $false, $u8) + $seq = [int64]0 + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('Id', 'KeyPad', 'Side', 'SeqPad') + + if ($hasUsersCandidate) { + $ur = [System.IO.StreamReader]::new($CandidateUsersCsv, $true) + try { + $uh = script:Read-PaxCsvRecord -Reader $ur + if ($null -eq $uh) { throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Users CSV has no header record.' } + $uId = & $findCol $uh @('PersonId_Normalized') + $uKey = & $findCol $uh @('UserKey') + if ($uId -lt 0 -or $uKey -lt 0) { + throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Users CSV does not carry both the normalized identity column and the UserKey column.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ur + if ($null -eq $row) { break } + $usersRows++ + $idRaw = if ($uId -lt $row.Length) { ([string]$row[$uId]).Trim() } else { '' } + $kStr = if ($uKey -lt $row.Length) { ([string]$row[$uKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($idRaw) -or [string]::IsNullOrWhiteSpace($kStr)) { continue } + $kInt = 0 + if (-not [int]::TryParse($kStr, [ref]$kInt)) { continue } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), '0', $seq.ToString('D19', $inv)) + $seq++ + } + } + finally { $ur.Dispose() } + } + + if ($CandidateFactCsv -and (Test-Path -LiteralPath $CandidateFactCsv -PathType Leaf)) { + $fr = [System.IO.StreamReader]::new($CandidateFactCsv, $true) + try { + $fh = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $fh) { throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Fact CSV has no header record.' } + $fId = & $findCol $fh @('User_Id_Normalized', 'Audit_UserId_Normalized') + $fKey = & $findCol $fh @('UserKey') + if ($fId -lt 0 -or $fKey -lt 0) { + throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Fact CSV does not carry both the normalized identity column and the UserKey column.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $row) { break } + $factRows++ + $kStr = if ($fKey -lt $row.Length) { ([string]$row[$fKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($kStr)) { continue } + $idRaw = if ($fId -lt $row.Length) { ([string]$row[$fId]).Trim() } else { '' } + $kInt = 0 + if (-not [int]::TryParse($kStr, [ref]$kInt)) { continue } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), '1', $seq.ToString('D19', $inv)) + $seq++ + } + } + finally { $fr.Dispose() } + } } - Write-LogHost "Fabric: 'deltalake' install returned exit code $LASTEXITCODE. Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red - return $false + finally { $pw.Flush(); $pw.Dispose() } + + # --- Key view: exactly one Users row per key, and every Fact row agrees. --- + $byKey = Join-Path $workDir 'by_key.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byKey -KeyColumn @('KeyPad', 'Side', 'SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + $kr = [System.IO.StreamReader]::new($byKey, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $kr + $haveK = $false; $kPad = ''; $kUsersRows = 0; $kUsersId = ''; $kFactMismatch = 0; $kFactRows = 0; $kFirstFactId = ''; $kFactIdConflict = $false + $rId = ''; $rPad = ''; $rSide = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $kr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId = [string]$row[0]; $rPad = [string]$row[1]; $rSide = [string]$row[2] } + if ($haveK -and ($isEnd -or ($rPad -ne $kPad))) { + if ($kUsersRows -gt 1) { $conflictingKeys++ } + elseif ($kUsersRows -eq 0 -and $kFactRows -gt 0) { + # With no Users candidate there is nothing to resolve against, so the + # only thing a key group can prove is that one key means one person. + if ($hasUsersCandidate) { $orphanFactKeys++ } + elseif ($kFactIdConflict) { $conflictingKeys++ } + } + $mismatchedFactRows += $kFactMismatch + $haveK = $false + } + if ($isEnd) { break } + if (-not $haveK) { $haveK = $true; $kPad = $rPad; $kUsersRows = 0; $kUsersId = ''; $kFactMismatch = 0; $kFactRows = 0; $kFirstFactId = ''; $kFactIdConflict = $false } + if ($rSide -eq '0') { + $kUsersRows++ + if ($kUsersRows -eq 1) { $kUsersId = $rId } + } + else { + $kFactRows++ + if ($kUsersRows -ge 1 -and ($cmpId.Compare($rId, $kUsersId) -ne 0)) { $kFactMismatch++ } + if ($kUsersRows -eq 0) { + if ($kFactRows -eq 1) { $kFirstFactId = $rId } + elseif ($cmpId.Compare($rId, $kFirstFactId) -ne 0) { $kFactIdConflict = $true } + } + } + } + } + finally { $kr.Dispose() } + + # --- Identity view: exactly one key per identity across the Users candidate. --- + $byId = Join-Path $workDir 'by_id.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byId -KeyColumn @('Id', 'Side', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + $dr = [System.IO.StreamReader]::new($byId, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $dr + $haveI = $false; $iId = ''; $iFirstPad = ''; $iSplit = $false + $rId2 = ''; $rPad2 = ''; $rSide2 = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $dr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId2 = [string]$row[0]; $rPad2 = [string]$row[1]; $rSide2 = [string]$row[2] } + if ($haveI -and ($isEnd -or ($cmpId.Compare($rId2, $iId) -ne 0))) { + if ($iSplit) { $splitIdentities++ } + $haveI = $false + } + if ($isEnd) { break } + if (-not $haveI) { $haveI = $true; $iId = $rId2; $iFirstPad = $rPad2; $iSplit = $false } + if ($rSide2 -eq '0' -or $rSide2 -eq '1') { + if ($rPad2 -ne $iFirstPad) { $iSplit = $true } + } + } + } + finally { $dr.Dispose() } } - catch { - Write-LogHost "Fabric: 'deltalake' install threw: $($_.Exception.Message). Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red - return $false + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch { } } + } + + $passed = (($conflictingKeys -eq 0) -and ($splitIdentities -eq 0) -and ($orphanFactKeys -eq 0) -and ($mismatchedFactRows -eq 0)) + return @{ + Passed = [bool]$passed + ConflictingUserKeys = [int]$conflictingKeys + SplitIdentities = [int]$splitIdentities + OrphanFactKeys = [int]$orphanFactKeys + MismatchedFactRows = [int]$mismatchedFactRows + UsersRows = [int64]$usersRows + FactRows = [int64]$factRows } } -function ConvertTo-UsersSeedMap { +function Invoke-PaxFactSeedAccelerator { <# .SYNOPSIS - Extract {PersonId_Normalized -> UserKey (INT)} from a target Users CSV and write it - as a compact JSON file the embedded CopilotInteraction processor consumes via - `--seed-userkey-map`. + Prepare and publish the pair of Fact continuity seed maps through the embedded + disk-backed seed builder. .DESCRIPTION - Pre-flight helper for -AppendUserInfo. Read the existing target Users CSV produced by a - prior run, build a dict of PersonId_Normalized -> UserKey, and write JSON. Skips rows - with empty PersonId_Normalized or non-integer UserKey. First occurrence of any - PersonId_Normalized wins (target is the source of truth). + Continuity seed preparation reads the whole append target once and folds it to + distinct first-occurrence keys. On a large target that fold dominates the run, so + this helper hands the fold to a small embedded builder that streams the CSV and + accumulates keys in a temporary, disk-backed database with a fixed cache bound. + Peak memory therefore does not grow with the target, and there is no row or key + limit. + + The builder writes TEMPORARY seed files only and reports a machine-readable result + file. This function reads that file rather than console text, and it publishes a + seed only after BOTH temporary seeds exist, are non-empty, parse as JSON objects, + and report non-negative counts. Any prior published seeds and the append target are + left byte-for-byte unchanged unless that full validation passes. + + Three outcomes are distinguished. 'Success' means both seeds are published. + 'CompatibilityFallbackRequired' means the target holds a key form the builder is + not proven to serialize identically; nothing is published, every temporary artifact + is removed, and the caller runs the accepted implementation instead. 'Failed' means + seed preparation must be abandoned so continuity is never silently changed. + + Only aggregate numeric progress reaches the log: rows read, key counts, and elapsed + seconds. No key, row, field, file name, path, URL, identifier, or credential is + emitted, and the temporary builder script is removed in a finally block. .OUTPUTS - Int count of entries written. + PSCustomObject with .Status, .MidCount, .ThreadCount, .RowsRead, .ElapsedSeconds, + .ExitCode and .Reason. #> [CmdletBinding()] param( - [Parameter(Mandatory)] [string] $TargetUsersCsv, - [Parameter(Mandatory)] [string] $OutJsonPath + [Parameter(Mandatory)] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $OutMidJsonPath, + [Parameter(Mandatory)] [string] $OutThreadJsonPath, + [Parameter(Mandatory)] [string] $PythonExe, + [string[]] $LauncherArgs = @(), + [Parameter(Mandatory)] [string] $ScriptDir, + [double] $HeartbeatIntervalSeconds = 0 ) - if (-not (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf)) { - throw "ConvertTo-UsersSeedMap: target Users CSV not found: '$TargetUsersCsv'" + + $pySource = @' +"""PAX Fact Continuity Seed Builder. + +Streams a target Fact CSV once and folds it to the two continuity seed maps that +keep Message_Id and thread surrogates stable across runs. Keys accumulate in a +temporary, disk-backed database with a fixed cache bound, so peak memory does not +grow with the target and no row or key limit applies. + +Only temporary outputs and one machine-readable result file are written; the +caller validates and publishes the pair. +""" + +import csv +import json +import os +import sqlite3 +import sys +import time + +STATUS_SUCCESS = 'success' +STATUS_COMPAT = 'compatibility-fallback-required' +STATUS_FAILED = 'failed' + +EXIT_SUCCESS = 0 +EXIT_FAILED = 1 +EXIT_COMPAT = 3 + +MIN_SUPPORTED = 0x20 +MAX_SUPPORTED = 0x7E + +INT32_MIN = -2147483648 +INT32_MAX = 2147483647 + +BATCH_ROWS = 5000 +FIELD_LIMIT = 16777216 +CACHE_KIB = 2048 +HEARTBEAT_ROW_STRIDE = 1000 + +_FOLD_TABLE = {c: c + 32 for c in range(0x41, 0x5B)} + + +def fold(text): + """Fold ASCII A-Z to a-z and leave every other character untouched.""" + return text.translate(_FOLD_TABLE) + + +def is_supported(text): + """Report whether every character lies in the range this builder is proven + to serialize identically to the accepted implementation. A key outside that + range is never normalized, rejected, or altered here; the whole build defers + to the accepted implementation instead.""" + for ch in text: + code = ord(ch) + if code < MIN_SUPPORTED or code > MAX_SUPPORTED: + return False + return True + + +def parse_surrogate(text): + """Accept an optional sign followed by ASCII digits, within signed 32-bit + range. Every other form is a skip, matching the accepted implementation.""" + if text is None: + return None + trimmed = text.strip() + if not trimmed: + return None + start = 0 + negative = False + if trimmed[0] in '+-': + negative = trimmed[0] == '-' + start = 1 + digits = trimmed[start:] + if not digits: + return None + for ch in digits: + if ch < '0' or ch > '9': + return None + value = int(digits) + if negative: + value = -value + if value < INT32_MIN or value > INT32_MAX: + return None + return value + + +def find_column(header, name): + """Resolve a column index case-insensitively, keeping the last match so the + resolution order matches the accepted implementation.""" + wanted = fold(name) + found = -1 + for index, cell in enumerate(header): + if fold(cell) == wanted: + found = index + return found + + +class Heartbeat(object): + """Aggregate-only progress, never more often than the requested interval.""" + + def __init__(self, interval, started): + self.interval = interval + self.started = started + self.last = started + + def emit(self, rows, mid_keys, thread_keys, now, force=False): + if self.interval <= 0: + return + if not force and (now - self.last) < self.interval: + return + self.last = now + sys.stdout.write( + 'PAXSEED heartbeat rowsRead=%d midKeys=%d threadKeys=%d elapsedSeconds=%.1f\n' + % (rows, mid_keys, thread_keys, now - self.started)) + sys.stdout.flush() + + +def remove_quietly(path): + try: + if path and os.path.exists(path): + os.remove(path) + except OSError: + pass + + +_RECORD_BREAKS = ',\r\n' + + +class QuoteAudit(object): + """Detect the one malformed quoting form the strict reader accepts but the + accepted reader rejects: a quote inside an unquoted field. A quoted field may + span physical lines, so the in-quotes state carries across them; a line that + holds no quote can neither open nor close one, which keeps the scan off the + hot path for ordinary rows.""" + + def __init__(self): + self.in_quotes = False + self.divergent = False + + def scan(self, line): + if not self.in_quotes and '"' not in line: + return + # 0 fresh, 1 unquoted, 2 inside quotes, 3 after a closing quote. + state = 2 if self.in_quotes else 0 + for ch in line: + if state == 0: + if ch == '"': + state = 2 + elif ch not in _RECORD_BREAKS: + state = 1 + elif state == 1: + if ch == '"': + self.divergent = True + return + elif ch in _RECORD_BREAKS: + state = 0 + elif state == 2: + if ch == '"': + state = 3 + else: + if ch == '"': + state = 2 + elif ch in _RECORD_BREAKS: + state = 0 + self.in_quotes = state == 2 + + +def audit_lines(handle, audit): + """Feed the reader while auditing the same characters, so a divergent record + is known as soon as the reader produces it and always before publication.""" + for line in handle: + audit.scan(line) + yield line + + +def write_seed(connection, table, path): + """Stream one compact seed map in first-occurrence order, UTF-8 without a + byte-order mark. An empty map is the literal two-character object.""" + count = 0 + with open(path, 'w', encoding='utf-8', newline='') as handle: + handle.write('{') + first = True + cursor = connection.execute( + 'SELECT raw_key, surrogate_value FROM %s ORDER BY source_sequence' % table) + for raw_key, surrogate in cursor: + if not first: + handle.write(',') + handle.write(json.dumps(raw_key, ensure_ascii=True)) + handle.write(':') + handle.write(str(int(surrogate))) + first = False + count += 1 + handle.write('}') + return count + + +def build(target_csv, mid_temp, thread_temp, work_dir, interval): + started = time.monotonic() + beat = Heartbeat(interval, started) + db_path = os.path.join(work_dir, 'pax_fact_seed.sqlite') + remove_quietly(db_path) + connection = sqlite3.connect(db_path, isolation_level=None) + try: + connection.execute('PRAGMA journal_mode = OFF') + connection.execute('PRAGMA synchronous = OFF') + connection.execute('PRAGMA temp_store = FILE') + connection.execute('PRAGMA cache_size = -%d' % CACHE_KIB) + for table in ('mid_seed', 'thread_seed'): + connection.execute( + 'CREATE TABLE %s (' + 'normalized_key TEXT PRIMARY KEY, ' + 'raw_key TEXT NOT NULL, ' + 'source_sequence INTEGER NOT NULL, ' + 'surrogate_value INTEGER NOT NULL)' % table) + + rows_read = 0 + mid_pairs = 0 + thread_pairs = 0 + with open(target_csv, 'r', encoding='utf-8-sig', newline='') as handle: + # Strict parsing rejects content after a closing quote and an unterminated + # quoted field at end of input, matching the accepted reader. The audit + # closes the remaining divergence: a quote inside an unquoted field, which + # the accepted reader rejects and the strict reader would accept. + audit = QuoteAudit() + reader = csv.reader(audit_lines(handle, audit), strict=True) + try: + header = next(reader) + except StopIteration: + header = None + + if header is not None: + if header: + header[0] = header[0].lstrip('\ufeff') + for cell in header: + if not is_supported(cell) or '"' in cell: + return (STATUS_COMPAT, 'unsupported-header-character', + 0, 0, 0, time.monotonic() - started) + seen = set() + for cell in header: + folded = fold(cell) + if folded in seen: + return (STATUS_FAILED, 'duplicate-header', + 0, 0, 0, time.monotonic() - started) + seen.add(folded) + + mid_raw_index = find_column(header, 'Message_Id_Raw') + mid_value_index = find_column(header, 'Message_Id') + thread_raw_index = find_column(header, 'ThreadId_Raw') + thread_key_index = find_column(header, 'ThreadKey') + thread_id_index = find_column(header, 'ThreadId') + if thread_key_index >= 0: + thread_value_index = thread_key_index + else: + thread_value_index = thread_id_index + + mid_batch = [] + thread_batch = [] + connection.execute('BEGIN') + for row in reader: + # Every character of this record has already been audited. + if audit.divergent: + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'unquoted-field-quote', + 0, 0, rows_read, time.monotonic() - started) + if 0 <= mid_raw_index < len(row): + raw_mid = row[mid_raw_index] + else: + raw_mid = '' + if raw_mid.strip(): + if not is_supported(raw_mid): + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'unsupported-key-character', + 0, 0, rows_read, time.monotonic() - started) + if 0 <= mid_value_index < len(row): + text = row[mid_value_index] + else: + text = '' + # A quote character inside a consumed value cannot be told apart + # from a quote the accepted reader rejects, so adjudication is + # left to that reader. + if '"' in raw_mid or '"' in text: + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'ambiguous-quoting', + 0, 0, rows_read, time.monotonic() - started) + value = parse_surrogate(text) + if value is not None: + mid_batch.append((fold(raw_mid), raw_mid, rows_read, value)) + mid_pairs += 1 + if 0 <= thread_raw_index < len(row): + raw_thread = row[thread_raw_index] + else: + raw_thread = '' + if raw_thread.strip(): + if not is_supported(raw_thread): + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'unsupported-key-character', + 0, 0, rows_read, time.monotonic() - started) + if 0 <= thread_value_index < len(row): + text = row[thread_value_index] + else: + text = '' + if '"' in raw_thread or '"' in text: + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'ambiguous-quoting', + 0, 0, rows_read, time.monotonic() - started) + value = parse_surrogate(text) + if value is not None: + thread_batch.append((fold(raw_thread), raw_thread, rows_read, value)) + thread_pairs += 1 + rows_read += 1 + if len(mid_batch) >= BATCH_ROWS: + connection.executemany( + 'INSERT OR IGNORE INTO mid_seed VALUES (?, ?, ?, ?)', mid_batch) + del mid_batch[:] + if len(thread_batch) >= BATCH_ROWS: + connection.executemany( + 'INSERT OR IGNORE INTO thread_seed VALUES (?, ?, ?, ?)', thread_batch) + del thread_batch[:] + if (rows_read % HEARTBEAT_ROW_STRIDE) == 0: + beat.emit(rows_read, mid_pairs, thread_pairs, time.monotonic()) + if mid_batch: + connection.executemany( + 'INSERT OR IGNORE INTO mid_seed VALUES (?, ?, ?, ?)', mid_batch) + del mid_batch[:] + if thread_batch: + connection.executemany( + 'INSERT OR IGNORE INTO thread_seed VALUES (?, ?, ?, ?)', thread_batch) + del thread_batch[:] + connection.execute('COMMIT') + + connection.execute('CREATE INDEX ix_mid_seq ON mid_seed(source_sequence)') + connection.execute('CREATE INDEX ix_thread_seq ON thread_seed(source_sequence)') + mid_count = write_seed(connection, 'mid_seed', mid_temp) + thread_count = write_seed(connection, 'thread_seed', thread_temp) + elapsed = time.monotonic() - started + beat.emit(rows_read, mid_count, thread_count, time.monotonic(), True) + return (STATUS_SUCCESS, '', mid_count, thread_count, rows_read, elapsed) + finally: + try: + connection.close() + finally: + remove_quietly(db_path) + remove_quietly(db_path + '-journal') + remove_quietly(db_path + '-wal') + remove_quietly(db_path + '-shm') + + +def main(argv): + if len(argv) != 7: + return EXIT_FAILED + target_csv = argv[1] + mid_temp = argv[2] + thread_temp = argv[3] + work_dir = argv[4] + try: + interval = float(argv[5]) + except ValueError: + interval = 0.0 + result_path = argv[6] + + status = STATUS_FAILED + reason = 'unexpected-error' + mid_count = 0 + thread_count = 0 + rows_read = 0 + elapsed = 0.0 + try: + csv.field_size_limit(FIELD_LIMIT) + status, reason, mid_count, thread_count, rows_read, elapsed = build( + target_csv, mid_temp, thread_temp, work_dir, interval) + except csv.Error: + status, reason = STATUS_FAILED, 'csv-error' + except sqlite3.Error: + status, reason = STATUS_FAILED, 'sqlite-error' + except UnicodeDecodeError: + status, reason = STATUS_FAILED, 'encoding-error' + except OSError: + status, reason = STATUS_FAILED, 'io-error' + except Exception: + status, reason = STATUS_FAILED, 'unexpected-error' + + if status != STATUS_SUCCESS: + mid_count = 0 + thread_count = 0 + remove_quietly(mid_temp) + remove_quietly(thread_temp) + + payload = { + 'status': status, + 'reason': reason, + 'rowsRead': rows_read, + 'midKeyCount': mid_count, + 'threadKeyCount': thread_count, + 'elapsedSeconds': round(elapsed, 3), + } + try: + with open(result_path, 'w', encoding='utf-8', newline='') as handle: + json.dump(payload, handle) + except OSError: + return EXIT_FAILED + + if status == STATUS_SUCCESS: + return EXIT_SUCCESS + if status == STATUS_COMPAT: + return EXIT_COMPAT + return EXIT_FAILED + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) +'@ + + if (-not (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { + throw "Invoke-PaxFactSeedAccelerator: target Fact CSV not found: '$TargetFactCsv'" + } + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_factseedpy_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $midTemp = $OutMidJsonPath + '.pyseedtmp' + $threadTemp = $OutThreadJsonPath + '.pyseedtmp' + $resultPath = Join-Path $workDir 'seed_result.json' + $tempPyPath = Join-Path $ScriptDir ('PAX_FactContinuitySeedBuilder_{0}.py' -f ([System.Guid]::NewGuid().ToString('N'))) + # Stage records are routed through a host lookup so the function stays inert when it + # is extracted and exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + + $status = 'Failed' + $reason = 'not-started' + $rowsRead = [int64]0 + $midCount = 0 + $threadCount = 0 + $elapsedSeconds = [double]0 + $exitCode = -1 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + if (-not (Test-Path -LiteralPath $ScriptDir -PathType Container)) { + [void][System.IO.Directory]::CreateDirectory($ScriptDir) + } + [System.IO.File]::WriteAllText($tempPyPath, $pySource, [System.Text.UTF8Encoding]::new($false)) + + $builderArgs = @($LauncherArgs) + @( + $tempPyPath + $TargetFactCsv + $midTemp + $threadTemp + $workDir + ([double]$HeartbeatIntervalSeconds).ToString($inv) + $resultPath + ) + $prevPyIoEnc = $env:PYTHONIOENCODING + try { + $env:PYTHONIOENCODING = 'utf-8' + # Only a line matching this exact aggregate shape is forwarded; every other + # byte the builder writes is dropped, so no key, row, path, or identifier can + # reach the log through this channel. + & $PythonExe @builderArgs 2>&1 | ForEach-Object { + $line = [string]$_ + if ($line -match '^PAXSEED heartbeat rowsRead=(\d+) midKeys=(\d+) threadKeys=(\d+) elapsedSeconds=(\d+(?:\.\d+)?)$') { + if ($__paxStage) { + try { + & $__paxStage -Stage 'fact-seed' -Status 'heartbeat' -Counters @{ + rowsRead = [int64]$Matches[1] + midKeys = [int64]$Matches[2] + threadKeys = [int64]$Matches[3] + elapsedSeconds = [double]::Parse($Matches[4], $inv) + } + } + catch { } + } + } + } + $exitCode = [int]$LASTEXITCODE + } + finally { + if ($null -eq $prevPyIoEnc) { Remove-Item Env:PYTHONIOENCODING -ErrorAction SilentlyContinue } + else { $env:PYTHONIOENCODING = $prevPyIoEnc } + } + + # Machine-readable outcome only; console text is never parsed. + $payload = $null + if (Test-Path -LiteralPath $resultPath -PathType Leaf) { + try { $payload = (Get-Content -LiteralPath $resultPath -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { $payload = $null } + } + if ($null -eq $payload) { + $status = 'Failed' + $reason = 'missing-result' + } + else { + $rowsRead = [int64]$payload.rowsRead + $elapsedSeconds = [double]$payload.elapsedSeconds + $reason = [string]$payload.reason + switch ([string]$payload.status) { + 'success' { + $status = if ($exitCode -eq 0) { 'Success' } else { 'Failed' } + if ($status -eq 'Failed') { $reason = 'exit-code-mismatch' } + $midCount = [int]$payload.midKeyCount + $threadCount = [int]$payload.threadKeyCount + } + 'compatibility-fallback-required' { $status = 'CompatibilityFallbackRequired' } + default { $status = 'Failed' } + } + } + + if ($status -eq 'Success') { + # Paired publication: verify BOTH temporary seeds before either is moved. + foreach ($tp in @($midTemp, $threadTemp)) { + if (-not (Test-Path -LiteralPath $tp -PathType Leaf)) { + $status = 'Failed'; $reason = 'missing-temp-seed'; break + } + if ([int64](Get-Item -LiteralPath $tp).Length -le 0) { + $status = 'Failed'; $reason = 'empty-temp-seed'; break + } + $parsed = $null + try { $parsed = (Get-Content -LiteralPath $tp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { $parsed = $null } + if ($null -eq $parsed -or -not ($parsed -is [System.Management.Automation.PSCustomObject])) { + $status = 'Failed'; $reason = 'invalid-temp-seed'; break + } + } + } + if ($status -eq 'Success' -and (($midCount -lt 0) -or ($threadCount -lt 0))) { + $status = 'Failed'; $reason = 'negative-count' + } + + if ($status -eq 'Success') { + if ($__paxStage) { + try { + & $__paxStage -Stage 'fact-seed-validate' -Status 'complete' -Counters @{ + midKeys = [int64]$midCount + threadKeys = [int64]$threadCount + midBytes = [int64](Get-Item -LiteralPath $midTemp).Length + threadBytes = [int64](Get-Item -LiteralPath $threadTemp).Length + } + } + catch { } + } + foreach ($pair in @(@($midTemp, $OutMidJsonPath), @($threadTemp, $OutThreadJsonPath))) { + $outDir = Split-Path -Parent $pair[1] + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force -ErrorAction Stop | Out-Null + } + } + # Publish as a pair. If the second move fails the first is rolled back, so a + # result never leaves one seed published against a stale partner. + $midRollback = $null + $midExisted = Test-Path -LiteralPath $OutMidJsonPath -PathType Leaf + if ($midExisted) { + $midRollback = $OutMidJsonPath + '.pyseedbak' + Copy-Item -LiteralPath $OutMidJsonPath -Destination $midRollback -Force -ErrorAction Stop + } + try { + Move-Item -LiteralPath $midTemp -Destination $OutMidJsonPath -Force -ErrorAction Stop + Move-Item -LiteralPath $threadTemp -Destination $OutThreadJsonPath -Force -ErrorAction Stop + } + catch { + if ($midRollback -and (Test-Path -LiteralPath $midRollback -PathType Leaf)) { + try { Move-Item -LiteralPath $midRollback -Destination $OutMidJsonPath -Force -ErrorAction Stop } catch { } + } + elseif ((-not $midExisted) -and (Test-Path -LiteralPath $OutMidJsonPath -PathType Leaf)) { + try { Remove-Item -LiteralPath $OutMidJsonPath -Force -ErrorAction SilentlyContinue } catch { } + } + throw + } + finally { + if ($midRollback -and (Test-Path -LiteralPath $midRollback)) { try { Remove-Item -LiteralPath $midRollback -Force -ErrorAction SilentlyContinue } catch { } } + } + } } - $rows = @(Import-Csv -LiteralPath $TargetUsersCsv -Encoding UTF8) - $map = [ordered]@{} - foreach ($r in $rows) { - $personId = $r.PersonId_Normalized - $uk = $r.UserKey - if ([string]::IsNullOrWhiteSpace($personId)) { continue } - if ([string]::IsNullOrWhiteSpace($uk)) { continue } - $ukInt = 0 - if (-not [int]::TryParse([string]$uk, [ref]$ukInt)) { continue } - if (-not $map.Contains($personId)) { $map[$personId] = $ukInt } - } - $json = if ($map.Count -gt 0) { ($map | ConvertTo-Json -Compress -Depth 2) } else { '{}' } - $utf8NoBom = [System.Text.UTF8Encoding]::new($false) - $outDir = Split-Path -Parent $OutJsonPath - if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { - New-Item -ItemType Directory -Path $outDir -Force | Out-Null + catch { + $status = 'Failed' + $reason = 'accelerator-error' + } + finally { + # The temporary builder script, the temporary database, and any unpublished + # temporary seed are always removed. Published seeds are never touched here. + if (Test-Path -LiteralPath $tempPyPath) { try { Remove-Item -LiteralPath $tempPyPath -Force -ErrorAction SilentlyContinue } catch {} } + foreach ($tp in @($midTemp, $threadTemp, ($OutMidJsonPath + '.pyseedbak'))) { + if (Test-Path -LiteralPath $tp) { try { Remove-Item -LiteralPath $tp -Force -ErrorAction SilentlyContinue } catch {} } + } + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + + if ($status -ne 'Success') { $midCount = 0; $threadCount = 0 } + return [PSCustomObject]@{ + Status = $status + MidCount = $midCount + ThreadCount = $threadCount + RowsRead = $rowsRead + ElapsedSeconds = $elapsedSeconds + ExitCode = $exitCode + Reason = $reason } - [System.IO.File]::WriteAllText($OutJsonPath, $json, $utf8NoBom) - return $map.Count } function ConvertTo-FactSeedMaps { @@ -10455,9 +16203,26 @@ function ConvertTo-FactSeedMaps { .DESCRIPTION Pre-flight helper for -AppendFile (CopilotInteraction rollup mode). Reads the - existing target Fact CSV produced by a prior run, builds two dicts keyed by the - raw GUID columns, and writes both JSON files. First occurrence wins. Rows with - blank raw GUID or non-integer surrogate are skipped. + existing target Fact CSV produced by a prior run and builds two seed maps keyed + by the raw GUID columns. First case-insensitive occurrence wins; rows with a + blank raw GUID or a non-integer surrogate are skipped. + + The target is never materialized in full. Each row streams exactly once through + the RFC-4180 reader; valid (raw-key, sequence, integer) pairs for each map are + spilled to bounded temporary CSVs and folded to distinct first-occurrence keys + using only the frozen external-sort machinery, then the compact seed JSON is + streamed one property at a time. Peak memory is governed by a fixed byte budget + rather than by the target size, and no row, key, or JSON property set is held + whole in memory. + + The seed JSON bytes are identical to the prior in-memory implementation: each + property is produced by ConvertTo-Json of a single-property ordered object with + its outer braces removed, so key escaping and integer formatting are unchanged, + an empty map is the literal '{}', and property order is first-occurrence order. + Neither seed file is published until both are complete and validated; on any + failure the function raises a terminating error, leaves any prior seed outputs + byte-for-byte unchanged, publishes no partial output, and removes its temporary + work. There is no in-memory fallback. .OUTPUTS PSCustomObject with .MidCount and .ThreadCount integer fields. @@ -10466,45 +16231,213 @@ function ConvertTo-FactSeedMaps { param( [Parameter(Mandatory)] [string] $TargetFactCsv, [Parameter(Mandatory)] [string] $OutMidJsonPath, - [Parameter(Mandatory)] [string] $OutThreadJsonPath + [Parameter(Mandatory)] [string] $OutThreadJsonPath, + # Fixed in-memory buffer budget for the bounded sorts (bytes). It governs only + # WHEN buffered pairs spill to run files; it never caps how many rows or keys + # are processed. A small value is used by tests to force multi-run spilling. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default. It is forwarded verbatim to + # the bounded sorts, which pass the callback an AGGREGATE snapshot only and swallow + # any exception it raises. Supplying it cannot change either seed JSON's bytes, the + # key sets, the ordering, or the paired fail-closed publication. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 ) if (-not (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { throw "ConvertTo-FactSeedMaps: target Fact CSV not found: '$TargetFactCsv'" } - $rows = @(Import-Csv -LiteralPath $TargetFactCsv -Encoding UTF8) - $midMap = [ordered]@{} - $threadMap = [ordered]@{} - foreach ($r in $rows) { - $midRaw = $r.Message_Id_Raw - $threadRaw = $r.ThreadId_Raw - $midInt = 0 - $threadInt = 0 - if (-not [string]::IsNullOrWhiteSpace($midRaw)) { - if ([int]::TryParse([string]$r.Message_Id, [ref]$midInt)) { - if (-not $midMap.Contains($midRaw)) { $midMap[$midRaw] = $midInt } + + # Build one compact seed JSON at $OutTempJsonPath from a normalized pairs CSV + # (header RawKey,SeqPad,Val; one row per source row). Distinct keys are folded + # case-insensitively (first original occurrence wins, first casing + first value + # kept) by two bounded external sorts and a streaming first-per-key pass; the + # JSON is streamed one single-property object at a time. Returns the distinct + # key count. Defined inline so the whole streaming contract lives in this one + # extractable function. + $buildSeedJson = { + param([string]$PairsCsvPath, [string]$OutTempJsonPath, [string]$WorkDir, [string]$Tag, [int64]$Budget, [scriptblock]$Progress, [double]$ProgressInterval) + $u8 = [System.Text.UTF8Encoding]::new($false) + $byKeyPath = Join-Path $WorkDir ($Tag + '_bykey.csv') + $firstPath = Join-Path $WorkDir ($Tag + '_first.csv') + $bySeqPath = Join-Path $WorkDir ($Tag + '_byseq.csv') + # 1) Sort by (RawKey, SeqPad) case-insensitively so equal keys are adjacent and + # ascending by original sequence within each key group. + [void](script:Invoke-PaxExternalSort -InputPath $PairsCsvPath -OutputPath $byKeyPath -KeyColumn @('RawKey', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + # 2) Emit the first row of each case-insensitive key group (lowest sequence), + # keeping that first row's original key casing and value. + $cmp = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $r1 = [System.IO.StreamReader]::new($byKeyPath, $true) + $w1 = [System.IO.StreamWriter]::new($firstPath, $false, $u8) + $distinct = [int64]0 + try { + $null = script:Read-PaxCsvRecord -Reader $r1 + script:Write-PaxCsvRecord -Writer $w1 -Fields @('RawKey', 'SeqPad', 'Val') + $haveLast = $false; $lastKey = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r1 + if ($null -eq $row) { break } + $k = if ($row.Length -ge 1) { [string]$row[0] } else { '' } + if ((-not $haveLast) -or ($cmp.Compare($k, $lastKey) -ne 0)) { + script:Write-PaxCsvRecord -Writer $w1 -Fields $row + $distinct++ + $lastKey = $k; $haveLast = $true + } + } + } + finally { $w1.Flush(); $w1.Dispose(); $r1.Dispose() } + # 3) Restore first-occurrence order (ascending original sequence). + [void](script:Invoke-PaxExternalSort -InputPath $firstPath -OutputPath $bySeqPath -KeyColumn @('SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + # 4) Stream the compact JSON object one property at a time. Each property's + # exact bytes come from ConvertTo-Json of a single-property ordered object + # with its outer braces removed, so escaping and integer formatting match the + # prior implementation byte-for-byte. An empty map emits the literal '{}'. + $r2 = [System.IO.StreamReader]::new($bySeqPath, $true) + $w2 = [System.IO.StreamWriter]::new($OutTempJsonPath, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $r2 + $w2.Write('{') + $wrote = $false + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r2 + if ($null -eq $row) { break } + $rawKey = [string]$row[0] + $valStr = if ($row.Length -ge 3) { [string]$row[2] } else { '0' } + $obj = [ordered]@{} + $obj[$rawKey] = [int]::Parse($valStr, [System.Globalization.CultureInfo]::InvariantCulture) + $oneLine = $obj | ConvertTo-Json -Compress -Depth 2 + $inner = $oneLine.Substring(1, $oneLine.Length - 2) + if ($wrote) { $w2.Write(',') } + $w2.Write($inner) + $wrote = $true + } + $w2.Write('}') + } + finally { $w2.Flush(); $w2.Dispose(); $r2.Dispose() } + return $distinct + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_factseed_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $midTemp = $OutMidJsonPath + '.seedtmp' + $threadTemp = $OutThreadJsonPath + '.seedtmp' + $midCount = 0 + $threadCount = 0 + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $midPairs = Join-Path $workDir 'mid_pairs.csv' + $threadPairs = Join-Path $workDir 'thread_pairs.csv' + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + + # --- Single streaming pass over the target: emit normalized (RawKey,SeqPad,Val) + # pair rows for each map. The complete target is never materialized. --- + $reader = [System.IO.StreamReader]::new($TargetFactCsv, $true) + $mw = [System.IO.StreamWriter]::new($midPairs, $false, $utf8NoBom) + $tw = [System.IO.StreamWriter]::new($threadPairs, $false, $utf8NoBom) + try { + script:Write-PaxCsvRecord -Writer $mw -Fields @('RawKey', 'SeqPad', 'Val') + script:Write-PaxCsvRecord -Writer $tw -Fields @('RawKey', 'SeqPad', 'Val') + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -ne $header) { + # Reject duplicate header columns (case-insensitive), matching the prior + # reader's fail-closed behavior on a duplicated member. + $seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($h in $header) { + if (-not $seen.Add([string]$h)) { throw "ConvertTo-FactSeedMaps: duplicate header column '$h' in target Fact CSV." } + } + $midRawIdx = -1; $midValIdx = -1; $threadRawIdx = -1; $threadKeyIdx = -1; $threadIdIdx = -1 + for ($i = 0; $i -lt $header.Length; $i++) { + $name = [string]$header[$i] + if ([string]::Equals($name, 'Message_Id_Raw', [System.StringComparison]::OrdinalIgnoreCase)) { $midRawIdx = $i } + elseif ([string]::Equals($name, 'Message_Id', [System.StringComparison]::OrdinalIgnoreCase)) { $midValIdx = $i } + elseif ([string]::Equals($name, 'ThreadId_Raw', [System.StringComparison]::OrdinalIgnoreCase)) { $threadRawIdx = $i } + elseif ([string]::Equals($name, 'ThreadKey', [System.StringComparison]::OrdinalIgnoreCase)) { $threadKeyIdx = $i } + elseif ([string]::Equals($name, 'ThreadId', [System.StringComparison]::OrdinalIgnoreCase)) { $threadIdIdx = $i } + } + # ThreadKey overrides ThreadId as the thread surrogate when present. + $threadValIdx = if ($threadKeyIdx -ge 0) { $threadKeyIdx } else { $threadIdIdx } + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $seq = [int64]0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $row) { break } + $seqPad = $seq.ToString('D19', $inv) + $midRaw = if ($midRawIdx -ge 0 -and $midRawIdx -lt $row.Length) { [string]$row[$midRawIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($midRaw)) { + $midValStr = if ($midValIdx -ge 0 -and $midValIdx -lt $row.Length) { [string]$row[$midValIdx] } else { '' } + $midInt = 0 + if ([int]::TryParse($midValStr, [ref]$midInt)) { + script:Write-PaxCsvRecord -Writer $mw -Fields @($midRaw, $seqPad, $midInt.ToString($inv)) + } + } + $threadRaw = if ($threadRawIdx -ge 0 -and $threadRawIdx -lt $row.Length) { [string]$row[$threadRawIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($threadRaw)) { + $threadValStr = if ($threadValIdx -ge 0 -and $threadValIdx -lt $row.Length) { [string]$row[$threadValIdx] } else { '' } + $threadInt = 0 + if ([int]::TryParse($threadValStr, [ref]$threadInt)) { + script:Write-PaxCsvRecord -Writer $tw -Fields @($threadRaw, $seqPad, $threadInt.ToString($inv)) + } + } + $seq++ + } } } - if (-not [string]::IsNullOrWhiteSpace($threadRaw)) { - $threadVal = if ($r.PSObject.Properties.Match('ThreadKey').Count) { $r.ThreadKey } else { $r.ThreadId } - if ([int]::TryParse([string]$threadVal, [ref]$threadInt)) { - if (-not $threadMap.Contains($threadRaw)) { $threadMap[$threadRaw] = $threadInt } + finally { + $mw.Flush(); $mw.Dispose(); $tw.Flush(); $tw.Dispose(); $reader.Dispose() + } + + # --- Build each seed JSON to a temp sibling using only bounded machinery. --- + $midCount = [int](& $buildSeedJson $midPairs $midTemp $workDir 'mid' $MemoryBudgetBytes $ProgressCallback $ProgressIntervalSeconds) + $threadCount = [int](& $buildSeedJson $threadPairs $threadTemp $workDir 'thread' $MemoryBudgetBytes $ProgressCallback $ProgressIntervalSeconds) + + # --- Validate both completed temp files parse before either is published. --- + foreach ($tp in @($midTemp, $threadTemp)) { + try { $null = (Get-Content -LiteralPath $tp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "ConvertTo-FactSeedMaps: seed JSON validation failed for '$tp': $($_.Exception.Message)" } + } + + # Aggregate-only stage record for the validated pair, emitted before publication. + # Routed through a host lookup so the function stays inert when it is extracted and + # exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + if ($__paxStage) { + $__midBytes = try { [int64](Get-Item -LiteralPath $midTemp).Length } catch { [int64]0 } + $__thrBytes = try { [int64](Get-Item -LiteralPath $threadTemp).Length } catch { [int64]0 } + try { + & $__paxStage -Stage 'fact-seed-validate' -Status 'complete' -Counters @{ + midKeys = [int64]$midCount + threadKeys = [int64]$threadCount + midBytes = $__midBytes + threadBytes = $__thrBytes + } + } catch { } + } + + # --- Publish only after both seeds are complete and valid. Prior published + # seeds are never touched before this point. --- + foreach ($pair in @(@($midTemp, $OutMidJsonPath), @($threadTemp, $OutThreadJsonPath))) { + $outDir = Split-Path -Parent $pair[1] + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null } } + Move-Item -LiteralPath $midTemp -Destination $OutMidJsonPath -Force -ErrorAction Stop + Move-Item -LiteralPath $threadTemp -Destination $OutThreadJsonPath -Force -ErrorAction Stop } - $utf8NoBom = [System.Text.UTF8Encoding]::new($false) - foreach ($pair in @(@($OutMidJsonPath, $midMap), @($OutThreadJsonPath, $threadMap))) { - $path = $pair[0] - $dict = $pair[1] - $outDir = Split-Path -Parent $path - if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { - New-Item -ItemType Directory -Path $outDir -Force | Out-Null + catch { + # Fail closed: remove any unpublished temp siblings so no partial seed is left + # behind, then re-raise. Prior published seed files are never modified before + # the paired publication step above. + foreach ($tp in @($midTemp, $threadTemp)) { + if (Test-Path -LiteralPath $tp) { try { Remove-Item -LiteralPath $tp -Force -ErrorAction SilentlyContinue } catch {} } } - $json = if ($dict.Count -gt 0) { ($dict | ConvertTo-Json -Compress -Depth 2) } else { '{}' } - [System.IO.File]::WriteAllText($path, $json, $utf8NoBom) + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } } return [PSCustomObject]@{ - MidCount = $midMap.Count - ThreadCount = $threadMap.Count + MidCount = $midCount + ThreadCount = $threadCount } } @@ -10675,177 +16608,294 @@ function Merge-UsersCsv { [Parameter(Mandatory)] [string] $TargetUsersCsv, [Parameter(Mandatory)] [string] $CurrentUsersCsv, [Parameter()] [string] $OutputPath, - [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd') + [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd'), + # Fixed in-memory buffer budget for the bounded sorts (bytes). Governs only + # WHEN buffered rows spill; never caps how many rows are processed. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default. It is forwarded verbatim to + # the bounded sorts, which pass the callback an AGGREGATE snapshot only and swallow + # any exception it raises. Supplying it cannot change the merged output, the row + # order, the classification counts, or the success of the merge. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 ) if (-not (Test-Path -LiteralPath $CurrentUsersCsv -PathType Leaf)) { throw "Merge-UsersCsv: current Users CSV not found: '$CurrentUsersCsv'" } if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentUsersCsv } - # Use script:Import-CsvDeduped instead of Import-Csv: a round-tripped Users CSV - # can contain duplicate header columns (e.g. both pre-rename 'userPrincipalName' - # and post-rename 'PersonId') which crashes Import-Csv with "member already - # present". The helper rewrites the header to dedupe (suffixing later dupes - # with _dup2/_dup3) before parsing. - $targetRows = @() - if (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf) { - $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetUsersCsv) - } - # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. - if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetUsersCsv).Length -gt 0)) { - throw "Merge-UsersCsv: target '$TargetUsersCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." - } - # Schema-narrowing warning. The only header we strictly require on the target - # is the dedup key (PersonId_Normalized) — without it the union cannot - # correctly classify Retained vs New vs Departed. Display / enrichment - # columns (userPrincipalName, mail, id, displayName, UserKey) are intentionally - # renamed or surrogate-replaced by the rollup processor (e.g. userPrincipalName - # → PersonId on the rolled-up Users CSV), so checking for them here would - # produce false-positive warnings on every rollup-shape seed. Header union - # handles legitimately absent display columns gracefully (target rows get - # blanks for current-only fields; current rows get blanks for target-only - # fields). - # - # Derive PersonId_Normalized on-the-fly when absent or empty: the raw - # 47-column EntraUsers CSV emitted by the M365 bundle path does NOT carry - # this column (it is added only by the CopilotInteraction Python rollup at - # L4651). Without derivation, both target and current dedup indexes stay - # empty on M365-bundle append runs and the seed content is silently dropped - # from the union. Fallback chain: PersonId_Normalized -> userPrincipalName - # -> PersonId, each lower+trim (matches the Python normalization exactly). - $_normPid = { - param($row) - $v = $row.PSObject.Properties['PersonId_Normalized'] - if ($v -and -not [string]::IsNullOrWhiteSpace([string]$v.Value)) { return ([string]$v.Value).Trim().ToLowerInvariant() } - $v = $row.PSObject.Properties['userPrincipalName'] - if ($v -and -not [string]::IsNullOrWhiteSpace([string]$v.Value)) { return ([string]$v.Value).Trim().ToLowerInvariant() } - $v = $row.PSObject.Properties['PersonId'] - if ($v -and -not [string]::IsNullOrWhiteSpace([string]$v.Value)) { return ([string]$v.Value).Trim().ToLowerInvariant() } - return '' - } - $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentUsersCsv) - - $targetByPid = @{} - $targetDerivedCnt = 0 - foreach ($r in $targetRows) { - $personId = & $_normPid $r - if ([string]::IsNullOrWhiteSpace($personId)) { continue } - $existing = $r.PSObject.Properties['PersonId_Normalized'] - if (-not $existing -or [string]::IsNullOrWhiteSpace([string]$existing.Value)) { - $r | Add-Member -NotePropertyName 'PersonId_Normalized' -NotePropertyValue $personId -Force - $targetDerivedCnt++ - } - if (-not $targetByPid.ContainsKey($personId)) { $targetByPid[$personId] = $r } - } - $currentByPid = @{} - $currentDerivedCnt = 0 - foreach ($r in $currentRows) { - $personId = & $_normPid $r - if ([string]::IsNullOrWhiteSpace($personId)) { continue } - $existing = $r.PSObject.Properties['PersonId_Normalized'] - if (-not $existing -or [string]::IsNullOrWhiteSpace([string]$existing.Value)) { - $r | Add-Member -NotePropertyName 'PersonId_Normalized' -NotePropertyValue $personId -Force - $currentDerivedCnt++ - } - if (-not $currentByPid.ContainsKey($personId)) { $currentByPid[$personId] = $r } - } - # Surface the derivation result so the run log is auditable. The warning - # only fires now when derivation truly failed for every target row (no - # usable dedup field anywhere on the seed) — in that case the union really - # does fall back to current-run-only and the operator needs to know. - if ($targetRows.Count -gt 0 -and $targetByPid.Count -eq 0) { - Write-LogHost ( - ("WARNING: Merge-UsersCsv: target Users CSV has no derivable dedup key (PersonId_Normalized / userPrincipalName / PersonId are all empty on every row). " + - "Cannot classify Retained / New / Departed; the union will contain only the current-run rows. " + - "Target: {0}") -f $TargetUsersCsv - ) -ForegroundColor Yellow - } - elseif ($targetDerivedCnt -gt 0 -or $currentDerivedCnt -gt 0) { - Write-LogHost ( - "Merge-UsersCsv: derived PersonId_Normalized on the fly (target rows: {0}, current rows: {1}) — column was absent on the seed/raw shape (expected for M365-bundle EntraUsers CSVs)." -f $targetDerivedCnt, $currentDerivedCnt - ) -ForegroundColor Cyan - } - # Header union: preserve current-run order, then append target-only columns, then provenance. - $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) - $hdrOrder = New-Object System.Collections.Generic.List[string] - if ($currentRows.Count -gt 0) { - foreach ($p in $currentRows[0].PSObject.Properties) { - if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + + # Read + dedupe a CSV header record (BOM strip / trim / blank->_blank / _dupN) + # exactly as script:Import-CsvDeduped rewrites it, returning the canonical names. + $readDedupHeader = { + param($reader) + $h = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $h) { return $null } + $seen = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + $out = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $h.Length; $i++) { + $n = [string]$h[$i] + if ($i -eq 0) { $n = $n -replace '^\uFEFF', '' } + $n = $n.Trim() + if ([string]::IsNullOrEmpty($n)) { $n = '_blank' } + if ($seen.ContainsKey($n)) { $seen[$n] = $seen[$n] + 1; $out.Add(('{0}_dup{1}' -f $n, $seen[$n])) } + else { $seen[$n] = 1; $out.Add($n) } + } + return , $out.ToArray() + } + # First index of a name in a header (OrdinalIgnoreCase) or -1. + $findIdx = { + param($header, $name) + for ($i = 0; $i -lt $header.Length; $i++) { if ([string]::Equals([string]$header[$i], $name, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_usersmerge_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $tmpPath = "$OutputPath.merging" + $retainedCount = 0; $newCount = 0; $departedCount = 0; $unionCount = 0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + + # --- Resolve headers (bounded: only the header record of each file). --- + $targetExists = Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf + $curHdrReader = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + $currentHeader = $null + try { $currentHeader = & $readDedupHeader $curHdrReader } finally { $curHdrReader.Dispose() } + if ($null -eq $currentHeader) { $currentHeader = @() } + $targetHeader = @() + if ($targetExists) { + $tgtHdrReader = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + try { $th = & $readDedupHeader $tgtHdrReader; if ($null -ne $th) { $targetHeader = $th } } finally { $tgtHdrReader.Dispose() } + } + + # --- Union header: current cols, PersonId_Normalized (derived-when-absent), + # target-only cols, then provenance cols. --- + $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $hdrOrder = New-Object System.Collections.Generic.List[string] + foreach ($c in $currentHeader) { if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } } + if ($hdrSet.Add('PersonId_Normalized')) { $hdrOrder.Add('PersonId_Normalized') } + foreach ($c in $targetHeader) { if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } } + foreach ($c in @('Date_Added', 'Latest_Append_Date', 'In_Latest_Append')) { if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } } + $unionCols = $hdrOrder.ToArray() + $U = $unionCols.Length + $uIndex = @{} + for ($i = 0; $i -lt $U; $i++) { $uIndex[$unionCols[$i]] = $i } + $idxPidNorm = [int]$uIndex['PersonId_Normalized'] + $idxUserKey = if ($uIndex.ContainsKey('UserKey')) { [int]$uIndex['UserKey'] } else { -1 } + $idxDateAdded = [int]$uIndex['Date_Added'] + $idxLatest = [int]$uIndex['Latest_Append_Date'] + $idxInLatest = [int]$uIndex['In_Latest_Append'] + $hasTotalEmp = $uIndex.ContainsKey('TotalEmployees') + $idxTotalEmp = if ($hasTotalEmp) { [int]$uIndex['TotalEmployees'] } else { -1 } + + # Source -> union position maps and PersonId source indices. + $cToU = New-Object 'int[]' $currentHeader.Length + for ($s = 0; $s -lt $currentHeader.Length; $s++) { $cToU[$s] = [int]$uIndex[$currentHeader[$s]] } + $cPN = & $findIdx $currentHeader 'PersonId_Normalized' + $cUPN = & $findIdx $currentHeader 'userPrincipalName' + $cPID = & $findIdx $currentHeader 'PersonId' + $tToU = New-Object 'int[]' $targetHeader.Length + for ($s = 0; $s -lt $targetHeader.Length; $s++) { $tToU[$s] = [int]$uIndex[$targetHeader[$s]] } + $tPN = & $findIdx $targetHeader 'PersonId_Normalized' + $tUPN = & $findIdx $targetHeader 'userPrincipalName' + $tPID = & $findIdx $targetHeader 'PersonId' + + # Build the union-normalized value vector for one source row; also return the + # derived normalized PersonId key and whether it was derived (PN absent/blank). + $normalizeRow = { + param($row, $srcToU, $iPN, $iUPN, $iPID) + $vals = New-Object 'string[]' $U + for ($j = 0; $j -lt $U; $j++) { $vals[$j] = '' } + for ($s = 0; $s -lt $srcToU.Length; $s++) { + $vals[$srcToU[$s]] = if ($s -lt $row.Length) { [string]$row[$s] } else { '' } + } + $pidVal = '' + if ($iPN -ge 0 -and $iPN -lt $row.Length -and -not [string]::IsNullOrWhiteSpace([string]$row[$iPN])) { $pidVal = ([string]$row[$iPN]).Trim().ToLowerInvariant() } + elseif ($iUPN -ge 0 -and $iUPN -lt $row.Length -and -not [string]::IsNullOrWhiteSpace([string]$row[$iUPN])) { $pidVal = ([string]$row[$iUPN]).Trim().ToLowerInvariant() } + elseif ($iPID -ge 0 -and $iPID -lt $row.Length -and -not [string]::IsNullOrWhiteSpace([string]$row[$iPID])) { $pidVal = ([string]$row[$iPID]).Trim().ToLowerInvariant() } + $pnBlank = ($iPN -lt 0) -or ($iPN -ge $row.Length) -or [string]::IsNullOrWhiteSpace([string]$row[$iPN]) + if ($pnBlank) { $vals[$idxPidNorm] = $pidVal } + return [pscustomobject]@{ Vals = $vals; Pid = $pidVal; Derived = ($pnBlank -and -not [string]::IsNullOrWhiteSpace($pidVal)) } + } + + # --- Stream both inputs to a bounded work file (never materialized whole). --- + $workPath = Join-Path $workDir 'work.csv' + $workHdr = @('__PID', '__SEQ', '__SIDE') + $unionCols + $ww = [System.IO.StreamWriter]::new($workPath, $false, $u8) + $seq = [int64]0 + $targetDataRows = [int64]0 + $targetDerived = 0; $currentDerived = 0; $anyTargetPid = $false + try { + script:Write-PaxCsvRecord -Writer $ww -Fields $workHdr + if ($targetExists) { + $tr = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $tr # skip header + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $tr + if ($null -eq $row) { break } + $targetDataRows++ + $n = & $normalizeRow $row $tToU $tPN $tUPN $tPID + if ($n.Derived) { $targetDerived++ } + if (-not [string]::IsNullOrWhiteSpace($n.Pid)) { + $anyTargetPid = $true + $fields = New-Object System.Collections.Generic.List[string] + $fields.Add($n.Pid); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('0') + foreach ($v in $n.Vals) { $fields.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + } + $seq++ + } + } + finally { $tr.Dispose() } + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetDataRows -eq 0 -and $targetExists -and ((Get-Item -LiteralPath $TargetUsersCsv).Length -gt 0)) { + throw "Merge-UsersCsv: target '$TargetUsersCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + $cr = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $cr # skip header + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr + if ($null -eq $row) { break } + $n = & $normalizeRow $row $cToU $cPN $cUPN $cPID + if ($n.Derived) { $currentDerived++ } + $fields = New-Object System.Collections.Generic.List[string] + $pidField = $n.Pid # may be '' (blank-pid current rows are still emitted) + $fields.Add($pidField); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('1') + foreach ($v in $n.Vals) { $fields.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + $seq++ + } + } + finally { $cr.Dispose() } + } + finally { $ww.Flush(); $ww.Dispose() } + + # Auditable derivation log (parity with the prior in-memory path). + if ($targetDataRows -gt 0 -and -not $anyTargetPid) { + Write-LogHost ( + ("WARNING: Merge-UsersCsv: target Users CSV has no derivable dedup key (PersonId_Normalized / userPrincipalName / PersonId are all empty on every row). " + + "Cannot classify Retained / New / Departed; the union will contain only the current-run rows. " + + "Target: {0}") -f $TargetUsersCsv + ) -ForegroundColor Yellow } - } - if ($targetRows.Count -gt 0) { - foreach ($p in $targetRows[0].PSObject.Properties) { - if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + elseif ($targetDerived -gt 0 -or $currentDerived -gt 0) { + Write-LogHost ( + "Merge-UsersCsv: derived PersonId_Normalized on the fly (target rows: {0}, current rows: {1}) — column was absent on the seed/raw shape (expected for M365-bundle EntraUsers CSVs)." -f $targetDerived, $currentDerived + ) -ForegroundColor Cyan } - } - foreach ($c in @('Date_Added','Latest_Append_Date','In_Latest_Append')) { - if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } - } - $merged = New-Object System.Collections.Generic.List[psobject] + # --- Sort work by (PID, SEQ): equal keys adjacent, target rows before current. --- + $sortedPath = Join-Path $workDir 'sorted.csv' + [void](script:Invoke-PaxExternalSort -InputPath $workPath -OutputPath $sortedPath -KeyColumn @('__PID', '__SEQ') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) - # 1. Current-run rows (retained + new). - foreach ($r in $currentRows) { - $personId = $r.PersonId_Normalized - $obj = [ordered]@{} - foreach ($c in $hdrOrder) { $obj[$c] = '' } - foreach ($p in $r.PSObject.Properties) { $obj[$p.Name] = $p.Value } - - if ($personId -and $targetByPid.ContainsKey($personId)) { - $tr = $targetByPid[$personId] - # Target's UserKey wins (continuity across runs). - if ($tr.UserKey -and -not [string]::IsNullOrWhiteSpace([string]$tr.UserKey)) { - $obj['UserKey'] = $tr.UserKey + # --- Join per PID group; emit ordered result rows [__ORD,__SEQ2]+union cols. + # __ORD '0' = current-section (current order), '1' = departed (target order). --- + $resultPath = Join-Path $workDir 'result.csv' + $sr = [System.IO.StreamReader]::new($sortedPath, $true) + $rw = [System.IO.StreamWriter]::new($resultPath, $false, $u8) + $section1 = 0 + try { + $null = script:Read-PaxCsvRecord -Reader $sr + script:Write-PaxCsvRecord -Writer $rw -Fields (@('__ORD', '__SEQ2') + $unionCols) + $emitRow = { + param($ord, $seq2, $vals) + $out = New-Object System.Collections.Generic.List[string] + $out.Add($ord); $out.Add($seq2) + foreach ($v in $vals) { $out.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $rw -Fields $out.ToArray() + } + $haveGroup = $false; $curPid = $null; $targetFirst = $null; $targetFirstSeq = $null; $sawCurrent = $false + $flush = { + if (-not $haveGroup) { return } + if (-not [string]::IsNullOrEmpty($curPid)) { + if ($sawCurrent) { + if ($null -ne $targetFirst) { $script:__uret++ } else { $script:__unew++ } + } + } + if ((-not $sawCurrent) -and ($null -ne $targetFirst)) { + $dv = $targetFirst.Clone() + if ([string]::IsNullOrWhiteSpace([string]$dv[$idxDateAdded])) { $dv[$idxDateAdded] = $RunDate } + $dv[$idxLatest] = $RunDate; $dv[$idxInLatest] = 'FALSE' + & $emitRow '1' $targetFirstSeq $dv + $script:__udep++ + } } - $tda = $tr.PSObject.Properties['Date_Added'] - if ($tda -and -not [string]::IsNullOrWhiteSpace([string]$tda.Value)) { - $obj['Date_Added'] = $tda.Value - } else { - # Target lacked Date_Added — first-seen today. - $obj['Date_Added'] = $RunDate + $script:__uret = 0; $script:__unew = 0; $script:__udep = 0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $sr + if ($null -eq $row) { break } + $k = [string]$row[0] + $rseq = [string]$row[1] + $side = [string]$row[2] + $vals = New-Object 'string[]' $U + for ($j = 0; $j -lt $U; $j++) { $vals[$j] = if (($j + 3) -lt $row.Length) { [string]$row[$j + 3] } else { '' } } + if ((-not $haveGroup) -or (-not [string]::Equals($k, $curPid, [System.StringComparison]::Ordinal))) { + & $flush + $curPid = $k; $targetFirst = $null; $targetFirstSeq = $null; $sawCurrent = $false; $haveGroup = $true + } + if ($side -eq '0') { + if ($null -eq $targetFirst) { $targetFirst = $vals; $targetFirstSeq = $rseq } + } + else { + $out = $vals.Clone() + if ($null -ne $targetFirst) { + if ($idxUserKey -ge 0 -and -not [string]::IsNullOrWhiteSpace([string]$targetFirst[$idxUserKey])) { $out[$idxUserKey] = $targetFirst[$idxUserKey] } + if (-not [string]::IsNullOrWhiteSpace([string]$targetFirst[$idxDateAdded])) { $out[$idxDateAdded] = $targetFirst[$idxDateAdded] } else { $out[$idxDateAdded] = $RunDate } + } + else { + $out[$idxDateAdded] = $RunDate + } + $out[$idxLatest] = $RunDate; $out[$idxInLatest] = 'TRUE' + & $emitRow '0' $rseq $out + $sawCurrent = $true + $section1++ + } } - } else { - $obj['Date_Added'] = $RunDate + & $flush + $retainedCount = [int]$script:__uret; $newCount = [int]$script:__unew; $departedCount = [int]$script:__udep + Remove-Variable -Name __uret, __unew, __udep -Scope Script -ErrorAction SilentlyContinue } - $obj['Latest_Append_Date'] = $RunDate - $obj['In_Latest_Append'] = 'TRUE' - $merged.Add([pscustomobject]$obj) - } + finally { $rw.Flush(); $rw.Dispose(); $sr.Dispose() } + + $unionCount = $section1 + $departedCount - # 2. Departed users (in target, not in current). - $departedCount = 0 - foreach ($personId in $targetByPid.Keys) { - if ($currentByPid.ContainsKey($personId)) { continue } - $tr = $targetByPid[$personId] - $obj = [ordered]@{} - foreach ($c in $hdrOrder) { $obj[$c] = '' } - foreach ($p in $tr.PSObject.Properties) { $obj[$p.Name] = $p.Value } - $tda = $tr.PSObject.Properties['Date_Added'] - if (-not $tda -or [string]::IsNullOrWhiteSpace([string]$tda.Value)) { - $obj['Date_Added'] = $RunDate + # --- Restore output order (current-section then departed) and publish. --- + $orderedPath = Join-Path $workDir 'ordered.csv' + [void](script:Invoke-PaxExternalSort -InputPath $resultPath -OutputPath $orderedPath -KeyColumn @('__ORD', '__SEQ2') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $unionColsLocal = $unionCols + $totalEmpStr = "$unionCount" + $objStream = { + $orr = [System.IO.StreamReader]::new($orderedPath, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $orr + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $orr + if ($null -eq $row) { break } + $o = [ordered]@{} + for ($j = 0; $j -lt $unionColsLocal.Length; $j++) { $o[$unionColsLocal[$j]] = if (($j + 2) -lt $row.Length) { [string]$row[$j + 2] } else { '' } } + if ($hasTotalEmp) { $o['TotalEmployees'] = $totalEmpStr } + [pscustomobject]$o + } + } + finally { $orr.Dispose() } } - $obj['Latest_Append_Date'] = $RunDate - $obj['In_Latest_Append'] = 'FALSE' - $merged.Add([pscustomobject]$obj) - $departedCount++ + & $objStream | Select-Object -Property $unionCols | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force -ErrorAction Stop } - - # 3. Recompute TotalEmployees over the union. - $unionCount = $merged.Count - if ($hdrSet.Contains('TotalEmployees')) { - foreach ($r in $merged) { $r.TotalEmployees = "$unionCount" } + catch { + if (Test-Path -LiteralPath $tmpPath) { try { Remove-Item -LiteralPath $tmpPath -Force -ErrorAction SilentlyContinue } catch {} } + throw } - - # 4. Atomic rewrite. - $tmpPath = "$OutputPath.merging" - $hdrArray = [string[]]$hdrOrder.ToArray() - $merged | Select-Object -Property $hdrArray | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 - Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force - - $retainedCount = 0 - $newCount = 0 - foreach ($personId in $currentByPid.Keys) { - if ($targetByPid.ContainsKey($personId)) { $retainedCount++ } else { $newCount++ } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } } return @{ @@ -10910,6 +16960,1028 @@ function Get-FactCompositeKeyColumns { # unrolls to nothing, which @(...) normalizes to an empty array). return $cols.ToArray() } +#region PAX-S2 bounded-memory external-sort primitives +# Shared bounded-memory CSV primitives. These helpers stream records through +# fixed-size buffers and spill to disk once a caller-supplied byte budget is +# reached, so peak memory stays bounded by the budget plus a single record +# rather than by the total row count. Nothing in the main flow calls them; they +# are self-contained building blocks. Every function is script-scoped to keep +# the session global namespace clean. The byte budget controls only WHEN data +# spills to disk; it never caps how many rows, fields, files, or dates are +# processed. A single record larger than the budget is still processed intact. + +function script:New-PaxSortMetrics { + # Return a fresh deterministic metrics record. Counters are Int64 aggregate + # numbers only; the object never carries field values, keys, or row contents. + # The run-descriptor and level-occupancy counters prove the sort retains a + # fixed-bounded amount of run bookkeeping regardless of total input size, and + # the managed-heap checkpoints record the live managed allocation at start, + # after every buffer has spilled, and after the final merge. + [CmdletBinding()] + param() + [pscustomobject]@{ + RowsRead = [int64]0 + RowsWritten = [int64]0 + RunCount = [int64]0 + MergePassCount = [int64]0 + MaxBufferedBytes = [int64]0 + MaxBufferedRecords = [int64]0 + MaxAccountedBufferBytes = [int64]0 + LargestRecordBytes = [int64]0 + LargestSerializedRecordBytes = [int64]0 + MaxRetainedRunDescriptors = [int64]0 + MaxLevelOccupancy = [int64]0 + MaxOpenReaders = [int64]0 + TempBytesWritten = [int64]0 + ManagedHeapBefore = [int64]0 + ManagedHeapAfterIngest = [int64]0 + ManagedHeapAfterMerge = [int64]0 + CleanupSucceeded = $true + FailureStage = $null + } +} + +function script:Read-PaxCsvRecord { + # Read the next logical CSV record from an open StreamReader and return its + # fields as an ordered [string[]], or $null at end of input. Behaviour is the + # authoritative RFC-4180 contract defined by Read-PaxCsvRecordFallback below. + # + # By default a compiled state machine (PaxCsvRecordReader) performs the read for + # speed; it implements the SAME four-state machine, the SAME record boundaries, + # the SAME doubled-quote / embedded CR-LF handling, and throws the SAME messages + # as the fallback. If dynamic compilation is unavailable in a restricted host, + # the original PowerShell parser is used instead with identical results. The + # backend is chosen ONCE; a parser error from the compiled backend is surfaced + # verbatim (never used to retry the fallback on an already-advanced stream). + # $script:PaxCsvBackendOverride is a TEST-ONLY seam ('Compiled' | 'Fallback'); + # the production default is automatic. + [CmdletBinding()] + param([Parameter(Mandatory)][System.IO.StreamReader]$Reader) + $backend = $script:PaxCsvBackendOverride + if ($backend -eq 'Fallback') { return , (script:Read-PaxCsvRecordFallback -Reader $Reader) } + if ($backend -ne 'Compiled') { + # Automatic: prefer the compiled backend; fall back ONLY when compilation is + # unavailable (an initialization decision, never a per-record parse retry). + if (-not (script:Initialize-PaxCsvCompiledReader)) { + if (-not $script:PaxCsvFallbackWarned) { + $script:PaxCsvFallbackWarned = $true + $lh = Microsoft.PowerShell.Core\Get-Command -Name 'Write-LogHost' -CommandType Function -ErrorAction SilentlyContinue + if ($lh) { try { & $lh -Message 'PAX CSV read acceleration is unavailable in this host; using the built-in parser (results are unchanged).' -ForegroundColor 'DarkGray' } catch { } } + } + return , (script:Read-PaxCsvRecordFallback -Reader $Reader) + } + } + else { + # Forced compiled (test seam): require the compiled backend. + if (-not (script:Initialize-PaxCsvCompiledReader)) { throw 'PaxCsvParse: compiled backend was forced but is unavailable in this host.' } + } + # Compiled read. Surface the parser's own message verbatim; NEVER fall back after + # the stream has advanced (that would corrupt record position). + try { + $rec = [PaxCsvRecordReader]::ReadRecord($Reader) + } + catch { + $pmsg = $_.Exception.Message + if ($_.Exception.InnerException -and $_.Exception.InnerException.Message) { $pmsg = $_.Exception.InnerException.Message } + throw $pmsg + } + return , $rec +} + +function script:Initialize-PaxCsvCompiledReader { + # Compile the RFC-4180 reader type ONCE per process and cache the outcome so + # routine record reads never re-probe assemblies, recompile, allocate source, + # or re-catch a prior initialization failure. Returns $true when the compiled + # backend is usable, $false when dynamic compilation is unavailable (restricted + # host / language-mode policy). Emits nothing to the pipeline. + if ($null -ne $script:PaxCsvCompiledReady) { return $script:PaxCsvCompiledReady } + try { + if (-not ('PaxCsvRecordReader' -as [type])) { + $paxCsvReaderSource = @' +using System.IO; +using System.Collections.Generic; +using System.Text; +public static class PaxCsvRecordReader { + // Field state: 0 fresh, 1 unquoted, 2 inside quotes, 3 after a closing quote. + public static string[] ReadRecord(StreamReader reader) { + if (reader.Peek() < 0) { return null; } + List fields = new List(); + StringBuilder sb = new StringBuilder(); + int state = 0; + while (true) { + int ci = reader.Read(); + if (ci < 0) { + if (state == 2) { throw new IOException("PaxCsvParse: unterminated quoted field at end of input."); } + fields.Add(sb.ToString()); + return fields.ToArray(); + } + char c = (char)ci; + if (state == 0) { + if (c == '"') { state = 2; } + else if (c == ',') { fields.Add(sb.ToString()); sb.Clear(); } + else if (c == '\r') { if (reader.Peek() == 10) { reader.Read(); } fields.Add(sb.ToString()); return fields.ToArray(); } + else if (c == '\n') { fields.Add(sb.ToString()); return fields.ToArray(); } + else { sb.Append(c); state = 1; } + } + else if (state == 1) { + if (c == '"') { throw new IOException("PaxCsvParse: quote inside an unquoted field."); } + else if (c == ',') { fields.Add(sb.ToString()); sb.Clear(); state = 0; } + else if (c == '\r') { if (reader.Peek() == 10) { reader.Read(); } fields.Add(sb.ToString()); return fields.ToArray(); } + else if (c == '\n') { fields.Add(sb.ToString()); return fields.ToArray(); } + else { sb.Append(c); } + } + else if (state == 2) { + if (c == '"') { + if (reader.Peek() == 34) { reader.Read(); sb.Append('"'); } + else { state = 3; } + } + else { sb.Append(c); } + } + else { + if (c == ',') { fields.Add(sb.ToString()); sb.Clear(); state = 0; } + else if (c == '\r') { if (reader.Peek() == 10) { reader.Read(); } fields.Add(sb.ToString()); return fields.ToArray(); } + else if (c == '\n') { fields.Add(sb.ToString()); return fields.ToArray(); } + else { throw new IOException("PaxCsvParse: unexpected content after a closing quote."); } + } + } + } +} +'@ + Add-Type -TypeDefinition $paxCsvReaderSource -Language CSharp -ErrorAction Stop | Out-Null + } + $script:PaxCsvCompiledReady = $true + } + catch { + $script:PaxCsvCompiledReady = $false + } + return $script:PaxCsvCompiledReady +} + +function script:Read-PaxCsvRecordFallback { + # AUTHORITATIVE RFC-4180 contract (original interpreted parser, preserved verbatim + # as a portable fallback). Read the next logical CSV record from an open + # StreamReader and return its fields as an ordered [string[]], or $null at end of + # input. A single logical record may span several physical lines when a field is + # quoted; quoted fields carry commas, CR, LF, and doubled quotes ("") as literal + # content. A record larger than any read buffer accumulates whole to its + # terminator. Malformed quoting is a terminal error: a quote inside an unquoted + # field, any content after a closing quote, or an unterminated quoted field at end + # of input all throw rather than guess. + [CmdletBinding()] + param([Parameter(Mandatory)][System.IO.StreamReader]$Reader) + if ($Reader.Peek() -lt 0) { return $null } + $fields = [System.Collections.Generic.List[string]]::new() + $sb = [System.Text.StringBuilder]::new() + # Field state: 0 fresh, 1 unquoted, 2 inside quotes, 3 after a closing quote. + $state = 0 + while ($true) { + $ci = $Reader.Read() + if ($ci -lt 0) { + if ($state -eq 2) { throw 'PaxCsvParse: unterminated quoted field at end of input.' } + $fields.Add($sb.ToString()) + return , $fields.ToArray() + } + $c = [char]$ci + switch ($state) { + 0 { + if ($c -eq '"') { $state = 2 } + elseif ($c -eq ',') { $fields.Add($sb.ToString()); [void]$sb.Clear() } + elseif ($c -eq "`r") { if ($Reader.Peek() -eq 10) { [void]$Reader.Read() }; $fields.Add($sb.ToString()); return , $fields.ToArray() } + elseif ($c -eq "`n") { $fields.Add($sb.ToString()); return , $fields.ToArray() } + else { [void]$sb.Append($c); $state = 1 } + } + 1 { + if ($c -eq '"') { throw 'PaxCsvParse: quote inside an unquoted field.' } + elseif ($c -eq ',') { $fields.Add($sb.ToString()); [void]$sb.Clear(); $state = 0 } + elseif ($c -eq "`r") { if ($Reader.Peek() -eq 10) { [void]$Reader.Read() }; $fields.Add($sb.ToString()); return , $fields.ToArray() } + elseif ($c -eq "`n") { $fields.Add($sb.ToString()); return , $fields.ToArray() } + else { [void]$sb.Append($c) } + } + 2 { + if ($c -eq '"') { + if ($Reader.Peek() -eq 34) { [void]$Reader.Read(); [void]$sb.Append('"') } + else { $state = 3 } + } + else { [void]$sb.Append($c) } + } + 3 { + if ($c -eq ',') { $fields.Add($sb.ToString()); [void]$sb.Clear(); $state = 0 } + elseif ($c -eq "`r") { if ($Reader.Peek() -eq 10) { [void]$Reader.Read() }; $fields.Add($sb.ToString()); return , $fields.ToArray() } + elseif ($c -eq "`n") { $fields.Add($sb.ToString()); return , $fields.ToArray() } + else { throw 'PaxCsvParse: unexpected content after a closing quote.' } + } + } + } +} + +function script:Write-PaxCsvRecord { + # Write one RFC-4180-safe record to an open StreamWriter. A field is quoted + # only when it contains a comma, a double-quote, a CR, or an LF; embedded + # quotes are doubled. Output is deterministic, and a parse -> write -> parse + # round trip preserves every field value exactly. Records are terminated with + # CRLF so the reader recognizes them regardless of platform. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamWriter]$Writer, + [Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]]$Fields + ) + $sb = [System.Text.StringBuilder]::new() + $special = [char[]]@(',', '"', "`r", "`n") + for ($i = 0; $i -lt $Fields.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append(',') } + $f = $Fields[$i]; if ($null -eq $f) { $f = '' } + if ($f.IndexOfAny($special) -ge 0) { + [void]$sb.Append('"'); [void]$sb.Append($f.Replace('"', '""')); [void]$sb.Append('"') + } + else { [void]$sb.Append($f) } + } + [void]$sb.Append("`r`n") + $Writer.Write($sb.ToString()) +} + +function script:Get-PaxComparer { + # Map a caller-selected comparer name to the matching ordinal StringComparer. + # The comparer is applied to whole composite keys; keys are never lowercased + # as a shortcut for case-insensitive comparison, so the caller stays in full + # control of case sensitivity. + [CmdletBinding()] + param([Parameter(Mandatory)][ValidateSet('Ordinal', 'OrdinalIgnoreCase')][string]$Name) + if ($Name -eq 'OrdinalIgnoreCase') { return [System.StringComparer]::OrdinalIgnoreCase } + return [System.StringComparer]::Ordinal +} + +function script:Get-PaxCompositeKey { + # Build the composite key for a record by joining the caller-selected columns, + # in caller order, with the U+001F unit separator. That separator is a control + # character absent from normal field text, so distinct column tuples map to + # distinct keys. A key index outside the record contributes an empty segment. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]]$Fields, + [Parameter(Mandatory)][int[]]$KeyIndices + ) + $sep = [char]0x1F + $sb = [System.Text.StringBuilder]::new() + for ($i = 0; $i -lt $KeyIndices.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append($sep) } + $idx = $KeyIndices[$i] + if ($idx -ge 0 -and $idx -lt $Fields.Length) { + $v = $Fields[$idx]; if ($null -ne $v) { [void]$sb.Append($v) } + } + } + return $sb.ToString() +} + +function script:ConvertTo-PaxCsvLine { + # Serialize a field array to a single RFC-4180 line with no terminator. A field + # is quoted only when it contains a comma, a double-quote, a CR, or an LF, and + # embedded quotes are doubled, so the result parses back to the same fields. + # The buffered item keeps this one compact string instead of the field array, + # and the run spill emits the sequence, a comma, and this string unchanged. + [CmdletBinding()] + param([Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]]$Fields) + $sb = [System.Text.StringBuilder]::new() + $special = [char[]]@(',', '"', "`r", "`n") + for ($i = 0; $i -lt $Fields.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append(',') } + $f = $Fields[$i]; if ($null -eq $f) { $f = '' } + if ($f.IndexOfAny($special) -ge 0) { + [void]$sb.Append('"'); [void]$sb.Append($f.Replace('"', '""')); [void]$sb.Append('"') + } + else { [void]$sb.Append($f) } + } + return $sb.ToString() +} + +function script:Get-PaxRetainedRecordBytes { + # Return a conservative upper bound on the RETAINED managed bytes of one + # buffered item: a fixed three-slot object array holding its composite key, its + # boxed Int64 sequence, and its serialized data line. A .NET char occupies two + # managed bytes, so the retained string payloads account for 2*Key.Length plus + # 2*Data.Length. The fixed 512-byte allowance covers both string headers, the + # object-array header and three reference slots, the boxed Int64, the list slot, + # unused list-capacity allowance, alignment, and runtime object bookkeeping. + # This governs + # only WHEN a buffer spills; it never caps how much data flows through, and an + # item larger than the whole budget is still measured and processed intact. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Key, + [Parameter(Mandatory)][AllowEmptyString()][string]$Data + ) + $overhead = [int64]512 + $k = if ($null -ne $Key) { [int64]$Key.Length } else { [int64]0 } + $d = if ($null -ne $Data) { [int64]$Data.Length } else { [int64]0 } + return ([int64]2 * $k) + ([int64]2 * $d) + $overhead +} + +function script:Read-PaxRunFront { + # Read the next row from an internally-sorted run stream and return its front + # view: the Int64 sequence carried in the first field, the original record + # fields, and the recomputed composite key. Returns $null at end of the run. + # A run row missing its sequence field is a terminal error. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamReader]$Reader, + [Parameter(Mandatory)][int[]]$KeyIndices + ) + $row = script:Read-PaxCsvRecord -Reader $Reader + if ($null -eq $row) { return $null } + if ($row.Length -lt 1) { throw 'PaxMerge: malformed run row (missing sequence field).' } + $seq = [int64]::Parse($row[0], [System.Globalization.CultureInfo]::InvariantCulture) + $orig = New-Object System.Collections.Generic.List[string] + for ($i = 1; $i -lt $row.Length; $i++) { $orig.Add($row[$i]) } + $of = $orig.ToArray() + $key = script:Get-PaxCompositeKey -Fields $of -KeyIndices $KeyIndices + [pscustomobject]@{ Seq = $seq; Fields = $of; Key = $key } +} + +function script:Merge-PaxRunGroup { + # Merge up to a fan-in's worth of internally-sorted runs by opening one + # bounded reader per run, holding only the current front record from each open + # reader, and repeatedly emitting the minimum by (composite key under the + # comparer, then Int64 sequence). It never opens more readers than the group + # size and never collects a key group, so a hot key streams through. When + # EmitSequence is set the sequence is carried forward so a later pass keeps + # global order; otherwise the original record fields are written. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$RunPaths, + [Parameter(Mandatory)][int[]]$KeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][System.IO.StreamWriter]$Writer, + [Parameter(Mandatory)][bool]$EmitSequence, + [Parameter(Mandatory)][scriptblock]$ReaderFactory, + [Parameter(Mandatory)][psobject]$Metrics, + # Optional observability-only heartbeat hook (default absent). When supplied it is invoked every + # few thousand emitted rows so a long single-group merge can report active progress. It receives + # no arguments, no data, no keys, and no paths; its exceptions are swallowed and it cannot alter + # ordering, the emitted stream, or the merge result. Absent by default so existing callers and the + # emitted bytes are unchanged. + [scriptblock]$RowObserver + ) + $readers = New-Object System.Collections.Generic.List[System.IO.StreamReader] + $fronts = New-Object System.Collections.Generic.List[object] + $rgN = [int64]0 + try { + foreach ($rp in $RunPaths) { + $r = & $ReaderFactory $rp + $readers.Add($r) + $fronts.Add((script:Read-PaxRunFront -Reader $r -KeyIndices $KeyIndices)) + } + if ([int64]$readers.Count -gt $Metrics.MaxOpenReaders) { $Metrics.MaxOpenReaders = [int64]$readers.Count } + while ($true) { + $minIdx = -1 + for ($i = 0; $i -lt $fronts.Count; $i++) { + $f = $fronts[$i] + if ($null -eq $f) { continue } + if ($minIdx -lt 0) { $minIdx = $i; continue } + $m = $fronts[$minIdx] + $c = $Comparer.Compare($f.Key, $m.Key) + if ($c -lt 0 -or ($c -eq 0 -and $f.Seq -lt $m.Seq)) { $minIdx = $i } + } + if ($minIdx -lt 0) { break } + $sel = $fronts[$minIdx] + if ($EmitSequence) { + $outFields = New-Object System.Collections.Generic.List[string] + $outFields.Add($sel.Seq.ToString([System.Globalization.CultureInfo]::InvariantCulture)) + foreach ($v in $sel.Fields) { $outFields.Add($v) } + script:Write-PaxCsvRecord -Writer $Writer -Fields $outFields.ToArray() + } + else { + script:Write-PaxCsvRecord -Writer $Writer -Fields $sel.Fields + $Metrics.RowsWritten++ + } + if ($RowObserver) { $rgN++; if (($rgN -band 4095) -eq 0) { try { & $RowObserver } catch { } } } + $fronts[$minIdx] = (script:Read-PaxRunFront -Reader $readers[$minIdx] -KeyIndices $KeyIndices) + } + } + finally { + foreach ($r in $readers) { if ($r) { $r.Close(); $r.Dispose() } } + } +} + +function script:Merge-PaxSortedRuns { + # Merge the caller's survivor run set into a single ordered stream while + # honoring a bounded fan-in. The survivor set is the fixed-bounded collection + # gathered from the compaction levels, never the total run count, so the pass + # lists below are bounded by that fixed constant. When the survivor count + # exceeds the fan-in the merge runs in deterministic multiple passes: it + # merges a fan-in's worth of runs at a time into larger runs, deletes each + # consumed run as soon as its group finishes, and repeats until the survivors + # fit one final merge. No pass opens more than the fan-in of readers and no + # pass materializes a whole run or a whole hot-key group. Ordering is stable + # on (composite key under the comparer, then Int64 sequence). The final pass + # writes the original record fields to the output writer; intermediate passes + # carry the sequence forward. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$RunPaths, + [Parameter(Mandatory)][int[]]$KeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][int]$FanIn, + [Parameter(Mandatory)][System.IO.StreamWriter]$OutputWriter, + [Parameter(Mandatory)][string]$WorkDir, + [Parameter(Mandatory)][scriptblock]$ReaderFactory, + [Parameter(Mandatory)][scriptblock]$WriterFactory, + [Parameter(Mandatory)][psobject]$Metrics, + # Optional observability-only observer (default absent). It is invoked as + # & $ProgressObserver at each intermediate-merge pass entry and once + # before the final merge, and (via a derived row hook) periodically during a long pass. It only + # ever receives an aggregate stage name and pass number, never a row, key, field, or path, and it + # cannot alter merge ordering, output, or the result. Absent by default so callers and emitted + # bytes are unchanged. + [scriptblock]$ProgressObserver + ) + if ($FanIn -lt 2) { throw 'PaxMerge: FanIn must be at least 2.' } + $current = New-Object System.Collections.Generic.List[string] + foreach ($p in $RunPaths) { $current.Add($p) } + if ($current.Count -eq 0) { return } + # Derive a row-level heartbeat hook that reports under the stage currently in flight. Absent unless + # an observer was supplied, so the default merge path allocates nothing extra and emits nothing. + $stageBox = [pscustomobject]@{ Stage = 'FinalMerge' } + $rowObs = $null + if ($ProgressObserver) { $rowObs = { & $ProgressObserver $stageBox.Stage -1 } } + $passIndex = 0 + while ($current.Count -gt $FanIn) { + $passIndex++ + $stageBox.Stage = 'IntermediateMerge' + if ($ProgressObserver) { & $ProgressObserver 'IntermediateMerge' $passIndex } + $next = New-Object System.Collections.Generic.List[string] + $groupNo = 0 + for ($i = 0; $i -lt $current.Count; $i += $FanIn) { + $group = New-Object System.Collections.Generic.List[string] + for ($j = $i; $j -lt [Math]::Min($i + $FanIn, $current.Count); $j++) { $group.Add($current[$j]) } + $groupNo++ + $outPath = Join-Path $WorkDir ("pass{0:d3}_grp{1:d4}.paxrun" -f $passIndex, $groupNo) + $w = & $WriterFactory $outPath + try { + script:Merge-PaxRunGroup -RunPaths $group.ToArray() -KeyIndices $KeyIndices -Comparer $Comparer -Writer $w -EmitSequence $true -ReaderFactory $ReaderFactory -Metrics $Metrics -RowObserver $rowObs + } + finally { $w.Flush(); $w.Close(); $w.Dispose() } + $fi = [System.IO.FileInfo]::new($outPath); if ($fi.Exists) { $Metrics.TempBytesWritten += [int64]$fi.Length } + foreach ($cp in $group) { if (Test-Path -LiteralPath $cp) { Remove-Item -LiteralPath $cp -Force -ErrorAction Stop } } + $next.Add($outPath) + } + $Metrics.MergePassCount++ + $current = $next + } + $stageBox.Stage = 'FinalMerge' + if ($ProgressObserver) { & $ProgressObserver 'FinalMerge' 0 } + script:Merge-PaxRunGroup -RunPaths $current.ToArray() -KeyIndices $KeyIndices -Comparer $Comparer -Writer $OutputWriter -EmitSequence $false -ReaderFactory $ReaderFactory -Metrics $Metrics -RowObserver $rowObs + foreach ($cp in $current) { if (Test-Path -LiteralPath $cp) { Remove-Item -LiteralPath $cp -Force -ErrorAction Stop } } + $Metrics.MergePassCount++ +} + +function script:Remove-PaxWorkspace { + # Remove a temp workspace and an optional in-progress publication file after a + # run finishes or fails. A cleanup problem never hides the primary outcome: it + # only clears CleanupSucceeded on the metrics so the caller can report it. + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$WorkDir, + [AllowNull()][AllowEmptyString()][string]$MergingPath, + [Parameter(Mandatory)][psobject]$Metrics + ) + $okAll = $true + if ($MergingPath -and (Test-Path -LiteralPath $MergingPath)) { + try { Remove-Item -LiteralPath $MergingPath -Force -ErrorAction Stop } catch { $okAll = $false } + } + if (Test-Path -LiteralPath $WorkDir) { + try { Remove-Item -LiteralPath $WorkDir -Recurse -Force -ErrorAction Stop } catch { $okAll = $false } + } + if (-not $okAll) { $Metrics.CleanupSucceeded = $false } +} + +function script:Select-PaxFirstRowPerKey { + # Stream the first row of each key from a (key, sequence)-sorted run stream. + # Because the input is ordered by key then ascending sequence, the first row + # seen for a key carries the lowest sequence and is emitted; later rows with + # the same key are skipped. A hot key with many rows streams through without + # ever collecting the group. Returns the number of rows emitted. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamReader]$Reader, + [Parameter(Mandatory)][int[]]$KeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][System.IO.StreamWriter]$Writer, + [bool]$EmitSequence = $false + ) + $haveLast = $false + $lastKey = $null + $emitted = [int64]0 + while ($true) { + $front = script:Read-PaxRunFront -Reader $Reader -KeyIndices $KeyIndices + if ($null -eq $front) { break } + $isNew = (-not $haveLast) -or ($Comparer.Compare($front.Key, $lastKey) -ne 0) + if ($isNew) { + if ($EmitSequence) { + $of = New-Object System.Collections.Generic.List[string] + $of.Add($front.Seq.ToString([System.Globalization.CultureInfo]::InvariantCulture)) + foreach ($v in $front.Fields) { $of.Add($v) } + script:Write-PaxCsvRecord -Writer $Writer -Fields $of.ToArray() + } + else { + script:Write-PaxCsvRecord -Writer $Writer -Fields $front.Fields + } + $emitted++ + $lastKey = $front.Key + $haveLast = $true + } + } + return $emitted +} + +function script:Merge-PaxKeyStreams { + # Co-merge two key-sorted, distinct-key run streams by advancing two bounded + # readers in lockstep. A key present in the current stream is written to the + # present writer; a key found only in the target stream is written to the + # target-only writer. Neither side is collected, so arbitrarily large streams + # classify in bounded memory. Returns present and target-only counts. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamReader]$CurrentReader, + [Parameter(Mandatory)][System.IO.StreamReader]$TargetReader, + [Parameter(Mandatory)][int[]]$CurrentKeyIndices, + [Parameter(Mandatory)][int[]]$TargetKeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][System.IO.StreamWriter]$PresentWriter, + [Parameter(Mandatory)][System.IO.StreamWriter]$TargetOnlyWriter + ) + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + $present = [int64]0; $targetOnly = [int64]0 + while ($true) { + if ($null -eq $cur -and $null -eq $tgt) { break } + if ($null -eq $tgt) { + script:Write-PaxCsvRecord -Writer $PresentWriter -Fields $cur.Fields; $present++ + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + continue + } + if ($null -eq $cur) { + script:Write-PaxCsvRecord -Writer $TargetOnlyWriter -Fields $tgt.Fields; $targetOnly++ + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + continue + } + $c = $Comparer.Compare($cur.Key, $tgt.Key) + if ($c -lt 0) { + script:Write-PaxCsvRecord -Writer $PresentWriter -Fields $cur.Fields; $present++ + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + } + elseif ($c -gt 0) { + script:Write-PaxCsvRecord -Writer $TargetOnlyWriter -Fields $tgt.Fields; $targetOnly++ + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + } + else { + script:Write-PaxCsvRecord -Writer $PresentWriter -Fields $cur.Fields; $present++ + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + } + } + return [pscustomobject]@{ Present = $present; TargetOnly = $targetOnly } +} + +function script:Invoke-PaxExternalSort { + <# + .SYNOPSIS + Sort a CSV by a caller-selected composite key using bounded memory and a + fixed-level online run compactor. + + .DESCRIPTION + Reads the header record of an input CSV, maps the requested key columns to + column indices, keys each data record by the U+001F join of those columns + under the caller-selected comparer, and writes the header plus the sorted + data to the output path. Records buffer in memory only until a caller + supplied byte budget is reached (or a single record alone exceeds it), then + spill to internally-sorted run files. Each buffered item retains only its + composite key, its Int64 sequence, and its serialized data line, so retained + memory is accounted honestly rather than by field payload alone. + + Runs are combined by an online fixed-level compactor rather than by + retaining every run path. A fresh run enters level zero; whenever a level + reaches the fan-in it merges into one promoted run at the next level, the + consumed runs are deleted, and the level clears. A level-L run subsumes at + least FanIn^L runs, so with a fan-in of at least two the sixty-four levels + cover the whole positive Int64 run domain (two to the sixty-third exceeds + Int64.MaxValue). At most fan-in-minus-one runs rest at each level, so the + surviving run set gathered for the final merge is bounded by a fixed + constant, never by the total run count. The final merge combines those + survivors with the same bounded, possibly multi-pass k-way merge, so peak + memory and retained run bookkeeping stay bounded by the budget, one record, + and that fixed constant rather than by total rows or total runs. The byte + budget controls only WHEN data spills; it never caps how many rows, fields, + or files are processed. Every record is preserved, including all duplicate + keys, and a hot key streams through without being collected. + + The output is published only after every read, spill, promotion, merge, + flush, and close succeeds: the sorted data is written to a temporary sibling + file and then moved onto the output path. Any failure surfaces a terminating + error tagged with the stage that failed, leaves an existing output file + byte-for-byte unchanged, and removes partial work where possible. There is + no in-memory fallback after a spill or merge failure. An optional fault hook + is invoked at named internal checkpoints so callers can verify failure + handling; it is inert when not supplied. + + .OUTPUTS + A metrics object with Int64 counters (RowsRead, RowsWritten, RunCount, + MergePassCount, MaxBufferedBytes, MaxBufferedRecords, MaxAccountedBufferBytes, + LargestRecordBytes, LargestSerializedRecordBytes, MaxRetainedRunDescriptors, + MaxLevelOccupancy, MaxOpenReaders, TempBytesWritten, ManagedHeapBefore, + ManagedHeapAfterIngest, ManagedHeapAfterMerge), a CleanupSucceeded flag, and + a FailureStage string. Metrics carry only aggregate numbers, never data. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$InputPath, + [Parameter(Mandatory)][string]$OutputPath, + [Parameter(Mandatory)][string[]]$KeyColumn, + [Parameter(Mandatory)][ValidateSet('Ordinal', 'OrdinalIgnoreCase')][string]$Comparer, + [Parameter(Mandatory)][int64]$MemoryBudgetBytes, + [Parameter(Mandatory)][int]$FanIn, + [string]$WorkRoot, + [scriptblock]$ReaderFactory, + [scriptblock]$WriterFactory, + [scriptblock]$FaultHook, + # Optional observability-only progress seam (default inert). ProgressCallback receives an AGGREGATE + # snapshot object (stage name, elapsed seconds, and Int64 counters copied from the metrics object); + # it is never passed a row, key, field, filename, or path. ProgressIntervalSeconds throttles periodic + # emissions; stage-entry and terminal emissions are always sent. A callback exception is swallowed and + # cannot alter or fail the sort. When ProgressCallback is not supplied the seam is fully inert and the + # output is byte-identical to a run without it. + [scriptblock]$ProgressCallback, + [double]$ProgressIntervalSeconds = 0, + # Optional starting counter values (default 0). Nonzero seeds let the checked + # Int64 overflow guards below be reached deterministically without materializing + # Int64.MaxValue records; at 0 the counters and all output are unchanged. + [int64]$InitialSequence = 0, + [int64]$InitialRunCounter = 0, + [int64]$InitialPromotionCounter = 0 + ) + if ($MemoryBudgetBytes -le 0) { throw 'PaxExternalSort: MemoryBudgetBytes must be positive.' } + if ($FanIn -lt 2) { throw 'PaxExternalSort: FanIn must be at least 2.' } + if (-not $ReaderFactory) { $ReaderFactory = { param($p) [System.IO.StreamReader]::new($p, $true) } } + if (-not $WriterFactory) { $WriterFactory = { param($p) [System.IO.StreamWriter]::new($p, $false, (New-Object System.Text.UTF8Encoding($false))) } } + $cmp = script:Get-PaxComparer -Name $Comparer + $metrics = script:New-PaxSortMetrics + $metrics.ManagedHeapBefore = [int64][System.GC]::GetTotalMemory($true) + # Progress state is allocated ONLY when a callback is supplied, so a run without one starts no + # Stopwatch, allocates no progress state, and never enters the emitter in the hot loops below; the + # managed-heap checkpoints therefore see the same allocations as a run compiled without the seam. + $prog = $null + if ($ProgressCallback) { + $prog = [pscustomobject]@{ + Sw = [System.Diagnostics.Stopwatch]::StartNew() + Last = [double](-1) + Seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + MergePass = $null + } + } + # Observability-only progress emitter (inert unless a callback is supplied). It reads aggregate + # metrics only and never mutates sort state, buffering, ordering, spill thresholds, or output. The + # first emission for each distinct stage is always sent (stage entry), a forced emission is always + # sent (terminal events), and other emissions are throttled by ProgressIntervalSeconds. Every + # callback stream (success, error, warning, verbose, debug, information) is discarded and a + # terminating callback exception is caught, so callback activity can neither become function output + # nor alter or fail the sort. No row, key, field, filename, or path is ever passed. + $emitProgress = { + param([bool]$Force, [string]$StageName) + if (-not $ProgressCallback) { return } + $el = $prog.Sw.Elapsed.TotalSeconds + $firstForStage = -not $prog.Seen.Contains($StageName) + if ($firstForStage) { [void]$prog.Seen.Add($StageName) } + if (-not ($Force -or $firstForStage)) { + if ($ProgressIntervalSeconds -le 0) { return } + if ($prog.Last -ge 0 -and ($el - $prog.Last) -lt $ProgressIntervalSeconds) { return } + } + $prog.Last = $el + $snapshot = [pscustomobject]@{ + Stage = $StageName + ElapsedSeconds = [Math]::Round($el, 3) + RowsRead = [int64]$metrics.RowsRead + RowsWritten = [int64]$metrics.RowsWritten + RunCount = [int64]$metrics.RunCount + MergePassCount = [int64]$metrics.MergePassCount + MergePass = $prog.MergePass + MaxOpenReaders = [int64]$metrics.MaxOpenReaders + TempBytesWritten = [int64]$metrics.TempBytesWritten + MaxBufferedBytes = [int64]$metrics.MaxBufferedBytes + } + try { & $ProgressCallback $snapshot 2>$null 3>$null 4>$null 5>$null 6>$null | Out-Null } + catch { } + } + # Merge-stage observer: forwards intermediate/final merge-pass entries and long-pass heartbeats + # from the merge primitive to the emitter as aggregate stage events. Inert without a callback. + $mergeObserver = { + param([string]$MergeStage, [int]$PassNo) + if (-not $ProgressCallback) { return } + if ($PassNo -ge 0) { $prog.MergePass = [int64]$PassNo } + & $emitProgress $false $MergeStage + } + # A level-L run subsumes at least FanIn^L runs; two to the sixty-third already + # exceeds Int64.MaxValue, so sixty-four fixed levels span the whole positive + # Int64 run-count domain for any fan-in of at least two. + $MaxLevels = 64 + if (-not $WorkRoot) { $WorkRoot = $env:TEMP; if (-not $WorkRoot) { $WorkRoot = [System.IO.Path]::GetTempPath() } } + $workDir = Join-Path $WorkRoot ('pax_extsort_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $mergingPath = $OutputPath + '.merging' + $st = [pscustomobject]@{ Stage = 'Init' } + $ctr = [pscustomobject]@{ Run = [int64]$InitialRunCounter; Promo = [int64]$InitialPromotionCounter } + $inReader = $null + $outWriter = $null + try { + $st.Stage = 'Workspace'; [void][System.IO.Directory]::CreateDirectory($workDir) + $st.Stage = 'Read' + $inReader = & $ReaderFactory $InputPath + $header = script:Read-PaxCsvRecord -Reader $inReader + if ($null -eq $header) { throw 'PaxExternalSort: input has no header record.' } + $st.Stage = 'KeyMapping' + $keyIndices = New-Object System.Collections.Generic.List[int] + foreach ($kc in $KeyColumn) { + $idx = -1 + for ($i = 0; $i -lt $header.Length; $i++) { if ([System.String]::Equals($header[$i], $kc, [System.StringComparison]::Ordinal)) { $idx = $i; break } } + if ($idx -lt 0) { throw 'PaxExternalSort: a requested key column is not present in the header.' } + $keyIndices.Add($idx) + } + $ki = $keyIndices.ToArray() + # Fixed-level compaction state: one bounded run-descriptor list per level. + $levels = New-Object 'System.Object[]' $MaxLevels + for ($li = 0; $li -lt $MaxLevels; $li++) { $levels[$li] = New-Object System.Collections.Generic.List[string] } + # Record the peak persistent run-descriptor count across the fixed levels and + # the final survivor owner. Fixed fan-in merge argument arrays are transient + # and never scale with total runs. + $updDesc = { + param([int64]$extra) + $sum = [int64]$extra + for ($di = 0; $di -lt $MaxLevels; $di++) { $sum += [int64]$levels[$di].Count } + if ($sum -gt $metrics.MaxRetainedRunDescriptors) { $metrics.MaxRetainedRunDescriptors = $sum } + } + # Insert a fresh run at level zero and cascade: a full level merges its + # fan-in runs into one promoted run, deletes the consumed runs, clears the + # level, and carries the promotion upward. + $insertRun = { + param($newRunPath) + $carry = $newRunPath + $lvl = 0 + while ($true) { + if ($lvl -ge $MaxLevels) { throw 'PaxExternalSort: run-level overflow (Int64 run domain exceeded).' } + $levels[$lvl].Add($carry) + if ([int64]$levels[$lvl].Count -gt $metrics.MaxLevelOccupancy) { $metrics.MaxLevelOccupancy = [int64]$levels[$lvl].Count } + & $updDesc 0 + if ($levels[$lvl].Count -lt $FanIn) { break } + if ($ctr.Promo -eq [System.Int64]::MaxValue) { $st.Stage = 'PromotionCounterOverflow'; throw 'PaxExternalSort: promotion counter reached Int64.MaxValue; refusing to wrap.' } + $ctr.Promo++ + $promoted = Join-Path $workDir ('lvl{0:d2}_{1:d6}.paxrun' -f ($lvl + 1), $ctr.Promo) + $consumed = $levels[$lvl].ToArray() + $st.Stage = 'PromoteWrite' + $pw = & $WriterFactory $promoted + try { + $st.Stage = 'PromoteRead' + script:Merge-PaxRunGroup -RunPaths $consumed -KeyIndices $ki -Comparer $cmp -Writer $pw -EmitSequence $true -ReaderFactory $ReaderFactory -Metrics $metrics + $st.Stage = 'PromoteWrite' + $pw.Flush() + } + finally { $pw.Close(); $pw.Dispose() } + $pfi = [System.IO.FileInfo]::new($promoted); if ($pfi.Exists) { $metrics.TempBytesWritten += [int64]$pfi.Length } + $st.Stage = 'PromoteDelete' + foreach ($cp in $consumed) { if (Test-Path -LiteralPath $cp) { Remove-Item -LiteralPath $cp -Force -ErrorAction Stop } } + $levels[$lvl].Clear() + if ($FaultHook) { $st.Stage = 'PromoteState'; & $FaultHook 'PromoteState' } + $metrics.MergePassCount++ + & $emitProgress $false 'Promote' + $carry = $promoted + $lvl++ + } + } + $spill = { + param($buf) + if ($buf.Count -eq 0) { return } + $buf.Sort([System.Comparison[object[]]] { + param($x, $y) + $k = $cmp.Compare([string]$x[0], [string]$y[0]) + if ($k -ne 0) { return $k } + return ([int64]$x[1]).CompareTo([int64]$y[1]) + }) + if ($ctr.Run -eq [System.Int64]::MaxValue) { $st.Stage = 'RunCounterOverflow'; throw 'PaxExternalSort: run counter reached Int64.MaxValue; refusing to wrap.' } + $ctr.Run++ + $st.Stage = 'RunCreate' + $rp = Join-Path $workDir ('run_{0:d6}.paxrun' -f $ctr.Run) + $w = & $WriterFactory $rp + $st.Stage = 'Spill' + try { + foreach ($it in $buf) { + $w.Write(([int64]$it[1]).ToString([System.Globalization.CultureInfo]::InvariantCulture)) + $w.Write(',') + $w.Write([string]$it[2]) + $w.Write("`r`n") + } + $w.Flush() + } + finally { $w.Close(); $w.Dispose() } + $fi = [System.IO.FileInfo]::new($rp); if ($fi.Exists) { $metrics.TempBytesWritten += [int64]$fi.Length } + $metrics.RunCount++ + & $emitProgress $false 'Spill' + & $insertRun $rp + } + $buffer = New-Object 'System.Collections.Generic.List[object[]]' + $bufferedBytes = [int64]0 + $seq = [int64]$InitialSequence + $st.Stage = 'Read' + & $emitProgress $true 'Ingest' + while ($true) { + $rec = script:Read-PaxCsvRecord -Reader $inReader + if ($null -eq $rec) { break } + $metrics.RowsRead++ + if ($ProgressCallback -and (($metrics.RowsRead -band 255) -eq 0)) { & $emitProgress $false 'Ingest' } + if ($seq -eq [System.Int64]::MaxValue) { $st.Stage = 'SequenceOverflow'; throw 'PaxExternalSort: ingest sequence reached Int64.MaxValue; refusing to wrap.' } + $seq++ + $key = script:Get-PaxCompositeKey -Fields $rec -KeyIndices $ki + $dataLine = script:ConvertTo-PaxCsvLine -Fields $rec + $seqStr = $seq.ToString([System.Globalization.CultureInfo]::InvariantCulture) + $serBytes = [int64][System.Text.Encoding]::UTF8.GetByteCount($seqStr + ',' + $dataLine) + if ($serBytes -gt $metrics.LargestSerializedRecordBytes) { $metrics.LargestSerializedRecordBytes = $serBytes } + $payloadBytes = [int64][System.Text.Encoding]::UTF8.GetByteCount($dataLine) + if ($payloadBytes -gt $metrics.LargestRecordBytes) { $metrics.LargestRecordBytes = $payloadBytes } + $accounted = script:Get-PaxRetainedRecordBytes -Key $key -Data $dataLine + if ($buffer.Count -gt 0 -and ($bufferedBytes + $accounted) -gt $MemoryBudgetBytes) { + & $spill $buffer + $buffer = New-Object 'System.Collections.Generic.List[object[]]' + $bufferedBytes = [int64]0 + $st.Stage = 'Read' + } + $buffer.Add([object[]]@($key, $seq, $dataLine)) + $bufferedBytes += $accounted + if ($bufferedBytes -gt $metrics.MaxBufferedBytes) { $metrics.MaxBufferedBytes = $bufferedBytes } + if ($bufferedBytes -gt $metrics.MaxAccountedBufferBytes) { $metrics.MaxAccountedBufferBytes = $bufferedBytes } + if ([int64]$buffer.Count -gt $metrics.MaxBufferedRecords) { $metrics.MaxBufferedRecords = [int64]$buffer.Count } + if ($accounted -gt $MemoryBudgetBytes) { + & $spill $buffer + $buffer = New-Object 'System.Collections.Generic.List[object[]]' + $bufferedBytes = [int64]0 + $st.Stage = 'Read' + } + } + if ($buffer.Count -gt 0) { & $spill $buffer; $buffer = $null } + $st.Stage = 'Read' + if ($inReader) { $inReader.Close(); $inReader.Dispose(); $inReader = $null } + $metrics.ManagedHeapAfterIngest = [int64][System.GC]::GetTotalMemory($true) + # Move the fixed-bounded surviving run descriptors out of the compaction + # levels. Clearing each source level as its paths transfer prevents retaining + # duplicate references in both the level set and the final survivor owner. + $survivors = New-Object System.Collections.Generic.List[string] + for ($li = 0; $li -lt $MaxLevels; $li++) { + foreach ($rp in $levels[$li]) { $survivors.Add($rp) } + $levels[$li].Clear() + & $updDesc ([int64]$survivors.Count) + } + $st.Stage = 'Publish' + $outWriter = & $WriterFactory $mergingPath + try { + script:Write-PaxCsvRecord -Writer $outWriter -Fields $header + if ($FaultHook) { $st.Stage = 'FinalMerge'; & $FaultHook 'FinalMerge' } + $st.Stage = 'Merge' + script:Merge-PaxSortedRuns -RunPaths $survivors.ToArray() -KeyIndices $ki -Comparer $cmp -FanIn $FanIn -OutputWriter $outWriter -WorkDir $workDir -ReaderFactory $ReaderFactory -WriterFactory $WriterFactory -Metrics $metrics -ProgressObserver $mergeObserver + $st.Stage = 'Publish' + $outWriter.Flush() + } + finally { $outWriter.Close(); $outWriter.Dispose(); $outWriter = $null } + $metrics.ManagedHeapAfterMerge = [int64][System.GC]::GetTotalMemory($true) + $st.Stage = 'Publish' + Move-Item -LiteralPath $mergingPath -Destination $OutputPath -Force -ErrorAction Stop + & $emitProgress $true 'Publish' + $st.Stage = 'Cleanup' + & $emitProgress $true 'Cleanup' + script:Remove-PaxWorkspace -WorkDir $workDir -MergingPath $null -Metrics $metrics + & $emitProgress $true 'Complete' + return $metrics + } + catch { + $primary = $_ + if (-not $metrics.FailureStage) { $metrics.FailureStage = $st.Stage } + & $emitProgress $true 'Failed' + if ($inReader) { try { $inReader.Close(); $inReader.Dispose() } catch { } } + if ($outWriter) { try { $outWriter.Close(); $outWriter.Dispose() } catch { } } + script:Remove-PaxWorkspace -WorkDir $workDir -MergingPath $mergingPath -Metrics $metrics + $ex = [System.Exception]::new(("PaxExternalSort failed at stage '{0}'." -f $metrics.FailureStage), $primary.Exception) + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'PaxExternalSortFailed', [System.Management.Automation.ErrorCategory]::NotSpecified, $metrics) + throw $er + } +} +#endregion PAX-S2 + +function script:Get-PaxDedupedFactHeader { + # Deduplicate a raw CSV header the same way the tolerant importer does: strip a + # leading byte-order mark from the first cell, trim each cell, replace a blank + # cell with _blank, and suffix later case-insensitive duplicates with _dup2, + # _dup3, and so on while the first occurrence keeps its name. The result is the + # stable column identity used for keying, cross-run continuity, and the header + # union. + [CmdletBinding()] + param([Parameter(Mandatory)][AllowEmptyCollection()][string[]]$Cells) + $seen = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + $out = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $Cells.Length; $i++) { + $n = [string]$Cells[$i] + if ($null -eq $n) { $n = '' } + if ($i -eq 0) { $n = $n -replace '^\uFEFF', '' } + $n = $n.Trim() + if ([string]::IsNullOrEmpty($n)) { $n = '_blank' } + if ($seen.ContainsKey($n)) { + $seen[$n] = $seen[$n] + 1 + $out.Add(('{0}_dup{1}' -f $n, $seen[$n])) + } + else { + $seen[$n] = 1 + $out.Add($n) + } + } + return , $out.ToArray() +} + +function script:ConvertTo-PaxFactNormalized { + # Stream a Fact CSV into a normalized work file whose header is the two + # synthetic columns __paxk and __paxseq followed by the deduplicated source + # header, and whose every data row carries its composite key, its zero-padded + # read-order sequence, and the source field values positionally aligned to the + # deduplicated header. The composite key joins the requested key columns with + # U+001F (a single key column contributes its value unseparated); a requested + # column absent from the header contributes an empty segment. Reading uses the + # bounded RFC-4180 record reader and writing uses the RFC-4180 record writer, so + # memory stays bounded regardless of file size. Returns the deduplicated header, + # the data-row count, and whether a header record was present. An absent header + # (empty input) yields a null header and zero rows. + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$InputPath, + [Parameter(Mandatory)][string]$OutputPath, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$KeyColumns + ) + $reader = [System.IO.StreamReader]::new($InputPath, $true) + try { + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $header) { + return [pscustomobject]@{ Header = $null; RowCount = [int64]0; HadHeader = $false } + } + $deduped = script:Get-PaxDedupedFactHeader -Cells $header + $idxMap = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + for ($i = 0; $i -lt $deduped.Length; $i++) { $idxMap[$deduped[$i]] = $i } + $keyIndices = New-Object System.Collections.Generic.List[int] + foreach ($kc in $KeyColumns) { + if ($idxMap.ContainsKey($kc)) { $keyIndices.Add($idxMap[$kc]) } else { $keyIndices.Add(-1) } + } + $ki = $keyIndices.ToArray() + $writer = [System.IO.StreamWriter]::new($OutputPath, $false, (New-Object System.Text.UTF8Encoding($false))) + try { + script:Write-PaxCsvRecord -Writer $writer -Fields (@('__paxk', '__paxseq') + $deduped) + $rowCount = [int64]0 + while ($true) { + $rec = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $rec) { break } + $paxk = script:Get-PaxCompositeKey -Fields $rec -KeyIndices $ki + $paxseq = $rowCount.ToString('D19') + $data = [string[]]::new($deduped.Length) + for ($j = 0; $j -lt $deduped.Length; $j++) { $data[$j] = if ($j -lt $rec.Length) { $rec[$j] } else { '' } } + script:Write-PaxCsvRecord -Writer $writer -Fields (@($paxk, $paxseq) + $data) + $rowCount++ + } + $writer.Flush() + } + finally { $writer.Dispose() } + return [pscustomobject]@{ Header = $deduped; RowCount = $rowCount; HadHeader = $true } + } + finally { $reader.Dispose() } +} + +function script:Read-PaxFactNextTargetGroup { + # Return the next distinct non-empty target key group from a sorted normalized + # target stream, or $null at end of input. Rows arrive grouped by key (a blank + # key sorts ahead of keyed rows) and, within a key, ordered by read sequence, so + # the first row seen for a group is that key's first original occurrence. No-key + # rows are skipped entirely and duplicate rows of the returned key are consumed + # and discarded; a single peeked row that starts the next thing is stashed on + # the state object so the following call resumes correctly. The state object + # carries the open reader plus a one-record pending slot; the returned value is + # a table with the group Key and the first-occurrence Fields. + [CmdletBinding()] + param([Parameter(Mandatory)][psobject]$State) + while ($true) { + if ($State.HasPending) { $row = $State.Pending; $State.Pending = $null; $State.HasPending = $false } + else { $row = script:Read-PaxCsvRecord -Reader $State.Reader } + if ($null -eq $row) { return $null } + $k = if ($row.Length -gt 0) { [string]$row[0] } else { '' } + if ([string]::IsNullOrWhiteSpace($k)) { continue } + $gk = $k + $first = $row + while ($true) { + $peek = script:Read-PaxCsvRecord -Reader $State.Reader + if ($null -eq $peek) { break } + $pk = if ($peek.Length -gt 0) { [string]$peek[0] } else { '' } + if ([string]::IsNullOrWhiteSpace($pk)) { $State.Pending = $peek; $State.HasPending = $true; break } + if ([System.String]::Equals($pk, $gk, [System.StringComparison]::OrdinalIgnoreCase)) { continue } + $State.Pending = $peek; $State.HasPending = $true; break + } + return @{ Key = $gk; Fields = $first } + } +} function Merge-FactCsv { <# @@ -10941,180 +18013,670 @@ function Merge-FactCsv { [Parameter()] [string] $OutputPath, [Parameter()] [string] $KeyColumn = 'Message_Id_Raw', [Parameter()] [string[]] $CompositeKeyColumn = @(), - [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd') + [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd'), + [Parameter()] [scriptblock] $ProgressCallback, + [Parameter()] [double] $ProgressIntervalSeconds = 0 ) if (-not (Test-Path -LiteralPath $CurrentFactCsv -PathType Leaf)) { throw "Merge-FactCsv: current Fact CSV not found: '$CurrentFactCsv'" } if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentFactCsv } - # Grain-composite dedup key. When -CompositeKeyColumn is - # supplied, a row's key is the U+001F-joined values of those columns (ASCII Unit - # Separator 0x1F is a control char that never appears in audit field data), so - # fan-out fact rows (many per Message_Id_Raw) are each keyed distinctly instead of - # collapsing to one. Otherwise the single -KeyColumn is used (RecordId raw-audit - # path, unchanged). $getRowKey is invoked for every target/current row below. + # Grain-composite dedup key. When -CompositeKeyColumn is supplied a row's key is + # the U+001F-joined values of those columns (ASCII Unit Separator 0x1F is a + # control char that never appears in audit field data), so fan-out fact rows + # (many per Message_Id_Raw) are each keyed distinctly instead of collapsing to + # one. Otherwise the single -KeyColumn is used (RecordId raw-audit path). The key + # is computed for every row while it is streamed into a normalized work file. $useCompositeKey = ($null -ne $CompositeKeyColumn -and @($CompositeKeyColumn).Count -gt 0) - $compositeSep = [char]0x1F - $getRowKey = { - param($row) - if ($useCompositeKey) { - $parts = foreach ($c in $CompositeKeyColumn) { - $pp = $row.PSObject.Properties[$c] - if ($pp) { [string]$pp.Value } else { '' } - } - return ($parts -join $compositeSep) - } - $kp = $row.PSObject.Properties[$KeyColumn] - if ($kp) { [string]$kp.Value } else { '' } - } - # Use script:Import-CsvDeduped for header-dupe safety (parallel to Merge-UsersCsv). - $targetRows = @() - if (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf) { - $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetFactCsv) - } - # APPEND SAFETY (data-loss guard): if the target file exists with content but parsed to - # zero rows, the read failed (parse/encoding/memory) or the key cannot be matched. - # Overwriting would replace the existing target file with current-run rows only (a - # row-count shrink). Abort instead so the existing target is left untouched; the caller keeps the - # fresh current-run CSV on disk for a manual merge. - if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetFactCsv).Length -gt 0)) { - throw "Merge-FactCsv: target '$TargetFactCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data). Verify the file's schema/key column ('$KeyColumn')." - } - # Schema-narrowing warning. The only header we strictly require on the target - # is the dedup key (-KeyColumn) — without it the union cannot correctly - # classify Retained vs New vs Departed. Display / enrichment columns - # (PersonId_Normalized, ThreadId, Message_Id, etc.) are intentionally renamed - # or surrogate-replaced by the rollup processor on the rolled-up Fact CSV, so - # checking for them here would produce false-positive warnings on every - # rollup-shape seed. Header union handles legitimately absent display columns - # gracefully (target rows get blanks for current-only fields; current rows - # get blanks for target-only fields). - if ($targetRows.Count -gt 0) { - $targetHeaders = @($targetRows[0].PSObject.Properties.Name) - $requiredKeyCols = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } - $missingKeyCols = @($requiredKeyCols | Where-Object { $_ -notin $targetHeaders }) - if ($missingKeyCols.Count -gt 0) { - Microsoft.PowerShell.Utility\Write-Host ( - ("WARNING: Merge-FactCsv: target Fact CSV is missing dedup key column(s) '{0}'. " + - "Cannot classify Retained / New / Departed rows reliably; treating ALL current-run rows as New. " + - "Target: {1}") -f ($missingKeyCols -join ', '), $TargetFactCsv - ) -ForegroundColor Yellow - } + $keyColumns = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } + + # All materialization is disk-backed and bounded: source rows are streamed into + # normalized work files, externally sorted by (key, read-order) under a fixed + # memory budget, joined in a single linear pass that holds at most one record per + # side, and streamed back out. Nothing accumulates a whole file in memory. + $memoryBudgetBytes = [int64]67108864 + $fanIn = 8 + $workDir = Join-Path $env:TEMP ('pax_factmerge_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $mergingPath = "$OutputPath.merging" + $cleanupMetrics = script:New-PaxSortMetrics + + # Coarse stage observability. Merge-FactCsv narrates its bounded pipeline as a + # fixed vocabulary of stage markers (Inspecting inputs, Normalizing existing + # target, Sorting existing target, Normalizing current run, Sorting current run, + # Reconciling rows, Restoring current order, Restoring departed order, Publishing, + # Cleanup, Complete, Failed). Each marker is surfaced through the run's normal + # host/log channel and recorded as a memory observation so a long field-scale + # reconciliation shows forward progress and per-stage memory. The logging helpers + # live in the full processor; when Merge-FactCsv is dot-sourced in isolation (unit + # extraction) they are absent, so emission resolves them once and stays fully + # inert if unavailable. + $logHostCmd = Microsoft.PowerShell.Core\Get-Command -Name 'Write-LogHost' -CommandType Function -ErrorAction SilentlyContinue + $memObsCmd = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxMemoryObservation' -CommandType Function -ErrorAction SilentlyContinue + $emitStage = { + param([string]$StageName, [string]$Detail = '') + $msg = if ($Detail) { "Fact append merge - $StageName ($Detail)" } else { "Fact append merge - $StageName" } + if ($logHostCmd) { try { & $logHostCmd -Message $msg -ForegroundColor 'Gray' } catch { } } + if ($memObsCmd) { try { & $memObsCmd -Stage ("FactMerge:" + $StageName) -TempDir $workDir -Force } catch { } } } - $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentFactCsv) + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + & $emitStage 'Inspecting inputs' + + # 1. Normalize the target: stream every physical data row into tgt_norm with + # its key and read-order sequence. targetRowCount counts ALL data rows + # (including no-key and duplicate-key rows) for the shrink guard. + $hasTarget = Test-Path -LiteralPath $TargetFactCsv -PathType Leaf + $targetLen = if ($hasTarget) { [int64](Get-Item -LiteralPath $TargetFactCsv).Length } else { [int64]0 } + $tgtNorm = Join-Path $workDir 'tgt_norm.csv' + $dedupedTargetHeader = $null + $targetRowCount = [int64]0 + if ($hasTarget) { + & $emitStage 'Normalizing existing target' + $tgtInfo = script:ConvertTo-PaxFactNormalized -InputPath $TargetFactCsv -OutputPath $tgtNorm -KeyColumns $keyColumns + $dedupedTargetHeader = $tgtInfo.Header + $targetRowCount = [int64]$tgtInfo.RowCount + } + + # APPEND SAFETY (data-loss guard): if the target file exists with content but parsed to + # zero rows, the read failed (parse/encoding/memory) or the key cannot be matched. + # Overwriting would replace the existing target file with current-run rows only (a + # row-count shrink). Abort instead so the existing target is left untouched; the caller keeps the + # fresh current-run CSV on disk for a manual merge. + if ($targetRowCount -eq 0 -and $hasTarget -and $targetLen -gt 0) { + throw "Merge-FactCsv: target '$TargetFactCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data). Verify the file's schema/key column ('$KeyColumn')." + } + + # Schema-narrowing warning. The only header we strictly require on the target + # is the dedup key (-KeyColumn) — without it the union cannot correctly + # classify Retained vs New vs Departed. Display / enrichment columns + # (PersonId_Normalized, ThreadId, Message_Id, etc.) are intentionally renamed + # or surrogate-replaced by the rollup processor on the rolled-up Fact CSV, so + # checking for them here would produce false-positive warnings on every + # rollup-shape seed. Header union handles legitimately absent display columns + # gracefully (target rows get blanks for current-only fields; current rows + # get blanks for target-only fields). $targetKeyMissing gates the later + # zero-overlap caution off this data-quality case (retained=0 there means the + # key could not be classified, not that the two nonempty sets were disjoint). + $targetKeyMissing = $false + if ($targetRowCount -gt 0) { + $requiredKeyCols = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } + $missingKeyCols = @($requiredKeyCols | Where-Object { $_ -notin $dedupedTargetHeader }) + if ($missingKeyCols.Count -gt 0) { + $targetKeyMissing = $true + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-FactCsv: target Fact CSV is missing dedup key column(s) '{0}'. " + + "Cannot classify Retained / New / Departed rows reliably; treating ALL current-run rows as New. " + + "Target: {1}") -f ($missingKeyCols -join ', '), $TargetFactCsv + ) -ForegroundColor Yellow + } + } + + # 2. Sort the normalized target by (key, read-order). Duplicate keys stay + # grouped with the first original occurrence first; no-key rows sort ahead + # of keyed rows and are skipped by the join. + $tgtSorted = Join-Path $workDir 'tgt_sorted.csv' + if ($targetRowCount -gt 0) { + & $emitStage 'Sorting existing target' + $null = script:Invoke-PaxExternalSort -InputPath $tgtNorm -OutputPath $tgtSorted -KeyColumn @('__paxk', '__paxseq') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + + # 3. Normalize the current run the same way. + $curNorm = Join-Path $workDir 'cur_norm.csv' + & $emitStage 'Normalizing current run' + $curInfo = script:ConvertTo-PaxFactNormalized -InputPath $CurrentFactCsv -OutputPath $curNorm -KeyColumns $keyColumns + $dedupedCurrentHeader = $curInfo.Header + $currentRowCount = [int64]$curInfo.RowCount + + # 4. Sort the normalized current run by (key, read-order). + $curSorted = Join-Path $workDir 'cur_sorted.csv' + if ($currentRowCount -gt 0) { + & $emitStage 'Sorting current run' + $null = script:Invoke-PaxExternalSort -InputPath $curNorm -OutputPath $curSorted -KeyColumn @('__paxk', '__paxseq') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + + # 5. Header union: preserve current-run order, then append target-only columns, + # then provenance — deduplicated case-insensitively. + $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $hdrOrder = New-Object System.Collections.Generic.List[string] + if ($currentRowCount -gt 0 -and $dedupedCurrentHeader) { + foreach ($n in $dedupedCurrentHeader) { if ($hdrSet.Add($n)) { $hdrOrder.Add($n) } } + } + if ($targetRowCount -gt 0 -and $dedupedTargetHeader) { + foreach ($n in $dedupedTargetHeader) { if ($hdrSet.Add($n)) { $hdrOrder.Add($n) } } + } + foreach ($c in @('Date_Added', 'Latest_Append_Date', 'In_Latest_Append')) { + if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } + } + $hdrArr = [string[]]$hdrOrder.ToArray() + $hdrIndex = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + for ($i = 0; $i -lt $hdrArr.Length; $i++) { $hdrIndex[$hdrArr[$i]] = $i } + $idxDateAdded = $hdrIndex['Date_Added'] + $idxLatest = $hdrIndex['Latest_Append_Date'] + $idxInLatest = $hdrIndex['In_Latest_Append'] + $hasMessageId = $hdrIndex.ContainsKey('Message_Id') + $idxMessageId = if ($hasMessageId) { $hdrIndex['Message_Id'] } else { -1 } + + # Map each normalized data column to its union position (case-insensitive), and + # locate the target's own Message_Id / Date_Added columns for continuity. + $curColToHdr = [int[]]::new(0) + if ($dedupedCurrentHeader) { + $curColToHdr = [int[]]::new($dedupedCurrentHeader.Length) + for ($j = 0; $j -lt $dedupedCurrentHeader.Length; $j++) { $curColToHdr[$j] = $hdrIndex[$dedupedCurrentHeader[$j]] } + } + $tgtColToHdr = [int[]]::new(0) + $tgtMidIdx = -1 + $tgtDaIdx = -1 + if ($dedupedTargetHeader) { + $tgtColToHdr = [int[]]::new($dedupedTargetHeader.Length) + for ($j = 0; $j -lt $dedupedTargetHeader.Length; $j++) { + $tgtColToHdr[$j] = $hdrIndex[$dedupedTargetHeader[$j]] + if ([System.String]::Equals($dedupedTargetHeader[$j], 'Message_Id', [System.StringComparison]::OrdinalIgnoreCase)) { $tgtMidIdx = $j } + if ([System.String]::Equals($dedupedTargetHeader[$j], 'Date_Added', [System.StringComparison]::OrdinalIgnoreCase)) { $tgtDaIdx = $j } + } + } + + # 6. Message_Id continuity applies when the dedup key is (or contains) + # Message_Id_Raw; then the target's Message_Id INT is carried forward on + # retained rows for cross-run continuity. For other single key columns + # (e.g. RecordId on the raw audit CSV) the current row's values stand. + $messageIdContinuity = ($KeyColumn -eq 'Message_Id_Raw') -or ($useCompositeKey -and (@($CompositeKeyColumn) -contains 'Message_Id_Raw')) + + # Union-row builders. Section-1 rows overlay the current row's own columns + # then apply continuity/provenance; departed rows overlay the target row's + # columns then apply provenance. Both return a full union-width field array. + $buildSection1 = { + param([string[]]$CurData, [bool]$Matched, [object]$HeldFields) + $u = [string[]]::new($hdrArr.Length) + for ($i = 0; $i -lt $u.Length; $i++) { $u[$i] = '' } + for ($j = 0; $j -lt $curColToHdr.Length; $j++) { if ($j -lt $CurData.Length) { $u[$curColToHdr[$j]] = $CurData[$j] } } + if ($Matched) { + if ($messageIdContinuity -and $hasMessageId) { + $tv = if ($tgtMidIdx -ge 0 -and (2 + $tgtMidIdx) -lt $HeldFields.Length) { [string]$HeldFields[2 + $tgtMidIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($tv)) { $u[$idxMessageId] = $tv } + } + $tda = if ($tgtDaIdx -ge 0 -and (2 + $tgtDaIdx) -lt $HeldFields.Length) { [string]$HeldFields[2 + $tgtDaIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($tda)) { $u[$idxDateAdded] = $tda } else { $u[$idxDateAdded] = $RunDate } + } + else { + $u[$idxDateAdded] = $RunDate + } + $u[$idxLatest] = $RunDate + $u[$idxInLatest] = 'TRUE' + return , $u + } + $buildDeparted = { + param([object]$HeldFields) + $u = [string[]]::new($hdrArr.Length) + for ($i = 0; $i -lt $u.Length; $i++) { $u[$i] = '' } + for ($j = 0; $j -lt $tgtColToHdr.Length; $j++) { $s = 2 + $j; if ($s -lt $HeldFields.Length) { $u[$tgtColToHdr[$j]] = [string]$HeldFields[$s] } } + if ([string]::IsNullOrWhiteSpace($u[$idxDateAdded])) { $u[$idxDateAdded] = $RunDate } + $u[$idxLatest] = $RunDate + $u[$idxInLatest] = 'FALSE' + return , $u + } + + # 7. Bounded left-join pass. Walk the sorted current rows, emitting every one + # to the section-1 side file tagged with its read-order sequence. Advance + # the sorted target one distinct key at a time (first occurrence only, + # no-key rows skipped); a target key never matched by a current key becomes + # one departed row. Stats count DISTINCT non-empty current keys. + $sec1Side = Join-Path $workDir 'sec1_side.csv' + $depSide = Join-Path $workDir 'dep_side.csv' + $sideHdr = @('__paxseq') + $hdrArr + $retainedCount = 0 + $newCount = 0 + $departedCount = 0 + & $emitStage 'Reconciling rows' + + $sec1Writer = [System.IO.StreamWriter]::new($sec1Side, $false, (New-Object System.Text.UTF8Encoding($false))) + $depWriter = [System.IO.StreamWriter]::new($depSide, $false, (New-Object System.Text.UTF8Encoding($false))) + $curReader = $null + $tgtState = $null + try { + script:Write-PaxCsvRecord -Writer $sec1Writer -Fields $sideHdr + script:Write-PaxCsvRecord -Writer $depWriter -Fields $sideHdr + + if ($targetRowCount -gt 0) { + $tgtReader = [System.IO.StreamReader]::new($tgtSorted, $true) + $null = script:Read-PaxCsvRecord -Reader $tgtReader + $tgtState = [pscustomobject]@{ Reader = $tgtReader; Pending = $null; HasPending = $false } + } + $held = if ($tgtState) { script:Read-PaxFactNextTargetGroup -State $tgtState } else { $null } + $heldMatched = $false + + $emitDeparted = { + param($Group) + $du = & $buildDeparted $Group.Fields + script:Write-PaxCsvRecord -Writer $depWriter -Fields (@([string]$Group.Fields[1]) + $du) + } - $targetByKey = @{} - foreach ($r in $targetRows) { - $k = [string](& $getRowKey $r) - if (-not [string]::IsNullOrWhiteSpace($k) -and -not $targetByKey.ContainsKey($k)) { - $targetByKey[$k] = $r + if ($currentRowCount -gt 0) { + $curReader = [System.IO.StreamReader]::new($curSorted, $true) + $null = script:Read-PaxCsvRecord -Reader $curReader + $havePrev = $false + $prevKey = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $curReader + if ($null -eq $row) { break } + $kc = [string]$row[0] + $curPaxseq = [string]$row[1] + $curData = if ($row.Length -gt 2) { $row[2..($row.Length - 1)] } else { @() } + if ([string]::IsNullOrWhiteSpace($kc)) { + $su = & $buildSection1 $curData $false $null + script:Write-PaxCsvRecord -Writer $sec1Writer -Fields (@($curPaxseq) + $su) + continue + } + while ($null -ne $held -and [System.StringComparer]::OrdinalIgnoreCase.Compare([string]$held.Key, $kc) -lt 0) { + if (-not $heldMatched) { & $emitDeparted $held; $departedCount++ } + $held = script:Read-PaxFactNextTargetGroup -State $tgtState + $heldMatched = $false + } + $matched = ($null -ne $held -and [System.String]::Equals([string]$held.Key, $kc, [System.StringComparison]::OrdinalIgnoreCase)) + if ($matched) { $heldMatched = $true } + $isNewDistinct = (-not $havePrev) -or (-not [System.String]::Equals($kc, $prevKey, [System.StringComparison]::OrdinalIgnoreCase)) + if ($isNewDistinct) { if ($matched) { $retainedCount++ } else { $newCount++ } } + $havePrev = $true + $prevKey = $kc + $heldFieldsForBuild = if ($matched) { $held.Fields } else { $null } + $su = & $buildSection1 $curData $matched $heldFieldsForBuild + script:Write-PaxCsvRecord -Writer $sec1Writer -Fields (@($curPaxseq) + $su) + } + } + # Flush the remaining unmatched target groups as departed rows. + while ($null -ne $held) { + if (-not $heldMatched) { & $emitDeparted $held; $departedCount++ } + $held = if ($tgtState) { script:Read-PaxFactNextTargetGroup -State $tgtState } else { $null } + $heldMatched = $false + } + + $sec1Writer.Flush() + $depWriter.Flush() + } + finally { + if ($curReader) { $curReader.Dispose() } + if ($tgtState -and $tgtState.Reader) { $tgtState.Reader.Dispose() } + $sec1Writer.Dispose() + $depWriter.Dispose() + } + + # 8. Restore original ordering: section-1 rows to current read order, departed + # rows to target first-occurrence order. + $sec1Sorted = Join-Path $workDir 'sec1_sorted.csv' + if ($currentRowCount -gt 0) { + & $emitStage 'Restoring current order' + $null = script:Invoke-PaxExternalSort -InputPath $sec1Side -OutputPath $sec1Sorted -KeyColumn @('__paxseq') -Comparer 'Ordinal' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + $depSorted = Join-Path $workDir 'dep_sorted.csv' + if ($departedCount -gt 0) { + & $emitStage 'Restoring departed order' + $null = script:Invoke-PaxExternalSort -InputPath $depSide -OutputPath $depSorted -KeyColumn @('__paxseq') -Comparer 'Ordinal' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + + # 9. APPEND SAFETY (shrink guard): a union must never be smaller than the existing target. + # If the target keyed but the dedup column did not match (Retained+Departed=0 while the + # target had rows), the union collapses to current-only rows and the file shrinks. Abort + # before any write so the target is preserved. + $unionCount = [int64]$currentRowCount + [int64]$departedCount + if ($targetRowCount -gt 0 -and $unionCount -lt $targetRowCount) { + throw ("Merge-FactCsv: refusing to write a smaller file than the target (union={0} < target={1}); key '{2}' likely did not match. Target left unchanged." -f $unionCount, $targetRowCount, $KeyColumn) + } + + # 10. Publish: write the header, then the ordered section-1 rows, then the + # ordered departed rows, each with the sequence column dropped. Write to a + # sibling .merging file and rename onto the output path so the target is + # byte-unchanged unless every step succeeds. + $pubWriter = [System.IO.StreamWriter]::new($mergingPath, $false, (New-Object System.Text.UTF8Encoding($false))) + & $emitStage 'Publishing' + try { + script:Write-PaxCsvRecord -Writer $pubWriter -Fields $hdrArr + if ($currentRowCount -gt 0) { + $sr = [System.IO.StreamReader]::new($sec1Sorted, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $sr + while ($true) { $rec = script:Read-PaxCsvRecord -Reader $sr; if ($null -eq $rec) { break }; script:Write-PaxCsvRecord -Writer $pubWriter -Fields ($rec[1..($rec.Length - 1)]) } + } + finally { $sr.Dispose() } + } + if ($departedCount -gt 0) { + $dr = [System.IO.StreamReader]::new($depSorted, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $dr + while ($true) { $rec = script:Read-PaxCsvRecord -Reader $dr; if ($null -eq $rec) { break }; script:Write-PaxCsvRecord -Writer $pubWriter -Fields ($rec[1..($rec.Length - 1)]) } + } + finally { $dr.Dispose() } + } + $pubWriter.Flush() + } + finally { $pubWriter.Dispose() } + # Fail closed if the rename cannot complete (e.g. the target is locked by a + # sync client or scanner): a non-terminating move would return success while + # the target still holds its old content and the merged output is discarded. + Move-Item -LiteralPath $mergingPath -Destination $OutputPath -Force -ErrorAction Stop + + # 11. Stats. + & $emitStage 'Cleanup' + & $emitStage 'Complete' + # Zero-overlap caution (count-only). A nonempty target and a nonempty current run + # that share NO keys is a VALID outcome for disjoint windows: every current row is + # New, every unmatched target row is Departed, and the lossless union is preserved. + # Surface it (aggregate counts only — never keys, identities, filenames, or paths) + # so a disjoint-window append is not mistaken for an error. Never thrown, never + # changes counts or provenance. Suppressed when the key column was missing from the + # target ($targetKeyMissing) — that data-quality case owns its own WARNING above and + # is not a genuine disjoint-set overlap. Emitted at most once, via the same resolved + # host helper as the stage markers (inert when Merge-FactCsv is dot-sourced alone). + if ($targetRowCount -gt 0 -and $currentRowCount -gt 0 -and $retainedCount -eq 0 -and -not $targetKeyMissing -and $logHostCmd) { + try { + & $logHostCmd -Message ( + ("Fact append merge found no overlapping keys between the existing target and this run. " + + "All current rows are classified as New and unmatched target rows as Departed. " + + "The lossless union was preserved. " + + "Target rows={0:N0} Current rows={1:N0} Retained={2:N0} New={3:N0} Departed={4:N0} Union={5:N0}") -f ` + [int64]$targetRowCount, [int64]$currentRowCount, [int64]$retainedCount, [int64]$newCount, [int64]$departedCount, [int64]$unionCount + ) -ForegroundColor 'Yellow' + } catch { } } - } - $currentByKey = @{} - foreach ($r in $currentRows) { - $k = [string](& $getRowKey $r) - if (-not [string]::IsNullOrWhiteSpace($k) -and -not $currentByKey.ContainsKey($k)) { - $currentByKey[$k] = $r + # Aggregate-only completion record so the fact merge appears in the unified rollup + # stage stream alongside the earlier preparation stages. Routed through a host + # lookup so the function stays inert when it is extracted and exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + if ($__paxStage) { + try { + & $__paxStage -Stage 'fact-merge' -Status 'complete' -Counters @{ + targetRows = [int64]$targetRowCount + currentRows = [int64]$currentRowCount + retained = [int64]$retainedCount + newRows = [int64]$newCount + departed = [int64]$departedCount + unionRows = [int64]$unionCount + } + } catch { } + } + return @{ + Retained = $retainedCount + New = $newCount + Departed = $departedCount + Union = $unionCount } } + catch { + & $emitStage 'Failed' $_.Exception.Message + throw + } + finally { + script:Remove-PaxWorkspace -WorkDir $workDir -MergingPath $mergingPath -Metrics $cleanupMetrics + } +} + +function script:Invoke-PaxKeyedAdditiveMerge { + <# + .SYNOPSIS + Bounded, disk-backed union-merge of a target and current CSV keyed by a + composite key with additive / min / max / OR aggregation. Shared by the two + M365Bundle append reducers. - # Header union: preserve current-run order, then append target-only columns, then provenance. - $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) - $hdrOrder = New-Object System.Collections.Generic.List[string] - if ($currentRows.Count -gt 0) { - foreach ($p in $currentRows[0].PSObject.Properties) { - if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + .DESCRIPTION + Streams both inputs through the frozen RFC-4180 reader, projects each row onto + a fixed canonical header (missing columns become '' or a supplied default), + spills a bounded work record (composite key, global sequence, side, projected + columns) per row, and folds one key group at a time via the external sort. The + first row of each key group (target rows precede current rows, each in original + order) seeds the group's values; later rows accumulate: summed columns add, + min/max columns take the lexicographic min/max of non-blank values, OR columns + become TRUE if any value is TRUE. The group's UserId-style casing therefore + comes from the first-seen (target-first) row. Output rows are restored to + first-appearance order with a bounded sequence sort and streamed to + '.merging' via Export-Csv, then published with a terminating move. The + complete inputs are never materialized and a hot key streams without being + collected. Any failure raises a terminating error, leaves a pre-existing target + byte-for-byte unchanged, publishes nothing partial, and cleans temporary work. + + .OUTPUTS + PSCustomObject with stats (Retained / New / Updated / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$TargetCsv, + [Parameter(Mandatory)][string]$CurrentCsv, + [Parameter(Mandatory)][string]$OutputPath, + [Parameter(Mandatory)][string[]]$Header, + [Parameter(Mandatory)][string[]]$KeyColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$FoldColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$SumColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$MinColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$MaxColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$OrColumns, + [Parameter(Mandatory)][hashtable]$Defaults, + [Parameter(Mandatory)][int64]$MemoryBudgetBytes, + [Parameter(Mandatory)][string]$Label, + [scriptblock]$OnMissingColumns + ) + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $hdrCount = $Header.Length + # Header name -> canonical index. + $hdrIndex = @{} + for ($i = 0; $i -lt $hdrCount; $i++) { $hdrIndex[$Header[$i]] = $i } + $keyIdx = @($KeyColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $foldSet = [System.Collections.Generic.HashSet[int]]::new() + foreach ($c in $FoldColumns) { [void]$foldSet.Add([int]$hdrIndex[$c]) } + $sumIdx = @($SumColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $minIdx = @($MinColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $maxIdx = @($MaxColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $orIdx = @($OrColumns | ForEach-Object { [int]$hdrIndex[$_] }) + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_kam_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $tmpPath = "$OutputPath.merging" + $u8 = [System.Text.UTF8Encoding]::new($false) + + # Resolve a source header record's index map onto the canonical header. + $mapSourceIndices = { + param($srcHeader) + $map = New-Object 'int[]' $hdrCount + for ($j = 0; $j -lt $hdrCount; $j++) { + $map[$j] = -1 + for ($s = 0; $s -lt $srcHeader.Length; $s++) { + if ([string]::Equals([string]$srcHeader[$s], $Header[$j], [System.StringComparison]::OrdinalIgnoreCase)) { $map[$j] = $s; break } + } } + return , $map } - if ($targetRows.Count -gt 0) { - foreach ($p in $targetRows[0].PSObject.Properties) { - if ($hdrSet.Add($p.Name)) { $hdrOrder.Add($p.Name) } + # Project a source row (fields + index map) onto canonical values with defaults. + $projectFields = { + param($row, $map) + $vals = New-Object 'string[]' $hdrCount + for ($j = 0; $j -lt $hdrCount; $j++) { + $idx = $map[$j] + $v = if ($idx -ge 0 -and $idx -lt $row.Length) { [string]$row[$idx] } else { '' } + if ([string]::IsNullOrWhiteSpace($v) -and $Defaults.ContainsKey($Header[$j])) { $v = [string]$Defaults[$Header[$j]] } + $vals[$j] = $v } + return , $vals } - foreach ($c in @('Date_Added','Latest_Append_Date','In_Latest_Append')) { - if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } + # Composite key of projected values (folded columns lowercased), '|'-joined. + $makeKey = { + param($vals) + $sb = [System.Text.StringBuilder]::new() + for ($i = 0; $i -lt $keyIdx.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append('|') } + $kv = [string]$vals[$keyIdx[$i]] + if ($foldSet.Contains($keyIdx[$i])) { $kv = $kv.ToLowerInvariant() } + [void]$sb.Append($kv) + } + return $sb.ToString() } - $merged = New-Object System.Collections.Generic.List[psobject] + $retained = 0; $newCount = 0; $updated = 0; $union = 0 + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $workPath = Join-Path $workDir 'work.csv' + $workHdr = @('__K', '__SEQ', '__SIDE') + $Header + $ww = [System.IO.StreamWriter]::new($workPath, $false, $u8) + $seq = [int64]0 + $targetDataRows = [int64]0 + $targetExists = Test-Path -LiteralPath $TargetCsv -PathType Leaf + try { + script:Write-PaxCsvRecord -Writer $ww -Fields $workHdr + # Target side (side '0'); may be absent. + if ($targetExists) { + $tr = [System.IO.StreamReader]::new($TargetCsv, $true) + try { + $th = script:Read-PaxCsvRecord -Reader $tr + if ($null -ne $th) { + $tmap = & $mapSourceIndices $th + if ($OnMissingColumns) { + $missing = @($Header | Where-Object { $hdrIndex.ContainsKey($_) -and $tmap[$hdrIndex[$_]] -lt 0 }) + if ($missing.Count -gt 0) { & $OnMissingColumns $missing } + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $tr + if ($null -eq $row) { break } + $targetDataRows++ + $vals = & $projectFields $row $tmap + $k = & $makeKey $vals + $fields = New-Object System.Collections.Generic.List[string] + $fields.Add($k); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('0') + foreach ($v in $vals) { $fields.Add($v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + $seq++ + } + } + } + finally { $tr.Dispose() } + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetDataRows -eq 0 -and $targetExists -and ((Get-Item -LiteralPath $TargetCsv).Length -gt 0)) { + throw "${Label}: target '$TargetCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + # Current side (side '1'). + $cr = [System.IO.StreamReader]::new($CurrentCsv, $true) + try { + $ch = script:Read-PaxCsvRecord -Reader $cr + if ($null -ne $ch) { + $cmap = & $mapSourceIndices $ch + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr + if ($null -eq $row) { break } + $vals = & $projectFields $row $cmap + $k = & $makeKey $vals + $fields = New-Object System.Collections.Generic.List[string] + $fields.Add($k); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('1') + foreach ($v in $vals) { $fields.Add($v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + $seq++ + } + } + } + finally { $cr.Dispose() } + } + finally { $ww.Flush(); $ww.Dispose() } - # 1. Current-run rows (retained + new). - foreach ($r in $currentRows) { - $k = [string](& $getRowKey $r) - $obj = [ordered]@{} - foreach ($c in $hdrOrder) { $obj[$c] = '' } - foreach ($p in $r.PSObject.Properties) { $obj[$p.Name] = $p.Value } + # Sort work by (key, sequence): equal keys adjacent, target rows before current. + $sortedPath = Join-Path $workDir 'sorted.csv' + [void](script:Invoke-PaxExternalSort -InputPath $workPath -OutputPath $sortedPath -KeyColumn @('__K', '__SEQ') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) - if ($k -and $targetByKey.ContainsKey($k)) { - $tr = $targetByKey[$k] - # When dedup-keyed on Message_Id_Raw (single or as part of the grain- - # composite key), target's Message_Id INT wins (continuity across runs; - # embedded Python seed-mid-map normally aligns this, but enforce here so a - # seed-prep failure still produces a continuous union). For other single key - # columns (e.g. RecordId on the raw audit CSV) there is no surrogate-INT - # continuity contract. - if ($KeyColumn -eq 'Message_Id_Raw' -or ($useCompositeKey -and (@($CompositeKeyColumn) -contains 'Message_Id_Raw'))) { - $trMid = $tr.PSObject.Properties['Message_Id'] - if ($trMid -and -not [string]::IsNullOrWhiteSpace([string]$trMid.Value)) { - $obj['Message_Id'] = $trMid.Value + # Reduce one key group at a time; emit [__FSEQ] + accumulated canonical cols. + $resultPath = Join-Path $workDir 'result.csv' + $sr = [System.IO.StreamReader]::new($sortedPath, $true) + $rw = [System.IO.StreamWriter]::new($resultPath, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $sr + script:Write-PaxCsvRecord -Writer $rw -Fields (@('__FSEQ') + $Header) + $haveGroup = $false + $curKey = $null; $acc = $null; $firstSeq = $null; $tCnt = 0; $cCnt = 0 + $flush = { + if (-not $haveGroup) { return } + $out = New-Object System.Collections.Generic.List[string] + $out.Add($firstSeq) + foreach ($v in $acc) { $out.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $rw -Fields $out.ToArray() + $script:__union++ + if ($tCnt -gt 0) { $script:__ret += $cCnt; $script:__upd += $cCnt } + else { if ($cCnt -ge 1) { $script:__new++ }; if ($cCnt -gt 1) { $script:__upd += ($cCnt - 1) } } + } + $script:__union = 0; $script:__ret = 0; $script:__new = 0; $script:__upd = 0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $sr + if ($null -eq $row) { break } + $k = [string]$row[0] + $rowSeq = [string]$row[1] + $side = [string]$row[2] + # Canonical values start at work index 3. + if ((-not $haveGroup) -or ($k -cne $curKey)) { + & $flush + $curKey = $k; $firstSeq = $rowSeq; $tCnt = 0; $cCnt = 0 + $acc = New-Object 'string[]' $hdrCount + for ($j = 0; $j -lt $hdrCount; $j++) { $acc[$j] = [string]$row[3 + $j] } + $haveGroup = $true } + else { + foreach ($sc in $sumIdx) { + $a = 0; [void][int]::TryParse([string]$acc[$sc], [ref]$a) + $b = 0; [void][int]::TryParse([string]$row[3 + $sc], [ref]$b) + $acc[$sc] = ($a + $b).ToString() + } + foreach ($mc in $minIdx) { + $rv = [string]$row[3 + $mc] + if (-not [string]::IsNullOrWhiteSpace($rv)) { + $av = [string]$acc[$mc] + if ([string]::IsNullOrWhiteSpace($av) -or ($rv -lt $av)) { $acc[$mc] = $rv } + } + } + foreach ($xc in $maxIdx) { + $rv = [string]$row[3 + $xc] + if (-not [string]::IsNullOrWhiteSpace($rv)) { + $av = [string]$acc[$xc] + if ([string]::IsNullOrWhiteSpace($av) -or ($rv -gt $av)) { $acc[$xc] = $rv } + } + } + foreach ($oc in $orIdx) { + if (([string]$row[3 + $oc]).Equals('TRUE', [System.StringComparison]::OrdinalIgnoreCase)) { $acc[$oc] = 'TRUE' } + } + } + if ($side -eq '0') { $tCnt++ } else { $cCnt++ } } - $tda = $tr.PSObject.Properties['Date_Added'] - if ($tda -and -not [string]::IsNullOrWhiteSpace([string]$tda.Value)) { - $obj['Date_Added'] = $tda.Value - } else { - $obj['Date_Added'] = $RunDate + & $flush + $union = [int]$script:__union; $retained = [int]$script:__ret; $newCount = [int]$script:__new; $updated = [int]$script:__upd + Remove-Variable -Name __union, __ret, __new, __upd -Scope Script -ErrorAction SilentlyContinue + } + finally { $rw.Flush(); $rw.Dispose(); $sr.Dispose() } + + # Restore first-appearance order. + $orderedPath = Join-Path $workDir 'ordered.csv' + [void](script:Invoke-PaxExternalSort -InputPath $resultPath -OutputPath $orderedPath -KeyColumn @('__FSEQ') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + + # Stream ordered result rows as objects to Export-Csv (exact prior format). + $hdrLocal = $Header + $objStream = { + $orr = [System.IO.StreamReader]::new($orderedPath, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $orr + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $orr + if ($null -eq $row) { break } + $o = [ordered]@{} + for ($j = 0; $j -lt $hdrLocal.Length; $j++) { $o[$hdrLocal[$j]] = [string]$row[$j + 1] } + [pscustomobject]$o + } } - } else { - $obj['Date_Added'] = $RunDate - } - $obj['Latest_Append_Date'] = $RunDate - $obj['In_Latest_Append'] = 'TRUE' - $merged.Add([pscustomobject]$obj) - } - - # 2. Departed rows (in target, not in current). - $departedCount = 0 - foreach ($k in $targetByKey.Keys) { - if ($currentByKey.ContainsKey($k)) { continue } - $tr = $targetByKey[$k] - $obj = [ordered]@{} - foreach ($c in $hdrOrder) { $obj[$c] = '' } - foreach ($p in $tr.PSObject.Properties) { $obj[$p.Name] = $p.Value } - $tda = $tr.PSObject.Properties['Date_Added'] - if (-not $tda -or [string]::IsNullOrWhiteSpace([string]$tda.Value)) { - $obj['Date_Added'] = $RunDate - } - $obj['Latest_Append_Date'] = $RunDate - $obj['In_Latest_Append'] = 'FALSE' - $merged.Add([pscustomobject]$obj) - $departedCount++ - } - - # 3. Atomic rewrite. - $unionCount = $merged.Count - # APPEND SAFETY (shrink guard): a union must never be smaller than the existing target. - # If the target keyed but the dedup column did not match (Retained+Departed=0 while the - # target had rows), the union collapses to current-only rows and the file shrinks. Abort - # before any write so the target is preserved. - if ($targetRows.Count -gt 0 -and $unionCount -lt $targetRows.Count) { - throw ("Merge-FactCsv: refusing to write a smaller file than the target (union={0} < target={1}); key '{2}' likely did not match. Target left unchanged." -f $unionCount, $targetRows.Count, $KeyColumn) + finally { $orr.Dispose() } + } + & $objStream | Select-Object -Property $Header | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force -ErrorAction Stop } - $tmpPath = "$OutputPath.merging" - $hdrArray = [string[]]$hdrOrder.ToArray() - $merged | Select-Object -Property $hdrArray | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 - Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force - - $retainedCount = 0 - $newCount = 0 - foreach ($k in $currentByKey.Keys) { - if ($targetByKey.ContainsKey($k)) { $retainedCount++ } else { $newCount++ } + catch { + if (Test-Path -LiteralPath $tmpPath) { try { Remove-Item -LiteralPath $tmpPath -Force -ErrorAction SilentlyContinue } catch {} } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } } - return @{ - Retained = $retainedCount + return [pscustomobject]@{ + Retained = $retained New = $newCount - Departed = $departedCount - Union = $unionCount + Updated = $updated + Union = $union } } @@ -11140,6 +18702,11 @@ function Merge-M365RollupCsv { Schema tolerance: target may be a legacy 9- or 10-column rollup. Missing columns are padded with empty strings; IsAgentInteraction defaults to FALSE. + Bounded rewrite: the union is computed by streaming both inputs through + bounded temporary storage (script:Invoke-PaxKeyedAdditiveMerge) instead of + loading both files into memory; output rows, ordering, accumulation, casing, + schema, guard, and returned stats are unchanged. + Atomic rewrite: writes to '.merging' and renames over -OutputPath. -OutputPath defaults to -TargetRollupCsv (in-place append semantics). @@ -11150,7 +18717,8 @@ function Merge-M365RollupCsv { param( [Parameter(Mandatory)] [string] $TargetRollupCsv, [Parameter(Mandatory)] [string] $CurrentRollupCsv, - [Parameter()] [string] $OutputPath + [Parameter()] [string] $OutputPath, + [int64] $MemoryBudgetBytes = 33554432 ) if (-not (Test-Path -LiteralPath $CurrentRollupCsv -PathType Leaf)) { @@ -11160,132 +18728,27 @@ function Merge-M365RollupCsv { # Canonical 14-column rollup header (order is contractual for downstream sidecar regen). $rollupHeader = @( - 'UserId','CreationDate','Operation','Workload','SourceFileExtension', - 'AppHost','EventCount','ItemsAccessedCount','CreationTime','MaxCreationTime', - 'AgentId','AgentName','ContextType','IsAgentInteraction' + 'UserId', 'CreationDate', 'Operation', 'Workload', 'SourceFileExtension', + 'AppHost', 'EventCount', 'ItemsAccessedCount', 'CreationTime', 'MaxCreationTime', + 'AgentId', 'AgentName', 'ContextType', 'IsAgentInteraction' ) - - # Project an incoming row onto the 14 canonical columns. Missing columns - # become '' (except IsAgentInteraction which defaults to 'FALSE' so legacy - # 9/10-col targets contribute correctly to the OR aggregate). - $projectRow = { - param($row) - $obj = [ordered]@{} - foreach ($c in $rollupHeader) { - $p = $row.PSObject.Properties[$c] - $obj[$c] = if ($p) { [string]$p.Value } else { '' } - } - if ([string]::IsNullOrWhiteSpace([string]$obj['IsAgentInteraction'])) { - $obj['IsAgentInteraction'] = 'FALSE' - } - return $obj - } - - # Composite 9-tuple key (case-folded UserId + SourceFileExtension). - $makeKey = { - param($obj) - $userLc = ([string]$obj['UserId']).ToLowerInvariant() - $extLc = ([string]$obj['SourceFileExtension']).ToLowerInvariant() - '{0}|{1}|{2}|{3}|{4}|{5}|{6}|{7}|{8}' -f ` - $userLc, $obj['CreationDate'], $obj['Operation'], $obj['Workload'], ` - $extLc, $obj['AppHost'], $obj['AgentId'], $obj['AgentName'], $obj['ContextType'] - } - - # Accumulate one (already-projected) row into an existing bucket entry. - $accumulateInto = { - param($acc, $obj) - $accEC = 0; [void][int]::TryParse([string]$acc['EventCount'], [ref]$accEC) - $rEC = 0; [void][int]::TryParse([string]$obj['EventCount'], [ref]$rEC) - $accIA = 0; [void][int]::TryParse([string]$acc['ItemsAccessedCount'], [ref]$accIA) - $rIA = 0; [void][int]::TryParse([string]$obj['ItemsAccessedCount'], [ref]$rIA) - $acc['EventCount'] = ($accEC + $rEC).ToString() - $acc['ItemsAccessedCount'] = ($accIA + $rIA).ToString() - $rCT = [string]$obj['CreationTime'] - $rMCT = [string]$obj['MaxCreationTime'] - if (-not [string]::IsNullOrWhiteSpace($rCT)) { - $accCT = [string]$acc['CreationTime'] - if ([string]::IsNullOrWhiteSpace($accCT) -or ($rCT -lt $accCT)) { $acc['CreationTime'] = $rCT } - } - if (-not [string]::IsNullOrWhiteSpace($rMCT)) { - $accMCT = [string]$acc['MaxCreationTime'] - if ([string]::IsNullOrWhiteSpace($accMCT) -or ($rMCT -gt $accMCT)) { $acc['MaxCreationTime'] = $rMCT } - } - if (([string]$obj['IsAgentInteraction']).Equals('TRUE', [System.StringComparison]::OrdinalIgnoreCase)) { - $acc['IsAgentInteraction'] = 'TRUE' - } - } - - # Load inputs. - $targetRows = @() - if (Test-Path -LiteralPath $TargetRollupCsv -PathType Leaf) { - $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetRollupCsv) - } - # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. - if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetRollupCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetRollupCsv).Length -gt 0)) { - throw "Merge-M365RollupCsv: target '$TargetRollupCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." - } - $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentRollupCsv) - - # Schema-narrowing warning for legacy 9/10-col targets. - if ($targetRows.Count -gt 0) { - $targetHeaders = @($targetRows[0].PSObject.Properties.Name) - $missing = @($rollupHeader | Where-Object { $_ -notin $targetHeaders }) - if ($missing.Count -gt 0) { - Microsoft.PowerShell.Utility\Write-Host ( - ("WARNING: Merge-M365RollupCsv: target rollup is missing {0} of the {1} canonical columns " + - "({2}). Padding with empty values; IsAgentInteraction defaults to FALSE. " + - "Target: {3}") -f $missing.Count, $rollupHeader.Count, ($missing -join ','), $TargetRollupCsv - ) -ForegroundColor Yellow - } - } - - # Bucket: composite key -> accumulator ordered hashtable. - # Seed target rows first so the target's UserId casing wins on retained groups. - $bucket = [ordered]@{} - foreach ($r in $targetRows) { - $obj = & $projectRow $r - $k = & $makeKey $obj - if ($bucket.Contains($k)) { - & $accumulateInto $bucket[$k] $obj - } else { - $bucket[$k] = $obj - } - } - - # Snapshot keys present after target-seed (to classify Retained vs New for current rows). - $targetKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) - foreach ($k in $bucket.Keys) { [void]$targetKeys.Add($k) } - - # Merge in current-run rows. - $newCount = 0 - $retainedCount = 0 - $updatedCount = 0 - foreach ($r in $currentRows) { - $obj = & $projectRow $r - $k = & $makeKey $obj - if ($bucket.Contains($k)) { - & $accumulateInto $bucket[$k] $obj - $updatedCount++ - if ($targetKeys.Contains($k)) { $retainedCount++ } - } else { - $bucket[$k] = $obj - $newCount++ - } - } - - # Atomic rewrite: write to .merging, rename over $OutputPath. - $unionCount = $bucket.Count - $tmpPath = "$OutputPath.merging" - [pscustomobject[]]$mergedRows = $bucket.Values | ForEach-Object { [pscustomobject]$_ } - $mergedRows | Select-Object -Property $rollupHeader | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 - Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force - - return [pscustomobject]@{ - Retained = $retainedCount - New = $newCount - Updated = $updatedCount - Union = $unionCount + $onMissing = { + param($missing) + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-M365RollupCsv: target rollup is missing {0} of the {1} canonical columns " + + "({2}). Padding with empty values; IsAgentInteraction defaults to FALSE. " + + "Target: {3}") -f $missing.Count, $rollupHeader.Count, ($missing -join ','), $TargetRollupCsv + ) -ForegroundColor Yellow } + return (script:Invoke-PaxKeyedAdditiveMerge ` + -TargetCsv $TargetRollupCsv -CurrentCsv $CurrentRollupCsv -OutputPath $OutputPath ` + -Header $rollupHeader ` + -KeyColumns @('UserId', 'CreationDate', 'Operation', 'Workload', 'SourceFileExtension', 'AppHost', 'AgentId', 'AgentName', 'ContextType') ` + -FoldColumns @('UserId', 'SourceFileExtension') ` + -SumColumns @('EventCount', 'ItemsAccessedCount') ` + -MinColumns @('CreationTime') -MaxColumns @('MaxCreationTime') -OrColumns @('IsAgentInteraction') ` + -Defaults @{ IsAgentInteraction = 'FALSE' } ` + -MemoryBudgetBytes $MemoryBudgetBytes -Label 'Merge-M365RollupCsv' -OnMissingColumns $onMissing) } function Merge-M365SessionStatsCsv { @@ -11295,7 +18758,7 @@ function Merge-M365SessionStatsCsv { .DESCRIPTION Post-Python helper for -AppendFile in M365Bundle mode. Performs a 3-tuple-keyed - union-merge with additive counter semantics on the SessionStats sidecar (v2.6.0+): + union-merge with additive counter semantics on the SessionStats sidecar: Composite key : (lower(UserId), CreationDate, AppHost) SessionCount : target + current @@ -11306,9 +18769,14 @@ function Merge-M365SessionStatsCsv { UserId : target's casing wins on retained groups (first-seen) Schema tolerance: target may be missing one or more counter columns (e.g. a - pre-2.6.0 placeholder). Missing counters are treated as 0 during accumulation + placeholder). Missing counters are treated as 0 during accumulation and emitted as the union value. + Bounded rewrite: the union is computed by streaming both inputs through + bounded temporary storage (script:Invoke-PaxKeyedAdditiveMerge) instead of + loading both files into memory; output rows, ordering, accumulation, casing, + schema, guard, and returned stats are unchanged. + Atomic rewrite: writes to '.merging' and renames over -OutputPath. -OutputPath defaults to -TargetSessionStatsCsv (in-place append semantics). @@ -11319,7 +18787,8 @@ function Merge-M365SessionStatsCsv { param( [Parameter(Mandatory)] [string] $TargetSessionStatsCsv, [Parameter(Mandatory)] [string] $CurrentSessionStatsCsv, - [Parameter()] [string] $OutputPath + [Parameter()] [string] $OutputPath, + [int64] $MemoryBudgetBytes = 33554432 ) if (-not (Test-Path -LiteralPath $CurrentSessionStatsCsv -PathType Leaf)) { @@ -11329,109 +18798,26 @@ function Merge-M365SessionStatsCsv { # Canonical 8-column SessionStats header (matches SESSIONSTATS_HEADER in the embedded processor). $ssHeader = @( - 'UserId','CreationDate','AppHost', - 'SessionCount','PromptCount','AgentPromptCount','ResponseCount','AgentSessionCount' + 'UserId', 'CreationDate', 'AppHost', + 'SessionCount', 'PromptCount', 'AgentPromptCount', 'ResponseCount', 'AgentSessionCount' ) - - # Project an incoming row onto the 8 canonical columns. Missing columns become ''. - $projectRow = { - param($row) - $obj = [ordered]@{} - foreach ($c in $ssHeader) { - $p = $row.PSObject.Properties[$c] - $obj[$c] = if ($p) { [string]$p.Value } else { '' } - } - return $obj - } - - # Composite 3-tuple key (case-folded UserId). - $makeKey = { - param($obj) - $userLc = ([string]$obj['UserId']).ToLowerInvariant() - '{0}|{1}|{2}' -f $userLc, $obj['CreationDate'], $obj['AppHost'] - } - - # Accumulate one (already-projected) row into an existing bucket entry. - $accumulateInto = { - param($acc, $obj) - foreach ($col in @('SessionCount','PromptCount','AgentPromptCount','ResponseCount','AgentSessionCount')) { - $accV = 0; [void][int]::TryParse([string]$acc[$col], [ref]$accV) - $rV = 0; [void][int]::TryParse([string]$obj[$col], [ref]$rV) - $acc[$col] = ($accV + $rV).ToString() - } - } - - # Load inputs. - $targetRows = @() - if (Test-Path -LiteralPath $TargetSessionStatsCsv -PathType Leaf) { - $targetRows = @(script:Import-CsvDeduped -LiteralPath $TargetSessionStatsCsv) - } - # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. - if ($targetRows.Count -eq 0 -and (Test-Path -LiteralPath $TargetSessionStatsCsv -PathType Leaf) -and ((Get-Item -LiteralPath $TargetSessionStatsCsv).Length -gt 0)) { - throw "Merge-M365SessionStatsCsv: target '$TargetSessionStatsCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." - } - $currentRows = @(script:Import-CsvDeduped -LiteralPath $CurrentSessionStatsCsv) - - # Schema-narrowing warning for legacy targets missing canonical columns. - if ($targetRows.Count -gt 0) { - $targetHeaders = @($targetRows[0].PSObject.Properties.Name) - $missing = @($ssHeader | Where-Object { $_ -notin $targetHeaders }) - if ($missing.Count -gt 0) { - Microsoft.PowerShell.Utility\Write-Host ( - ("WARNING: Merge-M365SessionStatsCsv: target SessionStats is missing {0} of the {1} canonical columns " + - "({2}). Padding with empty values; missing counters treated as 0. Target: {3}") ` + $onMissing = { + param($missing) + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-M365SessionStatsCsv: target SessionStats is missing {0} of the {1} canonical columns " + + "({2}). Padding with empty values; missing counters treated as 0. Target: {3}") ` -f $missing.Count, $ssHeader.Count, ($missing -join ','), $TargetSessionStatsCsv - ) -ForegroundColor Yellow - } - } - - # Bucket: composite key -> accumulator ordered hashtable. - # Seed target rows first so the target's UserId casing wins on retained groups. - $bucket = [ordered]@{} - foreach ($r in $targetRows) { - $obj = & $projectRow $r - $k = & $makeKey $obj - if ($bucket.Contains($k)) { - & $accumulateInto $bucket[$k] $obj - } else { - $bucket[$k] = $obj - } - } - - # Snapshot keys present after target-seed (to classify Retained vs New for current rows). - $targetKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) - foreach ($k in $bucket.Keys) { [void]$targetKeys.Add($k) } - - # Merge in current-run rows. - $newCount = 0 - $retainedCount = 0 - $updatedCount = 0 - foreach ($r in $currentRows) { - $obj = & $projectRow $r - $k = & $makeKey $obj - if ($bucket.Contains($k)) { - & $accumulateInto $bucket[$k] $obj - $updatedCount++ - if ($targetKeys.Contains($k)) { $retainedCount++ } - } else { - $bucket[$k] = $obj - $newCount++ - } - } - - # Atomic rewrite: write to .merging, rename over $OutputPath. - $unionCount = $bucket.Count - $tmpPath = "$OutputPath.merging" - [pscustomobject[]]$mergedRows = $bucket.Values | ForEach-Object { [pscustomobject]$_ } - $mergedRows | Select-Object -Property $ssHeader | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 - Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force - - return [pscustomobject]@{ - Retained = $retainedCount - New = $newCount - Updated = $updatedCount - Union = $unionCount + ) -ForegroundColor Yellow } + return (script:Invoke-PaxKeyedAdditiveMerge ` + -TargetCsv $TargetSessionStatsCsv -CurrentCsv $CurrentSessionStatsCsv -OutputPath $OutputPath ` + -Header $ssHeader ` + -KeyColumns @('UserId', 'CreationDate', 'AppHost') ` + -FoldColumns @('UserId') ` + -SumColumns @('SessionCount', 'PromptCount', 'AgentPromptCount', 'ResponseCount', 'AgentSessionCount') ` + -MinColumns @() -MaxColumns @() -OrColumns @() ` + -Defaults @{} ` + -MemoryBudgetBytes $MemoryBudgetBytes -Label 'Merge-M365SessionStatsCsv' -OnMissingColumns $onMissing) } function Convert-CsvToDelta { @@ -11534,16 +18920,27 @@ def _sanitize_col_names(cols): out.append(s) return out try: - # All-string read avoids per-column inference mismatches between runs; the - # Lakehouse semantic model casts at query time. - table = pacsv.read_csv( + import pyarrow as pa + # Stream the CSV as bounded RecordBatches instead of reading the whole file into + # one Arrow table. Each batch is cast to all-string (the Lakehouse model casts at + # query time) and handed to write_deltalake through a self-describing + # RecordBatchReader, so only one bounded batch is materialized at a time. + read_options = pacsv.ReadOptions(block_size=8 * 1024 * 1024) + src_reader = pacsv.open_csv( in_csv, + read_options=read_options, convert_options=pacsv.ConvertOptions(strings_can_be_null=False), ) - cols = table.column_names + cols = list(src_reader.schema.names) safe_cols = _sanitize_col_names(cols) - import pyarrow as pa - table = pa.Table.from_arrays([table.column(c).cast(pa.string()) for c in cols], names=safe_cols) + target_schema = pa.schema([pa.field(sc, pa.string()) for sc in safe_cols]) + counter = [0] + def _string_batches(): + for batch in src_reader: + counter[0] += batch.num_rows + arrays = [batch.column(i).cast(pa.string()) for i in range(batch.num_columns)] + yield pa.RecordBatch.from_arrays(arrays, names=safe_cols) + batch_reader = pa.RecordBatchReader.from_batches(target_schema, _string_batches()) storage_options = None if out_uri.startswith('https://') or out_uri.startswith('abfss://'): if not token: @@ -11552,12 +18949,12 @@ try: storage_options = {'bearer_token': token, 'use_fabric_endpoint': 'true'} write_deltalake( out_uri, - table, + batch_reader, mode=mode, schema_mode='merge', storage_options=storage_options, ) - print(f'Convert-CsvToDelta: wrote {table.num_rows} rows to {out_uri} (mode={mode})') + print(f'Convert-CsvToDelta: wrote {counter[0]} rows to {out_uri} (mode={mode}, streamed)') except Exception as e: print(f'Convert-CsvToDelta: write failed: {e}', file=sys.stderr) sys.exit(4) @@ -11932,9 +19329,9 @@ function Resolve-SharePointTarget { $diagLines.Add(' "Permissions Required for THIS run" table earlier in this output') $diagLines.Add(' for the full list of required scopes for SharePoint output.') } else { - $diagLines.Add('Cause: Graph rejected the token (401). For app-only auth this usually means') - $diagLines.Add(' application permissions (Sites.ReadWrite.All / Files.ReadWrite.All)') - $diagLines.Add(' have not been ADMIN-CONSENTED on the app registration / managed identity.') + $diagLines.Add('Cause: Graph rejected the token (401). For app-only auth this usually means the') + $diagLines.Add(' application permission (Sites.ReadWrite.All)') + $diagLines.Add(' has not been ADMIN-CONSENTED on the app registration / managed identity.') $diagLines.Add('Action: Have a Global Administrator grant admin consent to the application') $diagLines.Add(' permissions listed in the "Permissions Required for THIS run" table above.') } @@ -12486,6 +19883,30 @@ function Get-GraphErrorDetail { return ($raw -replace '\s+', ' ').Trim() } +function script:New-PaxGraphUploadSession { + # Direct-REST createUploadSession for large SharePoint uploads (files above the Graph simple-upload + # ceiling). The request body is built as a plain ordered object and serialized with ConvertTo-Json, + # then sent via Invoke-RestMethod with an explicit bearer header - deliberately BYPASSING the Graph SDK + # object serializer used by Invoke-MgGraphRequest. Handing a hashtable body that carries a PowerShell + # string to that serializer fails, because the string's extended-type '.Chars' member is a + # PSParameterizedProperty that triggers a self-referencing-loop serialization error at 'item.name.Chars' + # and no HTTP request is ever sent. Returns the parsed session object (with .uploadUrl); throws with + # Graph error detail on failure. Never logs the token or the returned uploadUrl. + param( + [Parameter(Mandatory)][string]$SessionUri, + [Parameter(Mandatory)][string]$RemoteFileName + ) + try { $null = Refresh-GraphTokenIfNeeded } catch { } + $token = Get-GraphAccessToken + if ([string]::IsNullOrWhiteSpace([string]$token)) { throw 'createUploadSession: no Graph access token available.' } + # Plain values only -> plain JSON. No PowerShell object reaches an SDK serializer. + $bodyObj = [ordered]@{ item = [ordered]@{ '@microsoft.graph.conflictBehavior' = 'replace'; name = [string]$RemoteFileName } } + $json = $bodyObj | ConvertTo-Json -Depth 4 -Compress + $jsonBytes = [System.Text.Encoding]::UTF8.GetBytes($json) + $headers = @{ Authorization = ('Bearer ' + [string]$token) } + return Invoke-RestMethod -Uri $SessionUri -Method POST -Headers $headers -ContentType 'application/json' -Body $jsonBytes -ErrorAction Stop +} + function Send-FileToSharePoint { [CmdletBinding()] param( @@ -12534,17 +19955,29 @@ function Send-FileToSharePoint { else { # Create upload session $sessUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/createUploadSession" - $sessBody = @{ item = @{ '@microsoft.graph.conflictBehavior' = 'replace'; name = $fileName } } | ConvertTo-Json -Depth 4 try { - $session = Invoke-MgGraphRequest -Method POST -Uri $sessUri -Body $sessBody -ContentType 'application/json' -ErrorAction Stop + # Direct REST (New-PaxGraphUploadSession): the body is never handed to the Graph SDK object + # serializer, which fails on a PowerShell string's ETS '.Chars' property before any HTTP send. + $session = script:New-PaxGraphUploadSession -SessionUri $sessUri -RemoteFileName $fileName } catch { $detail = Get-GraphErrorDetail -ErrorRecord $_ - throw "createUploadSession failed for '$relPath': $detail" + $httpStatus = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + $sizeGb = $fileInfo.Length / 1GB + throw ("createUploadSession failed for '{0}' (HTTP {1}; file size {2:N0} bytes / {3:N2} GiB): {4}" -f $relPath, $httpStatus, $fileInfo.Length, $sizeGb, $detail) } $uploadUrl = $session.uploadUrl if (-not $uploadUrl) { throw "SharePoint createUploadSession returned no uploadUrl for '$relPath'." } + # Aggregate upload progress heartbeat (large files only). Reports transfer status only - never + # the uploadUrl, bearer token, or any response secret. + $hbInterval = if ($StatusIntervalSeconds -gt 0) { [int]$StatusIntervalSeconds } else { 60 } + $uploadStart = Get-Date + $lastHb = $uploadStart + $chunkCount = 0 + $retryTotal = 0 + Write-LogHost (" SharePoint resumable upload session created: {0} ({1:N0} bytes / {2:N2} MiB); uploading in chunks..." -f $relPath, $fileInfo.Length, ($fileInfo.Length / 1MB)) -ForegroundColor Cyan + $chunkSize = 5MB # multiple of 320 KB $fs = [System.IO.File]::OpenRead($LocalPath) try { @@ -12587,6 +20020,7 @@ function Send-FileToSharePoint { $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } $attempt++ if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $retryTotal++ $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } Start-Sleep -Seconds $wait continue @@ -12595,6 +20029,15 @@ function Send-FileToSharePoint { } } $offset += $read + $chunkCount++ + $nowHb = Get-Date + if ((($nowHb - $lastHb).TotalSeconds -ge $hbInterval) -or ($offset -ge $fileInfo.Length)) { + $elapsedSec = ($nowHb - $uploadStart).TotalSeconds + $pct = if ($fileInfo.Length -gt 0) { [Math]::Round(100.0 * $offset / $fileInfo.Length, 1) } else { 100 } + $rateMb = if ($elapsedSec -gt 0) { [Math]::Round(($offset / 1MB) / $elapsedSec, 2) } else { 0 } + Write-LogHost (" upload progress: {0:N0}/{1:N0} bytes ({2}%); chunk {3:N0}; elapsed {4:N0}s; {5} MiB/s; retries {6:N0}" -f $offset, $fileInfo.Length, $pct, $chunkCount, $elapsedSec, $rateMb, $retryTotal) -ForegroundColor DarkGray + $lastHb = $nowHb + } } } finally { $fs.Dispose() } @@ -12603,13 +20046,32 @@ function Send-FileToSharePoint { } function Send-FileToOneLake { + <# + .SYNOPSIS + Uploads a local file to OneLake Files/ via the DFS create/append/flush sequence. + .PARAMETER FailIfExists + Create-if-absent publication mode for IMMUTABLE resume-protocol objects (generation + artifacts and authority records). The DFS Create File request carries an + If-None-Match: * precondition, so an already-existing target answers 409/412 and is + raised as a HARD COLLISION instead of being overwritten - an existence conflict is + never success. Callers retry by allocating a completely fresh unique target, never by + replacing the conflicted object. + + This is OPT-IN and scoped to resume-generation publication. Customer-facing output + uploads omit the switch and keep the existing overwrite semantics unchanged. + + ASSUMPTION: the OneLake/ADLS Gen2 surface honours the If-None-Match: * conditional + on Create File. Where it does not, the caller's nonce-bearing target names remain + mandatory and any collision that is detected still fails closed. + #> [CmdletBinding()] param( [Parameter(Mandatory)] [string] $LocalPath, [string] $RemoteFileName, # Optional per-data-type Fabric/OneLake parent URL. See Send-FileToSharePoint # for the rationale. When omitted, falls back to the script-wide Purview URL. - [string] $ParentOverride + [string] $ParentOverride, + [switch] $FailIfExists ) if (-not (Test-Path -LiteralPath $LocalPath)) { throw "Send-FileToOneLake: source not found: $LocalPath" } if ($ParentOverride) { @@ -12654,24 +20116,66 @@ function Send-FileToOneLake { } } - # Step 1: Create (PUT ?resource=file) - overwrites any existing file. + # Step 1: Create (PUT ?resource=file) - overwrites any existing file, unless + # -FailIfExists asks for create-if-absent semantics. $createUri = "$dfsPath`?resource=file" - $null = Invoke-FabricWebRequest -Uri $createUri -Method PUT + if ($FailIfExists) { + # If-None-Match: * makes the create succeed ONLY when the target does not already + # exist. A 409/412 means something is already published at this exact name; for an + # immutable resume-protocol object that is a hard collision, not a success, so it is + # raised and the caller allocates a fresh unique target instead of replacing bytes. + try { + $null = Invoke-FabricWebRequest -Uri $createUri -Method PUT -Headers @{ 'If-None-Match' = '*' } + } + catch { + $createStatus = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($createStatus -eq 409 -or $createStatus -eq 412) { + throw ("Send-FileToOneLake: create-if-absent conflict for {0} - the target object already exists and an immutable resume object is never overwritten." -f $relInItem) + } + throw + } + } + else { + $null = Invoke-FabricWebRequest -Uri $createUri -Method PUT + } # Step 2: Append in chunks (PATCH ?action=append&position=N). # Inner retry loop handles 429/5xx with exponential backoff. 401 is handled transparently # inside Invoke-FabricWebRequest (force-refresh + retry once); any 401 that escapes here # means refresh itself failed, which is fatal for this upload and propagates up. $chunkSize = 4MB - $fs = [System.IO.File]::OpenRead($LocalPath) + # Production ALWAYS streams from disk via File.OpenRead. $script:PaxOneLakeSourceStreamOverride + # is an internal test seam (null in every non-test run) that lets a focused test inject a + # stream whose Read() returns deliberately short fragments, exercising the fill loop below + # without changing the production read path. + $fs = if ($script:PaxOneLakeSourceStreamOverride) { $script:PaxOneLakeSourceStreamOverride } else { [System.IO.File]::OpenRead($LocalPath) } try { $buffer = New-Object byte[] $chunkSize $position = 0L while ($position -lt $fileInfo.Length) { - $toRead = [Math]::Min([int64]$chunkSize, $fileInfo.Length - $position) - $read = $fs.Read($buffer, 0, [int]$toRead) + $toRead = [int][Math]::Min([int64]$chunkSize, $fileInfo.Length - $position) + # FileStream.Read may return fewer bytes than requested; fill the fragment fully (or to EOF) + # so the PATCH Content-Length and the declared append position stay exactly consistent. + $read = 0 + while ($read -lt $toRead) { + $n = $fs.Read($buffer, $read, $toRead - $read) + if ($n -le 0) { break } + $read += $n + } if ($read -le 0) { break } - $slice = if ($read -eq $chunkSize) { $buffer } else { $buffer[0..($read-1)] } + # Send a TRUE byte[] body. PowerShell range-indexing ($buffer[0..N]) returns an Object[] of + # boxed bytes, which the web cmdlet stringifies instead of transmitting as raw bytes - the wire + # length then disagrees with the declared append position and OneLake rejects the PATCH with + # "uploaded data is not contiguous" / "append position does not equal current file length". + # Partial chunks (checkpoint JSON and small shards below the 4 MiB chunk size) are the NORMAL + # path, so a right-sized byte[] copy is required for correctness here. + if ($read -eq $chunkSize) { + $slice = $buffer + } + else { + $slice = [byte[]]::new($read) + [System.Array]::Copy($buffer, 0, $slice, 0, $read) + } $appendUri = "$dfsPath`?action=append&position=$position" $attempt = 0 while ($true) { @@ -12696,6 +20200,15 @@ function Send-FileToOneLake { } finally { $fs.Dispose() } + # Local integrity guard: the append loop must have transmitted every source byte before + # we ask OneLake to flush the declared length. A short/interrupted read (unexpected EOF, or + # the local file mutated mid-upload) leaves $position < length; flushing that length would + # commit a remote file whose tail was never sent. Fail locally instead of asking OneLake to + # flush a length that was not uploaded. Normal successful uploads satisfy $position == length. + if ($position -ne $fileInfo.Length) { + throw ("Send-FileToOneLake: incomplete source read for {0} - sent {1} of {2} bytes before flush; aborting upload." -f $relInItem, $position, $fileInfo.Length) + } + # Step 3: Flush (PATCH ?action=flush&position=) $flushUri = "$dfsPath`?action=flush&position=$($fileInfo.Length)" $null = Invoke-FabricWebRequest -Uri $flushUri -Method PATCH @@ -12721,7 +20234,18 @@ function Get-RemoteFile-SharePoint { $relName = [System.Uri]::EscapeDataString($RelativeName) $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$relName" } else { $relName } $dlUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/content" - Invoke-MgGraphRequest -Method GET -Uri $dlUri -OutputFilePath $DestinationPath -ErrorAction Stop | Out-Null + # Stream the response body straight to disk. The Graph SDK's + # Invoke-MgGraphRequest -OutputFilePath buffers the ENTIRE body in memory before + # writing it out, so a large append target (roughly 1 GB or more) exceeds the + # single-object memory ceiling and fails. Invoke-WebRequest -OutFile on PowerShell 7 + # streams to disk instead, keeping peak memory flat regardless of file size (this + # mirrors Get-RemoteFile-OneLake). Graph accepts a raw bearer token directly; a + # missing file still surfaces as an HTTP 404 whose + # $_.Exception.Response.StatusCode.value__ is 404, so every caller's + # first-run / not-found classification is unchanged. + $graphToken = Get-GraphAccessToken + if (-not $graphToken) { throw "Get-RemoteFile-SharePoint: no Microsoft Graph access token available." } + Invoke-WebRequest -Uri $dlUri -Method GET -Headers @{ Authorization = "Bearer $graphToken" } -OutFile $DestinationPath -UseBasicParsing -ErrorAction Stop | Out-Null } function Get-RemoteFile-OneLake { @@ -12751,14 +20275,357 @@ function Get-RemoteFile-OneLake { $null = Invoke-FabricWebRequest -Uri $dfsPath -Method GET -OutFile $DestinationPath } +function Get-RemoteAppendTargetSize { + <# + .SYNOPSIS + Return the byte size of an append/merge target WITHOUT downloading its body, + or $null when the size cannot be determined. + .DESCRIPTION + A cheap, read-only pre-flight probe used to warn when an append target is large + enough that the in-memory merge could strain RAM. Dispatches by tier: + * SharePoint : a driveItem metadata GET (the download URL WITHOUT the + ':/content' suffix) whose JSON carries a 'size' field. + * Fabric : a DFS HEAD to the same path Get-RemoteFile-OneLake reads, + returning Content-Length with no body. + * Local : (Get-Item).Length. + FAIL-OPEN: any error, unexpected response shape, or a not-found (404) target + returns $null so the caller falls through to normal first-run / merge handling + with no warning and no abort. A probe failure is never read as "huge" or + "missing" - it simply yields "size unknown". + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [ValidateSet('SharePoint', 'Fabric', 'Local')] [string] $Tier, + [string] $RelativeName, + [string] $Path, + [string] $ParentOverride + ) + try { + switch ($Tier) { + 'Local' { + if (-not $Path) { return $null } + return [int64](Get-Item -LiteralPath $Path -ErrorAction Stop).Length + } + 'SharePoint' { + if ($ParentOverride) { $resolved = Resolve-SharePointTarget -Url $ParentOverride } + else { + if (-not $script:SPResolved) { $script:SPResolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl } + $resolved = $script:SPResolved + } + $relName = [System.Uri]::EscapeDataString($RelativeName) + $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$relName" } else { $relName } + # Metadata GET: the download URL WITHOUT ':/content' returns the driveItem JSON. + $metaUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}" + $item = Invoke-MgGraphRequest -Method GET -Uri $metaUri -ErrorAction Stop + if ($null -ne $item -and $null -ne $item.size) { return [int64]$item.size } + return $null + } + 'Fabric' { + if ($ParentOverride) { $resolved = Resolve-FabricTarget -Url $ParentOverride } + else { + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + } + $relName = ($RelativeName -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/$relName" } else { $relName } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem" + $resp = Invoke-FabricWebRequest -Uri $dfsPath -Method HEAD + $cl = $null + if ($resp -and $resp.Headers) { $cl = $resp.Headers['Content-Length'] } + if ($cl -is [array]) { $cl = $cl[0] } + if ($null -ne $cl -and "$cl" -ne '') { return [int64]$cl } + return $null + } + } + } + catch { + # Fail-open: size unknown -> caller proceeds normally, no warning, no abort. + return $null + } + return $null +} + +function Get-AppendTargetSizeWarning { + <# + .SYNOPSIS + Given an append-target size in bytes (or $null) and a threshold, return a + human-readable advisory warning string when the size exceeds the threshold, + otherwise $null. Pure (no side effects) so the decision is independently testable. + #> + [CmdletBinding()] + param( + [AllowNull()] [System.Nullable[int64]] $SizeBytes, + [Parameter(Mandatory)] [int64] $ThresholdBytes, + [string] $Label + ) + if ($null -eq $SizeBytes) { return $null } + if ($SizeBytes -le $ThresholdBytes) { return $null } + $gb = [math]::Round($SizeBytes / 1GB, 2) + $name = if ($Label) { " '$Label'" } else { '' } + return ( + "WARNING: The append target$name is $gb GB. The merge step reads the entire " + + "target into memory to deduplicate against it, which can require several times " + + "that size in RAM. Large downloads themselves are streamed safely, but a very " + + "large target can still exhaust memory during the merge. Consider archiving or " + + "rolling the target to a fresh file, or ensure the machine has enough memory. " + + "This is an advisory only - the run will continue." + ) +} + +function script:Test-AuditPartitionConsistency { + <# + .SYNOPSIS + Bounded, deterministic consistency check for a REPEATED fetch of a single 'successful' audit + partition. Compares two RecordId samples (or counts) of the SAME partition and classifies the + repeat as 'Stable' (identical set) or 'Unstable' (any difference). It NEVER picks the larger + sample and NEVER unions the two - it only DETECTS/CLASSIFIES so the caller can refuse to + silently certify an unstable partition (data-safe reconciliation). Observed live: SharePoint + partition 26 returned 13,350 vs 15,465 (2,115 RecordIds silently lost, a strict subset). + .OUTPUTS + [pscustomobject] Consistent(bool), Verdict('Stable'|'Unstable'), FirstCount, RepeatCount, + MissingFromRepeat, ExtraInRepeat, Basis('Set'|'Count'). + #> + [CmdletBinding()] + param([AllowNull()] $FirstRecordIds, [AllowNull()] $RepeatRecordIds, [AllowNull()] $FirstCount = $null, [AllowNull()] $RepeatCount = $null) + $haveSets = ($null -ne $FirstRecordIds) -and ($null -ne $RepeatRecordIds) + if ($haveSets) { + $first = New-Object 'System.Collections.Generic.HashSet[string]' ([StringComparer]::Ordinal) + foreach ($id in @($FirstRecordIds)) { if ($null -ne $id) { [void]$first.Add([string]$id) } } + $repeat = New-Object 'System.Collections.Generic.HashSet[string]' ([StringComparer]::Ordinal) + foreach ($id in @($RepeatRecordIds)) { if ($null -ne $id) { [void]$repeat.Add([string]$id) } } + $missing = 0; foreach ($id in $first) { if (-not $repeat.Contains($id)) { $missing++ } } + $extra = 0; foreach ($id in $repeat) { if (-not $first.Contains($id)) { $extra++ } } + $consistent = ($missing -eq 0 -and $extra -eq 0) + return [pscustomobject]@{ Consistent = $consistent; Verdict = $(if ($consistent) { 'Stable' } else { 'Unstable' }); FirstCount = $first.Count; RepeatCount = $repeat.Count; MissingFromRepeat = $missing; ExtraInRepeat = $extra; Basis = 'Set' } + } + $fc = if ($null -ne $FirstCount) { [int]$FirstCount } else { -1 } + $rc = if ($null -ne $RepeatCount) { [int]$RepeatCount } else { -1 } + $consistent = ($fc -ge 0 -and $rc -ge 0 -and $fc -eq $rc) + return [pscustomobject]@{ Consistent = $consistent; Verdict = $(if ($consistent) { 'Stable' } else { 'Unstable' }); FirstCount = $fc; RepeatCount = $rc; MissingFromRepeat = $(if ($fc -gt $rc) { $fc - $rc } else { 0 }); ExtraInRepeat = $(if ($rc -gt $fc) { $rc - $fc } else { 0 }); Basis = 'Count' } +} + +function script:Get-PartitionStabilityRecordIds { + <# + .SYNOPSIS + FIRST-pass RecordId source for -VerifyPartitionStability's bounded repeat check. Materializes + the partition's original-fetch RecordIds from the run's persisted incremental JSONL (a disk + read of what the first fetch already wrote - NOT a new service query). The REPEAT pass is NO + LONGER a re-page of the partition's (now-DELETED) Graph audit QueryId; it is an INDEPENDENT + create->poll->page->delete query for the same window handled by + Invoke-PartitionStabilityRepeatQuery. Returns a [string[]] of RecordIds (identity resolved via + the codebase's RecordId/Identity/Id fallback), never the raw records, so no full set is kept. + .OUTPUTS + [string[]] RecordIds (possibly empty). + #> + [CmdletBinding()] + param([Parameter(Mandatory)] $PartitionStatus, [bool] $IsRepeat) + $resolveId = { param($r) if ($null -eq $r) { return $null }; if ($r.RecordId) { return [string]$r.RecordId }; if ($r.Identity) { return [string]$r.Identity }; if ($r.Id) { return [string]$r.Id }; return $null } + $ids = [System.Collections.Generic.List[string]]::new() + # This helper is FIRST-pass only. The independent repeat query lives in + # Invoke-PartitionStabilityRepeatQuery; $IsRepeat is retained only for the injectable -Fetcher + # shape and, if ever passed, still returns the persisted first-fetch set (never a QueryId re-page). + # FIRST pass: read the partition's persisted first-fetch JSONL (the data the first fetch wrote). + try { + if ($script:PartialOutputPath) { + $incDir = Join-Path (Split-Path $script:PartialOutputPath -Parent) '.pax_incremental' + if (Test-Path -LiteralPath $incDir) { + $pat = "Part$($PartitionStatus.Partition.Index)_$($global:ScriptRunTimestamp)_*.jsonl" + foreach ($f in @(Get-ChildItem -LiteralPath $incDir -Filter $pat -File -ErrorAction SilentlyContinue)) { + foreach ($line in [System.IO.File]::ReadLines($f.FullName)) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + try { $rec = $line | ConvertFrom-Json -ErrorAction Stop; $v = & $resolveId $rec; if ($v) { [void]$ids.Add($v) } } catch {} + } + } + } + } + } catch {} + return ,([string[]]$ids.ToArray()) +} + +function script:Invoke-PartitionStabilityRepeatQuery { + <# + .SYNOPSIS + REPEAT-pass executor for -VerifyPartitionStability. Runs ONE bounded, INDEPENDENT audit query + for a completed partition's EXACT window + operation/recordType/service filters (a NEW QueryId, + never a re-page of the partition's already-DELETED first-fetch QueryId) so it can detect the + CROSS-QUERY under-delivery a re-page can never see (observed live: a fresh query for the same + window returned a very different count). Flow: create -> bounded poll to 'succeeded' -> page the + complete record set -> best-effort DELETE the repeat query. Any create / poll (failed|cancelled| + timeout) / paging failure yields Outcome='Failed' (the caller treats that as Unstable/Unknown -> + exit 40); cleanup is housekeeping and cannot invalidate a completed comparison. Returns only compact + RecordId strings (identity resolved via the RecordId/Identity/Id fallback), never raw records. + .OUTPUTS + [pscustomobject] RecordIds([string[]]), Outcome('Complete'|'Failed'), FailureStage, + RepeatQueryId, CleanupSucceeded. + .PARAMETER CreateOp + Injectable create op ($DisplayName,$StartDate,$EndDate,$Operations,$RecordTypes,$ServiceTypes) + -> new queryId or $null. Defaults to the real Invoke-GraphAuditQuery (main-thread Graph wrapper). + .PARAMETER StatusOp + Injectable status op ($QueryId) -> @{Status;RecordCount}. Defaults to Get-GraphAuditQueryStatus. + .PARAMETER RecordsOp + Injectable records op ($QueryId) -> record objects. Defaults to Get-GraphAuditRecords. + .PARAMETER DeleteOp + Injectable delete op ($QueryId) -> void. Defaults to the real Graph queries/$id DELETE. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] $PartitionStatus, + [AllowNull()][string[]] $TargetUsers = $null, + [scriptblock] $CreateOp = $null, + [scriptblock] $StatusOp = $null, + [scriptblock] $RecordsOp = $null, + [scriptblock] $DeleteOp = $null, + [int] $MaxPollAttempts = 960, + [int] $PollIntervalSeconds = 15 + ) + if ($null -eq $CreateOp) { $CreateOp = { param($dn, $sd, $ed, $ops, $rt, $svc) Invoke-GraphAuditQuery -DisplayName $dn -StartDate $sd -EndDate $ed -Operations $ops -RecordTypes $rt -ServiceTypes $svc } } + if ($null -eq $StatusOp) { $StatusOp = { param($qid) Get-GraphAuditQueryStatus -QueryId $qid } } + if ($null -eq $RecordsOp) { $RecordsOp = { param($qid) Get-GraphAuditRecords -QueryId $qid } } + if ($null -eq $DeleteOp) { $DeleteOp = { param($qid) Invoke-MgGraphRequest -Method DELETE -Uri (Get-GraphAuditApiUri -Path "queries/$qid") -ErrorAction Stop | Out-Null } } + $resolveId = { param($r) if ($null -eq $r) { return $null }; if ($r.RecordId) { return [string]$r.RecordId }; if ($r.Identity) { return [string]$r.Identity }; if ($r.Id) { return [string]$r.Id }; return $null } + $fail = { param($stage, $qid) [pscustomobject]@{ RecordIds = [string[]]@(); Outcome = 'Failed'; FailureStage = $stage; RepeatQueryId = $qid; CleanupSucceeded = $false } } + $pt = $PartitionStatus.Partition + if ($null -eq $pt) { return (& $fail 'Create' $null) } + # Build the repeat create payload from the partition's OWN window + filters (exact parity, so a + # test can capture and assert the create args equal the original partition's window/filters). + $ops = if ($pt.Activities) { @($pt.Activities) } elseif ($pt.Activity) { @($pt.Activity) } else { @() } + $svc = $null; if ($pt.ServiceFilter) { $svc = @($pt.ServiceFilter) } + $rt = $null; if ($pt.RecordTypes) { $rt = @($pt.RecordTypes) } + $dn = "PAX_StabilityRepeat_P$($pt.Index)of$($pt.Total)_$([DateTime]::UtcNow.ToString('yyyyMMddHHmmss'))" + # CREATE an independent NEW query (never reuse / re-page $PartitionStatus.QueryId). + $repeatId = $null + try { $repeatId = & $CreateOp $dn $pt.PStart $pt.PEnd $ops $rt $svc } catch { return (& $fail 'Create' $null) } + if (-not $repeatId) { return (& $fail 'Create' $null) } + # POLL to 'succeeded' - bounded (finite attempts, NOT unbounded). + $succeeded = $false + for ($a = 0; $a -lt $MaxPollAttempts; $a++) { + $st = $null + try { $st = & $StatusOp $repeatId } catch { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Poll' $repeatId) } + $state = if ($st) { [string]$st.Status } else { $null } + if ($state -eq 'succeeded') { $succeeded = $true; break } + if ($state -in @('failed', 'cancelled')) { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Poll' $repeatId) } + if ($PollIntervalSeconds -gt 0) { Start-Sleep -Seconds $PollIntervalSeconds } + } + if (-not $succeeded) { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Timeout' $repeatId) } + # PAGE the complete record set, resolving compact RecordId strings (never retain raw records). + $ids = [System.Collections.Generic.List[string]]::new() + try { foreach ($r in @(& $RecordsOp $repeatId)) { if ($TargetUsers -and $TargetUsers.Count -gt 0) { $__upn = if ($null -ne $r) { [string]$r.userPrincipalName } else { $null }; if (-not ($TargetUsers -contains $__upn)) { continue } } $v = & $resolveId $r; if ($v) { [void]$ids.Add($v) } } } + catch { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Page' $repeatId) } + # DELETE is best-effort housekeeping after retrieval has completed. A delete failure does not + # change the retrieved RecordId set or invalidate the comparison. + $deleteOk = $true + try { & $DeleteOp $repeatId } catch { $deleteOk = $false } + return [pscustomobject]@{ RecordIds = [string[]]$ids.ToArray(); Outcome = 'Complete'; FailureStage = $null; RepeatQueryId = $repeatId; CleanupSucceeded = $deleteOk } +} + +function script:Invoke-PartitionStabilityVerification { + <# + .SYNOPSIS + Bounded, opt-in repeat-fetch stability verification for completed audit partitions + (-VerifyPartitionStability). For each 'Complete' partition it captures the first-fetch + RecordId set (.RecordIds) and runs EXACTLY ONE independent repeat fetch of the same partition + window (.StabilityRepeatIds), then classifies the repeat via Test-AuditPartitionConsistency. + Any 'Unstable' verdict (any RecordId-set difference - missing OR extra) is treated as a gap: + $script:HadTerminalFailures is set (exit 40) - never silently certified, never blind-accept + -larger, never unioned, never count-only (Set basis). Memory is bounded: RecordId sets are + materialized per-partition inside the loop and ONLY when enabled - default OFF performs no + repeat fetch and retains nothing (byte-identical to a run without the switch). + .PARAMETER Fetcher + Injectable FIRST-pass RecordId source scriptblock ($PartitionStatus, [bool]$IsRepeat) -> + [string[]] of RecordIds. Defaults to Get-PartitionStabilityRecordIds (persisted-JSONL first + pass only). Injected by tests to model the first-fetch persisted set. + .PARAMETER RepeatExecutor + Injectable REPEAT-pass executor scriptblock ($PartitionStatus) -> { RecordIds=[string[]]; + Outcome='Complete'|'Failed'; FailureStage; RepeatQueryId }. Defaults to + Invoke-PartitionStabilityRepeatQuery (one independent create->poll->page->delete query for the + SAME window/filters, a NEW QueryId). Injected by tests to model the repeat query without a + live tenant. + #> + [CmdletBinding()] + param( + [AllowNull()] $CompletedPartitions, + [AllowNull()][string[]] $TargetUsers = $null, + [scriptblock] $Fetcher = $null, + [scriptblock] $RepeatExecutor = $null, + [switch] $Enabled + ) + # Default OFF: no repeat query, no RecordId retention -> byte-identical to a run without the switch. + if (-not $Enabled) { return } + if ($null -eq $CompletedPartitions) { return } + if ($null -eq $Fetcher) { $Fetcher = ${function:script:Get-PartitionStabilityRecordIds} } + if ($null -eq $RepeatExecutor) { $RepeatExecutor = ${function:script:Invoke-PartitionStabilityRepeatQuery} } + foreach ($__ps in @($CompletedPartitions)) { + if ($null -eq $__ps) { continue } + $__pt = $__ps.Partition + # FIRST set: original-fetch RecordIds (a disk read of the first fetch's persisted output, not a + # new service query). Retained only now (switch ON) and only one partition at a time (bounded). + $__ps.RecordIds = @(& $Fetcher $__ps $false) + # REPEAT set: exactly ONE bounded INDEPENDENT query (a NEW QueryId) for the SAME window/filters + # via create->poll->page->delete. This detects the CROSS-QUERY under-delivery that a re-page of + # the (already-deleted) first-fetch QueryId never could. + $__rep = & $RepeatExecutor $__ps $TargetUsers + if ($null -eq $__rep -or $__rep.Outcome -ne 'Complete') { + # Any create/poll/page/cleanup failure => Unknown: never silently certified (gap -> exit 40). + $script:HadTerminalFailures = $true + $__ps.StabilityVerdict = 'Unknown' + $__ps.StabilityFirstCount = @($__ps.RecordIds).Count + $__ps.StabilityRepeatCount = -1 + $__ps.StabilityMissing = -1 + $__ps.StabilityExtra = -1 + $__stage = if ($__rep) { [string]$__rep.FailureStage } else { 'Executor' } + Write-LogHost (" [STABILITY] Partition {0}/{1} NOT CERTIFIED - independent repeat query failed at stage '{2}'; treated as a gap (exit 40)." -f $__pt.Index, $__pt.Total, $__stage) -ForegroundColor Red + # Bounded memory: drop RecordId arrays immediately; keep only compact verdict metadata. + $__ps.RecordIds = $null + $__ps.StabilityRepeatIds = $null + continue + } + $__ps.StabilityRepeatIds = @($__rep.RecordIds) + # Classify via the pure detector (Set basis: never blind-larger, never union, never count-only). + $__cons = script:Test-AuditPartitionConsistency -FirstRecordIds $__ps.RecordIds -RepeatRecordIds $__ps.StabilityRepeatIds + # Retain ONLY compact verdict/count metadata (no RecordId arrays held across partitions). + $__ps.StabilityVerdict = $__cons.Verdict + $__ps.StabilityFirstCount = $__cons.FirstCount + $__ps.StabilityRepeatCount = $__cons.RepeatCount + $__ps.StabilityMissing = $__cons.MissingFromRepeat + $__ps.StabilityExtra = $__cons.ExtraInRepeat + if (-not $__cons.Consistent) { + $script:HadTerminalFailures = $true + Write-LogHost (" [STABILITY] Partition {0}/{1} UNSTABLE on independent repeat verification: {2} RecordId(s) missing, {3} extra (first={4}, repeat={5}); NOT silently certified (gap -> exit 40)." -f $__pt.Index, $__pt.Total, $__cons.MissingFromRepeat, $__cons.ExtraInRepeat, $__cons.FirstCount, $__cons.RepeatCount) -ForegroundColor Red + } + # Bounded memory: drop RecordId arrays immediately after the verdict; keep only compact metadata. + $__ps.RecordIds = $null + $__ps.StabilityRepeatIds = $null + } +} + function Get-DataTypeForOutputFile { # Map a customer-visible artifact basename to its data-type key. Used by the # upload sweep to route each file to its per-data-type destination (-OutputPath*). - # Returns one of: 'Purview' | 'UserInfo' | 'Agent365Info' | 'Log'. + # Returns one of: 'Purview' | 'UserInfo' | 'Agent365Info' | 'DefenderUsage' | 'Log'. + # The twelve fixed-name AISID dashboard leaves (ai_activity_sessions.csv, + # ai_offhours_geo.csv, ai_file_proximity.csv, ai_oauth_consents.csv, + # ai_sso_signins.csv, ai_client_channel.csv, ai_copilot_usage_graph.csv, + # ai_solutions_catalog.csv, EntraUsers.csv) route to 'DefenderUsage' — matched + # against the authoritative $script:AISIDTableSchemas fixed-name set BEFORE the + # UserInfo pattern so the AISID EntraUsers.csv projection can never be misrouted + # to UserInfo (the timestamped '_Users.csv' stream) or fall through to + # Purview. The internal '.aisid_cache' delta-cache directory (and any file under + # it) is a working area, never a dashboard artifact, so it is excluded from + # DefenderUsage and treated as a non-dashboard Purview fallback. # Unknown / unmatched names fall back to 'Purview' (primary audit destination). [CmdletBinding()] param([Parameter(Mandatory)] [string] $FileName) $n = [System.IO.Path]::GetFileName($FileName) + # The '.aisid_cache' delta-cache is internal scratch, never a dashboard artifact — + # exclude any path segment named '.aisid_cache' from every dashboard classification. + if (($FileName -replace '\\', '/') -match '(^|/)\.aisid_cache(/|$)') { return 'Purview' } + # AISID dashboard leaves -> DefenderUsage (authoritative fixed-name set kept in + # sync with $script:AISIDTableSchemas), checked BEFORE UserInfo so EntraUsers.csv + # routes to DefenderUsage, never UserInfo/Purview. + $aisidLeaves = @() + if ($script:AISIDTableSchemas) { $aisidLeaves = @($script:AISIDTableSchemas.Values | ForEach-Object { $_.FileName }) } + if ($aisidLeaves -notcontains 'EntraUsers.csv') { $aisidLeaves += 'EntraUsers.csv' } + foreach ($aisidLeaf in $aisidLeaves) { if ($aisidLeaf -and $n -ieq $aisidLeaf) { return 'DefenderUsage' } } if ($n -like 'EntraUsers_*' -or $n -like '*_Users.csv' -or $n -like '*_Users_*.csv') { return 'UserInfo' } if ($n -like 'Agent365_*' -or $n -like 'Agent365.csv') { return 'Agent365Info' } if ($n -like '*.log' -or $n -like '*.partial.log') { return 'Log' } @@ -12831,6 +20698,8 @@ function Invoke-EmbeddedProcessor { Write-LogHost "Rollup: invoking embedded $label v$ver ($PythonExe)" -ForegroundColor Cyan Write-LogFile "Rollup: temp script -> $tempPyPath" Write-LogFile "Rollup: args -> $($ProcessorArgs -join ' ')" + # Additive memory/stage observation at the processor boundary (never fatal). + try { Write-PaxMemoryObservation -Stage ("EmbeddedProcessor:" + $label) -TempDir $IncrementalDir -Force } catch {} $exitCode = 1 # Force UTF-8 across the Python subprocess boundary so Unicode glyphs the @@ -13162,6 +21031,63 @@ $script:TrimStartDateUTC = if ($StartDate -ne '*') { [datetime]::SpecifyKind([da $script:TrimEndDateUTC = if ($EndDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } $script:DateTrimCount = 0 +# ============================================================ +# AISID date-window contract - fresh runs only +# ============================================================ +# -Dashboard AISID requires TWO concrete date boundaries after normal defaulting: both dates omitted +# -> the [yesterday..today) UTC default (both concrete) is accepted; both supplied -> the existing +# format/order validation applies; exactly one supplied -> the other is the '*' sentinel (unbounded) +# here, which is INVALID for AISID. Reject the one-sided shape now - before authentication, checkpoint +# creation, or any customer output - with a clear both-or-neither message and no prompt. Resume runs +# restore the original run's concrete dates from the checkpoint, so this fresh-run gate is skipped +# under -Resume. Non-AISID one-sided / unbounded runs are unaffected. +if ($Dashboard -eq 'AISID' -and $null -eq $Resume) { + if ($StartDate -eq '*' -or $EndDate -eq '*' -or $null -eq $script:TrimStartDateUTC -or $null -eq $script:TrimEndDateUTC) { + $missingBoundary = if ($StartDate -eq '*' -and $EndDate -eq '*') { 'both StartDate and EndDate' } elseif ($StartDate -eq '*') { 'StartDate' } else { 'EndDate' } + Write-LogHost "ERROR: -Dashboard AISID requires BOTH -StartDate and -EndDate to be concrete dates (missing: $missingBoundary)." -ForegroundColor Red + Write-LogHost " AISID intersects a bounded requested window with the 30-day retention window, so a one-sided (unbounded) range is not valid." -ForegroundColor Yellow + Write-LogHost " Supply BOTH -StartDate and -EndDate (yyyy-MM-dd), or omit BOTH to use the default [yesterday..today) UTC window." -ForegroundColor Yellow + Write-LogHost " No customer data files were written and no service connection or query was started." -ForegroundColor Yellow + if ($script:LogFile) { Write-LogHost (" Diagnostic log preserved at: {0}" -f $script:LogFile) -ForegroundColor Yellow } + exit 1 + } + # Freeze ONE reference instant for the whole run and resolve the requested-vs-available window once. + $script:AISIDReferenceNowUtc = (Get-Date).ToUniversalTime() + $script:AISIDWindow = Resolve-AISIDDateWindow -RequestedStartUtc $script:TrimStartDateUTC -RequestedEndUtc $script:TrimEndDateUTC -ReferenceNowUtc $script:AISIDReferenceNowUtc -RetentionDays (Get-AISIDEffectiveLookbackDays) + $fmtW = 'yyyy-MM-ddTHH:mm:ssZ' + $reqW = ('[{0} .. {1})' -f $script:AISIDWindow.RequestedStartUtc.ToString($fmtW), $script:AISIDWindow.RequestedEndUtc.ToString($fmtW)) + $availW = ('[{0} .. {1})' -f $script:AISIDWindow.AvailableStartUtc.ToString($fmtW), $script:AISIDWindow.AvailableEndUtc.ToString($fmtW)) + $effW = ('[{0} .. {1})' -f $script:AISIDWindow.EffectiveStartUtc.ToString($fmtW), $script:AISIDWindow.EffectiveEndUtc.ToString($fmtW)) + switch ($script:AISIDWindow.Status) { + 'NONE' { + # No overlap between the requested window and the 30-day retention window. Stop BEFORE any + # authentication, Purview/Entra/Defender query, licensing probe, checkpoint, rollup, or customer + # CSV. Controlled exit; this is a date-selection decision, not a Ctrl+C / interrupt. + Write-LogHost "AISID collection window: NONE - the requested date range does not overlap the available 30-day retention window; no collection was started." -ForegroundColor Red + Write-LogHost (" Requested window : {0}" -f $reqW) -ForegroundColor Yellow + Write-LogHost (" Available window : {0}" -f $availW) -ForegroundColor Yellow + Write-LogHost " Choose a -StartDate/-EndDate range that overlaps the last 30 days, then re-run." -ForegroundColor Yellow + Write-LogHost " No customer data files were written and no service connection or query was started." -ForegroundColor Yellow + if ($script:LogFile) { Write-LogHost (" Diagnostic log preserved at: {0}" -f $script:LogFile) -ForegroundColor Yellow } + exit 1 + } + 'PARTIAL' { + # The requested window extends beyond the 30-day retention window. Purview retains the FULL requested + # range; the AISID data sets use ONLY the requested-intersect-available intersection. One warning, no + # input, continue immediately. + Write-LogHost ("WARNING: AISID collection window PARTIAL - the requested range extends beyond the 30-day retention window; AISID uses only the intersection.") -ForegroundColor Yellow + Write-LogHost (" Requested window : {0}" -f $reqW) -ForegroundColor Yellow + Write-LogHost (" Available window : {0}" -f $availW) -ForegroundColor Yellow + Write-LogHost (" Effective window : {0}" -f $effW) -ForegroundColor Yellow + Write-LogHost (" Reason : {0}" -f $script:AISIDWindow.ClipReason) -ForegroundColor Yellow + Write-LogHost (" Purview retains the FULL requested range; only the AISID data sets use the intersection. No input is required; processing continues automatically.") -ForegroundColor Yellow + } + default { + # FULL - the requested window is entirely within retention. No clipping, no warning. + } + } +} + if ($BlockHours -le 0) { Write-Host "ERROR: BlockHours must be positive." -ForegroundColor Red; exit 1 } try { if ($PSVersionTable.PSEdition -eq 'Core' -and ($global:InformationPreference -in @('SilentlyContinue', 'Ignore'))) { $global:InformationPreference = 'Continue' } } catch {} @@ -13331,6 +21257,120 @@ function Assert-PartitionStatusEntry { } } +# Number of consecutive same-category failures (signature = partition index + +# failure category) a partition may accumulate before it is flagged as a terminal +# failure. Count only; there is no elapsed-time component. Maintained by +# Set-PartitionFailure. +$script:MaxConsecutiveIdenticalFailures = 5 + +# Defensive runaway guard: the maximum number of times a single collection window +# may be subdivided by record-limit auto-subdivision. A legitimate high-volume +# window reaches well under this depth; hitting it indicates a pathological or +# spurious limit signal. A window still over the record limit at this depth is +# stopped and reported as a DISTINCT subdivision-limit state (Status +# 'SubdivisionLimit'), separate from the terminal create/fetch failure class. This +# is a depth count only — not a time limit. +$script:MaxSubdivisionLevel = 20 + +function Get-PartitionFailureCategory { + <# + .SYNOPSIS + Maps a specific failure reason to the coarse category used for the + terminal-detection signature. + .DESCRIPTION + All "the attempt failed at runtime" reasons collapse to FAILED so the SAME + underlying failure accumulates toward terminal regardless of which + observation site (live monitoring, reconcile, retry) reports it (a job error + can surface as JOB_ERROR live and ERROR_STREAM at reconcile for one problem). + AUTH (token) and SLOW (query still running) stay distinct so genuine + transients reset the consecutive-failure streak instead of driving a false + terminal. + #> + param([string]$Reason) + switch ($Reason) { + '401' { 'AUTH'; break } + 'STILL_RUNNING' { 'SLOW'; break } + default { 'FAILED' } + } +} + +function Set-PartitionFailure { + <# + .SYNOPSIS + Central sink for marking a partition Failed while tracking consecutive + same-category failure runs. + .DESCRIPTION + Sets Status='Failed' (identical to the inline assignment it replaces) and, + in addition, maintains a per-partition consecutive-failure counter keyed on a + coarse signature of (partition index + failure category). The category + collapses all runtime-failure reasons to FAILED (see + Get-PartitionFailureCategory) so the same underlying failure accumulates + regardless of which observation site reports it; AUTH and SLOW stay distinct. + When the same category repeats $script:MaxConsecutiveIdenticalFailures times + in a row, IsTerminalFailure is set; a differing category resets the counter + to 1. The most-recent Stage and Reason are recorded (LastFailureStage / + LastFailureReason) for the diagnostic terminal-failure report — they are NOT + part of the counting signature. LastError is left to the caller (unchanged). + The tracking fields are additive and are not consumed by any re-queue or + retry predicate here. + .PARAMETER Index + Partition index key into $script:partitionStatus. + .PARAMETER Stage + Coarse phase where the failure was observed (e.g. CREATE, FETCH, RECONCILE). + Recorded for the report only. + .PARAMETER Reason + Specific failure token (e.g. 401, EMPTY_QUERYID, JOB_FAILED). Recorded for the + report and mapped to a category for the counting signature. + #> + param( + [Parameter(Mandatory)] $Index, + [Parameter(Mandatory)] [string]$Stage, + [Parameter(Mandatory)] [string]$Reason + ) + if (-not ($script:partitionStatus -and $script:partitionStatus.ContainsKey($Index))) { return } + $entry = $script:partitionStatus[$Index] + $category = Get-PartitionFailureCategory -Reason $Reason + $signature = "$Index|$category" + if ($entry.LastFailureSignature -eq $signature) { + $entry.ConsecutiveIdenticalFailures = [int]$entry.ConsecutiveIdenticalFailures + 1 + } else { + $entry.ConsecutiveIdenticalFailures = 1 + $entry.LastFailureSignature = $signature + } + $entry.LastFailureStage = $Stage + $entry.LastFailureReason = $Reason + # A persistent create denial permits one token-renewal retry before becoming terminal. + $terminalThreshold = if ($Reason -eq 'PersistentForbidden') { 2 } else { $script:MaxConsecutiveIdenticalFailures } + $entry.IsTerminalFailure = ($entry.ConsecutiveIdenticalFailures -ge $terminalThreshold) + # Structured lifecycle category preservation: map the coarse Reason + # to the bounded vocabulary and NEVER let a generic parent reconciliation result + # (JOB_FAILED / ERROR_STREAM / NO_DATA / INVALID_COMPLETE) overwrite a more + # specific worker-established category. LastError stays diagnostic-only and is + # not consulted here. Additive: does not affect the terminal-failure counting. + $incomingCategory = Get-GraphAuditReasonCategory -Reason $Reason + $existingCategory = if ($entry.ContainsKey('FailureCategory') -and $null -ne $entry.FailureCategory) { [string]$entry.FailureCategory } else { 'None' } + $entry.FailureCategory = Merge-GraphAuditLifecycleCategory -Existing $existingCategory -Incoming $incomingCategory + $entry.Status = 'Failed' +} + +function Test-TransientAuthSignal { + <# + .SYNOPSIS + Returns $true only when error text carries a genuine transient-authentication + signal (expired/invalid token), never a coincidental '401' substring. + .DESCRIPTION + Matches explicit auth tokens (Unauthorized, InvalidAuthenticationToken, + token expired, authentication failed, lifetime validation failed) and the + HTTP status 401 ONLY in a status context — "(401)", "HTTP 401", or + "401 Unauthorized" — so a bare number such as a record count ("4013 records") + is not mistaken for an auth failure. Fail-closed: ambiguous or empty text + returns $false, so the failure then counts normally against the retry limit. + #> + param([string]$Text) + if ([string]::IsNullOrWhiteSpace($Text)) { return $false } + return [bool]($Text -match '(?i)(unauthorized|invalidauthenticationtoken|token(?:\s+has)?\s+expired|authentication\s+failed|lifetime\s+validation\s+failed|\(401\)|http[ /]?401\b|401\s+unauthorized)') +} + $script:metrics = @{ StartTime = (Get-Date).ToUniversalTime() QueryMs = 0 @@ -13379,6 +21419,13 @@ $script:metrics = @{ FilteringExcludeAgents = 0 FilteringUserIds = 0 FilteringGroupNames = 0 + ScopeResolvedGroups = 0 + ScopeFailedGroups = 0 + ScopeExpandedMembers = 0 + ScopeFinalTargetUsers = 0 + DirectoryRowsBeforeScope = 0 + DirectoryRowsAfterScope = 0 + DirectoryRowsExcludedByScope = 0 FilteringOther = 0 AdaptiveEvents = @() AdaptiveMemoryReductions = 0 @@ -13521,6 +21568,48 @@ $script:consecutiveBlockFailures = 0 $script:circuitBreakerOpen = $false $script:circuitBreakerOpenUntil = $null +# Finalize-time "completed with gaps" signals for the process exit code. Set true in +# the query-submission summary when the run finalizes with one or more partitions +# terminally failed (Issue 3A) or windows that hit the subdivision-depth guard +# (Issue 3B). Drive exit code 40 (lowest precedence — existing 10/20/30 dominate). +$script:HadTerminalFailures = $false +$script:HadSubdivisionLimit = $false +$script:AISIDHadGaps = $false +# Set true when requested Agent 365 catalog output is incomplete: an incomplete catalog +# listing, a package whose detail could not be retrieved, a row that could not be built, or +# a listed entry without a usable identifier. Drives exit code 40 at the same lowest +# precedence as the other completed-with-gaps signals. +$script:Agent365HadGaps = $false +# Leaves of Agent 365 recovery material written when the catalog listing did not complete. +# Recovery material is never published as a completed catalog, so both the end-of-run local +# listing and the remote upload sweep exclude these leaves (same convention as the Fact-append +# recovery leaves). +$script:Agent365RecoveryLeafs = $null +$script:AISIDCompletionStatus = $null +# D3: set true when the CopilotInteraction rollup post-processor fails (throw / nonzero exit) or does not +# produce a required deliverable (Fact / Users). Consumed ONLY under -Dashboard AISID (a requested-output +# failure -> completed-with-gaps, exit 40); non-AISID best-effort rollup exit semantics are unchanged. +$script:RollupProcessorFailed = $false + +# 1C: a REQUESTED Fact-append (-AppendFile) reconciliation FAILED this run. This is a +# requested-output failure -> completed-with-gaps (exit 40, driven via HadTerminalFailures), +# and it must ALSO preserve the recovery artifacts (the unmerged current-run Fact CSV, the +# remote scratch dir, and the Fabric resume mirror) that the normal clean-run cleanup would +# otherwise delete. It is DELIBERATELY DISTINCT from AnyUploadFailed (no upload actually +# failed) so diagnostics never mislabel a local merge failure as an upload failure. The +# exact leaves registered in FactAppendRecoveryLeafs are excluded from the end-of-run upload +# sweep and the local output listing so the unmerged current-run Fact is never published as +# though it were the completed -AppendFile target (nor as a second normal Fact output). +$script:FactAppendFailed = $false +$script:FactAppendRecoveryLeafs = $null + +# Advisory append-target size guardrail. When an append/merge target exceeds this +# many bytes, a non-blocking warning is emitted before the merge (which reads the +# whole target into memory to deduplicate against it). Default 1 GB; override in +# bytes via the PAX_APPEND_WARN_BYTES environment variable. The size probe itself is +# fail-open (see Get-RemoteAppendTargetSize) so it can never block or mis-handle a run. +$script:AppendTargetSizeWarnBytes = if ($env:PAX_APPEND_WARN_BYTES -and (($env:PAX_APPEND_WARN_BYTES -as [int64]) -gt 0)) { [int64]$env:PAX_APPEND_WARN_BYTES } else { 1GB } + # ============================================== # GRAPH API VERSION DETECTION HELPER # ============================================== @@ -13556,27 +21645,2367 @@ function Get-GraphAuditApiUri { [string]$Path ) - # Auto-detect version on first use (cached for session) + # Select the audit-query lifecycle version on first use (cached for session). + # + # A successful collection GET does NOT prove that per-item status, records, + # pagination and cleanup all work on a given version, so it must never by + # itself select the version used for the full query lifecycle. Field evidence + # showed a version whose collection GET and create succeeded but whose + # item/status and records operations returned 404 for queries that had + # actually completed server-side, while the alternate version served the same + # lifecycle reliably. Cross-version query-id visibility is unproven, so the + # lifecycle is never mixed across versions within a single query. + # + # This release therefore pins the version with a proven end-to-end lifecycle + # in this service ($script:GraphAuditApiVersion_Previous). The other version + # ($script:GraphAuditApiVersion_Current) remains configured for a future + # capability-gated path that is enabled only after a read-only cross-version + # discriminator proves its per-operation support and cross-version visibility. if ($null -eq $script:GraphAuditApiVersion) { - $currentVer = $script:GraphAuditApiVersion_Current - $previousVer = $script:GraphAuditApiVersion_Previous - - try { - # Test if current version endpoint is available - $testUri = "https://graph.microsoft.com/$currentVer/security/auditLog/queries" - Invoke-MgGraphRequest -Method GET -Uri $testUri -ErrorAction Stop | Out-Null - $script:GraphAuditApiVersion = $currentVer - Write-LogHost "Graph API: security/auditLog endpoint using version $currentVer" -ForegroundColor Green - } catch { - # Current version not available, fallback to previous - $script:GraphAuditApiVersion = $previousVer - Write-LogHost "Graph API: security/auditLog endpoint using version $previousVer (fallback from $currentVer)" -ForegroundColor Yellow - } + $script:GraphAuditApiVersion = $script:GraphAuditApiVersion_Previous + Write-LogHost "Graph API: security/auditLog lifecycle pinned to version $($script:GraphAuditApiVersion) (proven end-to-end; collection availability alone does not select a version)" -ForegroundColor Green } return "https://graph.microsoft.com/$($script:GraphAuditApiVersion)/security/auditLog/$Path" } +# ============================================================================ +# GRAPH AUDIT-QUERY LIFECYCLE DECISION LAYER +# ---------------------------------------------------------------------------- +# One shared, injectable decision layer for the Graph security/auditLog query +# lifecycle (create / status / records / pagination / delete). These are pure +# functions driven by STRUCTURED inputs (HTTP status codes + exception TYPE), +# never by error-message text. They are exercised directly by +# tests/Purview.GraphAuditLifecycleStateMachine.Tests.ps1 with injected HTTP +# operations, so the tested logic is the shipped logic. The same rules govern +# both the parent path and the isolated ThreadJob worker runspace. +# +# Stage-A-grounded invariants (beta is the default; v1.0 CREATION stays disabled): +# * beta is the production default; new query CREATION always uses beta. +# * A single endpoint/version 404 is QueryNotVisible, never QueryConfirmedAbsent. +# * Version fallback is a same-ID GET on the alternate version only; it NEVER +# issues a POST. Status and records are decided INDEPENDENTLY (Stage A proved +# v1.0 status 404 while v1.0 records 200 for the same id). +# * 401/403, 429, network, and 5xx keep existing handling with NO version +# fallback. +# * @odata.nextLink is followed verbatim; no $top / cap / truncation is added. +# * A successful zero-record retrieval is Complete, never NO_DATA / JOB_FAILED. +# * Delete uses the effective version and is best-effort; cleanup failure is +# observable but never erases saved records or fabricates a data gap. +# ============================================================================ + +# Bounded failure-category vocabulary. Extending this list is a deliberate +# contract change and must be mirrored in the classifier + tests. +$script:GraphAuditFailureCategories = @( + 'None', 'EndpointUnavailable', 'QueryNotVisible', 'QueryConfirmedAbsent', + 'Authentication', 'Throttled', 'Network', 'Service', 'Cleanup', 'Other' +) +# Categories that carry a specific, worker-established meaning. A generic parent +# reconciliation result (JOB_FAILED / ERROR_STREAM / NO_DATA / INVALID_COMPLETE) +# must never overwrite one of these. +$script:GraphAuditSpecificFailureCategories = @( + 'EndpointUnavailable', 'QueryNotVisible', 'QueryConfirmedAbsent', + 'Authentication', 'Throttled', 'Network', 'Service', 'Cleanup' +) + +function Get-GraphAuditFailureCategory { + <# + .SYNOPSIS + Maps a structured HTTP status + exception type to the bounded lifecycle + failure vocabulary. NEVER inspects message text. + #> + param( + [Nullable[int]]$HttpStatus = $null, + [string]$ExceptionType = $null, + [ValidateSet('Create', 'Status', 'Records', 'Delete', 'List', '')] + [string]$Operation = '', + [switch]$ConfirmedAbsent + ) + # Delete/cleanup failures are always Cleanup (best-effort; never data-erasing). + # A 404 on delete means the query is already gone => successful cleanup (None). + if ($Operation -eq 'Delete') { + if ($HttpStatus -eq 404) { return 'None' } + if (($null -ne $HttpStatus -and $HttpStatus -ge 400) -or $ExceptionType) { return 'Cleanup' } + return 'None' + } + if ($null -ne $HttpStatus) { + switch ([int]$HttpStatus) { + 401 { return 'Authentication' } + 403 { return 'Authentication' } + 429 { return 'Throttled' } + 404 { if ($ConfirmedAbsent) { return 'QueryConfirmedAbsent' } else { return 'QueryNotVisible' } } + default { + if ($HttpStatus -ge 500 -and $HttpStatus -le 599) { return 'Service' } + if ($HttpStatus -ge 200 -and $HttpStatus -le 299) { return 'None' } + return 'Other' + } + } + } + # No HTTP status: network/transport-class exceptions (by TYPE, not text). + if ($ExceptionType) { + $net = @('HttpRequestException', 'WebException', 'SocketException', 'TaskCanceledException', 'TimeoutException', 'IOException') + foreach ($n in $net) { if ($ExceptionType -like "*$n*") { return 'Network' } } + return 'Other' + } + return 'None' +} + +function New-GraphAuditLifecycleState { + <# + .SYNOPSIS + Factory for the additive per-query structured lifecycle fields. Ordered so + checkpoint serialization is stable. + #> + param( + [string]$QueryApiVersion = $null, + [string]$LifecycleOperation = $null + ) + return [ordered]@{ + QueryApiVersion = $QueryApiVersion + LifecycleOperation = $LifecycleOperation + FailureCategory = 'None' + AlternateVersionChecked = $false + AlternateVersionResult = $null # 'Visible' | 'NotVisible' | $null + QueryConfirmedAbsent = $false + ReplacementAttempted = $false + ReplacementQueryId = $null + LastLifecycleHttpStatus = $null + LifecycleRetryCount = 0 + ZeroRecordComplete = $false + # Durable uncertain-create state. UncertainCreate is a boolean flag; when true the + # partition has an unproven (possibly-accepted) first create POST and must only ever + # be resolved by READ-ONLY exact-name reconciliation, never another POST. + # UncertainCreateInfo is a structured sub-object (Fingerprint, DisplayName, PStart, + # PEnd, Activities, RecordTypes, ServiceFilter, FirstSeenUtc, AttemptMarker) used to + # rebuild the exact fingerprinted name for reconciliation across attempts/resume. + UncertainCreate = $false + UncertainCreateInfo = $null + } +} + +function Set-GraphAuditLifecycleFields { + <# + .SYNOPSIS + Ensures the lifecycle fields exist on a partitionStatus/checkpoint entry + WITHOUT discarding existing values. + .DESCRIPTION + Legacy-safe: a missing field is added with its default; an existing field + (including a stored QueryId elsewhere on the entry) is NEVER overwritten or + cleared. Loading a legacy checkpoint that lacks these fields therefore can + never discard or recreate a stored QueryId. + #> + param([Parameter(Mandatory)] $Entry) + if ($null -eq $Entry) { return $Entry } + $defaults = New-GraphAuditLifecycleState + foreach ($k in $defaults.Keys) { + $has = $false + if ($Entry -is [hashtable]) { $has = $Entry.ContainsKey($k) } + else { $has = ($null -ne $Entry.PSObject.Properties[$k]) } + if (-not $has) { + if ($Entry -is [hashtable]) { $Entry[$k] = $defaults[$k] } + else { Add-Member -InputObject $Entry -NotePropertyName $k -NotePropertyValue $defaults[$k] -Force } + } + } + return $Entry +} + +function Merge-GraphAuditLifecycleCategory { + <# + .SYNOPSIS + Reconciliation preservation: a generic parent result never overwrites a + more specific worker-established category. Returns the category to keep. + #> + param([string]$Existing, [string]$Incoming) + if ([string]::IsNullOrWhiteSpace($Existing)) { return $Incoming } + if (($script:GraphAuditSpecificFailureCategories -contains $Existing) -and + (-not ($script:GraphAuditSpecificFailureCategories -contains $Incoming))) { + return $Existing + } + return $Incoming +} + +function Get-GraphAuditReasonCategory { + <# + .SYNOPSIS + Maps a coarse parent Reason token to the bounded category vocabulary. + .DESCRIPTION + Generic runtime reasons (JOB_FAILED / ERROR_STREAM / NO_DATA / + INVALID_COMPLETE) collapse to 'Other' (non-specific) so they cannot mask a + worker-established specific category. Driven by the structured token only. + #> + param([string]$Reason) + switch -Regex ($Reason) { + '^401$|^403$' { 'Authentication'; break } + '^429$' { 'Throttled'; break } + '^QUERY-?GONE$|^QueryNotVisible$' { 'QueryNotVisible'; break } + '^QueryConfirmedAbsent$' { 'QueryConfirmedAbsent'; break } + '^Cleanup$' { 'Cleanup'; break } + '^Network$' { 'Network'; break } + '^Service$|^5\d\d$' { 'Service'; break } + default { 'Other' } + } +} + +function Invoke-GraphAuditLifecycleOperation { + <# + .SYNOPSIS + Shared decision layer for ONE lifecycle operation (Status/Records/Delete). + .DESCRIPTION + Routes a single operation through: (1) call on the effective version; + (2) on 404, bounded eventual-consistency retry on the SAME version; + (3) then a same-ID GET on the alternate version (never a POST); (4) if the + alternate GET succeeds, retain the id, record the result, switch the + effective QueryApiVersion, and continue. 401/403/429/network/5xx keep their + existing meaning with NO version fallback. Status and records are decided + INDEPENDENTLY. The -Invoke callback isolates the HTTP transport so the same + function is exercised by injected tests and by production callers. + .PARAMETER Invoke + scriptblock param($Version,$QueryId,$Operation) -> + @{ HttpStatus=; Data=; ExceptionType= } + #> + param( + [Parameter(Mandatory)][ValidateSet('Status', 'Records', 'Delete')] [string]$Operation, + [Parameter(Mandatory)][string]$QueryId, + [Parameter(Mandatory)][scriptblock]$Invoke, + [string]$EffectiveVersion = $null, + [string]$AlternateVersion = $null, + [Parameter(Mandatory)] $State, + [int]$MaxConsistencyRetries = 3, + [bool]$EnableAlternateVersion = $true + ) + if ([string]::IsNullOrWhiteSpace($EffectiveVersion)) { $EffectiveVersion = $script:GraphAuditApiVersion_Previous } # legacy/unknown => beta + if ([string]::IsNullOrWhiteSpace($AlternateVersion)) { + $AlternateVersion = if ($EffectiveVersion -eq $script:GraphAuditApiVersion_Previous) { $script:GraphAuditApiVersion_Current } else { $script:GraphAuditApiVersion_Previous } + } + $State.LifecycleOperation = $Operation + $ver = $EffectiveVersion + $attempt = 0 + while ($true) { + $r = & $Invoke $ver $QueryId $Operation + $status = if ($r) { $r.HttpStatus } else { $null } + $exType = if ($r) { $r.ExceptionType } else { $null } + $State.LastLifecycleHttpStatus = $status + # Success. + if ($null -ne $status -and $status -ge 200 -and $status -le 299) { + $State.QueryApiVersion = $ver + $State.FailureCategory = 'None' + $data = if ($r) { $r.Data } else { $null } + if ($Operation -eq 'Records') { + $count = 0 + if ($data -and $data.value) { $count = @($data.value).Count } + if ($count -eq 0) { $State.ZeroRecordComplete = $true } + } + return @{ Success = $true; HttpStatus = $status; Version = $ver; Category = 'None'; Data = $data; State = $State } + } + # 404 => bounded eventual-consistency retry on the SAME version first. + if ($status -eq 404) { + if ($attempt -lt $MaxConsistencyRetries) { + $attempt++ + $State.LifecycleRetryCount = [int]$State.LifecycleRetryCount + 1 + continue + } + # Delete: a 404 after retries means already-gone => successful cleanup. + if ($Operation -eq 'Delete') { + $State.QueryApiVersion = $ver + $State.FailureCategory = 'None' + return @{ Success = $true; HttpStatus = 404; Version = $ver; Category = 'None'; Data = $null; State = $State } + } + # Same-ID GET on the alternate version (never a POST). + if ($EnableAlternateVersion -and $AlternateVersion -and $AlternateVersion -ne $ver) { + $State.AlternateVersionChecked = $true + $ra = & $Invoke $AlternateVersion $QueryId $Operation + $as = if ($ra) { $ra.HttpStatus } else { $null } + if ($null -ne $as -and $as -ge 200 -and $as -le 299) { + $State.AlternateVersionResult = 'Visible' + $State.QueryApiVersion = $AlternateVersion # switch effective version for subsequent ops + $State.FailureCategory = 'None' + $State.LastLifecycleHttpStatus = $as + $adata = if ($ra) { $ra.Data } else { $null } + if ($Operation -eq 'Records') { + $count = 0; if ($adata -and $adata.value) { $count = @($adata.value).Count } + if ($count -eq 0) { $State.ZeroRecordComplete = $true } + } + return @{ Success = $true; HttpStatus = $as; Version = $AlternateVersion; Category = 'None'; Data = $adata; State = $State } + } + $State.AlternateVersionResult = 'NotVisible' + } + # Both versions 404 (or alternate disabled): NOT visible. Absence is NOT + # confirmed here — confirmation additionally requires a fully-paginated + # exact-name miss (see Resolve-GraphAuditDuplicateSafeQuery). + $State.FailureCategory = 'QueryNotVisible' + return @{ Success = $false; HttpStatus = 404; Version = $ver; Category = 'QueryNotVisible'; Data = $null; State = $State } + } + # Non-404 failures: classify and NEVER version-fallback. + $cat = Get-GraphAuditFailureCategory -HttpStatus $status -ExceptionType $exType -Operation $Operation + $State.FailureCategory = $cat + return @{ Success = $false; HttpStatus = $status; Version = $ver; Category = $cat; Data = $null; State = $State } + } +} + +function Resolve-GraphAuditDuplicateSafeQuery { + <# + .SYNOPSIS + Duplicate-safe creation/replacement around a deterministic exact partition + displayName. + .DESCRIPTION + Rules: fully-paginated exact-name lookup (ordinal exact match) BEFORE any + recovery POST; exactly one match => adopt (no POST); multiple exact matches + => fail closed (no POST); confirmed absence requires both item versions 404 + for the same id AND an exact-name miss; at most ONE replacement POST, only + after absence is confirmed and persisted ReplacementAttempted is false + (persisted BEFORE the POST so a resume cannot double-create); an ambiguous + POST error is reconciled by exact-name lookup, never a blind re-POST. A + partition with no prior id may create its initial beta query but still uses + the exact-name + ambiguous-POST guard. + #> + param( + [Parameter(Mandatory)][string]$DisplayName, + [string]$ExistingQueryId = $null, + [Parameter(Mandatory)] $State, + [Parameter(Mandatory)][scriptblock]$ListQueries, # -> array of @{ id; displayName } (fully paginated) + [scriptblock]$CheckItem = $null, # param($Version,$QueryId) -> @{ HttpStatus } + [scriptblock]$CreateQuery = $null, # param($Version,$DisplayName) -> @{ HttpStatus; QueryId; Ambiguous } + [scriptblock]$PersistReplacementIntent = $null, # param($State) -> durably persist ReplacementAttempted; fired ONCE, immediately before the sole replacement POST + [string]$EffectiveVersion = $null, + [string]$AlternateVersion = $null, + [int]$MaxConsistencyRetries = 3 + ) + if ([string]::IsNullOrWhiteSpace($EffectiveVersion)) { $EffectiveVersion = $script:GraphAuditApiVersion_Previous } + if ([string]::IsNullOrWhiteSpace($AlternateVersion)) { + $AlternateVersion = if ($EffectiveVersion -eq $script:GraphAuditApiVersion_Previous) { $script:GraphAuditApiVersion_Current } else { $script:GraphAuditApiVersion_Previous } + } + $exactMatch = { + param($items) + @($items | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $DisplayName, [System.StringComparison]::Ordinal) }) + } + # 1) Fully-paginated exact-name lookup (established ordinal case semantics). + $all = @(& $ListQueries) + $matches = @(& $exactMatch $all) + if ($matches.Count -gt 1) { + # Multiple exact matches => fail closed. Never choose arbitrarily; never POST. + $State.FailureCategory = 'Other' + return @{ Action = 'FailClosed'; Reason = 'MultipleExactMatches'; MatchCount = $matches.Count; QueryId = $null } + } + if ($matches.Count -eq 1) { + # Exactly one existing exact match => adopt; no POST. + return @{ Action = 'Adopt'; Reason = 'ExactNameMatch'; QueryId = [string]$matches[0].id } + } + # Zero exact-name matches. + if (-not [string]::IsNullOrWhiteSpace($ExistingQueryId)) { + # Replacement path: only after positively-confirmed absence. + $absent = $true + if ($CheckItem) { + foreach ($v in @($EffectiveVersion, $AlternateVersion | Select-Object -Unique)) { + if ([string]::IsNullOrWhiteSpace($v)) { continue } + for ($i = 0; $i -le $MaxConsistencyRetries; $i++) { + $ci = & $CheckItem $v $ExistingQueryId + $cs = if ($ci) { $ci.HttpStatus } else { $null } + if ($null -ne $cs -and $cs -ge 200 -and $cs -le 299) { $absent = $false; break } + if ($cs -eq 404) { if ($i -ge $MaxConsistencyRetries) { break } else { continue } } + # Non-404, non-2xx (auth/throttle/5xx/network): cannot confirm absence. + $absent = $false; break + } + if (-not $absent) { break } + } + } + if (-not $absent) { + # The id is still visible somewhere => retain it, never replace. + $State.AlternateVersionChecked = $true + $State.AlternateVersionResult = 'Visible' + return @{ Action = 'Adopt'; Reason = 'StillVisible'; QueryId = $ExistingQueryId } + } + # Positively confirmed absent: both-version 404 + exact-name miss. + $State.QueryConfirmedAbsent = $true + if ($State.ReplacementAttempted) { + # At-most-once: a replacement was already persisted; never create again. + return @{ Action = 'NoReplacement'; Reason = 'ReplacementAlreadyAttempted'; QueryId = $null } + } + # Persist ReplacementAttempted BEFORE creating, so a crash/resume cannot double-create. + # Durable persistence is delegated to the caller-supplied intent callback and runs + # ONLY here: after confirmed absence + exact-name miss, immediately before the sole + # replacement POST (never for an Adopt/FailClosed/StillVisible outcome that POSTs nothing). + $State.ReplacementAttempted = $true + if ($PersistReplacementIntent) { & $PersistReplacementIntent $State } + if ($CreateQuery) { + $cr = & $CreateQuery $EffectiveVersion $DisplayName + if ($cr -and $cr.Ambiguous) { + $again = @(& $exactMatch (@(& $ListQueries))) + if ($again.Count -eq 1) { return @{ Action = 'Adopt'; Reason = 'AmbiguousResolvedByName'; QueryId = [string]$again[0].id } } + if ($again.Count -gt 1) { $State.FailureCategory = 'Other'; return @{ Action = 'FailClosed'; Reason = 'MultipleExactMatches'; MatchCount = $again.Count; QueryId = $null } } + return @{ Action = 'NoReplacement'; Reason = 'AmbiguousUnresolved'; QueryId = $null } + } + $newId = if ($cr) { [string]$cr.QueryId } else { $null } + $State.ReplacementQueryId = $newId + return @{ Action = 'Replaced'; Reason = 'ConfirmedAbsent'; QueryId = $newId } + } + return @{ Action = 'NoReplacement'; Reason = 'NoCreateCallback'; QueryId = $null } + } + # No existing id and no exact-name match => initial creation (beta). Still guard + # against duplicates via exact-name lookup + ambiguous-POST reconciliation. + if ($CreateQuery) { + $cr = & $CreateQuery $EffectiveVersion $DisplayName + if ($cr -and $cr.Ambiguous) { + $again = @(& $exactMatch (@(& $ListQueries))) + if ($again.Count -eq 1) { return @{ Action = 'Adopt'; Reason = 'AmbiguousResolvedByName'; QueryId = [string]$again[0].id } } + if ($again.Count -gt 1) { $State.FailureCategory = 'Other'; return @{ Action = 'FailClosed'; Reason = 'MultipleExactMatches'; MatchCount = $again.Count; QueryId = $null } } + return @{ Action = 'NoReplacement'; Reason = 'AmbiguousUnresolved'; QueryId = $null } + } + $newId = if ($cr) { [string]$cr.QueryId } else { $null } + return @{ Action = 'Created'; Reason = 'InitialCreate'; QueryId = $newId } + } + return @{ Action = 'NoReplacement'; Reason = 'NoCreateCallback'; QueryId = $null } +} + +function New-GraphAuditQueryFingerprint { + <# + .SYNOPSIS + Deterministic short contract fingerprint for a partition audit query. + .DESCRIPTION + Derives a stable hex fingerprint from the CANONICAL creation contract - the + UTC start/end window plus the operation, record-type, and service filters - + so two partitions that share a date window and partition index but differ in + ANY filter yield DIFFERENT fingerprints (and therefore different display + names). Each string array is normalized identically before hashing (null and + blank dropped, trimmed, invariant upper-cased, ordinal-sorted, joined with a + fixed unit separator); the two dates are rendered in invariant UTC; the parts + are concatenated in a fixed order with a fixed record separator; the whole is + hashed with SHA256 and the first twelve lower-case hex characters are returned. + The same contract always yields the same fingerprint regardless of input + order, element casing, or surrounding whitespace, so initial create, + replacement, checkpoint naming, and exact-name reconciliation all derive one + identity from the same producer. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + $Activities, + $RecordTypes, + $ServiceFilter + ) + $normArray = { + param($arr) + if ($null -eq $arr) { return '' } + $vals = @(@($arr) | ForEach-Object { if ($null -eq $_) { '' } else { ([string]$_).Trim().ToUpperInvariant() } } | Where-Object { $_ -ne '' }) + if ($vals.Count -eq 0) { return '' } + $arrSorted = [string[]]$vals + [System.Array]::Sort($arrSorted, [System.StringComparer]::Ordinal) + return ($arrSorted -join ([char]0x1F)) + } + $svc = if ($null -eq $ServiceFilter) { '' } else { ([string]$ServiceFilter).Trim().ToUpperInvariant() } + $parts = @( + $PStart.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture), + $PEnd.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture), + (& $normArray $Activities), + (& $normArray $RecordTypes), + $svc + ) + $canonical = $parts -join ([char]0x1E) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $hash = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($canonical)) + } + finally { $sha.Dispose() } + $hex = -join ($hash | ForEach-Object { $_.ToString('x2') }) + return $hex.Substring(0, 12) +} + +function Get-PaxCheckpointEntryFingerprint { + <# + .SYNOPSIS + Returns the durable contract fingerprint carried by a checkpoint bucket entry, or + $null when the entry predates fingerprint identity (a legacy checkpoint). + .DESCRIPTION + A fingerprinted entry stores the stable New-GraphAuditQueryFingerprint value in a + top-level `fingerprint` field; durable uncertain-create entries additionally carry it + inside their structured UncertainCreateInfo. This reader is tolerant of both the + in-memory hashtable shape and the ConvertFrom-Json object shape and NEVER throws on a + missing field (a legacy entry yields $null so callers fall back to the historical + index-only identity). + #> + param($Entry) + if ($null -eq $Entry) { return $null } + $fp = $Entry.fingerprint + if (-not [string]::IsNullOrWhiteSpace([string]$fp)) { return [string]$fp } + $uci = $Entry.UncertainCreateInfo + if ($uci) { + $ufp = $uci.fingerprint + if (-not [string]::IsNullOrWhiteSpace([string]$ufp)) { return [string]$ufp } + } + return $null +} + +function Get-PaxEntryIdentityKey { + <# + .SYNOPSIS + Bucket-exclusivity identity key for a checkpoint entry: its fingerprint when present + (the durable per-contract identity), otherwise its integer index (legacy fallback). + #> + param($Entry) + $fp = Get-PaxCheckpointEntryFingerprint $Entry + if (-not [string]::IsNullOrWhiteSpace($fp)) { return "fp:$fp" } + if ($null -ne $Entry -and $null -ne $Entry.index) { return "idx:$([int]$Entry.index)" } + return $null +} + +function Test-PaxCheckpointIdentityMatch { + <# + .SYNOPSIS + $true when a checkpoint bucket entry denotes the SAME durable partition identity as + the supplied (Index, Fingerprint). + .DESCRIPTION + Fingerprint is the authoritative identity: when BOTH the entry and the query carry a + non-empty fingerprint, ONLY equal fingerprints match, so two workload-pass contracts + that reuse the same partition index remain distinct identities and never collide. + Defect 2 (fail-closed identity): fingerprint is the SOLE durable identity in the + resume-classification and persistence paths. A legacy (fingerprint-less) checkpoint + entry is migrated to a proven-unique fingerprint by the pre-auth legacy-mapping + resolver (Invoke-PaxResumeLegacyContractMapping) BEFORE any partition is classified + or persisted, so an un-migrated legacy entry - or an identity-less caller - reaching + this matcher is NEVER silently index-matched onto a fingerprinted contract. Index-only + "matching" is exactly the cross-workload-pass collision this defect closes, so when a + fingerprint is not available on both sides this matcher fails closed (returns $false). + #> + param($Entry, [int]$Index, [string]$Fingerprint) + if ($null -eq $Entry) { return $false } + $efp = Get-PaxCheckpointEntryFingerprint $Entry + if (-not [string]::IsNullOrWhiteSpace($Fingerprint) -and -not [string]::IsNullOrWhiteSpace($efp)) { + return ($efp -eq [string]$Fingerprint) + } + # Tightened (defect 2, Option 1): no permissive index fallback. Legacy entries are + # migrated to a durable fingerprint pre-auth; an un-migrated legacy entry (or a caller + # with no contract fingerprint in hand) must not be silently index-matched onto a + # fingerprinted contract. Fail closed. + return $false +} + +function Test-PaxLegacyContractCandidate { + <# + .SYNOPSIS + $true when a current partition contract is a candidate match for a legacy + (fingerprint-less) checkpoint bucket entry, under the read-only pre-auth + legacy-mapping resolver. + .DESCRIPTION + A legacy entry identifies its partition only by integer index plus its saved + start/end window (and, when present, its stored contract fields). This tester matches + a reconstructed current contract to that entry by: + * integer partition index (when the entry carries one), AND + * exact UTC-instant equality of the start and end window, AND + * every stored contract field the entry actually carries (serviceFilter/service, + recordTypes) - a field absent from the legacy entry is not used to reject. + It NEVER computes or trusts a fingerprint; disambiguation among several contracts that + share a window/index is deliberately impossible here, so such a legacy entry yields + more than one candidate and the resolver fails closed rather than guessing. + #> + param($Entry, $Index, $Start, $End, $Contract) + if ($null -eq $Contract) { return $false } + if ($null -ne $Index -and $null -ne $Contract.Index -and ([int]$Contract.Index -ne [int]$Index)) { return $false } + if ($Start -and $Contract.PStart) { + if (([datetime]$Contract.PStart).ToUniversalTime() -ne ([datetime]$Start).ToUniversalTime()) { return $false } + } + if ($End -and $Contract.PEnd) { + if (([datetime]$Contract.PEnd).ToUniversalTime() -ne ([datetime]$End).ToUniversalTime()) { return $false } + } + $eSvc = $Entry.serviceFilter + if ($null -eq $eSvc) { $eSvc = $Entry.service } + if ($null -ne $eSvc -and -not [string]::IsNullOrWhiteSpace([string]$eSvc)) { + if ([string]$eSvc -ne [string]$Contract.ServiceFilter) { return $false } + } + $eRt = $Entry.recordTypes + if ($null -ne $eRt) { + $eRtJoin = (@($eRt) | Sort-Object) -join '|' + $cRtJoin = (@($Contract.RecordTypes) | Sort-Object) -join '|' + if ($eRtJoin -ne $cRtJoin) { return $false } + } + return $true +} + +function Test-PaxCheckpointEntryCompatible { + <# + .SYNOPSIS + $true when two checkpoint bucket entries that claim the SAME durable contract identity + carry PROVABLY equivalent stored metadata, so one may supersede the other inside a + single atomic migration. $false whenever equivalence cannot be proven. + .DESCRIPTION + Bucket exclusivity (completed > queryCreated > uncertainCreate) may only be enforced by + REMOVING a lower-precedence duplicate when that duplicate is provably the SAME work. A + shared fingerprint alone is NOT that evidence: two entries can share a contract + fingerprint yet carry different server query ids, indexes, windows, or filters, which + means the checkpoint records genuinely different state and silently discarding either + would lose it. + + Every field is compared ONLY when BOTH entries carry a usable value - an absent or + blank field on a legacy entry is not evidence of a conflict. Any field present on both + sides that DIFFERS (server query id, partition index, start / end instant, record + types, or service filter) is a CONFLICT and yields $false so the caller fails closed + instead of guessing. A start / end value present on both sides that cannot be parsed on + either side is likewise unproven, and therefore a conflict - never a match. + #> + param($A, $B) + if ($null -eq $A -or $null -eq $B) { return $false } + $looseEq = { + param($x, $y) + if ($null -eq $x -or $null -eq $y) { return $true } + $xs = ([string]$x).Trim(); $ys = ([string]$y).Trim() + if ($xs -eq '' -or $ys -eq '') { return $true } + return ([string]::Equals($xs, $ys, [System.StringComparison]::OrdinalIgnoreCase)) + } + if (-not (& $looseEq $A.queryId $B.queryId)) { return $false } + if ($null -ne $A.index -and $null -ne $B.index) { + if ([int]$A.index -ne [int]$B.index) { return $false } + } + foreach ($fld in @('start', 'end')) { + $av = $A.$fld + $bv = $B.$fld + if ([string]::IsNullOrWhiteSpace([string]$av) -or [string]::IsNullOrWhiteSpace([string]$bv)) { continue } + $ad = script:Parse-DateSafe $av + $bd = script:Parse-DateSafe $bv + if ($null -eq $ad -or $null -eq $bd) { return $false } + if (([datetime]$ad).ToUniversalTime() -ne ([datetime]$bd).ToUniversalTime()) { return $false } + } + if ($null -ne $A.recordTypes -and $null -ne $B.recordTypes) { + $aRt = (@($A.recordTypes) | Sort-Object) -join '|' + $bRt = (@($B.recordTypes) | Sort-Object) -join '|' + if ($aRt -ne $bRt) { return $false } + } + $aSvc = if ($null -ne $A.serviceFilter) { $A.serviceFilter } else { $A.service } + $bSvc = if ($null -ne $B.serviceFilter) { $B.serviceFilter } else { $B.service } + if (-not (& $looseEq $aSvc $bSvc)) { return $false } + return $true +} + +function Get-PaxResumeCurrentContractSet { + <# + .SYNOPSIS + READ-ONLY reconstruction of the COMPLETE set of current partition contracts across + ALL serviceRuns x query groups, from restored checkpoint parameters. No Graph call, + no authentication, no mutation. + .DESCRIPTION + Deterministically replays the main partition build (serviceRuns split, per-workload + operation/recordType alignment, partition-hours defaulting + MaxPartitions capping, + Get-QueryPlan grouping, and per-group time slicing) so a legacy (fingerprint-less) + checkpoint entry can be matched to exactly one durable contract BEFORE authentication. + It reproduces the single-partition variable-carry quirk of the main build byte-for-byte: + $partitionRecordTypes / $partitionServiceFilter are assigned ONLY inside the + multi-partition branch and otherwise carry across groups and service passes, so a + single-partition group inherits the prior multi-partition group's record/service + values exactly as the live build does. The normal per-group build remains the sole + producer of the contracts actually processed; this helper only ENUMERATES them. + #> + param( + [datetime]$StartDateObj, + [datetime]$EndDateObj, + [string[]]$ActivityTypes, + [string[]]$RecordTypes, + [string[]]$ServiceTypes, + [int]$PartitionHours = 0, + [int]$MaxPartitions = 50, + [int]$MaxConcurrency = 10, + [switch]$IncludeM365Usage, + [switch]$UseEOM, + [hashtable]$ServiceOperationMap, + [hashtable]$RecordTypeWorkloadMap + ) + $contracts = New-Object System.Collections.ArrayList + if (-not $UseEOM -and $ServiceTypes -and $ServiceTypes.Count -gt 0) { $serviceRuns = $ServiceTypes } else { $serviceRuns = @($null) } + # Variable-carry quirk (mirrors main build): declared outside both loops and assigned + # ONLY in the multi-partition branch, so a single-partition group carries the prior value. + $partitionRecordTypes = $null + $partitionServiceFilter = $null + foreach ($currentServiceFilter in $serviceRuns) { + $serviceActivities = $ActivityTypes + if ($currentServiceFilter -and $ServiceOperationMap -and $ServiceOperationMap.ContainsKey($currentServiceFilter)) { + $mappedOps = @($ServiceOperationMap[$currentServiceFilter] | Where-Object { $ActivityTypes -contains $_ }) + if ($mappedOps.Count -gt 0) { $serviceActivities = $mappedOps } + } + $serviceRecordTypes = $RecordTypes + if ($currentServiceFilter -and $RecordTypes) { + $matched = @() + foreach ($rt in $RecordTypes) { + $rtServices = $RecordTypeWorkloadMap[$rt] + if ($null -eq $rtServices -or $rtServices.Count -eq 0) { $matched += $rt; continue } + if ($rtServices -contains $currentServiceFilter) { $matched += $rt } + } + if ($matched.Count -gt 0) { $serviceRecordTypes = $matched | Select-Object -Unique } else { $serviceRecordTypes = $null } + } + $queryPlan = @(Get-QueryPlan -RequestedActivities $serviceActivities) + foreach ($grp in $queryPlan) { + $effectivePartitionHours = $PartitionHours + $userSpecifiedPartitionHours = ($PartitionHours -gt 0) + if (-not $UseEOM -and $effectivePartitionHours -eq 0) { $effectivePartitionHours = 12 } + if (-not $UseEOM -and $effectivePartitionHours -gt 0) { + $totalHours = ($EndDateObj - $StartDateObj).TotalHours + $calculatedPartitions = [Math]::Ceiling($totalHours / $effectivePartitionHours) + if ($calculatedPartitions -gt $MaxPartitions) { + if (-not $userSpecifiedPartitionHours) { + $effectivePartitionHours = 12 + $calculatedPartitions = [Math]::Ceiling($totalHours / $effectivePartitionHours) + } + if ($calculatedPartitions -gt $MaxPartitions) { + $calculatedPartitions = $MaxPartitions + $effectivePartitionHours = $totalHours / $calculatedPartitions + } + } + $degree = $calculatedPartitions + } else { + $degree = [Math]::Min($grp.Concurrency, $MaxConcurrency) + } + $totalPartitions = $degree + $activities = $grp.Activities + $activity = $grp.Activities[0] + if ($totalPartitions -gt 1) { + $totalHours = ($EndDateObj - $StartDateObj).TotalHours + if (-not $UseEOM -and $effectivePartitionHours -gt 0) { $sliceHours = $effectivePartitionHours } else { $sliceHours = $totalHours / $totalPartitions } + $partitionRecordTypes = $serviceRecordTypes + $partitionServiceFilter = $currentServiceFilter + if ($IncludeM365Usage) { $partitionRecordTypes = $null; $partitionServiceFilter = $null } + for ($pi = 0; $pi -lt $totalPartitions; $pi++) { + $pStart = $StartDateObj.AddHours($sliceHours * $pi) + $pEnd = if ($pi -eq ($totalPartitions - 1)) { $EndDateObj } else { $StartDateObj.AddHours($sliceHours * ($pi + 1)) } + [void]$contracts.Add([pscustomobject]@{ + Activities = $activities + Activity = $activity + PStart = $pStart + PEnd = $pEnd + Index = ($pi + 1) + Total = $totalPartitions + RecordTypes = $partitionRecordTypes + ServiceFilter = $partitionServiceFilter + }) + } + } else { + [void]$contracts.Add([pscustomobject]@{ + Activities = $activities + Activity = $activity + PStart = $StartDateObj + PEnd = $EndDateObj + Index = 1 + Total = 1 + RecordTypes = $partitionRecordTypes + ServiceFilter = $partitionServiceFilter + }) + } + } + } + return @($contracts.ToArray()) +} + +function Invoke-PaxResumeLegacyContractMapping { + <# + .SYNOPSIS + Pre-auth, read-only cross-run legacy-mapping resolver (defect 2). Migrates every + legacy (fingerprint-less) checkpoint bucket entry to a proven-unique durable + fingerprint, or fails closed WITHOUT authentication or mutation. + .DESCRIPTION + Runs on resume BEFORE authentication and the per-workload partition build. For each + legacy entry (completed / queryCreated / uncertainCreate WITHOUT a fingerprint) it + finds candidate current contracts (Test-PaxLegacyContractCandidate) from the complete + reconstructed set. EXACTLY ONE candidate stages a unique mapping (the contract's + New-GraphAuditQueryFingerprint); ZERO fails closed (never discard/recreate); MORE THAN + ONE fails closed as LegacyContractAmbiguous. + + BUCKET EXCLUSIVITY. A contract identity may occupy EXACTLY ONE lifecycle bucket, under + the precedence completed > queryCreated > uncertainCreate. Every candidate fingerprint + is therefore checked against ALL THREE buckets, not only its source bucket. A collision + is resolved only with EXPLICIT evidence (Test-PaxCheckpointEntryCompatible proves the + stored query id / index / window / record types / service filter are equivalent): a + legacy row already owned by a HIGHER-precedence entry is classified superseded and + removed as part of this same atomic migration, a LOWER-precedence duplicate of a + migrated row is normalized in the same transaction, a same-bucket collision stays + LegacyContractAmbiguous, and any CONFLICTING metadata fails closed as + LegacyContractConflict rather than guessing. Residual cross-bucket duplicates among + untouched identities are normalized under the same rule, so the resume classifier never + depends on a normalization that happens later. + + The ENTIRE migration + normalization plan is validated first; only on full success are + fingerprints stamped in-memory and published ATOMICALLY and EXACTLY ONCE (via the + supplied -Publish block, default Save-CheckpointStrict) BEFORE any fetch/reconcile/ + create. Publication must be PROVEN durable - an explicit $true, or a throw on failure; + any other outcome restores the exact pre-migration in-memory state (bucket membership + AND every fingerprint stamp), leaves the durable checkpoint byte-for-byte unchanged, + latches $script:GenericFatal, and throws PAX-FATAL. On any zero/ambiguous/collision/ + conflict outcome it likewise latches $script:GenericFatal and + throws a PAX-FATAL error so the caller can emit actionable output and exit nonzero with + no auth and no mutation. Returns a status object on success or when there is nothing to + migrate. + #> + param( + [Parameter(Mandatory = $true)]$CheckpointData, + [object[]]$CurrentContracts = @(), + [scriptblock]$Publish + ) + if ($null -eq $CheckpointData -or $null -eq $CheckpointData.partitions) { + return [pscustomobject]@{ Status = 'NoLegacy'; Migrated = 0; Reason = 'no checkpoint partitions present' } + } + $buckets = @('completed', 'queryCreated', 'uncertainCreate') + # Lifecycle precedence: a single contract identity may occupy EXACTLY ONE bucket, and a + # higher-precedence record always supersedes a lower-precedence one. + $rankOf = @{ completed = 3; queryCreated = 2; uncertainCreate = 1 } + $legacy = New-Object System.Collections.ArrayList + foreach ($b in $buckets) { + foreach ($e in @($CheckpointData.partitions.$b)) { + if ($null -eq $e) { continue } + $efp = Get-PaxCheckpointEntryFingerprint $e + if ([string]::IsNullOrWhiteSpace($efp)) { [void]$legacy.Add([pscustomobject]@{ Bucket = $b; Entry = $e }) } + } + } + if ($legacy.Count -eq 0) { + return [pscustomobject]@{ Status = 'NoLegacy'; Migrated = 0; Reason = 'all checkpoint entries already carry a durable fingerprint' } + } + # Validate the ENTIRE migration first - no mutation until every legacy entry maps uniquely. + $claimed = @{} + $stage = New-Object System.Collections.ArrayList + foreach ($item in $legacy) { + $e = $item.Entry + $idx = if ($null -ne $e.index) { [int]$e.index } else { $null } + $eStart = if ($e.start) { script:Parse-DateSafe $e.start } else { $null } + $eEnd = if ($e.end) { script:Parse-DateSafe $e.end } else { $null } + $cands = @($CurrentContracts | Where-Object { Test-PaxLegacyContractCandidate -Entry $e -Index $idx -Start $eStart -End $eEnd -Contract $_ }) + $where = "index $idx, window $($e.start)..$($e.end), bucket $($item.Bucket)" + if ($cands.Count -eq 0) { + $script:GenericFatal = $true + throw "PAX-FATAL-LEGACY-NOMATCH: a legacy (fingerprint-less) checkpoint entry ($where) matches NONE of the current partition contracts. Refusing to discard or recreate it. Confirm the resume parameters (date range, activity/record/service filters, partition hours) match the original run, then retry." + } + if ($cands.Count -gt 1) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractAmbiguous: a legacy (fingerprint-less) checkpoint entry ($where) matches $($cands.Count) current partition contracts and cannot be assigned a durable identity safely. This checkpoint predates per-contract fingerprints and spans multiple workload contracts at one index. Re-run the original collection to completion, or narrow -ServiceTypes/-RecordTypes so the entry maps to a single contract." + } + $c = $cands[0] + $fp = New-GraphAuditQueryFingerprint -PStart $c.PStart -PEnd $c.PEnd -Activities $c.Activities -RecordTypes $c.RecordTypes -ServiceFilter $c.ServiceFilter + if ([string]::IsNullOrWhiteSpace($fp)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LEGACY-FP: could not derive a durable fingerprint for the current contract matching a legacy checkpoint entry ($where)." + } + if ($claimed.ContainsKey($fp)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractAmbiguous: two legacy checkpoint entries resolve to the same current contract (fingerprint $fp); the mapping is not one-to-one. Re-run the original collection to completion." + } + # Bucket exclusivity is validated across ALL THREE lifecycle buckets, never only the + # SOURCE bucket: a fingerprint may occupy EXACTLY ONE bucket. Every existing owner of + # this candidate identity is located, its stored metadata is compared for PROVABLE + # equivalence, and the outcome is resolved by precedence - all BEFORE any mutation. + $owners = New-Object System.Collections.ArrayList + foreach ($ob in $buckets) { + foreach ($oe in @($CheckpointData.partitions.$ob)) { + if ($null -eq $oe) { continue } + if ([object]::ReferenceEquals($oe, $e)) { continue } + if ((Get-PaxCheckpointEntryFingerprint $oe) -eq $fp) { + [void]$owners.Add([pscustomobject]@{ Bucket = $ob; Entry = $oe }) + } + } + } + $legacyRank = [int]$rankOf[$item.Bucket] + $supersededLegacy = $false + $demoted = New-Object System.Collections.ArrayList + foreach ($o in $owners) { + if (-not (Test-PaxCheckpointEntryCompatible $e $o.Entry)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractConflict: migrating a legacy checkpoint entry ($where) to fingerprint $fp would collide with an existing '$($o.Bucket)' entry whose stored query id, partition index, window, or contract metadata DIFFERS. The two records are not provably the same work, so neither may be discarded. Re-run the original collection to completion, or narrow -ServiceTypes/-RecordTypes so the entry maps to a single contract." + } + $ownerRank = [int]$rankOf[$o.Bucket] + if ($ownerRank -eq $legacyRank) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractAmbiguous: migrating a legacy checkpoint entry ($where) to fingerprint $fp would collide with an existing durable entry in the same bucket. Re-run the original collection to completion." + } + if ($ownerRank -gt $legacyRank) { $supersededLegacy = $true } + else { [void]$demoted.Add($o) } + } + $claimed[$fp] = $idx + if ($supersededLegacy) { + # A HIGHER-precedence entry already owns this contract and is PROVABLY the same work, + # so the legacy lower-precedence row is superseded. It is removed ONLY as part of the + # atomic migration and is never stamped into a second bucket. + [void]$stage.Add([pscustomobject]@{ Entry = $e; Fingerprint = $fp; Bucket = $item.Bucket; Action = 'Supersede'; Demoted = @() }) + } + else { + # No owner, or only LOWER-precedence owners: stamp the legacy entry and normalize + # every lower-precedence duplicate inside the SAME atomic transaction. + [void]$stage.Add([pscustomobject]@{ Entry = $e; Fingerprint = $fp; Bucket = $item.Bucket; Action = 'Migrate'; Demoted = @($demoted.ToArray()) }) + } + } + # Residual cross-bucket duplicates among identities this migration does not touch are + # resolved under the SAME precedence and the SAME evidence rule, planned BEFORE any + # mutation so a fail-closed outcome still leaves the checkpoint untouched. This is what + # lets Get-PartitionsToProcess classify from an already-exclusive checkpoint instead of + # depending on a normalization that happens later. + $existingByFp = @{} + foreach ($b in $buckets) { + foreach ($xe in @($CheckpointData.partitions.$b)) { + if ($null -eq $xe) { continue } + $xfp = Get-PaxCheckpointEntryFingerprint $xe + if ([string]::IsNullOrWhiteSpace($xfp)) { continue } + if (-not $existingByFp.ContainsKey($xfp)) { $existingByFp[$xfp] = New-Object System.Collections.ArrayList } + [void]$existingByFp[$xfp].Add([pscustomobject]@{ Bucket = $b; Entry = $xe }) + } + } + $residual = New-Object System.Collections.ArrayList + foreach ($k in @($existingByFp.Keys)) { + $grp = @($existingByFp[$k].ToArray()) + if ($grp.Count -le 1) { continue } + if (@($grp | ForEach-Object { $_.Bucket } | Select-Object -Unique).Count -le 1) { continue } + $winner = @($grp | Sort-Object @{ Expression = { [int]$rankOf[$_.Bucket] }; Descending = $true })[0] + foreach ($g in $grp) { + if ([object]::ReferenceEquals($g.Entry, $winner.Entry)) { continue } + if ([int]$rankOf[$g.Bucket] -eq [int]$rankOf[$winner.Bucket]) { continue } + if (-not (Test-PaxCheckpointEntryCompatible $winner.Entry $g.Entry)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractConflict: contract $k is recorded in both the '$($winner.Bucket)' and '$($g.Bucket)' lifecycle buckets with DIFFERING stored query id, partition index, window, or contract metadata, so neither record may be discarded. Re-run the original collection to completion." + } + [void]$residual.Add([pscustomobject]@{ Bucket = $g.Bucket; Entry = $g.Entry }) + } + } + # Full migration + normalization validated. Snapshot every lifecycle bucket and the exact + # set of entries about to be stamped so an unproven publication rolls back to the EXACT + # pre-migration in-memory state, then mutate and publish EXACTLY ONCE. + $snapCompleted = @($CheckpointData.partitions.completed) + $snapQueryCreated = @($CheckpointData.partitions.queryCreated) + $snapUncertain = @($CheckpointData.partitions.uncertainCreate) + $stamped = New-Object System.Collections.ArrayList + $removals = New-Object System.Collections.ArrayList + foreach ($s in $stage) { + if ($s.Action -eq 'Supersede') { + [void]$removals.Add([pscustomobject]@{ Bucket = $s.Bucket; Entry = $s.Entry }) + continue + } + if ($s.Entry -is [System.Collections.IDictionary]) { $s.Entry['fingerprint'] = $s.Fingerprint } + else { $s.Entry | Add-Member -NotePropertyName fingerprint -NotePropertyValue $s.Fingerprint -Force } + [void]$stamped.Add($s.Entry) + foreach ($d in @($s.Demoted)) { if ($null -ne $d) { [void]$removals.Add([pscustomobject]@{ Bucket = $d.Bucket; Entry = $d.Entry }) } } + } + foreach ($r in @($residual.ToArray())) { [void]$removals.Add($r) } + foreach ($b in $buckets) { + $drop = @(@($removals.ToArray()) | Where-Object { $_.Bucket -eq $b } | ForEach-Object { $_.Entry }) + if ($drop.Count -eq 0) { continue } + $CheckpointData.partitions.$b = @(@($CheckpointData.partitions.$b) | Where-Object { + $candidate = $_ + $keep = $true + foreach ($d in $drop) { if ([object]::ReferenceEquals($candidate, $d)) { $keep = $false; break } } + $keep + }) + } + # Publish EXACTLY ONCE. Publication must be PROVEN durable (an explicit $true, or a throw + # on failure); anything else is a failed publication and rolls the checkpoint back. + $published = $false + $publishErr = $null + try { + if ($Publish) { + $publishResult = @(& $Publish) | Select-Object -Last 1 + $published = ($publishResult -is [bool]) -and ([bool]$publishResult) + } + elseif (Get-Command Save-CheckpointStrict -ErrorAction SilentlyContinue) { + $published = [bool](Save-CheckpointStrict) + } + } + catch { $publishErr = $_ } + if (-not $published) { + # Fail closed: restore the EXACT pre-migration in-memory state (bucket membership AND + # every fingerprint stamp). The strict publisher leaves the durable copy byte-for-byte + # unchanged, so memory and disk agree again and no fetch, reconcile, or create is + # authorized on a migration that was never proven durable. + $CheckpointData.partitions.completed = $snapCompleted + $CheckpointData.partitions.queryCreated = $snapQueryCreated + $CheckpointData.partitions.uncertainCreate = $snapUncertain + foreach ($se in @($stamped.ToArray())) { + if ($se -is [System.Collections.IDictionary]) { + if ($se.Contains('fingerprint')) { [void]$se.Remove('fingerprint') } + } + elseif ($null -ne $se.PSObject.Properties['fingerprint']) { + $se.PSObject.Properties.Remove('fingerprint') + } + } + $script:GenericFatal = $true + $publishDetail = if ($publishErr) { [string]$publishErr.Exception.Message } else { 'the publication callback did not return a proven durable publication.' } + if ($publishDetail -match 'PAX-FATAL-ROLLBACK') { + throw "PAX-FATAL-ROLLBACK: durable publication of the legacy contract migration failed AND the prior checkpoint could not be restored; disk state is indeterminate - manual checkpoint inspection required. $publishDetail" + } + throw "PAX-FATAL: durable publication of the legacy contract migration failed; fail-closed, the checkpoint is unchanged and no fetch, reconcile, or create is authorized. $publishDetail" + } + return [pscustomobject]@{ Status = 'Migrated'; Migrated = $stamped.Count; Superseded = @($stage | Where-Object { $_.Action -eq 'Supersede' }).Count; Normalized = @($removals.ToArray()).Count; Reason = 'legacy entries migrated to proven-unique fingerprints, normalized to one lifecycle bucket per contract, and published atomically' } +} + +function New-GraphAuditPartitionDisplayName { + <# + .SYNOPSIS + Single deterministic producer of a partition query displayName. + .DESCRIPTION + Every path that names a partition query (parent create/replacement, + exact-name lookup, checkpoint, and completion/retry display) routes through + this one function so the same partition always yields a byte-identical name. + The end boundary is exclusive, so the name shows the last INCLUDED minute + (PEnd minus one minute), matching the Purview UI and the created query. + + Identity is contract-inclusive: when the creation contract (Activities, + RecordTypes, ServiceFilter) or an explicit -Fingerprint is supplied, a + deterministic fingerprint of that contract is appended so two partitions with + the same date window and partition index but a DIFFERENT operation, record + type, service, or date filter produce different names and cannot cross-adopt. + An explicit -Fingerprint wins; otherwise the fingerprint is derived here from + the supplied contract through the single fingerprint producer. With no contract + and no fingerprint the legacy (fingerprint-less) name is returned unchanged, so + a pre-fingerprint checkpoint name can still be reproduced for legacy recovery. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + [Parameter(Mandatory)]$Index, + [Parameter(Mandatory)]$Total, + $Activities, + $RecordTypes, + $ServiceFilter, + [string]$Fingerprint + ) + $pEndDisplay = $PEnd.AddMinutes(-1) + $base = "PAX_Query_$($PStart.ToString('yyyyMMdd_HHmm'))-$($pEndDisplay.ToString('yyyyMMdd_HHmm'))_Part$Index/$Total" + $fp = $Fingerprint + if ([string]::IsNullOrWhiteSpace($fp) -and ($PSBoundParameters.ContainsKey('Activities') -or $PSBoundParameters.ContainsKey('RecordTypes') -or $PSBoundParameters.ContainsKey('ServiceFilter'))) { + $fp = New-GraphAuditQueryFingerprint -PStart $PStart -PEnd $PEnd -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + } + if (-not [string]::IsNullOrWhiteSpace($fp)) { return "${base}_$fp" } + return $base +} + +function New-GraphAuditQueryRequest { + <# + .SYNOPSIS + Single producer of a partition audit-query request body (filters only). + .DESCRIPTION + Every recovery/replacement create path routes its filter payload through + this one function so an initial create and a replacement create for the same + partition produce EQUIVALENT bodies. Emits filterStartDateTime and + filterEndDateTime in invariant-culture .fffZ, an unconditional + operationFilters array, and the recordTypeFilters / serviceFilter keys gated + exactly as the production create. The displayName is added by the caller (see + New-GraphAuditPartitionDisplayName), so the name and the filters each have a + single source. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + $Activities, + $RecordTypes, + $ServiceFilter + ) + $body = @{ + filterStartDateTime = $PStart.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + filterEndDateTime = $PEnd.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + operationFilters = @($Activities) + } + if ($RecordTypes -and $RecordTypes.Count -gt 0) { $body.recordTypeFilters = @($RecordTypes) } + if ($ServiceFilter) { $body.serviceFilter = $ServiceFilter } + return $body +} + +function Invoke-GraphAuditHardenedCreate { + <# + .SYNOPSIS + Parent-callable hardened creation of a single partition audit query. + .DESCRIPTION + Reproduces the per-partition create robustness in a parent-scope function so + query creation can run in the main runspace (where the token can be + refreshed) before workers are launched. The HTTP transport is injected via + -Invoke so the same logic serves both production (real Invoke-MgGraphRequest / + Invoke-RestMethod) and injected-transport validation. + + The name comes from New-GraphAuditPartitionDisplayName and the filters from + New-GraphAuditQueryRequest, so a parent create and a worker/replacement create + for the same partition are byte-equivalent. Creation always targets the beta + surface; v1.0 creation is disabled. + + Failure handling preserves the established caps exactly: + * transient 403: retry up to 3 times with 15s / 30s / 60s backoff; + * permanent 403 / CAE (InsufficientPrivileges | Authorization_RequestDenied | + AccessDenied | InvalidAuthenticationToken in the body, or a WWW-Authenticate + claims challenge): fast-fail, no retry; + * 429: honour Retry-After when present else exponential 60/120/240 capped at + 300s, abort once the shared attempt counter reaches 20; + * transient network / 5xx: retry inside a time-based outage window + (MaxNetworkOutageMinutes) with a 30-60s randomized delay, then abort; + * an ambiguous create response (name conflict) is reconciled by a + fully-paginated exact-name lookup: exactly one match is adopted, multiple + fail closed, none aborts - never a blind re-POST. + + The injected transport callback receives ($Version, $BodyJson, $ClientRequestId) + and returns @{ HttpStatus; Id; ResponseBody; Headers }. A connectivity failure + is surfaced as HttpStatus 0 (or null) or >= 500. Returns a structured result: + @{ QueryId; FailureCategory; HttpStatus; DisplayName; Version; Adopted }. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + [Parameter(Mandatory)]$Index, + [Parameter(Mandatory)]$Total, + $Activities, + $RecordTypes, + $ServiceFilter, + [Parameter(Mandatory)][scriptblock]$Invoke, # param($Version,$BodyJson,$ClientRequestId) -> @{ HttpStatus; Id; ResponseBody; Headers } + [string]$Version = $null, # creation always beta; v1.0 disabled + [int]$MaxNetworkOutageMinutes = 30, + [scriptblock]$ListQueries = $null, # -> array of @{ id; displayName } (fully paginated), for ambiguous-POST reconcile + [switch]$PreCreateReconcile, # with $ListQueries, run a fully-paginated exact-name lookup BEFORE the first POST (adopt one, fail closed on many, POST only on a miss) + [switch]$ReconcileOnly, # with $ListQueries, resolve a durable uncertain-create partition READ-ONLY: adopt one exact match, fail closed on many, stay unresolved on none - NEVER a POST + [scriptblock]$FetchItem = $null, # param($QueryId) -> the query item with its filter contract (operationFilters/recordTypeFilters/serviceFilter/filterStartDateTime/filterEndDateTime); required to safely adopt a legacy fingerprint-less same-name match + [scriptblock]$Sleep = $null, # param($Seconds) - injectable delay (defaults to Start-Sleep) + [scriptblock]$Log = $null # param($Message) - optional diagnostic sink + ) + if ([string]::IsNullOrWhiteSpace($Version)) { $Version = $script:GraphAuditApiVersion_Previous } + if (-not $Sleep) { $Sleep = { param($s) Start-Sleep -Seconds $s } } + $emit = { param($m) if ($Log) { & $Log $m } } + + # Contract-inclusive identity: the created name, every exact-name reconcile lookup, + # and the POST body all derive from the SAME contract, so a differing operation / + # record-type / service / date contract yields a different name and cannot cross-adopt. + $displayName = New-GraphAuditPartitionDisplayName -PStart $PStart -PEnd $PEnd -Index $Index -Total $Total -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + $bodyObj = New-GraphAuditQueryRequest -PStart $PStart -PEnd $PEnd -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + $bodyObj.displayName = $displayName + $bodyJson = $bodyObj | ConvertTo-Json -Depth 5 + $clientRequestId = [guid]::NewGuid().ToString() + + $new = { + param($queryId, $category, $status, $adopted, $requestId = $null, $responseBody = $null) + return @{ + QueryId = $queryId + FailureCategory = $category + HttpStatus = $status + DisplayName = $displayName + Version = $Version + Adopted = [bool]$adopted + ClientRequestId = $clientRequestId + RequestId = $requestId + ResponseBody = $responseBody + } + } + + # Reconcile an ambiguous create by exact-name lookup: adopt one, fail closed on + # many, abort on none. Never a blind re-POST. + $reconcile = { + param($status) + if (-not $ListQueries) { return & $new $null 'AmbiguousUnresolved' $status $false } + $all = @(& $ListQueries) + $exact = @($all | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $displayName, [System.StringComparison]::Ordinal) }) + if ($exact.Count -eq 1) { return & $new ([string]$exact[0].id) 'None' $status $true } + if ($exact.Count -gt 1) { return & $new $null 'MultipleExactMatches' $status $false } + return & $new $null 'AmbiguousUnresolved' $status $false + } + + # Exact-name lookup shared by the pre-create and uncertain-outcome reconcile + # paths: an ordinal displayName match against the fully-paginated collection. + $findExactByName = { + $all = @(& $ListQueries) + @($all | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $displayName, [System.StringComparison]::Ordinal) }) + } + + # Legacy fingerprint-less recovery. A pre-fingerprint checkpoint created queries + # whose names lack the contract fingerprint, so a query that matches the LEGACY + # (fingerprint-less) name may actually carry a DIFFERENT operation / record-type / + # service / date contract. Such a match is adopted ONLY after its COMPLETE server + # contract is retrieved and compared to the requested contract; a fingerprinted-name + # match already encodes the contract and never reaches this path. The compare reuses + # the deterministic fingerprint producer over the server fields, so it is stable and + # order/case-insensitive in exactly the same way the created name is. + $legacyDisplayName = New-GraphAuditPartitionDisplayName -PStart $PStart -PEnd $PEnd -Index $Index -Total $Total + $requestedFingerprint = New-GraphAuditQueryFingerprint -PStart $PStart -PEnd $PEnd -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + $serverContractFingerprint = { + param($item) + if ($null -eq $item) { return $null } + $fsRaw = if ($item -is [hashtable]) { $item['filterStartDateTime'] } else { $item.filterStartDateTime } + $feRaw = if ($item -is [hashtable]) { $item['filterEndDateTime'] } else { $item.filterEndDateTime } + if ([string]::IsNullOrWhiteSpace([string]$fsRaw) -or [string]::IsNullOrWhiteSpace([string]$feRaw)) { return $null } + $opsRaw = if ($item -is [hashtable]) { $item['operationFilters'] } else { $item.operationFilters } + $rtRaw = if ($item -is [hashtable]) { $item['recordTypeFilters'] } else { $item.recordTypeFilters } + $svcRaw = if ($item -is [hashtable]) { $item['serviceFilter'] } else { $item.serviceFilter } + $styles = [System.Globalization.DateTimeStyles]::AdjustToUniversal -bor [System.Globalization.DateTimeStyles]::AssumeUniversal + $fs = $null; $fe = $null + try { $fs = [datetime]::Parse([string]$fsRaw, [System.Globalization.CultureInfo]::InvariantCulture, $styles) } catch { return $null } + try { $fe = [datetime]::Parse([string]$feRaw, [System.Globalization.CultureInfo]::InvariantCulture, $styles) } catch { return $null } + return New-GraphAuditQueryFingerprint -PStart $fs -PEnd $fe -Activities $opsRaw -RecordTypes $rtRaw -ServiceFilter $svcRaw + } + # Classify a legacy fingerprint-less same-name candidate set against the requested + # contract. A candidate is adopted ONLY after its COMPLETE server contract is + # retrieved and its deterministic fingerprint matches. A candidate whose contract + # cannot be verified (missing filter fields with no fetch, a failed/throwing fetch, + # or a malformed/incomplete fetched item) is never adopted and forces a fail-closed + # outcome so an unverifiable candidate can never be duplicated by a fresh POST. + # Returns @{ Outcome; Id; Category } where Outcome is None | Match | Mismatch | + # Unresolved. + $findLegacyContractMatch = { + if (-not $ListQueries) { return @{ Outcome = 'None'; Id = $null; Category = $null } } + $all = @(& $ListQueries) + $legacy = @($all | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $legacyDisplayName, [System.StringComparison]::Ordinal) }) + if ($legacy.Count -eq 0) { return @{ Outcome = 'None'; Id = $null; Category = $null } } + $matched = New-Object System.Collections.Generic.List[string] + $anyUnverifiable = $false + foreach ($q in $legacy) { + $qid = if ($q -is [hashtable]) { $q['id'] } else { $q.id } + $sfp = & $serverContractFingerprint $q + if (-not $sfp -and $FetchItem) { + # Read-only item fetch with a bounded retry consistent with the create + # retry policy; a fetch that throws or never yields a complete contract + # leaves the candidate unverifiable. + $fetched = $null + $fetchOk = $false + for ($fa = 1; $fa -le 3; $fa++) { + try { $fetched = & $FetchItem ([string]$qid); $fetchOk = $true; break } + catch { $fetched = $null; $fetchOk = $false; if ($fa -lt 3) { & $Sleep 2 } } + } + if ($fetchOk) { $sfp = & $serverContractFingerprint $fetched } + } + if (-not $sfp) { $anyUnverifiable = $true; continue } + if ([string]::Equals([string]$sfp, [string]$requestedFingerprint, [System.StringComparison]::Ordinal)) { $matched.Add([string]$qid) } + } + if ($anyUnverifiable) { return @{ Outcome = 'Unresolved'; Id = $null; Category = 'LegacyContractUnverified' } } + if ($matched.Count -gt 1) { return @{ Outcome = 'Unresolved'; Id = $null; Category = 'MultipleExactMatches' } } + if ($matched.Count -eq 1) { return @{ Outcome = 'Match'; Id = $matched[0]; Category = $null } } + return @{ Outcome = 'Mismatch'; Id = $null; Category = $null } + } + + # Pre-create reconciliation: before the first POST, adopt an already-present + # query by exact name (zero POST), fail closed on multiple exact matches, and + # fall through to the create POST only on an exact-name miss. + if ($PreCreateReconcile -and $ListQueries) { + $preExact = @(& $findExactByName) + if ($preExact.Count -eq 1) { + & $emit "[PRE-RECONCILE] Partition $Index/$Total - adopting existing query by exact name (no POST)" + return & $new ([string]$preExact[0].id) 'None' $null $true + } + if ($preExact.Count -gt 1) { + & $emit "[PRE-RECONCILE] Partition $Index/$Total - multiple exact-name matches - failing closed (no POST)" + return & $new $null 'MultipleExactMatches' $null $false + } + # No fingerprinted-name match: consider a legacy fingerprint-less same-name query, + # adopting ONLY after its full server contract is fetched and matches (no POST on a + # contract mismatch, which falls through to a fresh create). + $preLegacy = & $findLegacyContractMatch + if ($preLegacy.Outcome -eq 'Match') { + & $emit "[PRE-RECONCILE] Partition $Index/$Total - adopting legacy same-name query after contract match (no POST)" + return & $new ([string]$preLegacy.Id) 'None' $null $true + } + if ($preLegacy.Outcome -eq 'Unresolved') { + # A legacy same-name candidate exists but its full contract could not be + # safely verified, or several candidates verify at once; fail closed with zero + # POST so an unverified candidate is never duplicated by a fresh create. + & $emit "[PRE-RECONCILE] Partition $Index/$Total - legacy same-name query could not be safely verified ($($preLegacy.Category)) - failing closed (no POST)" + return & $new $null $preLegacy.Category $null $false + } + # Outcome None (no legacy candidate) or Mismatch (every candidate verified to a + # different contract) falls through to exactly one fingerprinted create POST. + } + + # Read-only reconcile-only mode: NEVER POST. Resolves a partition that carries durable + # uncertain-create state (a possibly-accepted first POST) across a later attempt, + # subdivision, or resume, so the create is never repeated. One exact fingerprinted-name + # match is adopted (real id); multiple fail closed; none stays UncertainCreateUnresolved. + # Never QueryConfirmedAbsent - a collection miss cannot prove the first POST was rejected. + if ($ReconcileOnly) { + if (-not $ListQueries) { return & $new $null 'UncertainCreateUnresolved' $null $false } + $roExact = @(& $findExactByName) + if ($roExact.Count -eq 1) { + & $emit "[RECONCILE-ONLY] Partition $Index/$Total - uncertain create resolved by exact name (no POST)" + return & $new ([string]$roExact[0].id) 'None' $null $true + } + if ($roExact.Count -gt 1) { + & $emit "[RECONCILE-ONLY] Partition $Index/$Total - multiple exact-name matches - failing closed (no POST)" + return & $new $null 'MultipleExactMatches' $null $false + } + & $emit "[RECONCILE-ONLY] Partition $Index/$Total - uncertain create still not visible - remains unresolved (no POST)" + return & $new $null 'UncertainCreateUnresolved' $null $false + } + + $createRetries = 0 + $max403Retries = 3 + $maxNetworkOutageSeconds = $MaxNetworkOutageMinutes * 60 + $networkErrorStart = $null + + while ($true) { + $resp = & $Invoke $Version $bodyJson $clientRequestId + $status = if ($resp) { $resp.HttpStatus } else { $null } + $body = if ($resp) { [string]$resp.ResponseBody } else { $null } + $headers = if ($resp) { $resp.Headers } else { $null } + + # Success. + if ($null -ne $status -and $status -ge 200 -and $status -le 299) { + $newId = if ($resp) { [string]$resp.Id } else { $null } + return & $new $newId 'None' $status $false + } + + # Classify. Network first, then 429, then 403, then ambiguous, then other. + $isNetwork = $false + if ($null -eq $status -or $status -eq 0 -or $status -ge 500) { $isNetwork = $true } + if (-not $isNetwork -and $body -match 'timed out|connection|unable to connect|could not be resolved|Bad Gateway|Service Unavailable|Gateway Timeout|Internal Server Error') { $isNetwork = $true } + + $is429 = (-not $isNetwork) -and ($status -eq 429) + $is403 = (-not $isNetwork) -and (-not $is429) -and ($status -eq 403) + $isAmbiguous = (-not $isNetwork) -and (-not $is429) -and (-not $is403) -and ($status -eq 409) + + if ($is403) { + $wwwAuth = $null + $requestId = $null + if ($headers) { try { $wwwAuth = [string]$headers['WWW-Authenticate'] } catch {} } + if ($headers) { try { $requestId = [string]$headers['request-id'] } catch {} } + $isPermanent = $false + if ($body -match 'InsufficientPrivileges|Authorization_RequestDenied|AccessDenied|InvalidAuthenticationToken') { $isPermanent = $true } + if ($wwwAuth -match 'claims') { $isPermanent = $true } + & $emit "[403-DIAG] Partition $Index/$Total client-request-id: $clientRequestId permanent: $isPermanent" + if ($isPermanent) { + & $emit "[403-PERM] Partition $Index/$Total - permanent 403 (permissions/CAE) - failing" + return & $new $null 'PermanentForbidden' $status $false $requestId $body + } + $createRetries++ + if ($createRetries -le $max403Retries) { + $retryAfter = [Math]::Min(15 * [Math]::Pow(2, $createRetries - 1), 60) + & $emit "[403-CREATE] Partition $Index/$Total - transient 403 (Attempt $createRetries/$max403Retries) - retrying in ${retryAfter}s" + & $Sleep $retryAfter + continue + } + & $emit "[403-PERSISTENT] Partition $Index/$Total - generic 403 persisted after all create retries" + return & $new $null 'PersistentForbidden' $status $false $requestId $body + } + elseif ($is429) { + $createRetries++ + $retryAfter = $null + if ($headers) { try { if ($headers['Retry-After']) { $retryAfter = [int]$headers['Retry-After'] } } catch {} } + if (-not $retryAfter) { $retryAfter = [Math]::Min(60 * [Math]::Pow(2, $createRetries - 1), 300) } + & $emit "[THROTTLE] Partition $Index/$Total - throttled (Attempt $createRetries) - waiting ${retryAfter}s" + & $Sleep $retryAfter + if ($createRetries -ge 20) { + & $emit "[THROTTLE-ABORT] Partition $Index/$Total - throttled after $createRetries attempts - aborting" + return & $new $null 'ThrottleAbort' $status $false + } + continue + } + elseif ($isNetwork) { + if (-not $networkErrorStart) { + $networkErrorStart = Get-Date + & $emit "[NETWORK] Partition $Index/$Total - uncertain create outcome (HTTP $status) - starting read-only reconcile window (max ${MaxNetworkOutageMinutes}m), no further POST" + } + # An uncertain create outcome (status 0/null, timeout, connection reset, or a + # 5xx where acceptance cannot be disproven) may have been accepted by the + # server, so the create is NEVER re-POSTed for this partition on this attempt. + # Recovery is read-only: reconcile the fingerprinted exact name against the + # fully-paginated collection until a single match appears (adopt), several + # appear (fail closed), or the window expires with no match (fail closed as + # unresolved). A collection miss cannot prove the POST was rejected, so it can + # only fail closed - never issue another POST. + if (-not $ListQueries) { + & $emit "[NETWORK-UNRESOLVED] Partition $Index/$Total - no collection transport to reconcile an uncertain create - failing closed (no re-POST)" + return & $new $null 'UncertainCreateUnresolved' $status $false + } + while ($true) { + $netExact = @(& $findExactByName) + if ($netExact.Count -eq 1) { + & $emit "[NETWORK-RECONCILE] Partition $Index/$Total - uncertain create adopted by exact name (no re-POST)" + return & $new ([string]$netExact[0].id) 'None' $status $true + } + if ($netExact.Count -gt 1) { + & $emit "[NETWORK-RECONCILE] Partition $Index/$Total - multiple exact-name matches after uncertain create - failing closed" + return & $new $null 'MultipleExactMatches' $status $false + } + $elapsedOutageSeconds = ((Get-Date) - $networkErrorStart).TotalSeconds + if ($elapsedOutageSeconds -ge $maxNetworkOutageSeconds) { + & $emit "[NETWORK-UNRESOLVED] Partition $Index/$Total - uncertain create not visible within $MaxNetworkOutageMinutes minute window - failing closed (no re-POST)" + return & $new $null 'UncertainCreateUnresolved' $status $false + } + $retryDelay = 30 + (Get-Random -Minimum 10 -Maximum 30) + & $emit "[NETWORK] Partition $Index/$Total - uncertain create not yet visible (${elapsedOutageSeconds}s elapsed) - re-listing after delay (no re-POST)" + & $Sleep $retryDelay + } + } + elseif ($isAmbiguous) { + & $emit "[AMBIGUOUS] Partition $Index/$Total - ambiguous create response - reconciling by exact name" + return & $reconcile $status + } + else { + & $emit "[CREATE-FAILED] Partition $Index/$Total - non-retriable create failure (HTTP $status)" + return & $new $null 'Other' $status $false + } + } +} + +function Write-PaxCreatePoolHeartbeat { + <# + .SYNOPSIS + Sanitized aggregate progress heartbeat for the bounded parent-create pool. Emits at + most one line per interval and ONLY safe aggregate counts - never a QueryId, token, + identifier, display name, operation name, URL, request body, or header. + .DESCRIPTION + Clock-driven and side-effect-minimal: given the pass start, the last-emit time, and + the current time (all supplied by the caller's injectable now-provider) plus the + interval, it emits one heartbeat line through the injected sink ONLY when at least a + full interval has elapsed since the last emit, and returns the updated last-emit time. + Before the first full interval nothing is emitted and the last-emit time is returned + unchanged, so a pass that finishes quickly produces no heartbeat. It never touches the + jobs, scheduling, throttle, timeout, token refresh, result order, checkpoint state, or + POST count. + .PARAMETER Start + Wall-clock time the wait pass began (from the injectable now-provider). + .PARAMETER LastEmit + Time the previous heartbeat was emitted (Start when none yet). + .PARAMETER Now + Current time sample (from the injectable now-provider). + .PARAMETER IntervalSeconds + Minimum seconds between heartbeats; <= 0 disables emission. + .PARAMETER Total / Completed / Active / Queued / Uncertain + Safe aggregate partition counts for this pass. + .PARAMETER Emit + Injectable single-argument sink for the heartbeat line (defaults to Write-Host). + .OUTPUTS + The (possibly-updated) last-emit [datetime]; equal to -LastEmit when nothing was emitted. + #> + param( + [Parameter(Mandatory)][datetime]$Start, + [Parameter(Mandatory)][datetime]$LastEmit, + [Parameter(Mandatory)][datetime]$Now, + [Parameter(Mandatory)][int]$IntervalSeconds, + [Parameter()][int]$Total = 0, + [Parameter()][int]$Completed = 0, + [Parameter()][int]$Active = 0, + [Parameter()][int]$Queued = 0, + [Parameter()][int]$Uncertain = 0, + [Parameter()][scriptblock]$Emit = $null + ) + if ($IntervalSeconds -le 0) { return $LastEmit } + if (($Now - $LastEmit).TotalSeconds -lt $IntervalSeconds) { return $LastEmit } + $elapsedSeconds = [int][Math]::Floor(($Now - $Start).TotalSeconds) + if ($elapsedSeconds -lt 0) { $elapsedSeconds = 0 } + $elapsedText = if ($elapsedSeconds -ge 60) { "$([int][Math]::Floor($elapsedSeconds / 60))m" } else { "$($elapsedSeconds)s" } + # 'localQueued' is the count still waiting in THIS pool, and 'pending' is everything + # not yet finished. Wording only: the counts, the pass, and the work are unchanged. + $pendingCount = $Active + $Queued + $message = " [PARENT-CREATE] $elapsedText elapsed | total $Total | completed $Completed | active $Active | localQueued $Queued | pending $pendingCount" + if ($Uncertain -gt 0) { $message += " | uncertain $Uncertain" } + if ($Emit) { & $Emit $message } else { Write-Host $message -ForegroundColor DarkCyan } + return $Now +} + +function Invoke-PaxBoundedCreatePool { + <# + .SYNOPSIS + Bounded-parallel executor for parent-scope partition query creation. + .DESCRIPTION + Runs Invoke-GraphAuditHardenedCreate for each supplied partition inside + isolated ThreadJob runspaces under a fixed concurrency cap. Because a + ThreadJob runspace does not inherit parent-scope functions or variables, the + create functions, version constants, and HTTP transport are injected as + text/arguments into every runspace. The transport is injected via + -TransportText so the same executor serves production (a real HTTP POST) and + validation (a runspace-local mock). The concurrency cap is passed straight to + Start-ThreadJob -ThrottleLimit, so the supplied bound is never exceeded. + Returns one result object per partition: + @{ Index; QueryId; Version; FailureCategory; HttpStatus }. + .PARAMETER Items + Partition-shaped objects carrying Index, Total, PStart, PEnd, Activities, + RecordTypes, ServiceFilter. + .PARAMETER ThrottleLimit + Maximum number of concurrent create runspaces. + .PARAMETER TransportText + Text of the injected transport scriptblock: param($Version,$BodyJson, + $ClientRequestId) returning @{ HttpStatus; Id; ResponseBody; Headers }. + .PARAMETER FunctionTexts + Verbatim function definitions dot-sourced into each runspace. + .PARAMETER InjectVariables + Name/value pairs set at script scope inside each runspace (version constants, + token material). + .PARAMETER ListTransportText + Optional text of an injected collection-list scriptblock: returns the + fully-paginated array of @{ id; displayName } for exact-name reconciliation. + When supplied, every create routes through exact-name lookup before and around + its POST; when omitted, the executor behaves as a plain create pool. + .PARAMETER FetchTransportText + Optional text of an injected item-fetch scriptblock: param($QueryId) returning + the query item with its filter contract, so a legacy fingerprint-less same-name + match can be adopted ONLY after its full server contract is fetched and matched. + .PARAMETER TokenRefresh + Optional parent-scope scriptblock invoked periodically WHILE the create jobs run + so a token nearing expiry is refreshed on the shared auth reference mid-pool. + Because the shared auth state is a synchronized hashtable, a refresh landing here + is observed by an already-started retrying create job before its next request. + Defaults to Refresh-GraphTokenIfNeeded; pass an injected refresher for validation. + #> + param( + [Parameter(Mandatory)]$Items, + [Parameter(Mandatory)][int]$ThrottleLimit, + [Parameter(Mandatory)][string]$TransportText, + [Parameter()][string[]]$FunctionTexts = @(), + [Parameter()][hashtable]$InjectVariables = @{}, + [Parameter()][string]$ListTransportText = $null, + [Parameter()][string]$FetchTransportText = $null, + [Parameter()][scriptblock]$TokenRefresh = $null, + [Parameter()][int]$MaxNetworkOutageMinutes = 30, + [Parameter()][int]$HeartbeatIntervalSeconds = 60, + [Parameter()][scriptblock]$NowProvider = $null, + [Parameter()][scriptblock]$HeartbeatEmit = $null + ) + if ($ThrottleLimit -lt 1) { $ThrottleLimit = 1 } + # Injectable clock + emit sink for the create-pool progress heartbeat. Production uses the + # wall clock and the run log/host; validation injects a synthetic clock + collector so the + # heartbeat is exercised deterministically without waiting a real interval. Neither affects + # scheduling, throttle, timeout, token refresh, result order, checkpoint state, or POST count. + if (-not $NowProvider) { $NowProvider = { Get-Date } } + if (-not $HeartbeatEmit) { + $HeartbeatEmit = { + param($HeartbeatMessage) + if (Get-Command -Name Write-LogHost -ErrorAction SilentlyContinue) { Write-LogHost $HeartbeatMessage -ForegroundColor DarkCyan } + else { Write-Host $HeartbeatMessage -ForegroundColor DarkCyan } + } + } + # Synchronized per-partition PHASE MARKER shared into every worker runspace by + # reference (like the shared auth state). It lets an interrupted pool tell a + # ThreadJob that never began (provably no create POST) apart from one whose create + # POST was in flight, so a never-started partition is never conflated with a + # started-but-unknown one. Keyed by partition Index; a missing entry means the + # worker never ran. + $phaseMap = [hashtable]::Synchronized(@{}) + $poolWorker = { + param($it, $fnTexts, $vars, $transportText, $listTransportText, $fetchTransportText, $outageMin, $phaseMap) + # First worker action: record that this partition's worker actually began. A + # missing phase entry at the interrupt therefore proves the ThreadJob never + # started and no create POST could have been sent. + if ($phaseMap) { $phaseMap[$it.Index] = 'Started' } + foreach ($ft in $fnTexts) { . ([scriptblock]::Create($ft)) } + if ($vars) { foreach ($vk in $vars.Keys) { Set-Variable -Name $vk -Value $vars[$vk] -Scope Script } } + $rawTransport = [scriptblock]::Create($transportText) + # Wrap the injected create transport so a 'PostInFlight' marker is set IMMEDIATELY + # BEFORE the real create POST and 'PostReturned' AFTER the response. ThreadJob + # 'Running' alone does not prove a POST started - only this marker does. The marker + # is set conservatively (before the call), so a worker stopped around its POST + # classifies as outcome-unknown, never as never-started. + $transport = { + param($Version, $BodyJson, $ClientRequestId) + if ($phaseMap) { $phaseMap[$it.Index] = 'PostInFlight' } + try { return (& $rawTransport $Version $BodyJson $ClientRequestId) } + finally { if ($phaseMap) { $phaseMap[$it.Index] = 'PostReturned' } } + }.GetNewClosure() + # Injected alongside the create transport so every create routes through + # exact-name reconciliation before and around its POST (adopt one, fail closed + # on many, POST only on a miss). + $listQueries = if ($listTransportText) { [scriptblock]::Create($listTransportText) } else { $null } + # Item-fetch transport: retrieves a query's full contract so a legacy + # fingerprint-less same-name match is adopted only after a contract compare. + $fetchItem = if ($fetchTransportText) { [scriptblock]::Create($fetchTransportText) } else { $null } + $res = Invoke-GraphAuditHardenedCreate -PStart $it.PStart -PEnd $it.PEnd -Index $it.Index -Total $it.Total -Activities $it.Activities -RecordTypes $it.RecordTypes -ServiceFilter $it.ServiceFilter -Invoke $transport -ListQueries $listQueries -PreCreateReconcile:([bool]$listQueries) -FetchItem $fetchItem -MaxNetworkOutageMinutes $outageMin + # Per-partition EXECUTION STATE for a COMPLETED worker, mirroring the parent's + # own create classification exactly: a proven QueryId is a create; an + # unproven-acceptance outcome stays outcome-unknown (durable uncertain); every + # other completion is a determinable failure that provably produced no query. + $execState = if ($res.QueryId) { 'CompletedWithQueryId' } elseif ([string]$res.FailureCategory -eq 'UncertainCreateUnresolved') { 'StartedOutcomeUnknown' } else { 'CompletedFailureBeforePOST' } + [pscustomobject]@{ + Index = $it.Index + QueryId = $res.QueryId + Version = $res.Version + FailureCategory = $res.FailureCategory + HttpStatus = $res.HttpStatus + ExecutionState = $execState + ClientRequestId = $res.ClientRequestId + RequestId = $res.RequestId + ResponseBody = $res.ResponseBody + } + } + $poolJobs = @() + $poolJobIndex = @() + foreach ($item in $Items) { + $poolJobs += Start-ThreadJob -ThrottleLimit $ThrottleLimit -ScriptBlock $poolWorker -ArgumentList $item, $FunctionTexts, $InjectVariables, $TransportText, $ListTransportText, $FetchTransportText, $MaxNetworkOutageMinutes, $phaseMap + $poolJobIndex += $item.Index + } + # Token monitoring during a prolonged create pass: while any create job is still + # running, invoke the token refresher periodically instead of blocking on a single + # Wait-Job. Because the shared auth state is a synchronized hashtable, a refresh + # landing here is visible to an already-started retrying create job BEFORE its next + # request - so a job that started under a token that later neared expiry can still + # observe a fresh token mid-retry, not only jobs launched after a pre-pass refresh. + # A token refresher can signal an interactive quit ('Quit') or throw. Either outcome + # fails the pool closed: no further create work is awaited as success, still-active + # jobs are stopped, only already-completed job results are collected, and a sentinel + # row carries the quit/error decision back to the caller so the parent orchestration + # can enter its graceful authentication-exit path instead of silently reporting + # success. + $authQuit = $false + $refresherError = $null + if ($poolJobs.Count -gt 0) { + if ($TokenRefresh) { + # Sanitized aggregate progress heartbeat: at most one line per interval while the + # create pool is still active, emitting ONLY safe aggregate counts (elapsed, total, + # completed, active, queued) - never a QueryId, token, identifier, name, URL, or + # body. The clock and interval are injectable so the emit is exercised without a + # real wait; the heartbeat never changes scheduling, throttle, timeout, token + # refresh, result order, checkpoint behaviour, or POST count. + $hbStart = & $NowProvider + $hbLastEmit = $hbStart + while (@($poolJobs | Where-Object { $_.State -in 'Running', 'NotStarted' }).Count -gt 0) { + $refreshOutcome = $null + try { $refreshOutcome = & $TokenRefresh } catch { $refresherError = $_.Exception.Message; break } + if ($refreshOutcome -is [string] -and $refreshOutcome -eq 'Quit') { $authQuit = $true; break } + $null = $poolJobs | Wait-Job -Timeout 5 + $hbCompleted = @($poolJobs | Where-Object { $_.State -in 'Completed', 'Failed', 'Stopped' }).Count + $hbActive = @($poolJobs | Where-Object { $_.State -eq 'Running' }).Count + $hbQueued = @($poolJobs | Where-Object { $_.State -eq 'NotStarted' }).Count + if (Get-Command Write-PaxCreatePoolHeartbeat -ErrorAction SilentlyContinue) { + $hbLastEmit = Write-PaxCreatePoolHeartbeat -Start $hbStart -LastEmit $hbLastEmit -Now (& $NowProvider) -IntervalSeconds $HeartbeatIntervalSeconds -Total $poolJobs.Count -Completed $hbCompleted -Active $hbActive -Queued $hbQueued -Emit $HeartbeatEmit + } + } + if (-not $authQuit -and -not $refresherError) { + try { $finalOutcome = & $TokenRefresh; if ($finalOutcome -is [string] -and $finalOutcome -eq 'Quit') { $authQuit = $true } } catch { $refresherError = $_.Exception.Message } + } + } + else { + $null = $poolJobs | Wait-Job + } + } + if ($authQuit -or $refresherError) { + $stillActive = @($poolJobs | Where-Object { $_.State -in 'Running', 'NotStarted' }) + if ($stillActive.Count -gt 0) { $stillActive | Stop-Job -ErrorAction SilentlyContinue } + $poolResults = @() + foreach ($pj in $poolJobs) { if ($pj.State -eq 'Completed') { try { $poolResults += Receive-Job -Job $pj -ErrorAction SilentlyContinue } catch {} } } + if ($poolJobs.Count -gt 0) { $poolJobs | Remove-Job -Force -ErrorAction SilentlyContinue } + # Emit a per-partition row for every requested job that did NOT complete, using the + # shared phase marker to distinguish a never-started worker (no phase entry, or only + # 'Started' - provably no create POST) from one whose create POST was in flight + # ('PostInFlight'/'PostReturned' - outcome unknown). This prevents the interrupt path + # from conflating never-started partitions with started-but-unknown ones. + $completedIdx = New-Object System.Collections.Generic.HashSet[string] + foreach ($pr in $poolResults) { if ($null -ne $pr -and $null -ne $pr.Index) { $null = $completedIdx.Add([string]$pr.Index) } } + foreach ($ji in $poolJobIndex) { + if ($completedIdx.Contains([string]$ji)) { continue } + $ph = $null + if ($phaseMap -and $phaseMap.ContainsKey($ji)) { $ph = [string]$phaseMap[$ji] } + $execState = if ($ph -eq 'PostInFlight' -or $ph -eq 'PostReturned') { 'StartedOutcomeUnknown' } else { 'NeverStarted' } + $poolResults += [pscustomobject]@{ Index = $ji; QueryId = $null; Version = $null; FailureCategory = 'Interrupted'; HttpStatus = $null; ExecutionState = $execState } + } + $poolResults += [pscustomobject]@{ Index = $null; QueryId = $null; Version = $null; FailureCategory = $(if ($authQuit) { 'AuthQuit' } else { 'TokenRefreshError' }); HttpStatus = $null; AuthQuit = $authQuit; RefresherError = $refresherError } + return $poolResults + } + $poolResults = @() + foreach ($pj in $poolJobs) { + try { $poolResults += Receive-Job -Job $pj -ErrorAction SilentlyContinue } catch {} + } + if ($poolJobs.Count -gt 0) { $poolJobs | Remove-Job -Force -ErrorAction SilentlyContinue } + return $poolResults +} + +function Invoke-PaxParentQueryCreation { + <# + .SYNOPSIS + Parent-side bounded-parallel pre-pass that creates the audit query for each + pending partition before its worker is launched. + .DESCRIPTION + For every supplied partition this creates the query in the main runspace, + where the token can be refreshed, under the same concurrency bound used for + worker launch. Creation runs through an injectable runner: production uses a + real bounded ThreadJob pool (Invoke-PaxBoundedCreatePool) with the parent + create functions and a real HTTP transport injected into each runspace, while + validation supplies its own runner with an injected transport. Returns an + index-keyed result map: + @{ = @{ Index; QueryId; Version; FailureCategory; HttpStatus } }. + Persistence and worker wiring stay with the caller: a successful entry carries + a non-null QueryId to store and checkpoint before launch; a failed entry + carries a null QueryId and must not launch a worker. + .PARAMETER Partitions + Partition objects needing creation (each with Index, Total, PStart, PEnd, + RecordTypes, ServiceFilter). + .PARAMETER ThrottleLimit + Concurrency bound reused from the launch phase. + .PARAMETER Activities + Run-level activity list applied to every create body. + .PARAMETER Runner + Optional injected runner: param($Items,$Throttle,$OutageMinutes) returning the + per-partition result objects. When omitted a production bounded ThreadJob pool + is used. + #> + param( + [Parameter(Mandatory)]$Partitions, + [Parameter(Mandatory)][int]$ThrottleLimit, + [Parameter()]$Activities = @(), + [Parameter()][int]$MaxNetworkOutageMinutes = 30, + [Parameter()][scriptblock]$Runner = $null + ) + $items = @() + foreach ($p in $Partitions) { + $items += [pscustomobject]@{ + Index = $p.Index + Total = $p.Total + PStart = $p.PStart + PEnd = $p.PEnd + Activities = $Activities + RecordTypes = $p.RecordTypes + ServiceFilter = $p.ServiceFilter + } + } + if (-not $Runner) { + $Runner = { + param($RunnerItems, $RunnerThrottle, $RunnerOutage) + $fnTexts = @( + "function New-GraphAuditQueryFingerprint {`n$(${function:New-GraphAuditQueryFingerprint})`n}", + "function New-GraphAuditPartitionDisplayName {`n$(${function:New-GraphAuditPartitionDisplayName})`n}", + "function New-GraphAuditQueryRequest {`n$(${function:New-GraphAuditQueryRequest})`n}", + "function Resolve-GraphAuditDuplicateSafeQuery {`n$(${function:Resolve-GraphAuditDuplicateSafeQuery})`n}", + "function Invoke-GraphAuditHardenedCreate {`n$(${function:Invoke-GraphAuditHardenedCreate})`n}" + ) + $vars = @{ + GraphAuditApiVersion_Previous = $script:GraphAuditApiVersion_Previous + GraphAuditApiVersion_Current = $script:GraphAuditApiVersion_Current + PaxSharedAuthState = $script:SharedAuthState + } + $transportText = @' +param($Version, $BodyJson, $ClientRequestId) +$uri = "https://graph.microsoft.com/$Version/security/auditLog/queries" +$headers = @{ Authorization = "Bearer $($script:PaxSharedAuthState.Token)"; 'Content-Type' = 'application/json' } +if ($ClientRequestId) { $headers['client-request-id'] = $ClientRequestId } +try { + $r = Invoke-WebRequest -Method POST -Uri $uri -Headers $headers -Body $BodyJson -ErrorAction Stop + $obj = $null; try { $obj = $r.Content | ConvertFrom-Json } catch {} + return @{ HttpStatus = [int]$r.StatusCode; Id = $obj.id; ResponseBody = $r.Content; Headers = $r.Headers } +} catch { + $resp = $_.Exception.Response + $code = 0; $body = $_.Exception.Message; $rh = @{} + if ($resp) { + try { $code = [int]$resp.StatusCode } catch {} + try { $sr = New-Object System.IO.StreamReader($resp.GetResponseStream()); $body = $sr.ReadToEnd() } catch {} + try { foreach ($hn in $resp.Headers) { $rh[$hn] = $resp.Headers[$hn] } } catch {} + } + return @{ HttpStatus = $code; Id = $null; ResponseBody = $body; Headers = $rh } +} +'@ + # Fully-paginated exact-name collection transport. Reads the live token from + # the shared auth state so a token refreshed by the parent is picked up by + # queued and retrying create jobs, and walks every page of the audit-query + # collection so exact-name reconciliation sees the full server state. + $listTransportText = @' +$__lver = $script:GraphAuditApiVersion_Previous +$__luri = "https://graph.microsoft.com/$__lver/security/auditLog/queries" +$__acc = @() +while ($__luri) { + $__lheaders = @{ Authorization = "Bearer $($script:PaxSharedAuthState.Token)" } + $__lr = Invoke-WebRequest -Method GET -Uri $__luri -Headers $__lheaders -ErrorAction Stop + $__lobj = $null; try { $__lobj = $__lr.Content | ConvertFrom-Json } catch {} + if ($__lobj -and $__lobj.value) { foreach ($__q in $__lobj.value) { $__acc += @{ id = $__q.id; displayName = $__q.displayName } } } + $__luri = if ($__lobj) { $__lobj.'@odata.nextLink' } else { $null } +} +, @($__acc) +'@ + # Single-item fetch transport. Retrieves a query's full filter contract so a + # legacy fingerprint-less same-name match is adopted only after its server + # contract matches the requested one. Reads the live shared-auth token. + $fetchTransportText = @' +param($QueryId) +$__fver = $script:GraphAuditApiVersion_Previous +$__furi = "https://graph.microsoft.com/$__fver/security/auditLog/queries/$QueryId" +$__fheaders = @{ Authorization = "Bearer $($script:PaxSharedAuthState.Token)" } +try { + $__fr = Invoke-WebRequest -Method GET -Uri $__furi -Headers $__fheaders -ErrorAction Stop + $__fobj = $null; try { $__fobj = $__fr.Content | ConvertFrom-Json } catch {} + return $__fobj +} catch { return $null } +'@ + # Refresh the shared-auth token WHILE the create jobs run so an already-started + # retrying job observes a fresh token before its next request. The refresher + # outcome (including an interactive 'Quit' or a thrown error) is propagated so + # the pool can fail closed instead of discarding a quit decision. + $tokenRefresh = { Refresh-GraphTokenIfNeeded -BufferMinutes 5 } + Invoke-PaxBoundedCreatePool -Items $RunnerItems -ThrottleLimit $RunnerThrottle -TransportText $transportText -FunctionTexts $fnTexts -InjectVariables $vars -ListTransportText $listTransportText -FetchTransportText $fetchTransportText -TokenRefresh $tokenRefresh -MaxNetworkOutageMinutes $RunnerOutage + } + } + $results = @(& $Runner $items $ThrottleLimit $MaxNetworkOutageMinutes) + $map = @{} + foreach ($r in $results) { + if ($null -ne $r) { + # A pool that failed closed on an interactive quit or a token-refresh error + # returns a sentinel row instead of a partition result; surface it under a + # reserved key so the caller enters its graceful authentication-exit path and + # never persists a partition as created off a quit/error decision. + $isQuit = if ($r -is [hashtable]) { [bool]$r['AuthQuit'] } else { [bool]$r.AuthQuit } + $rerr = if ($r -is [hashtable]) { $r['RefresherError'] } else { $r.RefresherError } + if ($isQuit) { $map['__AuthQuit'] = $true; continue } + if ($rerr) { $map['__RefresherError'] = [string]$rerr; continue } + $idx = if ($r -is [hashtable]) { $r['Index'] } else { $r.Index } + if ($null -ne $idx) { $map[[int]$idx] = $r } + } + } + return $map +} + +function Save-PaxInterruptedCreatePool { + <# + .SYNOPSIS + On a create-pool auth-quit / refresher-error interrupt, durably persist the + partitions the pool actually completed AND the partitions still in flight, + BEFORE the graceful exit. + .DESCRIPTION + Invoke-PaxBoundedCreatePool returns the results of jobs that completed BEFORE + the interrupt as integer-keyed entries in the map, alongside the reserved + __AuthQuit / __RefresherError sentinel. Discarding that map on the interrupt + would (a) throw away a query that was actually created (forcing a duplicate + create on resume) and (b) leave a possibly-accepted in-flight POST unrecorded. + + This helper reconciles the interrupt map against the list of partitions the + pass requested ($Requested): + + * A requested partition whose map entry carries a real QueryId is persisted + via the SAME success path a normal completion uses (status QueryId + + effective QueryApiVersion + Create operation + a QueryCreated checkpoint). + * A partition whose worker never began (ExecutionState 'NeverStarted' - + provably no create POST was ever issued) is left untouched so a resume + creates it fresh; it is NEVER persisted as UncertainCreate. This is the + correction for conflating never-started jobs with started-but-unknown ones. + * A partition whose worker completed with a determinable non-accepting failure + (ExecutionState 'CompletedFailureBeforePOST') is recorded via + Set-PartitionFailure, matching the normal-completion classification. + * Every OTHER requested partition was still in flight at the interrupt + (ExecutionState 'StartedOutcomeUnknown', or a legacy map entry with no + ExecutionState) and is persisted as durable UncertainCreate, so a resume + routes it to a READ-ONLY reconcile instead of a duplicate create. + + A stopped / incomplete job is NEVER persisted as successful: only a map entry + with a non-null QueryId is treated as completed. + #> + param( + [Parameter(Mandatory)]$CreateMap, + [Parameter(Mandatory)]$Requested, + [Parameter()]$Activities = @(), + [Parameter(Mandatory)][ValidateSet('initial', 'retry')][string]$Phase + ) + $phaseTag = if ($Phase -eq 'retry') { 'retry ' } else { '' } + $completedIndices = [System.Collections.Generic.HashSet[int]]::new() + # 1) Persist every partition the pool actually completed (has a real QueryId). + foreach ($cp in $Requested) { + $cr = $CreateMap[[int]$cp.Index] + if ($cr -and $cr.QueryId) { + $cp | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue ([string]$cr.QueryId) -Force + $script:partitionStatus[$cp.Index].QueryId = [string]$cr.QueryId + $script:partitionStatus[$cp.Index].QueryApiVersion = $cr.Version + $script:partitionStatus[$cp.Index].LifecycleOperation = 'Create' + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -QueryId ([string]$cr.QueryId) -State 'QueryCreated' -PartitionStart $cp.PStart -PartitionEnd $cp.PEnd -LifecycleState $script:partitionStatus[$cp.Index] + } + [void]$completedIndices.Add([int]$cp.Index) + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) query created before ${phaseTag}interrupt (QueryId: $($cr.QueryId)) - persisted, not discarded" -ForegroundColor DarkCyan + } + } + # 2) Persist the remaining requested partitions by execution state: never-started + # stays fresh (no create POST happened), a determinable failure is recorded as a + # failure, and only a genuinely in-flight partition becomes durable UncertainCreate. + foreach ($cp in $Requested) { + if ($completedIndices.Contains([int]$cp.Index)) { continue } + $cr = $CreateMap[[int]$cp.Index] + $execState = if ($cr) { [string]$cr.ExecutionState } else { '' } + if ($execState -eq 'NeverStarted') { + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) never started before ${phaseTag}interrupt - no create POST issued, left fresh for create on resume" -ForegroundColor DarkGray + continue + } + if ($execState -eq 'CompletedFailureBeforePOST') { + Set-PartitionFailure -Index $cp.Index -Stage 'CREATE' -Reason ([string]$cr.FailureCategory) + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) failed before ${phaseTag}interrupt (no query created) - recorded as failure" -ForegroundColor DarkYellow + continue + } + $script:partitionStatus[$cp.Index].UncertainCreate = $true + $script:partitionStatus[$cp.Index].UncertainCreateInfo = @{ + displayName = New-GraphAuditPartitionDisplayName -PStart $cp.PStart -PEnd $cp.PEnd -Index $cp.Index -Total $cp.Total -Activities $Activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + fingerprint = New-GraphAuditQueryFingerprint -PStart $cp.PStart -PEnd $cp.PEnd -Activities $Activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + pStart = $cp.PStart.ToString('o') + pEnd = $cp.PEnd.ToString('o') + attempt = [int]$script:partitionStatus[$cp.Index].AttemptNumber + phase = $Phase + } + $script:partitionStatus[$cp.Index].LastError = "Parent-side query creation in-flight at ${phaseTag}interrupt - durable reconcile pending" + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -State 'UncertainCreate' -PartitionStart $cp.PStart -PartitionEnd $cp.PEnd -LifecycleState $script:partitionStatus[$cp.Index] + } + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) in flight at ${phaseTag}interrupt - persisted for read-only reconcile, no re-POST" -ForegroundColor Yellow + } +} + +function Invoke-PaxResumeUncertainReconcile { + <# + .SYNOPSIS + Resume-side READ-ONLY reconcile of durable uncertain-create partitions (never a POST). + .DESCRIPTION + Consumes the ToReconcileOnly bucket produced by Get-PartitionsToProcess on a + resume. Each such partition carries durable uncertain-create state: a first + create POST that MAY have been accepted but was never proven. In the MAIN + runspace (where the token can refresh) this runs a READ-ONLY exact-name + reconcile via Invoke-GraphAuditHardenedCreate -ReconcileOnly against the + fully-paginated audit-query collection - it NEVER issues a POST. + + Per partition: + * exactly one exact fingerprinted-name match -> ADOPT: store the real QueryId, + checkpoint it as QueryCreated, vacate the durable uncertain entry, and route + the partition to fetch-only; + * multiple exact matches -> FAIL CLOSED: no adopt, no POST, + the durable uncertain entry is preserved for an operator decision; + * none after the bounded collection window -> REMAIN UNCERTAIN: no adopt, no + POST, never QueryConfirmedAbsent, the durable uncertain entry is preserved. + Only adopted partitions are returned for launch; unresolved and fail-closed + partitions are deliberately withheld, so no worker is launched for an unresolved + uncertain create (worker-launch gating). + + The identity is recomputed from the partition's contract (operation / record-type + / service / date), so a differing contract yields a different name and can never + cross-adopt a different query. Transports are injectable: production builds a real + fully-paginated GET collection and a single-item fetch that read the live + shared-auth token; validation injects list / fetch / invoke scriptblocks. + .PARAMETER Partitions + The ToReconcileOnly partition objects (each with Index, Total, PStart, PEnd, + RecordTypes, ServiceFilter, UncertainCreateState). + .PARAMETER Activities + Run-level activity list applied to the reconciled contract identity. + .PARAMETER ListQueries + Optional injected fully-paginated collection transport (-> array of + @{ id; displayName }). Omitted in production to build the real transport. + .PARAMETER FetchItem + Optional injected single-item fetch transport for legacy contract verification. + .PARAMETER Invoke + Optional injected transport required by the shared Invoke-GraphAuditHardenedCreate + signature; it is NEVER called in reconcile-only mode (a fail-closed no-op default). + .PARAMETER Persist + Optional callback param($PartitionIndex,$QueryId,$PStart,$PEnd,$Fingerprint) used to + persist an adopted partition with its EXACT reconciled contract fingerprint. + + CONTRACT (fail-closed): the callback MUST stage the complete bucket transition and + publish it durably, then return the literal boolean $true as its LAST output to signal a + PROVEN durable publication - or throw. Any other outcome (no output, $null, $false, a + non-boolean value) is treated as a FAILED publication: the pre-adoption in-memory state + is restored exactly, the partition is NOT adopted and NOT routed to fetch-only, no POST + is authorized, $script:GenericFatal is latched, and a PAX-FATAL error is thrown. The + default callback stages the transition in memory only (Save-Checkpoint -DeferDiskWrite) + and publishes it EXACTLY ONCE through the strict local+Fabric transaction + (Save-CheckpointStrict), whose verify-after-write result is returned; the warning-only + Save-CheckpointToDisk path is deliberately never used here because a swallowed local + write failure would let the run proceed as though the adoption were durable. + .PARAMETER Log + Optional diagnostic sink param($Message). + .OUTPUTS + @{ AdoptedFetchOnly = @(); Unresolved = @(); FailedClosed = @() } + #> + param( + [Parameter(Mandatory)]$Partitions, + [Parameter()]$Activities = @(), + [Parameter()][scriptblock]$ListQueries = $null, + [Parameter()][scriptblock]$FetchItem = $null, + [Parameter()][scriptblock]$Invoke = $null, + [Parameter()][scriptblock]$Persist = $null, + [Parameter()][scriptblock]$Log = $null + ) + $emit = { param($m) if ($Log) { & $Log $m } } + $result = @{ AdoptedFetchOnly = @(); Unresolved = @(); FailedClosed = @() } + if (-not $Partitions -or @($Partitions).Count -eq 0) { return $result } + + # Reconcile-only NEVER posts; this default transport fails closed if ever reached. + if (-not $Invoke) { $Invoke = { param($v, $b, $c) @{ HttpStatus = 0; Id = $null; ResponseBody = 'reconcile-only-no-post'; Headers = @{} } } } + + # Production list / fetch transports: a fully-paginated exact-name collection and a + # single-item contract fetch, both reading the live shared-auth token so a parent + # token refresh is observed. Injected transports (validation) take precedence. + if (-not $ListQueries) { + $ListQueries = { + $__lver = $script:GraphAuditApiVersion_Previous + $__luri = "https://graph.microsoft.com/$__lver/security/auditLog/queries" + $__acc = @() + while ($__luri) { + $__lheaders = @{ Authorization = "Bearer $($script:SharedAuthState.Token)" } + $__lr = Invoke-WebRequest -Method GET -Uri $__luri -Headers $__lheaders -ErrorAction Stop + $__lobj = $null; try { $__lobj = $__lr.Content | ConvertFrom-Json } catch {} + if ($__lobj -and $__lobj.value) { foreach ($__q in $__lobj.value) { $__acc += @{ id = $__q.id; displayName = $__q.displayName } } } + $__luri = if ($__lobj) { $__lobj.'@odata.nextLink' } else { $null } + } + , @($__acc) + } + } + if (-not $FetchItem) { + $FetchItem = { + param($QueryId) + $__fver = $script:GraphAuditApiVersion_Previous + $__furi = "https://graph.microsoft.com/$__fver/security/auditLog/queries/$QueryId" + $__fheaders = @{ Authorization = "Bearer $($script:SharedAuthState.Token)" } + try { + $__fr = Invoke-WebRequest -Method GET -Uri $__furi -Headers $__fheaders -ErrorAction Stop + $__fobj = $null; try { $__fobj = $__fr.Content | ConvertFrom-Json } catch {} + return $__fobj + } catch { return $null } + } + } + if (-not $Persist) { + $Persist = { + param($PartitionIndex, $QueryId, $PStart, $PEnd, $Fingerprint) + # Adoption persists the proven QueryId with the EXACT reconciled contract fingerprint so + # the queryCreated entry preserves per-contract identity even when this runs BEFORE + # $script:partitionStatus is initialized on resume (Save-Checkpoint cannot re-derive the + # fingerprint from partitionStatus at that point). A legacy fingerprint-less contract + # passes no fingerprint and retains the historical index identity. + # + # The bucket transition is staged IN MEMORY ONLY (-DeferDiskWrite) and then published + # EXACTLY ONCE through the STRICT verify-after-write local+Fabric transaction, so the + # adoption can only affect processing after its durability has been PROVEN. Returning + # the strict publisher's boolean result is what makes an unproven publication fail + # closed in the caller. + if ([string]::IsNullOrWhiteSpace([string]$Fingerprint)) { + Save-Checkpoint -PartitionIndex $PartitionIndex -State 'QueryCreated' -QueryId $QueryId -PartitionStart $PStart -PartitionEnd $PEnd -DeferDiskWrite + } + else { + Save-Checkpoint -PartitionIndex $PartitionIndex -State 'QueryCreated' -QueryId $QueryId -PartitionStart $PStart -PartitionEnd $PEnd -Fingerprint $Fingerprint -DeferDiskWrite + } + return [bool](Save-CheckpointStrict) + } + } + + foreach ($p in @($Partitions)) { + $res = Invoke-GraphAuditHardenedCreate -PStart $p.PStart -PEnd $p.PEnd -Index $p.Index -Total $p.Total -Activities $Activities -RecordTypes $p.RecordTypes -ServiceFilter $p.ServiceFilter -Invoke $Invoke -ListQueries $ListQueries -FetchItem $FetchItem -ReconcileOnly + if ($res.Adopted -and $res.QueryId) { + # Adopt: store the proven real id, checkpoint as created, and vacate the durable + # uncertain entry so a later resume sees a normal fetch-only (never a re-POST). + # + # ONE identity drives BOTH the persist and the vacate so the adopted queryCreated entry + # and the vacated uncertain entry are the SAME contract. Prefer the exact fingerprint + # carried on the partition's durable uncertain state (so the adopted entry PRESERVES that + # precise identity and a sibling same-index contract is never vacated); else reconstruct + # it from the SAME contract the reconcile matched on - the RUN-LEVEL $Activities plus the + # partition record-type / service / date window (NOT $p.Activities, which is absent before + # partitionStatus is initialized on resume). A fingerprinted durable entry whose + # fingerprint cannot be resolved FAILS CLOSED for this partition - it is NEVER adopted + # with an index-only identity, which would persist an identity-less entry and vacate a + # sibling same-index contract. + $__ucState = if ($p.PSObject.Properties['UncertainCreateState']) { $p.UncertainCreateState } else { $null } + $__storedFp = if ($null -ne $__ucState) { Get-PaxCheckpointEntryFingerprint $__ucState } else { $null } + $__entryFingerprinted = (-not [string]::IsNullOrWhiteSpace([string]$__storedFp)) + $__fp = $null + if ($__entryFingerprinted) { + $__fp = [string]$__storedFp + } + else { + try { $__fp = New-GraphAuditQueryFingerprint -PStart $p.PStart -PEnd $p.PEnd -Activities $Activities -RecordTypes $p.RecordTypes -ServiceFilter $p.ServiceFilter } catch { $__fp = $null } + } + if ($__entryFingerprinted -and [string]::IsNullOrWhiteSpace([string]$__fp)) { + # Fingerprinted entry, unresolved fingerprint: never index-fallback (a sibling same-index + # contract would be destroyed). Hold this partition, do not adopt, never POST. + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - fingerprinted uncertain entry could not resolve its contract identity - failing closed (no adopt, no POST)" + $result.FailedClosed += $p + continue + } + # ONE transaction owns the COMPLETE bucket transition. The post-adoption state is built + # in memory FIRST (vacate the matching uncertain contract here, add the queryCreated + # entry inside the persist callback), then published EXACTLY ONCE through the STRICT + # local+Fabric transaction. An EXACT snapshot of every lifecycle bucket is captured + # BEFORE the mutation so a failed publication is rolled back to the precise pre-adoption + # state rather than a best-effort re-derivation. + $__cpParts = if ($script:CheckpointData) { $script:CheckpointData.partitions } else { $null } + $__snapCompleted = @() + $__snapQueryCreated = @() + $__snapUncertain = @() + if ($__cpParts) { + # Assigned as statements (never as an `if` expression) so an EMPTY bucket snapshots + # as a real empty array rather than collapsing to $null, which would restore a + # one-element @($null) bucket instead of an empty one. + $__snapCompleted = @($__cpParts.completed) + $__snapQueryCreated = @($__cpParts.queryCreated) + $__snapUncertain = @($__cpParts.uncertainCreate) + } + if ($__cpParts -and $__cpParts.uncertainCreate) { + # Vacate ONLY the adopted contract, using the SAME single fingerprint used to persist, + # so a sibling same-index contract is preserved byte-for-byte. + $__cpParts.uncertainCreate = @( + $__cpParts.uncertainCreate | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $p.Index $__fp) } + ) + } + $__persistProven = $false + $__persistErr = $null + try { + $__persistResult = @(& $Persist $p.Index ([string]$res.QueryId) $p.PStart $p.PEnd $__fp) | Select-Object -Last 1 + $__persistProven = ($__persistResult -is [bool]) -and ([bool]$__persistResult) + } + catch { $__persistErr = $_ } + if (-not $__persistProven) { + # Fail closed: the adoption's durable publication was NOT proven. Restore the EXACT + # pre-adoption in-memory state (the strict publisher leaves the durable copy + # byte-for-byte unchanged), so the uncertain contract survives in memory AND on disk; + # withhold the partition (never adopted, never routed to fetch-only, ZERO POSTs); + # latch the classified fatal and throw so the caller terminates nonzero. + if ($__cpParts) { + $__cpParts.completed = $__snapCompleted + $__cpParts.queryCreated = $__snapQueryCreated + $__cpParts.uncertainCreate = $__snapUncertain + } + $script:GenericFatal = $true + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - durable publication of the adopted create was NOT proven - failing closed (no adopt, no fetch, no POST)" + $__persistDetail = if ($__persistErr) { [string]$__persistErr.Exception.Message } else { 'the persistence callback did not return a proven durable publication.' } + if ($__persistDetail -match 'PAX-FATAL-ROLLBACK') { + throw "PAX-FATAL-ROLLBACK: durable publication of the adopted uncertain create failed for partition $([int]$p.Index) AND the prior checkpoint could not be restored; disk state is indeterminate - manual checkpoint inspection required. $__persistDetail" + } + throw "PAX-FATAL: durable publication of the adopted uncertain create failed for partition $([int]$p.Index); fail-closed, the durable uncertain contract is preserved and no fetch or fresh create is authorized. $__persistDetail" + } + # PROVEN durable publication: only NOW may the adoption affect processing - the real id + # is stamped on the partition and the partition is classified fetch-only. + $p | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue ([string]$res.QueryId) -Force + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - durable uncertain create adopted by exact name (no POST)" + $result.AdoptedFetchOnly += $p + } + elseif ($res.FailureCategory -eq 'MultipleExactMatches') { + # Fail closed: several exact matches; never adopt, never POST, keep the durable + # uncertain state for an operator decision (-ClearUncertainCreate). + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - multiple exact-name matches - failing closed (no POST, uncertain preserved)" + $result.FailedClosed += $p + } + else { + # Remain uncertain: not visible after the bounded window. Never POST, never + # QueryConfirmedAbsent (a collection miss cannot prove the first POST was rejected); + # the durable uncertain entry is preserved. + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - uncertain create still not visible - remains unresolved (no POST, uncertain preserved)" + $result.Unresolved += $p + } + } + return $result +} + +function Clear-PaxUncertainCreateState { + <# + .SYNOPSIS + Operator-directed SELECTIVE clear of durable uncertain-create checkpoint state. + .DESCRIPTION + Durable uncertain-create entries are NEVER auto-cleared - not by elapsed time, a + collection miss, a process restart, or failure-count exhaustion. The ONLY way to + clear them is an explicit operator request that names specific partition indexes (the + top-level -ClearUncertainCreate parameter). The clear is always + SELECTIVE and bucket-exclusive: only the durable entries whose index is in + -SelectedIndexes are removed; every other uncertain entry is preserved exactly. A + selection that is empty, contains a duplicate, or names an index that is not currently + in the durable uncertain-create set is REJECTED and nothing is cleared. With + -ValidateOnly the selection is validated and returned WITHOUT mutating the checkpoint + (used at resume time to fail an invalid selection before any auth or create). + .PARAMETER CheckpointData + The in-memory checkpoint hashtable ($script:CheckpointData is used when omitted). + .PARAMETER Requested + $true only when the operator explicitly passed -ClearUncertainCreate. + .PARAMETER SelectedIndexes + The explicit operator-supplied list of partition indexes to clear. + .PARAMETER SelectedContracts + The explicit operator-supplied list of ":" tokens naming durable + uncertain-create contracts to clear (the -ClearUncertainContract companion). Used to + disambiguate an index that maps to more than one durable uncertain contract: each token + must resolve to EXACTLY one durable entry by (index, fingerprint); a token that is + malformed, unknown, matches more than one, or is duplicated is REJECTED and nothing is + cleared. When supplied it takes precedence over -SelectedIndexes and clears strictly by + contract identity (bucket-exclusive by fingerprint); every unselected contract is kept. + .PARAMETER ValidateOnly + Validate and return the selection without removing anything from the checkpoint. + .OUTPUTS + A pscustomobject: Rejected (bool), Reason (string), Selected (int[] validated indexes), + Cleared (int[] indexes actually removed), DurableIndexes (int[] indexes present at call), + SelectedContracts (string[] validated ":" tokens), ClearedContracts + (string[] tokens actually removed), DurableContracts (string[] tokens present at call). + #> + param( + [Parameter()]$CheckpointData = $null, + [Parameter()][bool]$Requested, + [Parameter()][int[]]$SelectedIndexes = @(), + [Parameter()][string[]]$SelectedContracts = @(), + [Parameter()][switch]$ValidateOnly + ) + $outcome = [pscustomobject]@{ Rejected = $false; Reason = ''; Selected = @(); Cleared = @(); DurableIndexes = @(); SelectedContracts = @(); ClearedContracts = @(); DurableContracts = @() } + if (-not $Requested) { return $outcome } + $cp = if ($null -ne $CheckpointData) { $CheckpointData } else { $script:CheckpointData } + $durable = @() + if ($cp -and $cp.partitions -and $cp.partitions.uncertainCreate) { + $durable = @(@($cp.partitions.uncertainCreate) | Where-Object { $null -ne $_ } | ForEach-Object { [int]$_.index }) + } + $outcome.DurableIndexes = $durable + # EXPLICIT-CONTRACT selection (-ClearUncertainContract companion): each token + # ":" names EXACTLY one durable uncertain contract by (index, + # fingerprint) identity, so an index that maps to more than one contract can be + # disambiguated. This path takes precedence when supplied: zero/multiple/unknown/malformed/ + # duplicate tokens fail closed WITHOUT mutation, and a valid clear removes strictly the + # named contract(s) (bucket-exclusive by fingerprint) leaving every unselected entry exact. + if (@($SelectedContracts).Count -gt 0) { + $durableEntries = @() + if ($cp -and $cp.partitions -and $cp.partitions.uncertainCreate) { + $durableEntries = @(@($cp.partitions.uncertainCreate) | Where-Object { $null -ne $_ }) + } + $outcome.DurableContracts = @($durableEntries | ForEach-Object { "$([int]$_.index):$(Get-PaxCheckpointEntryFingerprint $_)" }) + $normTokens = @() + $matchedEntries = @() + foreach ($tok in @($SelectedContracts)) { + $raw = [string]$tok + $ci = $raw.IndexOf(':') + if ($ci -lt 1 -or $ci -ge ($raw.Length - 1)) { + $outcome.Rejected = $true; $outcome.Reason = "malformed explicit contract selector '$raw' (expected ':')"; return $outcome + } + $idxRaw = $raw.Substring(0, $ci).Trim() + $fpRaw = $raw.Substring($ci + 1).Trim() + $idxParsed = 0 + if (-not [int]::TryParse($idxRaw, [ref]$idxParsed)) { + $outcome.Rejected = $true; $outcome.Reason = "malformed explicit contract selector '$raw' (index is not an integer)"; return $outcome + } + if ([string]::IsNullOrWhiteSpace($fpRaw)) { + $outcome.Rejected = $true; $outcome.Reason = "malformed explicit contract selector '$raw' (missing fingerprint)"; return $outcome + } + $normTok = "$($idxParsed):$fpRaw" + $hits = @($durableEntries | Where-Object { ([int]$_.index -eq $idxParsed) -and ([string](Get-PaxCheckpointEntryFingerprint $_) -eq $fpRaw) }) + if (@($hits).Count -eq 0) { + $outcome.Rejected = $true; $outcome.Reason = "explicit contract selector '$normTok' matches no durable uncertain-create contract"; return $outcome + } + if (@($hits).Count -gt 1) { + $outcome.Rejected = $true; $outcome.Reason = "explicit contract selector '$normTok' matches more than one durable uncertain-create contract"; return $outcome + } + $normTokens += $normTok + $matchedEntries += $hits[0] + } + $dupeTok = @($normTokens | Group-Object | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name }) + if (@($dupeTok).Count -gt 0) { + $outcome.Rejected = $true; $outcome.Reason = "duplicate explicit contract selector token ($(@($dupeTok).Count) duplicate value(s))"; return $outcome + } + $outcome.SelectedContracts = $normTokens + if ($ValidateOnly) { return $outcome } + $cp.partitions.uncertainCreate = @( + @($cp.partitions.uncertainCreate) | Where-Object { + $entry = $_ + $drop = $false + foreach ($m in $matchedEntries) { + if (([int]$entry.index -eq [int]$m.index) -and ([string](Get-PaxCheckpointEntryFingerprint $entry) -eq [string](Get-PaxCheckpointEntryFingerprint $m))) { $drop = $true; break } + } + -not $drop + } + ) + $outcome.ClearedContracts = $normTokens + return $outcome + } + $selected = @($SelectedIndexes | ForEach-Object { [int]$_ }) + if ($selected.Count -eq 0) { + $outcome.Rejected = $true; $outcome.Reason = 'no partition index was supplied (selection is empty)'; return $outcome + } + $dupes = @($selected | Group-Object | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name }) + if ($dupes.Count -gt 0) { + $outcome.Rejected = $true; $outcome.Reason = "duplicate partition index selected ($($dupes.Count) duplicate value(s))"; return $outcome + } + $unknown = @($selected | Where-Object { $_ -notin $durable }) + if ($unknown.Count -gt 0) { + $outcome.Rejected = $true; $outcome.Reason = "$($unknown.Count) selected index(es) are not in the durable uncertain-create set of $($durable.Count) partition(s)"; return $outcome + } + # Ambiguity guard: because Graph workload passes RESTART partition indexes at 1, a single + # partition index can map to MORE THAN ONE durable uncertain contract (distinct + # fingerprints). An index-only selector cannot deterministically name which contract to + # clear, so REJECT before any mutation - clearing the wrong contract's durable uncertain + # state is unrecoverable. An explicit fingerprint selector is required to disambiguate. + $ambiguous = @() + foreach ($si in $selected) { + $cnt = @($durable | Where-Object { $_ -eq $si }).Count + if ($cnt -gt 1) { $ambiguous += $si } + } + if ($ambiguous.Count -gt 0) { + $cand = @() + foreach ($ai in @($ambiguous | Sort-Object -Unique)) { + foreach ($e in @($cp.partitions.uncertainCreate | Where-Object { $null -ne $_ -and ([int]$_.index -eq [int]$ai) })) { + $efp = Get-PaxCheckpointEntryFingerprint $e + if (-not [string]::IsNullOrWhiteSpace([string]$efp)) { $cand += "$([int]$ai):$efp" } + } + } + $hint = if (@($cand).Count -gt 0) { " re-run with -ClearUncertainContract naming the exact contract token(s): $(@($cand) -join ', ')" } else { ' the durable entries carry no fingerprint (legacy) and cannot be safely disambiguated - no clear is possible' } + $outcome.Rejected = $true; $outcome.Reason = "$($ambiguous.Count) selected index(es) map to more than one durable uncertain contract (ambiguous index-only selection);$hint"; return $outcome + } + $outcome.Selected = $selected + if ($ValidateOnly) { return $outcome } + $cp.partitions.uncertainCreate = @( + @($cp.partitions.uncertainCreate) | Where-Object { $null -ne $_ -and ([int]$_.index) -notin $selected } + ) + $outcome.Cleared = $selected + return $outcome +} + +function Resolve-PaxOperatorClearedUncertain { + <# + .SYNOPSIS + Selective operator recovery for a single NOT-VISIBLE durable-uncertain partition on + resume. Returns $true only when the operator explicitly selected this partition index + via -ClearUncertainCreate, in which case its durable uncertain marker is cleared + SELECTIVELY and atomically (single durable write, bucket-exclusive) BEFORE any fresh + create and the partition is authorized for a fresh create this run. + .DESCRIPTION + This is only ever called for a partition the READ-ONLY reconcile could NOT resolve by + exact name (Unresolved / not visible). A still-visible partition is adopted or fails + closed by the reconcile before this point, so a visible query is never cleared into a + duplicate POST. When the partition is not in the operator selection this is a no-op + that returns $false and preserves the durable uncertain state exactly. + .PARAMETER Index + The partition index being considered for operator-selected clear. + .PARAMETER Fingerprint + The contract fingerprint of the partition being considered, used to match an explicit + -ClearUncertainContract ":" selection so a clear removes EXACTLY that + contract even when another contract shares the index. + .PARAMETER Selection + The staged operator selection set (defaults to $script:ClearUncertainSelection). + .PARAMETER ContractSelection + The staged explicit-contract selection set of ":" tokens (defaults to + $script:ClearUncertainContractSelection). + .PARAMETER CheckpointData + The in-memory checkpoint (defaults to $script:CheckpointData). + .PARAMETER Persist + Optional durable-write callback (defaults to Save-CheckpointStrict when checkpointing + is enabled) invoked immediately after the selective clear, BEFORE any fresh create. The + write must be PROVEN durable: a failure rolls back the clear and fails closed. + .OUTPUTS + $true when the partition was operator-cleared for a fresh create; otherwise $false. + #> + param( + [Parameter(Mandatory)][int]$Index, + [Parameter()][string]$Fingerprint = $null, + [Parameter()]$Selection = $null, + [Parameter()]$ContractSelection = $null, + [Parameter()]$CheckpointData = $null, + [Parameter()][scriptblock]$Persist = $null + ) + $sel = $Selection + if ($null -eq $sel) { $sel = $script:ClearUncertainSelection } + $csel = $ContractSelection + if ($null -eq $csel) { $csel = $script:ClearUncertainContractSelection } + # This partition is operator-selected either by an explicit contract token + # (":", the precise identity that disambiguates two contracts sharing an + # index) or by an unambiguous index-only selection. The explicit contract selector is checked + # first so a fingerprinted clear removes EXACTLY that contract; index selection is only ever + # staged for an UNAMBIGUOUS index (ambiguous index selectors are rejected at resume time). + $contractToken = if (-not [string]::IsNullOrWhiteSpace($Fingerprint)) { "$([int]$Index):$Fingerprint" } else { $null } + $byContract = ($null -ne $contractToken -and $null -ne $csel -and $csel.Contains($contractToken)) + $byIndex = ($null -ne $sel -and $sel.Contains([int]$Index)) + if (-not $byContract -and -not $byIndex) { return $false } + $cp = if ($null -ne $CheckpointData) { $CheckpointData } else { $script:CheckpointData } + # Copy-on-write snapshot of the durable uncertain bucket so a failed durable write + # can be rolled back to EXACTLY the pre-clear state (fail-closed, no data loss). + $origUncertain = @($cp.partitions.uncertainCreate) + # Selective + atomic durable clear of ONLY this partition's uncertain marker - by the exact + # contract identity when a fingerprint selection matched, else by unambiguous index. + if ($byContract) { + $null = Clear-PaxUncertainCreateState -CheckpointData $cp -Requested:$true -SelectedContracts @($contractToken) + } + else { + $null = Clear-PaxUncertainCreateState -CheckpointData $cp -Requested:$true -SelectedIndexes @([int]$Index) + } + # Single durable write BEFORE any fresh create POST for this partition. The write + # must be PROVEN durable: the default path uses the strict verify-after-write + # publisher, and any failure (default OR injected callback) fails closed below. + $persistOk = $false + $persistError = $null + try { + if ($Persist) { & $Persist } + elseif ($script:CheckpointEnabled) { $null = Save-CheckpointStrict } + $persistOk = $true + } + catch { + $persistOk = $false + $persistError = $_ + } + if (-not $persistOk) { + # Fail-closed: durable persistence of the operator-cleared uncertain marker was + # NOT proven. Roll back the in-memory clear, leave the operator selection + # UNCONSUMED (no fresh create authorized), latch the classified fatal so the run + # exits nonzero, and re-throw so the caller loop stops without adding a + # create-capable partition. + $cp.partitions.uncertainCreate = $origUncertain + $script:GenericFatal = $true + # If the strict publisher could NOT restore the prior checkpoint bytes, the disk state + # is INDETERMINATE - surface that distinct rollback-failure category rather than the + # normal fail-closed message (which implies the prior state was restored intact). + if ($persistError -and ([string]$persistError.Exception.Message) -match 'PAX-FATAL-ROLLBACK') { + throw "PAX-FATAL-ROLLBACK: durable persistence of operator-cleared uncertain create failed for partition $([int]$Index) AND the prior checkpoint could not be restored; disk state is indeterminate - manual checkpoint inspection required. $([string]$persistError.Exception.Message)" + } + throw "PAX-FATAL: durable persistence of operator-cleared uncertain create failed for partition $([int]$Index); fail-closed, no fresh create authorized." + } + # Only a PROVEN durable write consumes the selection and authorizes the fresh create. + if ($byContract) { [void]$csel.Remove($contractToken) } + else { [void]$sel.Remove([int]$Index) } + return $true +} + # ============================================== # CTRL+C GRACEFUL EXIT HANDLER # ============================================== @@ -13589,6 +24018,43 @@ $script:EarlyExit = $false # this run. When true, local run files are preserved at end of run - treated the same as # Ctrl+C / early-exit / crash - so nothing is lost when a destination upload did not complete. $script:AnyUploadFailed = $false +$script:ResumeValidationFailed = $false +# 1D-a: a GENERIC, unexpected terminating error reached the top-level catch (i.e. NOT a +# Ctrl+C / graceful early-exit, and NOT one of the classified failure conditions that already +# drive a specific exit code). Without this flag such a crash falls through the finally exit +# selector with every failure flag false and reports exit 0 (false success) to ACA/automation, +# contradicting the exit contract (1 = Fatal / aborted). It maps to exit 1 at top precedence, +# suppresses the finally's Ctrl+C mis-inference, and (like any non-clean end) preserves the +# durable recovery artifacts the clean-run cleanup would otherwise delete. +$script:GenericFatal = $false +$script:AISIDCleanTail = $false + +function script:Invoke-PaxGraphDisconnect { + param([System.Management.Automation.ActionPreference]$ErrorAction = [System.Management.Automation.ActionPreference]::Stop) + $writer = $null + $originalWarning = $null + try { + try { + $writer = [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.OutputWriter + if ($writer -and $writer.WriteWarning) { + $originalWarning = $writer.WriteWarning + $warningPrefix = 'Failed to clear the persisted MSAL token cache during Disconnect-MgGraph:' + $writer.WriteWarning = [System.Action[string]]{ + param($message) + if (-not ([string]$message).StartsWith($warningPrefix, [System.StringComparison]::Ordinal)) { + $originalWarning.Invoke([string]$message) + } + }.GetNewClosure() + } + } catch {} + Disconnect-MgGraph -ErrorAction $ErrorAction | Out-Null + } + finally { + if ($writer -and $originalWarning) { + $writer.WriteWarning = $originalWarning + } + } +} # Register exit handler that ALWAYS runs when PowerShell exits # This works even when Ctrl+C is pressed before the try block (e.g., during module loading) @@ -13634,7 +24100,7 @@ function Invoke-GracefulExit { # Disconnect from Microsoft Graph - ALWAYS attempt disconnect Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Cyan try { - Disconnect-MgGraph -ErrorAction Stop | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction Stop Write-Host " Microsoft Graph disconnected" -ForegroundColor Green } catch { @@ -14028,7 +24494,7 @@ function Connect-PurviewAudit { # This forces a fresh Connect-MgGraph with a new token, preventing issues where MSAL # silently returns a cached expired token or a token from a different user account. Write-LogHost "Clearing any previous Graph session..." -ForegroundColor Gray - try { Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null } catch { } + try { script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue } catch { } Write-LogHost "Connecting to Microsoft Graph Security API..." -ForegroundColor Cyan @@ -14050,7 +24516,7 @@ function Connect-PurviewAudit { # Organization.Read.All (read /subscribedSkus for license SKU lookup) # -GroupNames (Graph API mode only): # GroupMember.Read.All (least-privilege scope per Microsoft Graph docs - # for GET /groups and GET /groups/{id}/members; + # for GET /groups and GET /groups/{id}/transitiveMembers; # used by Expand-PurviewGroupMembership) $RequiredScopes = [System.Collections.Generic.List[string]]::new() if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { @@ -14094,9 +24560,11 @@ function Connect-PurviewAudit { } # SharePoint remote-output destination needs delegated/app drive write scopes. # Fabric/OneLake uses a separate storage-audience token (Az.Accounts), not a Graph scope. + # Sites.ReadWrite.All alone authorizes every SharePoint drive/file endpoint used here + # (site/drive resolve, folder create via POST children, PUT / createUploadSession, download) + # for both delegated and application auth, so Files.ReadWrite.All is not requested. if ($script:RemoteOutputMode -eq 'SharePoint') { if ($RequiredScopes -notcontains 'Sites.ReadWrite.All') { [void]$RequiredScopes.Add('Sites.ReadWrite.All') } - if ($RequiredScopes -notcontains 'Files.ReadWrite.All') { [void]$RequiredScopes.Add('Files.ReadWrite.All') } } $RequiredScopes = $RequiredScopes.ToArray() @@ -14320,10 +24788,18 @@ function Connect-PurviewAudit { # Build a context-aware connection-success message $connectMsg = if ($AuthMethod.ToLower() -eq 'appregistration') { if ($IncludeAgent365Info -or $OnlyAgent365Info) { - "Successfully connected to Microsoft Graph (Phase 1 of 2: app-only audit context). The Phase 2 (Agent 365) interactive sign-in will be requested NEXT, up-front, before audit work begins." + # App-only + Agent 365 reuses this application context; the Agent 365 phase runs + # app-only against pre-consented application permissions with no interactive sign-in. + "Successfully connected to Microsoft Graph (app-only context). The Agent 365 phase reuses this application context - no interactive sign-in is required." } else { "Successfully connected to Microsoft Graph (app-only context)." } + } elseif ($AuthMethod.ToLower() -eq 'managedidentity') { + if ($IncludeAgent365Info -or $OnlyAgent365Info) { + "Successfully connected to Microsoft Graph (managed-identity app-only context). The Agent 365 phase reuses this application context - no interactive sign-in is required." + } else { + "Successfully connected to Microsoft Graph (managed-identity app-only context)." + } } else { "Successfully connected to Microsoft Graph (delegated user context)." } @@ -14627,7 +25103,7 @@ function Invoke-TokenRefresh { try { # Disconnect first to ensure clean state try { - Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue } catch { } # Re-authenticate based on stored credential type @@ -14740,6 +25216,44 @@ function Invoke-TokenRefresh { # ============================================== # PROACTIVE TOKEN REFRESH FOR LONG-RUNNING OPERATIONS # ============================================== +# PAX-1E token-refresh lifecycle state (parent-owned; UTC). Defined early so every +# Refresh-GraphTokenIfNeeded call site can record truthful state. Holds ONLY lifecycle +# state + timing - never token or credential material. Set 'Refreshing' before a real +# synchronous refresh attempt; resolve on every outcome path and in a finally so the +# state can never stay stuck at 'Refreshing' after a throw. +function script:Initialize-PaxTokenRefreshState { + if (-not $script:PaxTokenRefreshState) { + $script:PaxTokenRefreshState = [pscustomobject]@{ + State = 'Idle'; LastReason = $null; LastAuthMode = $null; StartedUtc = $null + LastSuccessUtc = $null; LastElapsedSeconds = $null; LastError = $null + } + } +} +function script:Set-PaxTokenRefreshing { + param([string]$Reason, [string]$AuthMode, [Nullable[datetime]]$StartUtc = $null) + Initialize-PaxTokenRefreshState + $script:PaxTokenRefreshState.State = 'Refreshing' + $script:PaxTokenRefreshState.LastReason = $Reason + $script:PaxTokenRefreshState.LastAuthMode = $AuthMode + $script:PaxTokenRefreshState.StartedUtc = if ($StartUtc) { [datetime]$StartUtc } else { (Get-Date).ToUniversalTime() } + $script:PaxTokenRefreshState.LastError = $null +} +function script:Resolve-PaxTokenRefresh { + param([bool]$Success, [Nullable[datetime]]$CompletionUtc = $null, [string]$ErrorCategory = $null) + Initialize-PaxTokenRefreshState + $done = if ($CompletionUtc) { [datetime]$CompletionUtc } else { (Get-Date).ToUniversalTime() } + $started = $script:PaxTokenRefreshState.StartedUtc + if ($started) { $script:PaxTokenRefreshState.LastElapsedSeconds = [Math]::Round(($done - [datetime]$started).TotalSeconds, 1) } + if ($Success) { + $script:PaxTokenRefreshState.State = 'Succeeded' + $script:PaxTokenRefreshState.LastSuccessUtc = $done + $script:PaxTokenRefreshState.LastError = $null + } else { + $script:PaxTokenRefreshState.State = 'Failed' + $script:PaxTokenRefreshState.LastError = $ErrorCategory + } +} + function Refresh-GraphTokenIfNeeded { <# .SYNOPSIS @@ -14809,7 +25323,14 @@ function Refresh-GraphTokenIfNeeded { } } $script:LastProactiveRefreshAttempt = Get-Date - + + # PAX-1E: mark token-refresh state 'Refreshing' for the duration of this REAL attempt + # (never on the no-refresh-needed / cooldown early-returns above). Resolved on every + # outcome path and in the finally so it can never stay stuck after a throw. + Set-PaxTokenRefreshing -Reason $(if ($needsProactiveRefresh) { 'proactive-age' } else { 'proactive-buffer' }) -AuthMode ("" + $script:AuthConfig.Method) -StartUtc (Get-Date).ToUniversalTime() + $script:PaxTokenRefreshResolved = $false + try { + # Log appropriate message based on trigger reason if (-not $needsProactiveRefresh) { Write-LogHost " [TOKEN] Token expires in $([Math]::Round($minutesRemaining, 1)) minutes - attempting proactive refresh..." -ForegroundColor Yellow @@ -14835,6 +25356,7 @@ function Refresh-GraphTokenIfNeeded { Write-LogHost " [TOKEN] Token refreshed silently (expires: $($tokenInfo.ExpiresOn.ToString('HH:mm:ss')) UTC, refresh #$($script:SharedAuthState.RefreshCount))" -ForegroundColor Green Write-LogHost " [TOKEN] Note: In-flight queries may still require re-auth before this expiration" -ForegroundColor DarkGray + Resolve-PaxTokenRefresh -Success $true -CompletionUtc (Get-Date).ToUniversalTime(); $script:PaxTokenRefreshResolved = $true return $true } } @@ -14850,6 +25372,7 @@ function Refresh-GraphTokenIfNeeded { $script:AuthConfig.TokenIssueTime = Get-Date # Reset age timer for proactive refresh Write-LogHost " [TOKEN] Token refreshed via AppRegistration (refresh #$($script:SharedAuthState.RefreshCount))" -ForegroundColor Green + Resolve-PaxTokenRefresh -Success $true -CompletionUtc (Get-Date).ToUniversalTime(); $script:PaxTokenRefreshResolved = $true return $true } } @@ -14864,6 +25387,7 @@ function Refresh-GraphTokenIfNeeded { if ($script:AuthConfig.Method -eq 'AppRegistration' -and $Force) { Write-LogHost " [TOKEN] FATAL: AppRegistration token refresh failed. Cannot continue headless (-Force mode)." -ForegroundColor Red Write-LogHost " [TOKEN] Check: client secret expiration, certificate validity, or API permissions." -ForegroundColor Yellow + Resolve-PaxTokenRefresh -Success $false -CompletionUtc (Get-Date).ToUniversalTime() -ErrorCategory 'Fatal'; $script:PaxTokenRefreshResolved = $true return 'Quit' } @@ -14871,12 +25395,18 @@ function Refresh-GraphTokenIfNeeded { $refreshResult = Invoke-TokenRefreshPrompt if ($refreshResult -eq 'Quit') { # User chose to quit - return special value for callers to handle + Resolve-PaxTokenRefresh -Success $false -CompletionUtc (Get-Date).ToUniversalTime() -ErrorCategory 'UserQuit'; $script:PaxTokenRefreshResolved = $true return 'Quit' } # User pressed R and successfully re-authenticated # Invoke-TokenRefreshPrompt already updated SharedAuthState and reset AuthFailureDetected + Resolve-PaxTokenRefresh -Success $true -CompletionUtc (Get-Date).ToUniversalTime(); $script:PaxTokenRefreshResolved = $true return $true + } + finally { + if (-not $script:PaxTokenRefreshResolved) { Resolve-PaxTokenRefresh -Success $false -CompletionUtc (Get-Date).ToUniversalTime() -ErrorCategory 'RefreshError' } + } } # ============================================== @@ -15040,6 +25570,19 @@ function Initialize-CheckpointForNewRun { # CopilotInteraction run restores the correct --profile (AIO vs AIBV cannot # be re-derived from -IncludeM365Usage alone). Last-write-wins on resume. rollupDashboard = if ($script:RollupDashboard) { $script:RollupDashboard } else { 'None' } + # Dashboard value (AISID etc.), distinct from the embedded rollup processor + # profile above. Persisted so a resume restores the dashboard the run was launched with + # (an AISID run embeds the AIO processor, so rollupDashboard alone cannot recover it). + dashboard = if ([string]$AllParameters.Dashboard -ieq 'ValueLens') { 'ValueLens' } elseif ($AllParameters.Dashboard) { [string]$AllParameters.Dashboard } else { 'None' } + # AISID (Defender) output set destination + delta-cache switch - restored on resume so + # an AISID resume re-targets and re-caches exactly as the original run did. + outputPathDefenderUsage = if ($AllParameters.OutputPathDefenderUsage) { [string]$AllParameters.OutputPathDefenderUsage } else { '' } + appendDefenderUsage = if ($AllParameters.AppendDefenderUsage) { [string]$AllParameters.AppendDefenderUsage } else { '' } + disableAISIDDeltaCache = [bool]$AllParameters.DisableAISIDDeltaCache + # Resolved AISID (DefenderUsage) output folder — the same Resolve-DataTypePaths + # 'DefenderUsage' .EffectiveDir value shown in the parameter snapshot / metrics JSON, + # persisted so a resume checkpoint records where the AISID CSV set was routed. + resolvedDefenderUsageDir = (script:Get-AISIDResolvedOutputDir) # Auth (method only - no secrets) auth = if ($AllParameters.Auth) { $AllParameters.Auth } else { 'WebLogin' } @@ -15064,6 +25607,11 @@ function Initialize-CheckpointForNewRun { blockHours = if ($AllParameters.BlockHours) { $AllParameters.BlockHours } else { 0.5 } completed = @() queryCreated = @() + # Durable uncertain-create bucket: partitions whose first create POST may have + # been accepted but cannot be proven. Absent on legacy checkpoints; treated as + # empty on load. A partition here is reconciled read-only on resume and NEVER + # re-POSTed. + uncertainCreate = @() } statistics = @{ totalRecordsSaved = 0 @@ -15077,6 +25625,29 @@ function Initialize-CheckpointForNewRun { rowsGenerated = 0 lastUpdateTime = $null } + # AISID rolling-window intersection state; present only for -Dashboard AISID runs, + # else $null. Persists the frozen ReferenceNowUtc plus requested/available/effective bounds and + # clip status so a resume replays the run's frozen window instead of the resume-time clock. + aisidWindow = if ($script:AISIDWindow) { + @{ + referenceNowUtc = $script:AISIDWindow.ReferenceNowUtc.ToString('o') + requestedStartUtc = $script:AISIDWindow.RequestedStartUtc.ToString('o') + requestedEndUtc = $script:AISIDWindow.RequestedEndUtc.ToString('o') + availableStartUtc = $script:AISIDWindow.AvailableStartUtc.ToString('o') + availableEndUtc = $script:AISIDWindow.AvailableEndUtc.ToString('o') + effectiveStartUtc = $script:AISIDWindow.EffectiveStartUtc.ToString('o') + effectiveEndUtc = $script:AISIDWindow.EffectiveEndUtc.ToString('o') + status = [string]$script:AISIDWindow.Status + wasClipped = [bool]$script:AISIDWindow.WasClipped + clipReason = [string]$script:AISIDWindow.ClipReason + retentionDays = [int]$script:AISIDWindow.RetentionDays + } + } else { $null } + # AISID resume-state version marker. Present (=1) only when this run resolved an AISID window + # AND persisted the dashboard + DefenderUsage destination + cache state above. Its presence + # marks a checkpoint that carries the full AISID resume state (dashboard + destination), versus + # one that stores only the collection window. + aisidResumeStateVersion = if ($script:AISIDWindow) { 1 } else { $null } } # Save initial checkpoint @@ -15085,22 +25656,871 @@ function Initialize-CheckpointForNewRun { return $script:PartialOutputPath } -function Sync-FabricResumeMirror { +# Contract version stamped into every resume commit manifest. A manifest whose +# contractVersion does not match is treated as unverifiable and is never selected. +# Version 2 adds the mandatory nonce-bearing generationId, so remote generation and +# authority-record identity no longer depends on a numeric sequence alone. +# Version 3 adds the mandatory per-artifact CONTENT identity (sha256 + role). A +# self-digest proves the manifest is the complete object the writer intended; it says +# nothing about the bytes the manifest REFERS to. Recording the writer-computed digest +# of each artifact is what lets a reader re-derive the digest from the bytes it actually +# downloaded and prove the artifact set is the one that was committed. +$script:FabricResumeMirrorContractVersion = 3 + +function Get-FabricResumeArtifactSha256 { + <# + .SYNOPSIS + Lowercase-hex SHA256 of a local file, streamed so an arbitrarily large shard or + partial output is never materialised in memory. + .DESCRIPTION + This is the ONLY content-identity primitive the resume protocol uses. A + size/mtime stamp is a change HINT, not an identity: two different payloads can + share a length, and a stamp says nothing about the bytes a remote object holds. + The writer records this digest for every artifact it publishes and the reader + re-derives it from the bytes it downloaded, so a same-length substitution is + detected instead of silently resumed. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Path) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $stream = [System.IO.File]::Open($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite) + try { $hash = $sha.ComputeHash($stream) } + finally { $stream.Dispose() } + return (($hash | ForEach-Object { $_.ToString('x2') }) -join '') + } + finally { $sha.Dispose() } +} + +function Test-FabricResumeArtifactPath { + <# + .SYNOPSIS + Returns $true only for a safe run-relative resume artifact path. + .DESCRIPTION + An artifact path arrives from a remote manifest, so it is untrusted input that is + about to be joined onto a local directory. A rooted path, a drive qualifier, a + backslash or duplicated separator, a trailing separator, or any '.'/'..' segment + could place a hydrated file OUTSIDE the resume directory, so every one of them + fails closed. Only plain forward-slash-separated relative segments are accepted. + #> + [CmdletBinding()] + param([string] $Path) + $p = [string]$Path + if ([string]::IsNullOrWhiteSpace($p)) { return $false } + if ($p.Contains('\') -or $p.Contains(':')) { return $false } + if ($p.StartsWith('/') -or $p.EndsWith('/') -or $p.Contains('//')) { return $false } + foreach ($seg in $p.Split('/')) { + if ($seg -eq '' -or $seg -eq '.' -or $seg -eq '..') { return $false } + } + return $true +} + +function New-FabricResumeCommitIntegrity { + <# + .SYNOPSIS + Deterministic integrity digest over the identity-bearing fields of a resume + commit manifest. + .DESCRIPTION + The commit manifest is the single object whose successful write promotes a + staged generation to authoritative. OneLake file creation is a multi-request + PUT/PATCH/flush sequence, so a torn or truncated manifest object IS possible. + This digest lets a reader prove the manifest it downloaded is the complete + object the writer intended: it is computed over contractVersion, runTimestamp, + generation, and the ordered artifact tuples, and is embedded in the manifest. + A manifest whose recomputed digest disagrees is torn/ambiguous and is rejected. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Manifest) + $sb = [System.Text.StringBuilder]::new() + [void]$sb.Append([string]$Manifest.contractVersion) + [void]$sb.Append('|'); [void]$sb.Append([string]$Manifest.runTimestamp) + [void]$sb.Append('|'); [void]$sb.Append([string]$Manifest.generation) + # The nonce-bearing generation identity is part of the digested identity, so two + # authority records that share a sequence but name different generations can never + # produce the same digest and are always distinguishable on read. + [void]$sb.Append('|'); [void]$sb.Append([string]$Manifest.generationId) + foreach ($a in @($Manifest.artifacts)) { + [void]$sb.Append('|') + [void]$sb.Append([string]$a.localPath); [void]$sb.Append('>') + [void]$sb.Append([string]$a.remotePath); [void]$sb.Append('>') + [void]$sb.Append([string]$a.generation); [void]$sb.Append('>') + [void]$sb.Append([string]$a.length); [void]$sb.Append('>') + # The artifact role and its CONTENT digest are part of the digested identity, so a + # manifest cannot be re-pointed at different bytes (or have its checkpoint role + # moved) while still presenting a valid self-digest. + [void]$sb.Append([string]$a.role); [void]$sb.Append('>') + [void]$sb.Append([string]$a.sha256) + } + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $hash = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($sb.ToString())) + return (($hash | ForEach-Object { $_.ToString('x2') }) -join '') + } + finally { $sha.Dispose() } +} + +function Test-FabricResumeCommitManifest { + <# + .SYNOPSIS + Validates a downloaded resume commit manifest. Returns $true only for a + complete, self-consistent, current-contract manifest. + .DESCRIPTION + FAIL CLOSED by design: any missing field, wrong contract version, non-positive + generation, malformed artifact entry, or integrity-digest mismatch returns + $false so the caller falls back to an OLDER verified generation instead of + guessing. Never throws - an unreadable manifest is simply not verified. + + The artifact list is validated as a SET, not merely element by element: every + local path and every remote path must be unique, every remote path must live in + the generation folder its own owning generation names, every artifact must carry + a well-formed content digest and a non-negative length, and exactly one artifact + may carry the checkpoint role. Structural validation here proves the manifest is + internally consistent and complete; it does NOT prove the referenced objects hold + those bytes. That proof is re-derived from the downloaded bytes in + Copy-FabricResumeCommitArtifacts, and a generation is usable only when BOTH pass. + #> + [CmdletBinding()] + param([object] $Manifest) + if (-not $Manifest) { return $false } + $names = @($Manifest.PSObject.Properties | ForEach-Object { $_.Name }) + foreach ($f in @('contractVersion', 'runTimestamp', 'generation', 'generationId', 'artifacts', 'integritySha256')) { + if ($names -notcontains $f) { return $false } + } + $contract = 0 + if (-not [int]::TryParse([string]$Manifest.contractVersion, [ref]$contract)) { return $false } + if ($contract -ne $script:FabricResumeMirrorContractVersion) { return $false } + if ([string]::IsNullOrWhiteSpace([string]$Manifest.runTimestamp)) { return $false } + $gen = 0 + if (-not [int]::TryParse([string]$Manifest.generation, [ref]$gen)) { return $false } + if ($gen -lt 1) { return $false } + # The generation identity must carry BOTH the ordering sequence and a nonce, and the + # sequence it carries must agree with the numeric generation field. A sequence-only + # name is rejected: object identity may never rely on a reusable sequence alone. + $genId = [string]$Manifest.generationId + if ($genId -notmatch '^gen-(\d+)-[0-9a-f]{16,}$') { return $false } + if ([int]$Matches[1] -ne $gen) { return $false } + if ($null -eq $Manifest.artifacts) { return $false } + $artifactList = @($Manifest.artifacts) + if ($artifactList.Count -lt 1) { return $false } + $seenLocal = @{} + $seenRemote = @{} + $checkpointCount = 0 + foreach ($a in $artifactList) { + if (-not $a) { return $false } + $an = @($a.PSObject.Properties | ForEach-Object { $_.Name }) + foreach ($af in @('localPath', 'remotePath', 'generation', 'length', 'sha256', 'role')) { + if ($an -notcontains $af) { return $false } + } + # The local path is untrusted input that is about to be joined onto a local + # directory, so traversal, rooted and alternate-separator forms fail closed. + $lp = [string]$a.localPath + if (-not (Test-FabricResumeArtifactPath -Path $lp)) { return $false } + if ($seenLocal.ContainsKey($lp)) { return $false } + $seenLocal[$lp] = $true + + $aGen = 0 + if (-not [int]::TryParse([string]$a.generation, [ref]$aGen)) { return $false } + # An artifact may be owned by this generation or INHERITED from an earlier one, + # but never from a generation that does not exist yet. + if ($aGen -lt 1 -or $aGen -gt $gen) { return $false } + + # An artifact must live inside the generation folder its OWN owning generation + # names, and the path inside that folder must be exactly its local path. A bare + # (non-generational) or disagreeing remote path would let a reader hydrate from + # unstaged, unknown, or another generation's state. + $rp = [string]$a.remotePath + if (-not $rp) { return $false } + $slash = $rp.IndexOf('/') + if ($slash -lt 1) { return $false } + $genFolder = $rp.Substring(0, $slash) + $genRest = $rp.Substring($slash + 1) + if ($genFolder -notmatch '^gen-(\d+)-[0-9a-f]{16,}$') { return $false } + if ([int]$Matches[1] -ne $aGen) { return $false } + if ($genRest -cne $lp) { return $false } + if ($seenRemote.ContainsKey($rp)) { return $false } + $seenRemote[$rp] = $true + + $len = [int64]0 + if (-not [int64]::TryParse([string]$a.length, [ref]$len)) { return $false } + if ($len -lt 0) { return $false } + + # Content identity is mandatory: without it a manifest can only prove its own + # shape, never that the objects it names hold the committed bytes. + if (([string]$a.sha256) -cnotmatch '^[0-9a-f]{64}$') { return $false } + + $role = [string]$a.role + if ($role -ceq 'checkpoint') { $checkpointCount++ } + elseif ($role -cne 'artifact') { return $false } + } + # A resume generation without exactly one checkpoint is not a resumable transaction. + if ($checkpointCount -ne 1) { return $false } + if ((New-FabricResumeCommitIntegrity -Manifest $Manifest) -cne [string]$Manifest.integritySha256) { return $false } + return $true +} + +function New-FabricResumeGenerationNonce { + <# + .SYNOPSIS + Collision-resistant lowercase-hex nonce appended to every resume generation + folder name and authority-record name. + .DESCRIPTION + A numeric sequence is USEFUL FOR ORDERING but is NOT an identity: it is derived + from a listing that may be stale or eventually consistent, and two processes (or + one process before and after a restart) can compute the same next sequence. The + nonce makes the remote object NAME unique regardless, so a stale listing can + never cause one generation to overwrite another. + #> + [CmdletBinding()] + param() + return ([guid]::NewGuid().ToString('n').Substring(0, 16)) +} + +function Get-FabricResumeNameSequence { + <# + .SYNOPSIS + Parses the ordering sequence out of a resume generation folder name or authority + record leaf name. Returns -1 when the name is not a resume protocol name. + .DESCRIPTION + Accepts BOTH the nonce-bearing current form (gen--, + commit--.json) and the earlier sequence-only form (gen-, + commit-.json). The earlier form is accepted HERE ONLY so that such a name + still RESERVES its sequence for high-water purposes; it is never written and, + because the manifest contract requires a nonce-bearing generationId, it is never + selected as authoritative. + #> + [CmdletBinding()] + param([string] $Name) + $n = [string]$Name + if ($n -match '^gen-(\d+)(-[0-9a-f]+)?$') { return [int]$Matches[1] } + if ($n -match '^commit-(\d+)(-[0-9a-f]+)?\.json$') { return [int]$Matches[1] } + return -1 +} + +function Get-FabricResumeRemoteHighWater { + <# + .SYNOPSIS + Highest sequence observed anywhere in a run's remote mirror listing. + .DESCRIPTION + Considers EVERY generation folder name and EVERY authority-record name, including + orphaned staging generations that carry no authority record and authority records + that fail verification. An orphan still RESERVES its name, so allocating strictly + above this value is what keeps a retry (or a restarted process) from re-staging + into a name that already exists remotely. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [AllowNull()] [AllowEmptyCollection()] [string[]] $RelativePaths) + $high = 0 + foreach ($p in @($RelativePaths)) { + $rest = [string]$p + if (-not $rest) { continue } + $seq = if ($rest.StartsWith('commits/')) { + Get-FabricResumeNameSequence -Name ($rest.Substring(8)) + } + else { + Get-FabricResumeNameSequence -Name ($rest.Split('/')[0]) + } + if ($seq -gt $high) { $high = $seq } + } + return $high +} + +function Get-FabricResumeRunListing { + <# + .SYNOPSIS + Lists everything under Files/.pax_resume// and returns the paths + RELATIVE to that run folder (directories included). + .DESCRIPTION + Directories are deliberately INCLUDED: a staging generation whose very first + artifact upload failed can exist as a folder with no files, and its name still + reserves a sequence. A 404 means the run has no mirror yet and yields an empty + listing; any other listing failure THROWS, because a publication that cannot + observe the remote high-water mark must fail closed rather than guess. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $RunTimestamp) + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume/$RunTimestamp" } else { ".pax_resume/$RunTimestamp" } + $dirParam = [System.Uri]::EscapeDataString("Files/$relInItem") + $listUri = "$($resolved.FilesystemBase)?resource=filesystem&recursive=true&directory=$($resolved.ItemFull)/$dirParam" + + $resp = $null + try { + $resp = Invoke-FabricWebRequest -Uri $listUri -Method GET + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return @() } + throw + } + if (-not $resp -or -not $resp.Content) { return @() } + $parsed = $resp.Content | ConvertFrom-Json -ErrorAction Stop + if (-not $parsed -or -not $parsed.paths) { return @() } + + $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + $out = New-Object System.Collections.Generic.List[string] + foreach ($entry in $parsed.paths) { + $name = "$($entry.name)" + $rest = if ($name.StartsWith($prefix)) { $name.Substring($prefix.Length) } else { Split-Path -Leaf $name } + if ($rest) { [void]$out.Add($rest) } + } + return $out.ToArray() +} + +function ConvertTo-FabricResumeOwnership { <# .SYNOPSIS - Mirrors local resume artifacts to OneLake Files/.pax_resume//. + Converts a VERIFIED commit manifest's artifact list into an ownership map. .DESCRIPTION - Uploads the checkpoint JSON, every changed .pax_incremental/*.jsonl shard, - and the *_PARTIAL.csv companion to a durable OneLake folder so a Fabric - container restart can rehydrate the run via Restore-FabricResumeMirror. + Ownership reconstructed from a remote manifest ALONE is recorded with + InheritEligible = $false. The manifest states the digest the writer computed; it + is not proof that the remote object still holds those bytes, and a local + size/mtime stamp is not content identity either. Inheritance therefore requires a + digest that was RE-DERIVED from bytes actually downloaded and verified - that map + is produced by Copy-FabricResumeCommitArtifacts and recorded through + Register-FabricResumeRestoredState, never inferred here. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Manifest) + $map = @{} + foreach ($a in @($Manifest.artifacts)) { + $map[[string]$a.localPath] = @{ + RemotePath = [string]$a.remotePath + Generation = [int]$a.generation + Length = [int64]$a.length + Sha256 = [string]$a.sha256 + Role = [string]$a.role + InheritEligible = $false + } + } + return $map +} + +function Register-FabricResumeRestoredState { + <# + .SYNOPSIS + Records the verified authority manifest and artifact ownership produced by a + successful hydration, keyed by run timestamp. + .DESCRIPTION + Hydration and publication are separate concerns, and a restore that "succeeded" + while publication state stayed reset to zero is exactly the hidden coupling that + let a restarted process reuse generation 1. This record makes the reconstructed + state observable per run and never mixes ownership between runs. It is a RECORD, + not the allocation authority: Sync-FabricResumeMirror independently re-derives the + remote high-water mark before it stages anything. + + When the caller supplies VerifiedOwnership - the map produced by a hydration that + re-derived every artifact digest from the bytes it downloaded - that map is + recorded instead of the manifest-only reconstruction, because only a re-derived + digest is content identity. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [object] $Manifest, + [Parameter(Mandatory)] [AllowNull()] [AllowEmptyCollection()] [string[]] $RelativePaths, + [hashtable] $VerifiedOwnership + ) + if (-not $script:FabricResumeRestoredState) { $script:FabricResumeRestoredState = @{} } + $high = Get-FabricResumeRemoteHighWater -RelativePaths $RelativePaths + if ([int]$Manifest.generation -gt $high) { $high = [int]$Manifest.generation } + $ownership = if ($VerifiedOwnership -and $VerifiedOwnership.Count -gt 0) { $VerifiedOwnership } else { ConvertTo-FabricResumeOwnership -Manifest $Manifest } + $script:FabricResumeRestoredState[[string]$RunTimestamp] = @{ + Generation = [int]$Manifest.generation + GenerationId = [string]$Manifest.generationId + HighWater = [int]$high + Artifacts = $ownership + Manifest = $Manifest + } +} + +function Initialize-FabricResumeCommitStateFromRemote { + <# + .SYNOPSIS + Reconstructs remote generation state for a run from the REMOTE mirror before any + artifact is staged. Runs once per process per run. + .DESCRIPTION + $script:FabricResumeCommitState is an in-process CACHE, never the source of truth + for remote generation allocation: it does not survive a container restart, so a + restarted process previously computed generation 1 again and overwrote the + existing gen-1 objects and authority record. This function makes the remote store + the authority - it reads the run's listing, takes the high-water sequence across + committed generations, ORPHANED staging generations and every authority record, + and reconstructs the current authoritative generation plus artifact ownership from + the newest VERIFIED manifest. The next allocation is then strictly above every + remotely observed sequence. + + Fails closed: a listing failure other than 404 throws, because a publication that + cannot observe the remote high-water mark must not guess at a name. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $RunTimestamp) + if (-not $script:FabricResumeCommitState) { + $script:FabricResumeCommitState = @{ Generation = 0; Counter = 0; Artifacts = @{} } + } + if ($script:FabricResumeCommitState['RemoteReconciled'] -eq $true -and + ([string]$script:FabricResumeCommitState['RunTimestamp']) -ceq [string]$RunTimestamp) { + return + } + + $listing = @(Get-FabricResumeRunListing -RunTimestamp $RunTimestamp) + $high = Get-FabricResumeRemoteHighWater -RelativePaths $listing + $commitLeaves = New-Object System.Collections.Generic.List[string] + foreach ($rel in @($listing)) { + $r = [string]$rel + if ($r.StartsWith('commits/')) { + $leaf = $r.Substring(8) + if ($leaf -and -not $leaf.Contains('/')) { [void]$commitLeaves.Add($leaf) } + } + } + + $generation = 0 + $artifacts = @{} + if ($commitLeaves.Count -gt 0) { + $manifest = Get-FabricResumeAuthoritativeCommit -RunTimestamp $RunTimestamp -CommitLeafNames $commitLeaves.ToArray() + if ($manifest) { + $generation = [int]$manifest.generation + $artifacts = ConvertTo-FabricResumeOwnership -Manifest $manifest + # If THIS process hydrated exactly this generation and re-derived every + # artifact digest from the downloaded bytes, that verified ownership replaces + # the manifest-only reconstruction: it is the one map whose digests are content + # identity rather than the writer's claim. Anything else stays + # non-inherit-eligible and is re-staged into the fresh generation. + $restored = $null + if ($script:FabricResumeRestoredState) { $restored = $script:FabricResumeRestoredState[[string]$RunTimestamp] } + if ($restored -and ([string]$restored.GenerationId) -ceq ([string]$manifest.generationId) -and $restored.Artifacts) { + $artifacts = $restored.Artifacts + } + if ($generation -gt $high) { $high = $generation } + } + } + + $counter = [int]$script:FabricResumeCommitState['Counter'] + if ($high -gt $counter) { $counter = $high } + $script:FabricResumeCommitState = @{ + Generation = $generation + Counter = $counter + Artifacts = $artifacts + RemoteReconciled = $true + RunTimestamp = [string]$RunTimestamp + } +} + +function Get-FabricResumeAuthoritativeCommit { + <# + .SYNOPSIS + Resolves the authoritative (most recent VERIFIED) resume commit manifest for a + run from the list of commit object leaf names discovered under + .pax_resume//commits/. + .DESCRIPTION + Commit objects are IMMUTABLE and monotonically numbered (commit-.json); a + commit is never overwritten, so a torn write can only ever affect the NEWEST + object and can never destroy an already-established authority. Candidates are + evaluated newest-first and the first one that downloads, parses, matches its own + generation/run identity, and passes the integrity digest wins. A torn, missing, + or ambiguous newest commit is SKIPPED and the search falls back to the most + recent verified older generation. Returns $null when nothing verifies - the + caller must then hydrate nothing rather than guess. + + Staging generations (gen-/ folders with no committed manifest) are structurally + unreachable here: selection follows manifests only, never a directory listing. + + AMBIGUITY IS FATAL TO SELECTION. Candidates are grouped by sequence and evaluated + highest sequence first. If TWO OR MORE candidates at the SAME sequence both verify + but name different generation identities (or carry different integrity digests), + nothing is returned - the reader never resolves a conflict by listing order, and it + never falls back to a LOWER sequence to escape the conflict. A lower sequence is + reached only when every higher candidate is torn or unverifiable. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $CommitLeafNames + ) + $candidates = New-Object System.Collections.Generic.List[object] + foreach ($n in @($CommitLeafNames)) { + $leaf = [string]$n + $seq = Get-FabricResumeNameSequence -Name $leaf + if ($seq -ge 1) { + [void]$candidates.Add([pscustomobject]@{ Generation = $seq; Leaf = $leaf }) + } + } + foreach ($grp in @($candidates | Group-Object -Property Generation | Sort-Object -Property { [int]$_.Name } -Descending)) { + $seq = [int]$grp.Name + $verified = New-Object System.Collections.Generic.List[object] + foreach ($c in @($grp.Group | Sort-Object -Property Leaf)) { + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_commit_read_{0}.json" -f [guid]::NewGuid().ToString('N')) + $parsed = $null + try { + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$RunTimestamp/commits/$($c.Leaf)" -DestinationPath $tmp + $parsed = (Get-Content -LiteralPath $tmp -Raw -ErrorAction Stop) | ConvertFrom-Json -ErrorAction Stop + } + catch { $parsed = $null } + finally { if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } } + if (-not (Test-FabricResumeCommitManifest -Manifest $parsed)) { + Write-Verbose ("Resume commit {0} for run {1} is torn or unverifiable; it cannot be selected." -f $c.Leaf, $RunTimestamp) + continue + } + if ([int]$parsed.generation -ne $seq) { continue } + if ([string]$parsed.runTimestamp -cne [string]$RunTimestamp) { continue } + [void]$verified.Add($parsed) + } + if ($verified.Count -eq 0) { continue } + if ($verified.Count -gt 1) { + $ids = @(@($verified | ForEach-Object { [string]$_.generationId }) | Sort-Object -Unique) + $digests = @(@($verified | ForEach-Object { [string]$_.integritySha256 }) | Sort-Object -Unique) + if ($ids.Count -gt 1 -or $digests.Count -gt 1) { + Write-Verbose ("Resume commit sequence {0} for run {1} is AMBIGUOUS - {2} verified authority records with differing identities or contents; hydrating nothing (fail closed)." -f $seq, $RunTimestamp, $verified.Count) + return $null + } + } + return $verified[0] + } + return $null +} - No-op unless the Purview destination tier is Fabric. Failures throw so the - calling try/catch (in Save-CheckpointToDisk) can abort the run and keep the - local + mirror artifact set consistent. +function Copy-FabricResumeCommitArtifacts { + <# + .SYNOPSIS + Downloads, VERIFIES, and then atomically publishes exactly the artifact set named + by a VERIFIED resume commit manifest. + .DESCRIPTION + Each artifact carries the generation that owns it, so an artifact that was + unchanged (and therefore not re-uploaded) in the committed generation is + resolved from the earlier generation that does own it. Prior committed + generations are retained for the life of the run precisely so these inherited + references always resolve. Returns the number of files published. + + TWO-PHASE, ALL-OR-NOTHING. A manifest self-digest proves the manifest is the + complete object the writer intended; it proves NOTHING about the bytes the + manifest refers to, and a download that is hash-checked AFTER being written into + the live resume directory has already made a partial set discoverable. So: + + PHASE 1 (isolated) - every artifact is downloaded into a transaction-scoped + temporary hydration directory that auto-discovery cannot see. Each one must + exist, match its manifest LENGTH exactly, and match a SHA256 RE-DERIVED from the + bytes just downloaded (ordinal comparison; a same-length substitution therefore + fails). The checkpoint artifact must additionally parse as JSON. The staged set + must equal the manifest set exactly - no missing, extra, or duplicated entry. + The live LocalDir is not created, read, or written during this phase. + + PHASE 2 (promote) - only once the ENTIRE set has verified is the live directory + touched, and then as ONE exact-set replacement of the run's OWNED artifacts: + every owned path is SNAPSHOT to a transaction-scoped backup, the prior checkpoint + is WITHDRAWN FIRST (from that moment until the last step no checkpoint of this run + is discoverable, so a torn intermediate set can never be resumed), every owned + file the authority record does not name is withdrawn, the verified artifacts are + moved in, and the CHECKPOINT IS PUBLISHED LAST. Find-Checkpoints discovers a run + by its checkpoint file, so a candidate checkpoint becomes visible only once every + companion artifact is already in place. + + OWNED RESUME-ARTIFACT BOUNDARY. LocalDir is normally the customer's -OutputPath, so + a whole-directory replacement is NOT an option. Exactly three narrow, name-anchored + sets are owned, and nothing else in LocalDir is enumerated, moved or deleted: + 1. /.pax_checkpoint_.json - this run's canonical + checkpoint, the name this run publishes and Find-Checkpoints discovers. + 2. every file under /.pax_incremental/ - the resume protocol's own + private shard directory; nothing outside the protocol writes there. + 3. every *_PARTIAL* file directly in whose name carries this run's + timestamp - this run's partial companion. That timestamp scoping is what keeps + another run's partial companion OUT of the boundary. + Plus every localPath the authority record names, which is owned by definition. A + customer report, workbook, log, sub-folder, or another run's checkpoint is OUTSIDE + the boundary and is therefore untouchable. PreserveLocalPaths additionally exempts + paths an earlier run in the same multi-run sweep already published, so hydrating + run B never withdraws run A's verified shards from the shared shard directory. + + Any failure throws. On a phase-1 failure LocalDir is byte-for-byte untouched and + stays ABSENT if it did not already exist. On a phase-2 failure every candidate byte + this call placed is withdrawn and the snapshot is restored, so the prior verified + set - checkpoint AND every companion - is put back byte-for-byte and stays + discoverable; a previously verified artifact is NEVER deleted in the name of + cleanup. When LocalDir did not exist beforehand it is removed again. The temporary + hydration and snapshot directories are always removed. + + When the caller supplies VerifiedOwnership it is populated with the RE-DERIVED + per-artifact digests, which is the only form of ownership that may later be + inherited instead of re-staged. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [object] $Manifest, + [Parameter(Mandatory)] [string] $LocalDir, + [hashtable] $VerifiedOwnership, + [string[]] $PreserveLocalPaths + ) + $artifactList = @($Manifest.artifacts) + $generationId = [string]$Manifest.generationId + if ($artifactList.Count -lt 1) { + throw "Resume generation '$generationId' for run $RunTimestamp names no artifacts; hydrating nothing." + } - Per-file change tracking lives in $script:FabricResumeMirrorState keyed by - full local path with "{size}:{ticks}" stamps to avoid re-uploading - unchanged shards on every checkpoint write. + $stageDir = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_hydrate_{0}" -f [guid]::NewGuid().ToString('N')) + New-Item -Path $stageDir -ItemType Directory -Force | Out-Null + try { + # ---- PHASE 1: download and verify in ISOLATION. LocalDir is never touched. ---- + $staged = New-Object System.Collections.Generic.List[object] + $verified = @{} + $seenLocal = @{} + foreach ($a in $artifactList) { + $lp = [string]$a.localPath + if (-not (Test-FabricResumeArtifactPath -Path $lp)) { + throw "Resume generation '$generationId' for run $RunTimestamp names an unsafe artifact path; hydrating nothing." + } + if ($seenLocal.ContainsKey($lp)) { + throw "Resume generation '$generationId' for run $RunTimestamp names artifact '$lp' more than once; hydrating nothing." + } + $seenLocal[$lp] = $true + + $dest = Join-Path $stageDir $lp + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$RunTimestamp/$([string]$a.remotePath)" -DestinationPath $dest + if (-not (Test-Path -LiteralPath $dest -PathType Leaf)) { + throw "Resume artifact '$lp' of generation '$generationId' for run $RunTimestamp did not download; hydrating nothing." + } + $actualLength = [int64](Get-Item -LiteralPath $dest).Length + if ($actualLength -ne [int64]$a.length) { + throw "Resume artifact '$lp' of generation '$generationId' for run $RunTimestamp has length $actualLength but the authority record declares $([int64]$a.length); hydrating nothing." + } + # Content identity is RE-DERIVED from the bytes that actually arrived - the one + # check a same-length substitution cannot survive. + $actualSha = Get-FabricResumeArtifactSha256 -Path $dest + if ($actualSha -cne ([string]$a.sha256)) { + throw "Resume artifact '$lp' of generation '$generationId' for run $RunTimestamp failed content verification; hydrating nothing." + } + if (([string]$a.role) -ceq 'checkpoint') { + try { $null = (Get-Content -LiteralPath $dest -Raw -ErrorAction Stop) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "Resume checkpoint '$lp' of generation '$generationId' for run $RunTimestamp is not parseable JSON; hydrating nothing." } + } + [void]$staged.Add([pscustomobject]@{ LocalPath = $lp; StagePath = $dest; Role = [string]$a.role }) + $verified[$lp] = @{ + RemotePath = [string]$a.remotePath + Generation = [int]$a.generation + Length = $actualLength + Sha256 = $actualSha + Role = [string]$a.role + InheritEligible = $true + } + } + # The staged set must be exactly the manifest set - an extra file would mean the + # generation is not the one the authority record describes. + $stagedOnDisk = @(Get-ChildItem -LiteralPath $stageDir -Recurse -File -Force -ErrorAction SilentlyContinue) + if ($stagedOnDisk.Count -ne $artifactList.Count) { + throw "Resume generation '$generationId' for run $RunTimestamp staged $($stagedOnDisk.Count) files for $($artifactList.Count) declared artifacts; hydrating nothing." + } + + # ---- PHASE 2: promote as ONE exact-set replacement of the OWNED boundary. ---- + $localDirExisted = (Test-Path -LiteralPath $LocalDir -PathType Container) + $manifestPaths = @($staged | ForEach-Object { [string]$_.LocalPath }) + $checkpointPaths = @(@($staged | Where-Object { $_.Role -ceq 'checkpoint' } | ForEach-Object { [string]$_.LocalPath }) + ".pax_checkpoint_$RunTimestamp.json") + $preserved = @{} + foreach ($p in @($PreserveLocalPaths)) { if ($p) { $preserved[[string]$p] = $true } } + + # Enumerate the OWNED RESUME-ARTIFACT BOUNDARY (see .DESCRIPTION). Nothing outside + # it is listed here, so nothing outside it can be withdrawn - a customer report or + # sub-folder sitting in the same -OutputPath is structurally unreachable. + $owned = New-Object System.Collections.Generic.List[string] + $ownedSeen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($rel in @($manifestPaths + $checkpointPaths)) { + if ($rel -and $ownedSeen.Add([string]$rel)) { [void]$owned.Add([string]$rel) } + } + if ($localDirExisted) { + $incLive = Join-Path $LocalDir '.pax_incremental' + if (Test-Path -LiteralPath $incLive -PathType Container) { + foreach ($f in @(Get-ChildItem -LiteralPath $incLive -Recurse -File -Force -ErrorAction SilentlyContinue)) { + $relInc = '.pax_incremental/' + $f.FullName.Substring($incLive.Length + 1).Replace('\', '/') + if ($ownedSeen.Add($relInc)) { [void]$owned.Add($relInc) } + } + } + foreach ($f in @(Get-ChildItem -LiteralPath $LocalDir -File -Force -ErrorAction SilentlyContinue)) { + $leaf = [string]$f.Name + if ($leaf -like '*_PARTIAL*' -and $leaf -like "*$RunTimestamp*" -and $ownedSeen.Add($leaf)) { [void]$owned.Add($leaf) } + } + } + + $backupDir = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_prior_{0}" -f [guid]::NewGuid().ToString('N')) + $backedUp = New-Object System.Collections.Generic.List[string] + $published = New-Object System.Collections.Generic.List[string] + $createdDirs = New-Object System.Collections.Generic.List[string] + try { + # 2a) SNAPSHOT the prior owned set. This - not a delete-on-failure sweep - is + # what makes a failed promotion recoverable: deleting a target that already + # held a previously VERIFIED artifact destroys it instead of restoring it. + New-Item -Path $backupDir -ItemType Directory -Force | Out-Null + foreach ($rel in $owned) { + $live = Join-Path $LocalDir $rel + if (-not (Test-Path -LiteralPath $live -PathType Leaf)) { continue } + $bak = Join-Path $backupDir $rel + $bakDir = Split-Path -Parent $bak + if ($bakDir -and -not (Test-Path -LiteralPath $bakDir -PathType Container)) { + New-Item -Path $bakDir -ItemType Directory -Force | Out-Null + } + [System.IO.File]::Copy($live, $bak, $true) + [void]$backedUp.Add($rel) + } + + if (-not $localDirExisted) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + [void]$createdDirs.Add($LocalDir) + } + + # 2b) WITHDRAW the prior checkpoint FIRST. From here until 2d no checkpoint of + # this run is discoverable, so an interruption exposes no candidate set at + # all rather than a checkpoint whose companions are mid-replacement. + foreach ($rel in $owned) { + if ($preserved.ContainsKey($rel)) { continue } + if ($checkpointPaths -notcontains $rel) { continue } + $live = Join-Path $LocalDir $rel + if (Test-Path -LiteralPath $live -PathType Leaf) { [System.IO.File]::Delete($live) } + } + + # 2c) EXACT SET: withdraw every owned file the authority record does not name, + # so no stale shard, stale partial, or prior-generation artifact survives + # into the promoted set and downstream enumeration sees exactly one set. + foreach ($rel in $owned) { + if ($preserved.ContainsKey($rel)) { continue } + if ($manifestPaths -contains $rel) { continue } + $live = Join-Path $LocalDir $rel + if (Test-Path -LiteralPath $live -PathType Leaf) { [System.IO.File]::Delete($live) } + } + + # 2d) Publish the verified set, CHECKPOINT LAST. + $ordered = @(@($staged | Where-Object { $_.Role -cne 'checkpoint' }) + @($staged | Where-Object { $_.Role -ceq 'checkpoint' })) + foreach ($s in $ordered) { + $target = Join-Path $LocalDir ([string]$s.LocalPath) + $targetDir = Split-Path -Parent $target + if ($targetDir -and -not (Test-Path -LiteralPath $targetDir -PathType Container)) { + New-Item -Path $targetDir -ItemType Directory -Force | Out-Null + [void]$createdDirs.Add($targetDir) + } + Move-PaxFileOverwriteAtomic -SourcePath ([string]$s.StagePath) -DestinationPath $target + [void]$published.Add($target) + } + } + catch { + # ROLLBACK: withdraw every candidate byte this call ACTUALLY PLACED, then + # RESTORE the snapshot. Withdrawal names $published rather than the INTENDED + # set $manifestPaths, because withdrawal and restore must cover the SAME + # files: restore can only put back what the snapshot loop captured, so a + # manifest-wide withdrawal is broader than the restore and permanently loses + # a prior verified file that the snapshot loop had not reached when it threw. + # At a snapshot-window failure $published is empty, so the prior live set is + # left completely untouched. The prior verified set is put back - it is never + # deleted in the name of cleanup, which is what made a failed publication + # destructive before. + foreach ($live in $published) { + if (Test-Path -LiteralPath $live -PathType Leaf) { Remove-Item -LiteralPath $live -Force -ErrorAction SilentlyContinue } + } + foreach ($rel in $backedUp) { + $live = Join-Path $LocalDir $rel + $liveDir = Split-Path -Parent $live + if ($liveDir -and -not (Test-Path -LiteralPath $liveDir -PathType Container)) { + New-Item -Path $liveDir -ItemType Directory -Force -ErrorAction SilentlyContinue | Out-Null + } + [System.IO.File]::Copy((Join-Path $backupDir $rel), $live, $true) + } + for ($i = $createdDirs.Count - 1; $i -ge 0; $i--) { + $cd = $createdDirs[$i] + if (-not (Test-Path -LiteralPath $cd -PathType Container)) { continue } + if (-not $localDirExisted) { Remove-Item -LiteralPath $cd -Recurse -Force -ErrorAction SilentlyContinue; continue } + if (@(Get-ChildItem -LiteralPath $cd -Force -ErrorAction SilentlyContinue).Count -eq 0) { + Remove-Item -LiteralPath $cd -Force -ErrorAction SilentlyContinue + } + } + throw + } + finally { + if (Test-Path -LiteralPath $backupDir) { Remove-Item -LiteralPath $backupDir -Recurse -Force -ErrorAction SilentlyContinue } + } + + if ($null -ne $VerifiedOwnership) { + foreach ($k in @($verified.Keys)) { $VerifiedOwnership[$k] = $verified[$k] } + } + return $published.Count + } + finally { + if (Test-Path -LiteralPath $stageDir) { Remove-Item -LiteralPath $stageDir -Recurse -Force -ErrorAction SilentlyContinue } + } +} + +function Sync-FabricResumeMirror { + <# + .SYNOPSIS + Publishes the run's resume artifact set to OneLake as a TRANSACTIONAL, + generation-scoped commit under Files/.pax_resume//. + .DESCRIPTION + A sequential multi-file upload is NOT a transaction. Overwriting a single + remote checkpoint object in place meant a checkpoint upload that succeeded + before a later shard/partial upload failed left the NEW checkpoint remotely + authoritative while the local transaction rolled back - a container restart + then rehydrated the repudiated remote state and erased the safety the local + rollback achieved. This function replaces that with an explicit commit protocol: + + 1. STAGE - every artifact that is new or changed is uploaded into a fresh, + transaction-scoped generation folder gen--/. Nothing already committed + is overwritten, so a failure here cannot damage the prior generation. + 2. INHERIT - an unchanged artifact is NOT re-uploaded; the manifest records the + earlier generation that OWNS it. Prior committed generations are retained for + the life of the run (cleanup happens only in Remove-FabricResumeMirror after a + clean finish), so every inherited reference resolves at hydration time. + Inheritance requires ownership this PROCESS published; ownership reconstructed + from a remote manifest after a restart is not inherit-eligible, because a local + size/mtime stamp is not content identity. + 3. COMMIT - after the ENTIRE required set is staged successfully, ONE immutable + manifest object commits/commit--.json is written. That single write is + the ONLY authoritative-generation transition; there is no in-place pointer to + overwrite and therefore no way to expose a partially committed checkpoint. + + Failure semantics: + * any staging upload fails -> throws BEFORE the commit; generation is an + orphaned staging folder that hydration structurally cannot select, and the + prior committed generation remains authoritative. + * the commit write fails -> throws; the prior committed generation remains + authoritative (a torn commit-.json fails its integrity digest on read). + * on either failure the per-file change stamps are NOT advanced, so the next + transaction re-uploads everything the failed one staged. + + CONSISTENCY ASSUMPTIONS (stated plainly). OneLake/ADLS Gen2 offers no atomic + directory promotion through the API surface this script uses, so the commit point + is a single small object write. This protocol does NOT assume that write is atomic: + commit objects are immutable and monotonically numbered, and each carries an + integrity digest over its own identity-bearing fields, so a torn or truncated + commit object is DETECTED on read and the reader fails closed to the most recent + VERIFIED older generation. It does assume that a flushed object is subsequently + readable - a property this script already relies on for every customer-facing upload. + + IMMUTABILITY MECHANISM (what this actually relies on). Object names are NOT assumed + to be unique by convention, and the in-memory counter is NOT trusted to survive a + restart. Two independent mechanisms enforce immutability: + * NAME UNIQUENESS - the sequence is allocated strictly above the remote high-water + mark (Initialize-FabricResumeCommitStateFromRemote) AND every generation folder + and authority record carries a collision-resistant nonce, so identity never rests + on a sequence that a stale listing could cause two writers to compute. + * CREATE-IF-ABSENT - every resume-protocol upload uses + Send-FileToOneLake -FailIfExists, which issues the DFS Create File request with an + If-None-Match: * precondition. An existing target answers 409/412 and is raised as + a HARD COLLISION, never silently overwritten and never treated as success. A retry + then allocates a completely fresh identity rather than replacing the conflicted + object. If the storage surface ever stopped honouring the precondition, the nonce + alone still makes a name collision vanishingly unlikely, and any collision that + did occur would still fail closed. + Customer-facing output uploads are untouched by this: they keep the default + overwrite semantics because a customer output file is intentionally replaceable. + + ORDERING. Save-CheckpointStrict verifies the LOCAL publication (atomic replace + + re-read + parse) BEFORE calling this function, so a remote generation is never + promoted ahead of the local state it describes. + + No-op unless the Purview destination tier is Fabric. Local and SharePoint tiers + keep resume artifacts local-only. + + Per-file change tracking lives in $script:FabricResumeMirrorState keyed by full + local path with "{size}:{ticks}" stamps (advanced only on a committed + transaction); remote ownership lives in $script:FabricResumeCommitState. #> [CmdletBinding()] param() @@ -15109,47 +26529,163 @@ function Sync-FabricResumeMirror { if (-not $global:ScriptRunTimestamp) { return } if (-not $script:FabricResumeMirrorState) { $script:FabricResumeMirrorState = @{} } - $mirrorBase = ".pax_resume/$global:ScriptRunTimestamp" + # The in-memory commit state is a CACHE and does not survive a container restart, so + # remote generation identity is reconstructed from the REMOTE store before anything is + # staged. Without this a restarted process recomputed generation 1 and overwrote the + # existing gen-1 artifacts and authority record. + Initialize-FabricResumeCommitStateFromRemote -RunTimestamp $global:ScriptRunTimestamp - # Checkpoint JSON — always upload (small, mutates every call). - $cpName = Split-Path -Leaf $script:CheckpointPath - Send-FileToOneLake -LocalPath $script:CheckpointPath -RemoteFileName "$mirrorBase/$cpName" + $mirrorBase = ".pax_resume/$global:ScriptRunTimestamp" + # Every ATTEMPT claims a sequence strictly ABOVE every remotely observed sequence - + # committed generations, orphaned staging generations and authority records alike - + # AND a collision-resistant nonce. The sequence orders generations; the nonce is what + # makes the remote NAME unique, so neither a retry after a failed transaction, nor a + # process restart, nor a stale/eventually-consistent listing can re-stage into (or + # overwrite) an existing generation folder or authority record. + $generation = [int]$script:FabricResumeCommitState.Counter + 1 + $script:FabricResumeCommitState.Counter = $generation + $generationNonce = New-FabricResumeGenerationNonce + $generationId = "gen-$generation-$generationNonce" + $commitLeaf = "commit-$generation-$generationNonce.json" + + # Required artifact set for THIS transaction: the checkpoint JSON (always changed), + # every .pax_incremental/*.jsonl shard, and the *_PARTIAL.csv companion when present. + $required = New-Object System.Collections.Generic.List[object] + $cpItem = Get-Item -LiteralPath $script:CheckpointPath + [void]$required.Add([pscustomobject]@{ + LocalFullPath = $cpItem.FullName + LocalPath = $cpItem.Name + Length = [int64]$cpItem.Length + Stamp = ("{0}:{1}" -f $cpItem.Length, $cpItem.LastWriteTimeUtc.Ticks) + Sha256 = (Get-FabricResumeArtifactSha256 -Path $cpItem.FullName) + Role = 'checkpoint' + AlwaysUpload = $true + }) - # .pax_incremental shards — upload only those whose size or mtime changed since last sync. $cpDir = Split-Path -Parent $script:CheckpointPath $incDir = Join-Path $cpDir ".pax_incremental" if (Test-Path -LiteralPath $incDir -PathType Container) { foreach ($f in (Get-ChildItem -LiteralPath $incDir -Filter '*.jsonl' -File -ErrorAction SilentlyContinue)) { - $key = $f.FullName - $stamp = "{0}:{1}" -f $f.Length, $f.LastWriteTimeUtc.Ticks - if ($script:FabricResumeMirrorState[$key] -eq $stamp) { continue } - Send-FileToOneLake -LocalPath $f.FullName -RemoteFileName "$mirrorBase/.pax_incremental/$($f.Name)" - $script:FabricResumeMirrorState[$key] = $stamp + [void]$required.Add([pscustomobject]@{ + LocalFullPath = $f.FullName + LocalPath = ".pax_incremental/$($f.Name)" + Length = [int64]$f.Length + Stamp = ("{0}:{1}" -f $f.Length, $f.LastWriteTimeUtc.Ticks) + Sha256 = (Get-FabricResumeArtifactSha256 -Path $f.FullName) + Role = 'artifact' + AlwaysUpload = $false + }) } } - # *_PARTIAL.csv — upload if present and changed. if ($script:PartialOutputPath -and (Test-Path -LiteralPath $script:PartialOutputPath)) { $pf = Get-Item -LiteralPath $script:PartialOutputPath - $pkey = $pf.FullName - $pstamp = "{0}:{1}" -f $pf.Length, $pf.LastWriteTimeUtc.Ticks - if ($script:FabricResumeMirrorState[$pkey] -ne $pstamp) { - Send-FileToOneLake -LocalPath $pf.FullName -RemoteFileName "$mirrorBase/$($pf.Name)" - $script:FabricResumeMirrorState[$pkey] = $pstamp + [void]$required.Add([pscustomobject]@{ + LocalFullPath = $pf.FullName + LocalPath = $pf.Name + Length = [int64]$pf.Length + Stamp = ("{0}:{1}" -f $pf.Length, $pf.LastWriteTimeUtc.Ticks) + Sha256 = (Get-FabricResumeArtifactSha256 -Path $pf.FullName) + Role = 'artifact' + AlwaysUpload = $false + }) + } + + # 1/2) STAGE + INHERIT. Any throw here aborts BEFORE the commit point. + $artifacts = New-Object System.Collections.Generic.List[object] + foreach ($item in $required) { + $prior = $script:FabricResumeCommitState.Artifacts[[string]$item.LocalPath] + # Inheritance is decided on CONTENT IDENTITY, never on a size/mtime stamp: the + # recorded digest must be one this process either published itself or re-derived + # from bytes it downloaded and verified, and it must equal the digest of the local + # file as it stands right now. Anything else is staged into the fresh generation. + $inheritEligible = ($prior -and ($prior['InheritEligible'] -eq $true) -and + ([string]$prior['Sha256']) -and (([string]$prior['Sha256']) -ceq ([string]$item.Sha256))) + if ((-not $item.AlwaysUpload) -and $inheritEligible) { + [void]$artifacts.Add([pscustomobject]@{ + localPath = [string]$item.LocalPath + remotePath = [string]$prior.RemotePath + generation = [int]$prior.Generation + length = [int64]$prior.Length + role = [string]$item.Role + sha256 = [string]$prior.Sha256 + }) + continue + } + $remotePath = "$generationId/$([string]$item.LocalPath)" + Send-FileToOneLake -LocalPath $item.LocalFullPath -RemoteFileName "$mirrorBase/$remotePath" -FailIfExists + [void]$artifacts.Add([pscustomobject]@{ + localPath = [string]$item.LocalPath + remotePath = $remotePath + generation = $generation + length = [int64]$item.Length + role = [string]$item.Role + sha256 = [string]$item.Sha256 + }) + } + + # 3) COMMIT. One immutable manifest object; its successful write - and nothing + # before it - promotes generation to authoritative. + $manifest = [pscustomobject]@{ + contractVersion = $script:FabricResumeMirrorContractVersion + runTimestamp = [string]$global:ScriptRunTimestamp + generation = $generation + generationId = $generationId + createdUtc = (Get-Date).ToUniversalTime().ToString('o') + artifacts = $artifacts.ToArray() + integritySha256 = '' + } + $manifest.integritySha256 = New-FabricResumeCommitIntegrity -Manifest $manifest + $commitLocal = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_commit_{0}.json" -f [guid]::NewGuid().ToString('N')) + try { + ($manifest | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath $commitLocal -Encoding UTF8 -Force -ErrorAction Stop + Send-FileToOneLake -LocalPath $commitLocal -RemoteFileName "$mirrorBase/commits/$commitLeaf" -FailIfExists + } + finally { + if (Test-Path -LiteralPath $commitLocal) { Remove-Item -LiteralPath $commitLocal -Force -ErrorAction SilentlyContinue } + } + + # Promotion proven. ONLY now is the new generation recorded as authoritative and the + # per-file change stamps advanced. + $ownership = @{} + foreach ($a in $artifacts) { $ownership[[string]$a.localPath] = $a } + foreach ($item in $required) { + $a = $ownership[[string]$item.LocalPath] + $script:FabricResumeCommitState.Artifacts[[string]$item.LocalPath] = @{ + RemotePath = [string]$a.remotePath + Generation = [int]$a.generation + Length = [int64]$a.length + Sha256 = [string]$a.sha256 + Role = [string]$a.role + InheritEligible = $true } + $script:FabricResumeMirrorState[[string]$item.LocalFullPath] = [string]$item.Stamp } + $script:FabricResumeCommitState.Generation = $generation + $script:FabricResumeCommitState.GenerationId = $generationId } function Restore-FabricResumeMirror { <# .SYNOPSIS - Hydrates local resume artifacts from OneLake Files/.pax_resume//. + Hydrates local resume artifacts from the authoritative COMMITTED generation of + OneLake Files/.pax_resume//. .DESCRIPTION Used on startup of a Fabric run when a resume is requested and the local working copy is missing or partial (typical after a container restart). - Lists the mirror folder via the ADLS Gen2 List Paths API and downloads - every file into the supplied LocalDir, preserving the .pax_incremental - subfolder layout. Returns the number of files downloaded. + Lists the mirror folder via the ADLS Gen2 List Paths API, resolves the most + recent VERIFIED commit manifest, and downloads exactly the artifact set that + manifest names, resolving each artifact from the generation that owns it. + + Hydration NEVER enumerates gen-/ folders directly, so a staging generation + left behind by a failed transaction is structurally unreachable. A torn or + unverifiable newest commit falls back to the most recent verified older + generation; when nothing verifies, 0 is returned and nothing is hydrated + (fail closed) rather than guessing. + + A run mirror that carries NO commits/ folder at all predates the transactional + protocol; it is hydrated from its flat legacy layout, still excluding any + gen-/ or commits/ path. Throws on partial-download failure so callers can abort rather than resume from a torn artifact set. @@ -15179,18 +26715,61 @@ function Restore-FabricResumeMirror { $parsed = try { $resp.Content | ConvertFrom-Json -ErrorAction Stop } catch { $null } if (-not $parsed -or -not $parsed.paths) { return 0 } - if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { - New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null - } - - $downloaded = 0 + # Classify the listing into commit objects vs legacy (pre-transactional) artifacts. + # gen-/ entries are deliberately ignored: only a verified manifest may name them. $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + $commitLeaves = New-Object System.Collections.Generic.List[string] + $legacy = New-Object System.Collections.Generic.List[string] + # Every observed relative path, DIRECTORIES INCLUDED, is retained so the restored-state + # record carries the true remote high-water mark - an orphaned staging generation + # reserves its sequence even though it can never be selected as authoritative. + $allRel = New-Object System.Collections.Generic.List[string] + # Any gen-/ folder or commits/ object proves this run uses the transactional + # protocol. The flat legacy layout is then NEVER a fallback: doing so on a merely + # absent or not-yet-visible authority record would resume from unstaged state. + $transactional = $false foreach ($entry in $parsed.paths) { $isDir = $entry.isDirectory - if ($isDir -eq 'true' -or $isDir -eq $true) { continue } $name = "$($entry.name)" $rest = if ($name.StartsWith($prefix)) { $name.Substring($prefix.Length) } else { Split-Path -Leaf $name } + if ($rest) { [void]$allRel.Add($rest) } + if ($rest -and ($rest.StartsWith('gen-') -or $rest.StartsWith('commits/') -or $rest -eq 'commits')) { $transactional = $true } + if ($isDir -eq 'true' -or $isDir -eq $true) { continue } if (-not $rest) { continue } + if ($rest.StartsWith('commits/')) { [void]$commitLeaves.Add($rest.Substring(8)); continue } + if ($rest.StartsWith('gen-')) { continue } + [void]$legacy.Add($rest) + } + + if ($commitLeaves.Count -gt 0) { + $manifest = Get-FabricResumeAuthoritativeCommit -RunTimestamp $RunTimestamp -CommitLeafNames $commitLeaves + if (-not $manifest) { + Write-Verbose ("No verified resume commit for run {0}; hydrating nothing (fail closed)." -f $RunTimestamp) + return 0 + } + $verifiedOwnership = @{} + $downloaded = Copy-FabricResumeCommitArtifacts -RunTimestamp $RunTimestamp -Manifest $manifest -LocalDir $LocalDir -VerifiedOwnership $verifiedOwnership + # Record the verified authority and its artifact ownership for THIS run only, so a + # successful hydration never leaves publication state silently reset to zero. The + # ownership carries RE-DERIVED digests, so it is the one form that may be inherited. + Register-FabricResumeRestoredState -RunTimestamp $RunTimestamp -Manifest $manifest -RelativePaths $allRel -VerifiedOwnership $verifiedOwnership + return $downloaded + } + + if ($transactional) { + # Generation folders exist but no authority record does: a first transaction that + # has not committed, or a listing that has not caught up. Either way there is no + # authoritative state to hydrate, and the flat layout is not a substitute. + Write-Verbose ("Run {0} uses the transactional resume protocol but exposes no authority record; hydrating nothing (fail closed)." -f $RunTimestamp) + return 0 + } + + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + } + + $downloaded = 0 + foreach ($rest in $legacy) { $dest = Join-Path $LocalDir $rest $destDir = Split-Path -Parent $dest if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { @@ -15221,6 +26800,13 @@ function Remove-FabricResumeMirror { try { $null = Invoke-FabricWebRequest -Uri $dfsPath -Method DELETE if ($script:FabricResumeMirrorState) { $script:FabricResumeMirrorState.Clear() } + # Every committed generation AND any orphaned staging generation lives under the + # folder just deleted, so the recorded remote ownership is now meaningless. This is + # the ONLY place resume generations are removed: staging orphans are deliberately + # retained during the run so a failed transaction never destroys recovery evidence. + $script:FabricResumeCommitState = @{ Generation = 0; Counter = 0; Artifacts = @{} } + # The recorded hydration state describes objects that no longer exist. + $script:FabricResumeRestoredState = @{} } catch { Write-Verbose ("Resume mirror cleanup failed: {0}" -f $_.Exception.Message) } @@ -15232,11 +26818,18 @@ function Restore-AllFabricResumeMirrors { Discovers all in-progress run mirrors under OneLake Files/.pax_resume/ and downloads their artifacts to a local directory for auto-discover resume. .DESCRIPTION - Lists Files/.pax_resume/ recursively, then for each / folder - downloads every file (checkpoint JSON, *_PARTIAL.csv, .pax_incremental/*.jsonl) - into LocalDir, flattening the / prefix because every artifact - filename already embeds the run timestamp. Returns the number of files - downloaded. No-op on Local/SharePoint tiers. + Lists Files/.pax_resume/ recursively, then for each / folder resolves + that run's most recent VERIFIED commit manifest and downloads exactly the artifact + set it names (checkpoint JSON, *_PARTIAL.csv, .pax_incremental/*.jsonl) into + LocalDir. Artifact local paths are already relative to the run folder, so the + / prefix is naturally flattened - every artifact filename already + embeds the run timestamp. Returns the number of files downloaded. + + A staging generation left behind by a failed transaction is never selected: only a + verified commit manifest may name a generation. A run whose commits are all torn or + unverifiable contributes NOTHING (fail closed). A run mirror with no commits/ folder + at all predates the transactional protocol and is hydrated from its flat legacy + layout, still excluding any gen-/ or commits/ path. No-op on Local/SharePoint tiers. #> [CmdletBinding()] param([Parameter(Mandatory)] [string] $LocalDir) @@ -15264,27 +26857,80 @@ function Restore-AllFabricResumeMirrors { New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null } + # Group the flat listing by run timestamp, separating commit objects from legacy + # (pre-transactional) artifacts. gen-/ entries are deliberately ignored here: + # only a verified commit manifest is allowed to name a generation. $prefix = "$($resolved.ItemFull)/Files/$relInItem/" - $count = 0 + $byRun = @{} foreach ($e in $parsed.paths) { $isDir = $e.isDirectory - if ($isDir -eq 'true' -or $isDir -eq $true) { continue } $n = "$($e.name)" if (-not $n.StartsWith($prefix)) { continue } $rest = $n.Substring($prefix.Length) if (-not $rest) { continue } - # rest = "/" or "/.pax_incremental/". - # Flatten the / prefix; filenames already embed it. + # rest = "/<...>". $slash = $rest.IndexOf('/') if ($slash -lt 0) { continue } - $localRel = $rest.Substring($slash + 1) - $dest = Join-Path $LocalDir $localRel - $destDir = Split-Path -Parent $dest - if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { - New-Item -Path $destDir -ItemType Directory -Force | Out-Null + $run = $rest.Substring(0, $slash) + $sub = $rest.Substring($slash + 1) + if (-not $sub) { continue } + if (-not $byRun.ContainsKey($run)) { + $byRun[$run] = [pscustomobject]@{ + Commits = (New-Object System.Collections.Generic.List[string]) + Legacy = (New-Object System.Collections.Generic.List[string]) + All = (New-Object System.Collections.Generic.List[string]) + Transactional = $false + } + } + # Retained PER RUN (directories included) so each run's high-water mark is derived + # only from its own remote state - ownership is never mixed between runs. + [void]$byRun[$run].All.Add($sub) + if ($sub.StartsWith('gen-') -or $sub.StartsWith('commits/') -or $sub -eq 'commits') { $byRun[$run].Transactional = $true } + if ($isDir -eq 'true' -or $isDir -eq $true) { continue } + if ($sub.StartsWith('commits/')) { [void]$byRun[$run].Commits.Add($sub.Substring(8)); continue } + if ($sub.StartsWith('gen-')) { continue } + [void]$byRun[$run].Legacy.Add($sub) + } + + $count = 0 + # Every path an earlier run in THIS sweep already published. Exact-set promotion is + # scoped to ONE run's owned boundary and the .pax_incremental shard directory is + # shared across runs, so run B is told explicitly which files run A already verified + # and must therefore never withdraw. + $sweepPublished = New-Object System.Collections.Generic.List[string] + foreach ($run in @($byRun.Keys | Sort-Object)) { + $bucket = $byRun[$run] + if ($bucket.Commits.Count -gt 0) { + $manifest = Get-FabricResumeAuthoritativeCommit -RunTimestamp $run -CommitLeafNames $bucket.Commits + if (-not $manifest) { + Write-Verbose ("No verified resume commit for run {0}; skipping it (fail closed)." -f $run) + continue + } + # Hydration is all-or-nothing PER RUN: a run whose artifact set fails content + # verification throws and contributes nothing, and the runs already restored + # keep exactly the verified sets they were given. + $verifiedOwnership = @{} + $count += (Copy-FabricResumeCommitArtifacts -RunTimestamp $run -Manifest $manifest -LocalDir $LocalDir -VerifiedOwnership $verifiedOwnership -PreserveLocalPaths $sweepPublished.ToArray()) + foreach ($a in @($manifest.artifacts)) { [void]$sweepPublished.Add([string]$a.localPath) } + Register-FabricResumeRestoredState -RunTimestamp $run -Manifest $manifest -RelativePaths $bucket.All -VerifiedOwnership $verifiedOwnership + continue + } + if ($bucket.Transactional) { + # Staging generations with no authority record: nothing is authoritative yet and + # the flat legacy layout is never a fallback for a transactional run. + Write-Verbose ("Run {0} uses the transactional resume protocol but exposes no authority record; skipping it (fail closed)." -f $run) + continue + } + foreach ($localRel in $bucket.Legacy) { + $dest = Join-Path $LocalDir $localRel + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$run/$localRel" -DestinationPath $dest + [void]$sweepPublished.Add([string]$localRel) + $count++ } - Get-RemoteFile-OneLake -RelativeName ".pax_resume/$rest" -DestinationPath $dest - $count++ } return $count } @@ -15534,6 +27180,243 @@ function Save-CheckpointToDisk { } } +function Move-PaxFileOverwriteAtomic { + <# + .SYNOPSIS + Same-directory overwrite-rename that keeps the DESTINATION path continuously + present, with a bounded retry for transient Windows sharing failures. + .DESCRIPTION + [System.IO.File]::Move(..., overwrite) maps to MoveFileEx with + MOVEFILE_REPLACE_EXISTING - one directory-entry replacement, so the destination + path never disappears. That is the property a checkpoint swap needs and that + neither remove-then-move nor [System.IO.File]::Replace provides: both rename or + delete the destination BEFORE putting the replacement in place, leaving a window + in which an interruption exposes no file at all. + + MoveFileEx can transiently fail while another agent (indexer, antivirus scan, a + just-closed reader) still holds the destination open, so a bounded retry with a + short backoff is applied. Every attempt is a COMPLETE replacement: a failed + attempt leaves the destination holding its previous content - never nothing - so + retrying is always safe. The last failure is rethrown when the budget is + exhausted so callers fail closed instead of proceeding on an unproven swap. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $SourcePath, + [Parameter(Mandatory)] [string] $DestinationPath, + [int] $MaxAttempts = 10, + [int] $BackoffMilliseconds = 25 + ) + $attempt = 0 + while ($true) { + try { + [System.IO.File]::Move($SourcePath, $DestinationPath, $true) + return + } + catch { + $attempt++ + if ($attempt -ge $MaxAttempts) { throw } + Start-Sleep -Milliseconds $BackoffMilliseconds + } + } +} + +function Save-CheckpointStrict { + <# + .SYNOPSIS + Durable, verify-after-write checkpoint publication for the operator-cleared + uncertain-create transition. Unlike Save-CheckpointToDisk, a local write or + replace failure is NEVER downgraded to a soft warning: any failure THROWS so + the caller can fail closed and refuse to authorize a fresh create. + .DESCRIPTION + Serializes the current checkpoint to a temp file with terminating error + semantics (-ErrorAction Stop, so a nonterminating write error cannot be + silently ignored), then atomically replaces the destination WITHOUT a + remove-then-move loss window (a retained backup copy plus a single + overwrite-rename when the destination already exists, else a plain move into + place), and finally re-reads the + destination from disk and confirms it parses before returning $true. Any + failure in write, replace, or re-read verification throws. The Fabric-tier + durable-or-abort mirror contract of Save-CheckpointToDisk is preserved. + + ORDERING (local publication vs remote transaction). These are two separate + systems, so the order is fixed and explicit: the LOCAL publication is completed + and verified FIRST (atomic replace, then re-read + parse), and only then is the + Fabric-tier remote transaction committed by Sync-FabricResumeMirror. A remote + generation therefore never becomes authoritative before the local publication it + describes has been proven, so a local post-publication failure can never require + "un-publishing" a remotely authoritative checkpoint. Conversely, a remote + transaction that fails before its commit point leaves the PRIOR remote generation + authoritative, so the local rollback below has nothing remote to undo. + + ROLLBACK ATOMICITY. The rollback restores the retained prior copy with a single + overwrite-rename via Move-PaxFileOverwriteAtomic (MoveFileEx / + MOVEFILE_REPLACE_EXISTING), so the canonical checkpoint path stays continuously + present for the whole rollback - the earlier remove-then-move sequence opened a + window in which an interruption left NO checkpoint at the canonical path, and + System.IO.File::Replace opens a smaller but real window of the same kind because + ReplaceFile renames the destination aside before moving the replacement in. The + repudiated candidate is retained as recovery material until the restoration has been + proven byte-exact. + .OUTPUTS + $true on a proven durable publication; otherwise throws. + #> + + # Self-gate identically to Save-CheckpointToDisk. A disabled or unconfigured + # checkpoint is not a durable-write failure; it is simply nothing to persist. + if (-not $script:CheckpointEnabled) { return $false } + if (-not $script:CheckpointPath -or -not $script:CheckpointData) { return $false } + + # Update timestamp + statistics (same shape as Save-CheckpointToDisk). + $script:CheckpointData.lastUpdated = (Get-Date).ToUniversalTime().ToString('o') + $script:CheckpointData.statistics.partitionsComplete = $script:CheckpointData.partitions.completed.Count + $script:CheckpointData.statistics.partitionsQueryCreated = $script:CheckpointData.partitions.queryCreated.Count + $script:CheckpointData.statistics.partitionsRemaining = $script:CheckpointData.partitions.total - + $script:CheckpointData.statistics.partitionsComplete - + $script:CheckpointData.statistics.partitionsQueryCreated + + $json = $script:CheckpointData | ConvertTo-Json -Depth 10 + $tempPath = "$($script:CheckpointPath).strict.tmp" + + # 1) Write the temp file with terminating semantics so a nonterminating write + # error surfaces as a failure instead of being silently ignored. + Set-Content -Path $tempPath -Value $json -Encoding UTF8 -Force -ErrorAction Stop + + # 2) Publish as a REAL transaction. The full post-publication verification (local + # re-read + parse, then the Fabric-tier durable-or-abort mirror) runs INSIDE the + # transaction, and the prior copy is RETAINED until that verification has fully + # succeeded. A post-publication failure is NOT allowed to leave the destination in + # the new state while memory rolls back (a split-brain): the prior bytes are + # restored atomically from the retained backup (dest-exists) or the newly published + # candidate is removed (new-destination), the restore is itself proven, and only + # then is the original failure re-thrown so the caller fails closed with the durable + # checkpoint byte-for-byte unchanged. A restore that itself fails raises the DISTINCT + # PAX-FATAL-ROLLBACK category so an indeterminate disk state is never hidden as a + # normal fail-closed success. Never bare-remove an existing destination before the + # swap (that opens a data-loss window). + try { + if (Test-Path -LiteralPath $script:CheckpointPath) { + # Capture the exact prior bytes BEFORE the swap so the restore can be proven to + # reproduce them byte-for-byte. A locked/unreadable destination throws here, + # BEFORE any mutation, leaving the prior checkpoint untouched. + $priorBytes = [System.IO.File]::ReadAllBytes($script:CheckpointPath) + $backupPath = "$($script:CheckpointPath).strict.bak" + # RETAIN the prior copy first, then swap with a SINGLE overwrite-rename so the + # canonical path is continuously present. [System.IO.File]::Replace is deliberately + # NOT used: the underlying ReplaceFile renames the destination ASIDE before renaming + # the replacement into place, which leaves an observable window in which the + # canonical path does not exist. Move-PaxFileOverwriteAtomic uses MoveFileEx with + # MOVEFILE_REPLACE_EXISTING - one directory-entry replacement, no absence window. + # The prior copy is RETAINED in the backup until the whole post-publication + # verification has committed. + [System.IO.File]::Copy($script:CheckpointPath, $backupPath, $true) + Move-PaxFileOverwriteAtomic -SourcePath $tempPath -DestinationPath $script:CheckpointPath + try { + # 3) Durability proof: re-read the destination and confirm it exists and parses. + if (-not (Test-Path -LiteralPath $script:CheckpointPath)) { + throw "Strict checkpoint publish failed: destination missing after replace." + } + $roundTrip = Get-Content -LiteralPath $script:CheckpointPath -Raw -ErrorAction Stop + $null = $roundTrip | ConvertFrom-Json -ErrorAction Stop + # Fabric tier: mirror inside the same durable-or-abort contract as + # Save-CheckpointToDisk (a mirror failure aborts to preserve resume integrity). + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + Sync-FabricResumeMirror + } + } + catch { + $__pubErr = $_ + # Post-publication failure: restore the prior bytes from the retained backup and + # PROVE the restore reproduced them exactly, then re-throw so the caller fails + # closed with the durable checkpoint byte-for-byte unchanged. A restore that + # cannot be proven raises the distinct rollback-failure category. + $rollbackBackupPath = "$($script:CheckpointPath).strict.rbk" + try { + if (Test-Path -LiteralPath $backupPath) { + # Restore the prior bytes from the retained backup WITHOUT ever removing the + # canonical destination first. The repudiated candidate is copied aside as + # recovery material (BEST EFFORT - failing to take that extra copy must never + # block the restoration itself), then a single overwrite-rename puts the prior + # bytes back. Move-PaxFileOverwriteAtomic is one directory-entry replacement, so + # the canonical checkpoint path is CONTINUOUSLY PRESENT for the whole rollback: + # a process termination, host restart, or storage fault can never observe an + # absent checkpoint. (The prior remove-then-move sequence had exactly that loss + # window, and [System.IO.File]::Replace has a smaller but real one because + # ReplaceFile renames the destination aside first.) + if (Test-Path -LiteralPath $script:CheckpointPath) { + try { [System.IO.File]::Copy($script:CheckpointPath, $rollbackBackupPath, $true) } + catch { Write-Verbose ("Repudiated checkpoint candidate could not be retained: {0}" -f $_.Exception.Message) } + } + Move-PaxFileOverwriteAtomic -SourcePath $backupPath -DestinationPath $script:CheckpointPath + } + else { + throw "retained backup is missing; the prior checkpoint bytes cannot be restored." + } + $restoredBytes = [System.IO.File]::ReadAllBytes($script:CheckpointPath) + $restoredMatches = ($restoredBytes.Length -eq $priorBytes.Length) + if ($restoredMatches) { + for ($__i = 0; $__i -lt $priorBytes.Length; $__i++) { + if ($restoredBytes[$__i] -ne $priorBytes[$__i]) { $restoredMatches = $false; break } + } + } + if (-not $restoredMatches) { + throw "the restored destination does not byte-match the pre-transaction checkpoint." + } + # Proven byte-exact restoration: the repudiated candidate retained by the atomic + # swap is no longer recovery material, so it is discarded HERE (success path only). + if (Test-Path -LiteralPath $rollbackBackupPath) { Remove-Item -LiteralPath $rollbackBackupPath -Force -ErrorAction SilentlyContinue } + } + catch { + $script:GenericFatal = $true + throw "PAX-FATAL-ROLLBACK: strict checkpoint publish failed and the prior checkpoint could NOT be restored for '$($script:CheckpointPath)'; disk state is indeterminate: $($_.Exception.Message)" + } + throw $__pubErr + } + # Proven durable publication: the prior copy is no longer needed. + Remove-Item -LiteralPath $backupPath -Force -ErrorAction SilentlyContinue + } + else { + # New destination (no prior checkpoint to preserve). Publish, then verify + mirror + # inside the transaction; a post-publication failure REMOVES the newly published + # candidate so no false durable state survives, then fails closed. + [System.IO.File]::Move($tempPath, $script:CheckpointPath) + try { + if (-not (Test-Path -LiteralPath $script:CheckpointPath)) { + throw "Strict checkpoint publish failed: destination missing after move." + } + $roundTrip = Get-Content -LiteralPath $script:CheckpointPath -Raw -ErrorAction Stop + $null = $roundTrip | ConvertFrom-Json -ErrorAction Stop + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + Sync-FabricResumeMirror + } + } + catch { + $__pubErr = $_ + # No prior checkpoint existed: remove the newly published candidate to restore + # the no-checkpoint pre-state, then fail closed. A removal that itself fails + # raises the distinct rollback-failure category (disk state indeterminate). + try { + if (Test-Path -LiteralPath $script:CheckpointPath) { + [System.IO.File]::Delete($script:CheckpointPath) + } + } + catch { + $script:GenericFatal = $true + throw "PAX-FATAL-ROLLBACK: strict checkpoint publish failed and the newly published candidate could NOT be removed for new destination '$($script:CheckpointPath)'; disk state is indeterminate: $($_.Exception.Message)" + } + throw $__pubErr + } + } + } + finally { + # Clean the strict temp without ever touching the prior valid checkpoint. After a + # successful Replace/Move the temp is already consumed; this only removes a leftover + # temp from a pre-publication failure. + if (Test-Path -LiteralPath $tempPath) { Remove-Item -LiteralPath $tempPath -Force -ErrorAction SilentlyContinue } + } + return $true +} + function Save-Checkpoint { <# .SYNOPSIS @@ -15541,7 +27424,7 @@ function Save-Checkpoint { .PARAMETER PartitionIndex The partition index (1-based). .PARAMETER State - 'QueryCreated' or 'Completed' + 'QueryCreated', 'Completed', or 'UncertainCreate' .PARAMETER QueryId The server-assigned query ID. .PARAMETER PartitionStart @@ -15550,6 +27433,12 @@ function Save-Checkpoint { Partition end time (optional - looked up from partitionStatus if not provided). .PARAMETER RecordCount Number of records (only for Completed state). + .PARAMETER DeferDiskWrite + Perform the in-memory bucket transition ONLY and skip the warning-only disk write. + Reserved for a caller that owns a STRICT single-publication transaction: it stages the + COMPLETE post-transition state here, then publishes it exactly once through + Save-CheckpointStrict and fails closed unless that publication is proven durable. + Ignored by -Force. .PARAMETER Force Just save current checkpoint state to disk without updating partition info. #> @@ -15558,7 +27447,7 @@ function Save-Checkpoint { [int]$PartitionIndex, [Parameter()] - [ValidateSet('QueryCreated', 'Completed')] + [ValidateSet('QueryCreated', 'Completed', 'UncertainCreate')] [string]$State, [Parameter()] @@ -15573,6 +27462,15 @@ function Save-Checkpoint { [Parameter()] [int]$RecordCount = 0, + [Parameter()] + [hashtable]$LifecycleState, + + [Parameter()] + [string]$Fingerprint, + + [Parameter()] + [switch]$DeferDiskWrite, + [Parameter()] [switch]$Force ) @@ -15587,8 +27485,10 @@ function Save-Checkpoint { return } - # For normal calls, require the mandatory parameters - if (-not $PartitionIndex -or -not $State -or -not $QueryId) { + # For normal calls, require the mandatory parameters. An UncertainCreate entry is + # durable state for a partition that has NO queryId yet (a possibly-accepted create + # whose acceptance is unproven), so it is exempt from the QueryId requirement. + if (-not $PartitionIndex -or -not $State -or (-not $QueryId -and $State -ne 'UncertainCreate')) { Write-Verbose "Save-Checkpoint: Missing required parameters (PartitionIndex, State, QueryId) - skipping" return } @@ -15615,12 +27515,56 @@ function Save-Checkpoint { end = $PartitionEnd.ToUniversalTime().ToString('o') queryId = $QueryId } + # Optionally persist structured per-query lifecycle state. These + # fields are ABSENT on legacy checkpoints; when absent on load the resume path + # treats each field as its default and NEVER discards or recreates the stored + # queryId. Only stable, resume-relevant fields are persisted. + if ($LifecycleState) { + foreach ($lk in @('QueryApiVersion', 'FailureCategory', 'AlternateVersionChecked', 'AlternateVersionResult', 'QueryConfirmedAbsent', 'ReplacementAttempted', 'ReplacementQueryId', 'ZeroRecordComplete', 'UncertainCreate', 'UncertainCreateInfo')) { + if ($LifecycleState.ContainsKey($lk)) { $partitionEntry[$lk] = $LifecycleState[$lk] } + } + } + + # Durable per-contract identity. The fingerprint (New-GraphAuditQueryFingerprint over the + # canonical creation contract) is the STABLE identity of this entry across workload passes + # that reuse the same partition index. Prefer an explicitly supplied fingerprint; else + # derive it from the partition contract carried on partitionStatus; else inherit one already + # present on this entry (an uncertain-create entry carries it via UncertainCreateInfo). + # Absent (a legacy caller with no contract in hand) it is simply not stamped and identity + # falls back to the historical index-only behavior. + $entryFingerprint = $null + if (-not [string]::IsNullOrWhiteSpace($Fingerprint)) { + $entryFingerprint = $Fingerprint + } + else { + $pInfo = if ($null -ne $script:partitionStatus) { $script:partitionStatus[$PartitionIndex] } else { $null } + if ($pInfo -and $pInfo.Partition -and $pInfo.Partition.PStart -and $pInfo.Partition.PEnd) { + try { + $entryFingerprint = New-GraphAuditQueryFingerprint -PStart $pInfo.Partition.PStart -PEnd $pInfo.Partition.PEnd -Activities $pInfo.Partition.Activities -RecordTypes $pInfo.Partition.RecordTypes -ServiceFilter $pInfo.Partition.ServiceFilter + } catch { $entryFingerprint = $null } + } + if ([string]::IsNullOrWhiteSpace($entryFingerprint)) { + $inherited = Get-PaxCheckpointEntryFingerprint $partitionEntry + if (-not [string]::IsNullOrWhiteSpace($inherited)) { $entryFingerprint = $inherited } + } + } + if (-not [string]::IsNullOrWhiteSpace($entryFingerprint)) { $partitionEntry.fingerprint = $entryFingerprint } if ($State -eq 'QueryCreated') { $partitionEntry.createdAt = (Get-Date).ToUniversalTime().ToString('o') + # A confirmed QueryCreated (an adopted or freshly created query) SUPERSEDES any + # durable uncertain-create entry for the SAME contract identity: vacate it here, in the + # SAME in-memory mutation, BEFORE the atomic disk write, so an adopted partition is never + # persisted in both queryCreated and uncertainCreate at once (a possibly-accepted create + # that is now proven created is no longer uncertain). Identity is the fingerprint when + # present so a different workload contract reusing this index is NOT wrongly vacated. + $script:CheckpointData.partitions.uncertainCreate = @( + $script:CheckpointData.partitions.uncertainCreate | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint) } + ) + # Add to queryCreated list (if not already there) - $existing = $script:CheckpointData.partitions.queryCreated | Where-Object { $_.index -eq $PartitionIndex } + $existing = $script:CheckpointData.partitions.queryCreated | Where-Object { Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint } if (-not $existing) { $script:CheckpointData.partitions.queryCreated += $partitionEntry } @@ -15628,21 +27572,72 @@ function Save-Checkpoint { elseif ($State -eq 'Completed') { $partitionEntry.records = $RecordCount - # Remove from queryCreated if present + # Remove from queryCreated if present (same contract identity) $script:CheckpointData.partitions.queryCreated = @( - $script:CheckpointData.partitions.queryCreated | Where-Object { $_.index -ne $PartitionIndex } + $script:CheckpointData.partitions.queryCreated | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint) } + ) + # A Completed partition also SUPERSEDES any durable uncertain-create entry for the + # SAME contract identity (same atomic mutation, before the disk write) so completion + # never leaves a stale uncertain duplicate behind. + $script:CheckpointData.partitions.uncertainCreate = @( + $script:CheckpointData.partitions.uncertainCreate | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint) } ) # Add to completed list (if not already there) - $existing = $script:CheckpointData.partitions.completed | Where-Object { $_.index -eq $PartitionIndex } + $existing = $script:CheckpointData.partitions.completed | Where-Object { Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint } if (-not $existing) { $script:CheckpointData.partitions.completed += $partitionEntry $script:CheckpointData.statistics.totalRecordsSaved += $RecordCount } } - - # Save to disk - Save-CheckpointToDisk + elseif ($State -eq 'UncertainCreate') { + # Durable uncertain-create state: the first create POST may have been accepted but + # cannot be proven. Persist the fingerprinted display name + contract (carried on + # UncertainCreateInfo via the lifecycle allow-list above) so ANY later attempt + # reconciles read-only and never re-POSTs. No real queryId is stored. + if ($partitionEntry.ContainsKey('queryId')) { $partitionEntry.Remove('queryId') } + $partitionEntry.UncertainCreate = $true + $partitionEntry.uncertainAt = (Get-Date).ToUniversalTime().ToString('o') + # Dedupe by contract identity (fingerprint when present, else index). + $existing = $script:CheckpointData.partitions.uncertainCreate | Where-Object { Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint } + if (-not $existing) { + $script:CheckpointData.partitions.uncertainCreate += $partitionEntry + } + } + + # Final normalization guard: a single partition IDENTITY must never be persisted in more + # than one bucket. Identity is the durable fingerprint when present (so two workload-pass + # contracts that reuse the same index remain distinct and both survive) and the integer + # index for legacy entries. This guard now legitimately CALLS the shared identity helpers + # (Get-PaxEntryIdentityKey / Get-PaxCheckpointEntryFingerprint), so an isolated dot-source + # of Save-Checkpoint must ALSO load those helpers or the calls throw at runtime. + # Enforce precedence completed > queryCreated > uncertainCreate BEFORE the atomic disk + # write, removing any duplicated identity from the lower-precedence bucket(s) in this same + # in-memory mutation. + if ($script:CheckpointData -and $script:CheckpointData.partitions) { + $bxCompletedKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($bxC in @($script:CheckpointData.partitions.completed)) { + $bxk = Get-PaxEntryIdentityKey $bxC + if ($null -ne $bxk) { [void]$bxCompletedKeys.Add($bxk) } + } + # completed supersedes queryCreated + $script:CheckpointData.partitions.queryCreated = @(@($script:CheckpointData.partitions.queryCreated) | Where-Object { $null -eq $_ -or $null -eq (Get-PaxEntryIdentityKey $_) -or -not $bxCompletedKeys.Contains((Get-PaxEntryIdentityKey $_)) }) + # completed OR queryCreated supersedes uncertainCreate + $bxQueryCreatedKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($bxQ in @($script:CheckpointData.partitions.queryCreated)) { + $bxk2 = Get-PaxEntryIdentityKey $bxQ + if ($null -ne $bxk2) { [void]$bxQueryCreatedKeys.Add($bxk2) } + } + $script:CheckpointData.partitions.uncertainCreate = @(@($script:CheckpointData.partitions.uncertainCreate) | Where-Object { $null -eq $_ -or $null -eq (Get-PaxEntryIdentityKey $_) -or -not ($bxCompletedKeys.Contains((Get-PaxEntryIdentityKey $_)) -or $bxQueryCreatedKeys.Contains((Get-PaxEntryIdentityKey $_))) }) + } + + # Save to disk. A caller that owns a STRICT single-publication transaction stages the + # COMPLETE bucket transition in memory (-DeferDiskWrite) and publishes it itself, so the + # warning-only writer is never allowed to turn an unproven write into a silent success and + # the transition is never published twice. + if (-not $DeferDiskWrite) { + Save-CheckpointToDisk + } } function Test-CheckpointCompatibility { @@ -15994,6 +27989,7 @@ function Get-PartitionsToProcess { ToSkip = @() # Already completed - skip entirely ToFetchOnly = @() # Query exists on server - just fetch records ToCreateAndFetch = @() # Start fresh - create query then fetch + ToReconcileOnly = @() # Durable uncertain-create - reconcile READ-ONLY, never POST } if (-not $script:CheckpointData) { @@ -16002,32 +27998,49 @@ function Get-PartitionsToProcess { return $result } - $completedIndices = @{} - $queryCreatedIndices = @{} - - # Build lookup tables from checkpoint (use string keys for reliable comparison) - foreach ($cp in $script:CheckpointData.partitions.completed) { - $completedIndices["$($cp.index)"] = $cp - } - foreach ($qc in $script:CheckpointData.partitions.queryCreated) { - $queryCreatedIndices["$($qc.index)"] = $qc - } + # Categorize each partition by DURABLE CONTRACT IDENTITY (the fingerprint when the entry + # carries one, else the partition index for a legacy checkpoint). Because Graph workload + # passes RESTART partition indexes at 1, a bucket can now hold MULTIPLE entries that share + # an index but denote different contracts; each partition is matched to the entry whose + # fingerprint equals its OWN, so it resumes against its own stored query and a + # possibly-accepted first POST is never repeated across a resume boundary. A durable + # uncertain-create partition is routed to a READ-ONLY reconcile bucket BEFORE the + # fresh-create fallback. + $cpCompleted = @($script:CheckpointData.partitions.completed | Where-Object { $null -ne $_ }) + $cpQueryCreated = @($script:CheckpointData.partitions.queryCreated | Where-Object { $null -ne $_ }) + $cpUncertain = @($script:CheckpointData.partitions.uncertainCreate | Where-Object { $null -ne $_ }) - # Categorize each partition foreach ($partition in $AllPartitions) { - $idx = "$($partition.Index)" # Convert to string for reliable comparison + $pIdx = [int]$partition.Index + $pFp = $null + if ($partition.PStart -and $partition.PEnd) { + try { + $pFp = New-GraphAuditQueryFingerprint -PStart $partition.PStart -PEnd $partition.PEnd -Activities $partition.Activities -RecordTypes $partition.RecordTypes -ServiceFilter $partition.ServiceFilter + } + catch { $pFp = $null } + } - if ($completedIndices.ContainsKey($idx)) { + $matchCompleted = @($cpCompleted | Where-Object { Test-PaxCheckpointIdentityMatch $_ $pIdx $pFp })[0] + if ($matchCompleted) { $result.ToSkip += $partition + continue } - elseif ($queryCreatedIndices.ContainsKey($idx)) { + $matchQuery = @($cpQueryCreated | Where-Object { Test-PaxCheckpointIdentityMatch $_ $pIdx $pFp })[0] + if ($matchQuery) { # Add QueryId to partition for fetch-only processing - $partition | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue $queryCreatedIndices[$idx].queryId -Force + $partition | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue $matchQuery.queryId -Force $result.ToFetchOnly += $partition + continue } - else { - $result.ToCreateAndFetch += $partition + $matchUncertain = @($cpUncertain | Where-Object { Test-PaxCheckpointIdentityMatch $_ $pIdx $pFp })[0] + if ($matchUncertain) { + # Attach the durable uncertain state so the read-only reconcile path can rebuild + # the fingerprinted exact name; this partition is NEVER routed to create. + $partition | Add-Member -NotePropertyName 'UncertainCreateState' -NotePropertyValue $matchUncertain -Force + $result.ToReconcileOnly += $partition + continue } + $result.ToCreateAndFetch += $partition } return $result @@ -16130,7 +28143,7 @@ function Invoke-TokenRefreshPrompt { try { # Disconnect and reconnect - Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue Connect-PurviewAudit -AuthMethod $Auth -UseEOMMode $false # Update token timing and reset auth failure flags @@ -17338,15 +29351,52 @@ function Invoke-GraphAuditQuery { $bodyJson = $body | ConvertTo-Json -Depth 10 Write-LogHost $bodyJson -ForegroundColor DarkGray - # Submit query via Graph API (auto-detects v1.0 or beta) - $uri = Get-GraphAuditApiUri -Path 'queries' - $response = Invoke-MgGraphRequest -Method POST -Uri $uri -Body $body -ErrorAction Stop - - if ($response -and $response.id) { - return $response.id + # Submit query via Graph API through the duplicate-safe decision layer. + # A fully-paginated exact-name lookup runs BEFORE any POST: an existing + # identically-named query is adopted (no duplicate), multiple exact matches + # fail closed (no POST), otherwise a single beta creation proceeds under the + # ambiguous-POST guard. Version pinning is honoured by the decision layer. + $__createState = New-GraphAuditLifecycleState + $__listQueries = { + $__items = New-Object System.Collections.Generic.List[object] + $__lu = Get-GraphAuditApiUri -Path 'queries' + while ($__lu) { + $__page = Invoke-MgGraphRequest -Method GET -Uri $__lu -ErrorAction Stop + if ($__page -and $__page.value) { + foreach ($__q in $__page.value) { + $__items.Add(@{ id = $__q.id; displayName = $__q.displayName }) + } + } + $__lu = $null + if ($__page) { $__lu = $__page.'@odata.nextLink' } + } + return $__items.ToArray() + } + $__createQuery = { + param($Version, $Name) + try { + $__cu = "https://graph.microsoft.com/$Version/security/auditLog/queries" + $__resp = Invoke-MgGraphRequest -Method POST -Uri $__cu -Body $body -ErrorAction Stop + $__id = $null + if ($__resp) { $__id = $__resp.id } + return @{ HttpStatus = 201; QueryId = $__id; Ambiguous = $false } + } + catch { + $__sc = $null + if ($_.Exception.Response) { + try { $__sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $__sc) { try { $__sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + Write-LogHost "ERROR: Failed to submit Graph audit query: $($_.Exception.Message)" -ForegroundColor Red + return @{ HttpStatus = $__sc; QueryId = $null; Ambiguous = ($__sc -eq 409) } + } + } + $__resolve = Resolve-GraphAuditDuplicateSafeQuery -DisplayName $DisplayName -State $__createState -ListQueries $__listQueries -CreateQuery $__createQuery + if ($__resolve -and $__resolve.QueryId) { + return $__resolve.QueryId } else { - Write-LogHost "WARNING: Graph API query submitted but no ID returned" -ForegroundColor Yellow + Write-LogHost "WARNING: Graph API query not created (Action=$($__resolve.Action); Reason=$($__resolve.Reason))" -ForegroundColor Yellow return $null } } @@ -17397,8 +29447,34 @@ function Get-GraphAuditQueryStatus { ) try { - $uri = Get-GraphAuditApiUri -Path "queries/$QueryId" - $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + # Route the status GET through the shared decision layer so a transient 404 gets a + # bounded same-id eventual-consistency retry and then a same-id GET on the alternate + # version (read-only; never a POST) before status is judged unavailable. Status and + # records are decided independently. + $__lcState = New-GraphAuditLifecycleState + $__statusInvoke = { + param($Version, $Qid, $Op) + try { + $u = "https://graph.microsoft.com/$Version/security/auditLog/queries/$Qid" + $resp = Invoke-MgGraphRequest -Method GET -Uri $u -ErrorAction Stop + return @{ HttpStatus = 200; Data = $resp; ExceptionType = $null } + } + catch { + $sc = $null + if ($_.Exception.Response) { + try { $sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $sc) { try { $sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + if ($null -eq $sc -and ($_.Exception.Message -match '404|Not Found')) { $sc = 404 } + return @{ HttpStatus = $sc; Data = $null; ExceptionType = $_.Exception.GetType().Name } + } + } + $__lc = Invoke-GraphAuditLifecycleOperation -Operation 'Status' -QueryId $QueryId -Invoke $__statusInvoke -State $__lcState + if (-not $__lc.Success) { + Write-LogHost "ERROR: Failed to get Graph query status (category $($__lc.Category), HTTP $($__lc.HttpStatus))" -ForegroundColor Red + return $null + } + $response = $__lc.Data $result = @{ QueryId = $QueryId @@ -17450,9 +29526,44 @@ function Get-GraphAuditRecords { try { $allRecords = @() - $uri = Get-GraphAuditApiUri -Path "queries/$QueryId/records" + # Route the first records GET through the shared decision layer (independent of the + # status decision): a transient 404 gets a bounded same-id retry then a same-id GET on + # the alternate version (read-only; never a POST). A successful ZERO-record page is a + # complete result, never a failure. Remaining pages follow @odata.nextLink verbatim on + # whichever version served the first page (no $top / cap / truncation added). + $__recState = New-GraphAuditLifecycleState + $__recInvoke = { + param($Version, $Qid, $Op) + try { + $u = "https://graph.microsoft.com/$Version/security/auditLog/queries/$Qid/records" + $resp = Invoke-MgGraphRequest -Method GET -Uri $u -ErrorAction Stop + return @{ HttpStatus = 200; Data = $resp; ExceptionType = $null } + } + catch { + $sc = $null + if ($_.Exception.Response) { + try { $sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $sc) { try { $sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + if ($null -eq $sc -and ($_.Exception.Message -match '404|Not Found')) { $sc = 404 } + return @{ HttpStatus = $sc; Data = $null; ExceptionType = $_.Exception.GetType().Name } + } + } + $__lc = Invoke-GraphAuditLifecycleOperation -Operation 'Records' -QueryId $QueryId -Invoke $__recInvoke -State $__recState + if (-not $__lc.Success) { + Write-LogHost "ERROR: Failed to retrieve Graph audit records (category $($__lc.Category), HTTP $($__lc.HttpStatus))" -ForegroundColor Red + return @() + } + $response = $__lc.Data + if ($response -and $response.value) { + $allRecords += $response.value + if ($MaxRecords -gt 0 -and $allRecords.Count -ge $MaxRecords) { + return ($allRecords | Select-Object -First $MaxRecords) + } + } + $uri = $response.'@odata.nextLink' - do { + while ($uri) { $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop if ($response -and $response.value) { @@ -17467,8 +29578,7 @@ function Get-GraphAuditRecords { # Check for pagination $uri = $response.'@odata.nextLink' - - } while ($uri) + } return $allRecords } @@ -17671,16 +29781,14 @@ function Invoke-Agent365EarlyInteractiveSignIn { No-op retained for call-site stability. .DESCRIPTION - Historically this performed an eager up-front interactive DELEGATED sign-in for the - Agent 365 phase under -Auth AppRegistration, because the catalog endpoint was assumed to - have no app-only Graph scope. The Agent Package Management API is read with + The Agent Package Management API is read with the APPLICATION app-role CopilotPackages.Read.All (+ Application.Read.All) on the app / managed-identity service principal, so NO interactive sign-in is required in ANY auth mode: - app-only modes (AppRegistration cert/secret, ManagedIdentity) reuse the existing application context (see Connect-Agent365InteractiveContext); - delegated modes (WebLogin / DeviceCode / Credential / Silent) already consented the Agent 365 scopes at their initial sign-in. - This function is intentionally a no-op so the startup wiring that calls it stays stable. + This function is a no-op. .OUTPUTS $true always (no early sign-in is ever needed). @@ -17746,34 +29854,72 @@ function Test-Agent365FrontierAccess { function Get-Agent365Packages { <# .SYNOPSIS - Returns all Agent 365 catalog packages (list view) with paging. + Returns the Agent 365 catalog packages (list view) together with an explicit + completeness verdict. + .DESCRIPTION + The walk follows every @odata.nextLink the service supplies and ends only when + the service stops supplying one. There is no page, package, or row cap. + + Completeness fails closed. A repeated next link (a paging cycle), a page that + carries no value collection, or a page request that fails all end the walk with + Complete = $false and a reason, instead of silently returning a partial array. + Callers must refuse to publish the canonical catalog from an incomplete listing: + with an unknown universe, departed-agent classification cannot be trusted. + .OUTPUTS + PSCustomObject with .Packages, .Complete, .Reason and .PageCount. #> $results = New-Object System.Collections.Generic.List[object] $uri = Get-Agent365PackagesUri $pageNum = 0 + $complete = $true + $reason = '' + $seenLinks = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) while ($uri) { + if (-not $seenLinks.Add([string]$uri)) { + $complete = $false + $reason = "the catalog repeated a paging link after page $pageNum (paging cycle)" + break + } $pageNum++ try { # Suppress the return value ($true/$false/'Quit') so it does not leak into this # function's output pipeline and pollute the returned package array. $null = Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue } catch {} + $resp = $null try { $resp = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop } catch { - Write-LogHost (" WARNING: Agent 365 list page {0} failed: {1}" -f $pageNum, $_.Exception.Message) -ForegroundColor Yellow + $complete = $false + $reason = ("catalog page {0} could not be retrieved: {1}" -f $pageNum, $_.Exception.Message) break } - if ($resp -and $resp.value) { - foreach ($p in $resp.value) { [void]$results.Add($p) } + $hasValue = $false + if ($null -ne $resp) { + try { + if ($resp -is [System.Collections.IDictionary]) { $hasValue = $resp.Contains('value') } + else { $hasValue = [bool](@($resp.PSObject.Properties.Name) -contains 'value') } + } catch { $hasValue = $false } } - $uri = $resp.'@odata.nextLink' - if ($pageNum -gt 500) { - Write-LogHost " WARNING: Agent 365 paging safety abort (>500 pages)" -ForegroundColor Yellow + if (-not $hasValue) { + $complete = $false + $reason = ("catalog page {0} was malformed (no result collection)" -f $pageNum) break } + $pageValue = $resp.value + if ($null -ne $pageValue) { + foreach ($p in $pageValue) { [void]$results.Add($p) } + } + $next = $null + try { $next = $resp.'@odata.nextLink' } catch { $next = $null } + $uri = $next + } + return [PSCustomObject]@{ + Packages = $results.ToArray() + Complete = $complete + Reason = $reason + PageCount = $pageNum } - return $results.ToArray() } function Invoke-Agent365GraphWithRetry { @@ -17808,7 +29954,17 @@ function Invoke-Agent365GraphWithRetry { function Get-Agent365PackageDetail { <# .SYNOPSIS - Returns the full detail object for a single agent package, including elementDetails. + Returns a structured detail outcome for a single agent package. + .DESCRIPTION + The result always distinguishes the outcome rather than collapsing every failure to + $null, so the catalog phase can reconcile every listed package exactly. Outcome is + 'Success' (Detail carries the package detail including elementDetails), + 'FailedDependency' (the service reported the FailedDependency reason phrase), or + 'DetailFailed' (any other retrieval failure). Per-package diagnostics are written to + the run log at real severity; the console carries only the end-of-phase aggregate so + package identifiers never flood it. + .OUTPUTS + PSCustomObject with .Outcome, .Detail and .Reason. #> param([Parameter(Mandatory = $true)][string]$PackageId) $uri = Get-Agent365PackagesUri -PackageId $PackageId @@ -17818,10 +29974,15 @@ function Get-Agent365PackageDetail { $null = Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue } catch {} try { - return Invoke-Agent365GraphWithRetry -Uri $uri + $a365Detail = Invoke-Agent365GraphWithRetry -Uri $uri + return [PSCustomObject]@{ Outcome = 'Success'; Detail = $a365Detail; Reason = '' } } catch { - Write-LogHost (" WARNING: Agent 365 detail fetch failed for '{0}': {1}" -f $PackageId, $_.Exception.Message) -ForegroundColor Yellow - return $null + $a365Reason = [string]$_.Exception.Message + $a365StatusName = '' + try { $a365StatusName = [string]$_.Exception.Response.StatusCode } catch { $a365StatusName = '' } + $a365Outcome = if ($a365StatusName -eq 'FailedDependency' -or $a365Reason -match 'FailedDependency') { 'FailedDependency' } else { 'DetailFailed' } + Write-LogFile ("Agent 365 detail retrieval failed for package '{0}' (outcome {1}): {2}" -f $PackageId, $a365Outcome, $a365Reason) -Level 'WARNING' + return [PSCustomObject]@{ Outcome = $a365Outcome; Detail = $null; Reason = $a365Reason } } } @@ -18161,6 +30322,42 @@ function Export-Agent365Csv { } } +function Save-Agent365RecoveryCsv { + <# + .SYNOPSIS + Writes the rows retrieved during an INCOMPLETE catalog listing to a clearly-named + recovery file that is never treated as the canonical Agent 365 catalog. + .DESCRIPTION + An incomplete listing means the catalog universe is unknown, so the canonical target + (and any -AppendAgent365Info union) must be left byte-for-byte unchanged. The rows + that WERE retrieved are still valuable, so they are preserved locally under a + distinct leaf. That leaf is registered as recovery material so the end-of-run local + listing and the remote upload sweep both exclude it: recovery material is never + published as a completed catalog. + .OUTPUTS + Full path to the recovery file, or $null when there was nothing to preserve. + #> + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Rows + ) + if (-not $Rows -or $Rows.Count -eq 0) { return $null } + $ts = $global:ScriptRunTimestamp + if (-not $ts) { $ts = (Get-Date).ToString('yyyyMMdd_HHmmss') } + $recoveryLeaf = "Agent365_IncompleteListing_${ts}_recovery.csv" + $recoveryPath = Join-Path $OutputPath $recoveryLeaf + try { + Save-CsvAtomic -InputObject $Rows -Path $recoveryPath -NoTypeInformation -Encoding UTF8 + if (-not $script:Agent365RecoveryLeafs) { $script:Agent365RecoveryLeafs = New-Object System.Collections.Generic.List[string] } + if (-not $script:Agent365RecoveryLeafs.Contains($recoveryLeaf)) { [void]$script:Agent365RecoveryLeafs.Add($recoveryLeaf) } + Write-LogHost (" Agent 365: {0:N0} retrieved row(s) preserved as recovery material (excluded from upload): {1}" -f $Rows.Count, (Get-DisplayPath -LocalPath $recoveryPath)) -ForegroundColor Yellow + return $recoveryPath + } + catch { + Write-LogHost (" WARNING: Agent 365 recovery file could not be written: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + return $null + } +} + function Add-Agent365WorkbookTab { <# .SYNOPSIS @@ -18232,36 +30429,112 @@ function Invoke-Agent365Phase { # List + per-package detail Write-LogHost " Listing Agent 365 packages..." -ForegroundColor DarkGray - $listed = Get-Agent365Packages + $listResult = Get-Agent365Packages + $listed = @($listResult.Packages) + $a365ListComplete = [bool]$listResult.Complete + if (-not $a365ListComplete) { + Write-LogHost (" WARNING: Agent 365 catalog listing is INCOMPLETE after {0:N0} page(s): {1}" -f $listResult.PageCount, $listResult.Reason) -ForegroundColor Yellow + Write-LogFile ("Agent 365 catalog listing incomplete after {0} page(s): {1}" -f $listResult.PageCount, $listResult.Reason) -Level 'ERROR' + } if (-not $listed -or $listed.Count -eq 0) { - Write-LogHost " No Agent 365 packages returned by the catalog." -ForegroundColor Yellow - return @{ CsvPath = $null; Rows = @() } + if (-not $a365ListComplete) { + $script:Agent365HadGaps = $true + Write-LogHost " Agent 365: no packages could be listed and the listing did not complete; the existing catalog target is left unchanged (completed with gaps, exit 40)." -ForegroundColor Red + } else { + Write-LogHost " No Agent 365 packages returned by the catalog." -ForegroundColor Yellow + } + return @{ CsvPath = $null; Rows = @(); Listed = 0; Emitted = 0; DetailFailed = 0; FailedDependency = 0; RowBuildFailed = 0; SkippedNoId = 0; Reconciled = $a365ListComplete; ListComplete = $a365ListComplete; RecoveryPath = $null } } Write-LogHost (" {0} package(s) listed; fetching details..." -f $listed.Count) -ForegroundColor DarkGray $rows = New-Object System.Collections.Generic.List[object] $idx = 0 + # Complete outcome accounting. Every listed package resolves to exactly one bucket, so + # listed == emitted + detailFailed + rowBuildFailed + skippedNoId holds for the walk. + $a365Emitted = 0 + $a365DetailFailed = 0 + $a365FailedDependency = 0 + $a365RowBuildFailed = 0 + $a365SkippedNoId = 0 foreach ($p in $listed) { $idx++ $pkgId = $null try { $pkgId = $p.id } catch {} if (-not $pkgId) { try { $pkgId = $p.titleId } catch {} } - if (-not $pkgId) { continue } - $detail = Get-Agent365PackageDetail -PackageId $pkgId - if (-not $detail) { continue } + if (-not $pkgId) { + $a365SkippedNoId++ + Write-LogFile ("Agent 365: listed entry {0} carries no usable package identifier; no row emitted." -f $idx) -Level 'WARNING' + continue + } + $detailResult = Get-Agent365PackageDetail -PackageId $pkgId + if ($null -eq $detailResult -or $detailResult.Outcome -ne 'Success') { + $a365DetailFailed++ + if ($detailResult -and $detailResult.Outcome -eq 'FailedDependency') { $a365FailedDependency++ } + continue + } try { - $row = ConvertTo-Agent365Row -Package $detail -AuditEnrichment $script:Agent365AuditEnrichment + $row = ConvertTo-Agent365Row -Package $detailResult.Detail -AuditEnrichment $script:Agent365AuditEnrichment [void]$rows.Add($row) + $a365Emitted++ } catch { - Write-LogHost (" WARNING: Row build failed for package '{0}': {1}" -f $pkgId, $_.Exception.Message) -ForegroundColor Yellow + $a365RowBuildFailed++ + Write-LogFile ("Agent 365: row build failed for package '{0}': {1}" -f $pkgId, $_.Exception.Message) -Level 'ERROR' } if (($idx % 25) -eq 0) { Write-LogHost (" ... {0}/{1} packages processed" -f $idx, $listed.Count) -ForegroundColor DarkGray } } + $a365Failed = $a365DetailFailed + $a365RowBuildFailed + $a365Accounted = $a365Emitted + $a365DetailFailed + $a365RowBuildFailed + $a365SkippedNoId + $a365Reconciled = ($a365Accounted -eq $listed.Count) + # One concise completion summary. Per-package diagnostics stay in the run log. + Write-LogHost (" Agent 365: {0:N0} listed, {1:N0} emitted, {2:N0} failed, {3:N0} skipped." -f $listed.Count, $a365Emitted, $a365Failed, $a365SkippedNoId) -ForegroundColor $(if ($a365Failed -gt 0 -or $a365SkippedNoId -gt 0) { 'Yellow' } else { 'Green' }) + Write-LogFile ("Agent 365 accounting: listed={0} emitted={1} detailFailed={2} (FailedDependency={3}) rowBuildFailed={4} skippedNoId={5} reconciled={6}" -f $listed.Count, $a365Emitted, $a365DetailFailed, $a365FailedDependency, $a365RowBuildFailed, $a365SkippedNoId, $a365Reconciled) -Level $(if ($a365Failed -gt 0 -or $a365SkippedNoId -gt 0 -or -not $a365Reconciled) { 'ERROR' } else { 'INFO' }) + if (-not $a365Reconciled) { + Write-LogFile ("Agent 365 accounting did not reconcile: {0} listed vs {1} accounted." -f $listed.Count, $a365Accounted) -Level 'ERROR' + } + # Requested Agent 365 output is incomplete whenever any listed package produced no row. + if ($a365Failed -gt 0 -or $a365SkippedNoId -gt 0 -or -not $a365Reconciled -or -not $a365ListComplete) { + $script:Agent365HadGaps = $true + Write-LogHost " Agent 365: requested catalog output is INCOMPLETE (completed with gaps, exit 40)." -ForegroundColor Yellow + } + + if (-not $a365ListComplete) { + # The catalog universe is unknown, so departed-agent classification cannot be trusted. + # Never publish or overwrite the canonical catalog target from a partial listing: leave + # any existing target byte-for-byte unchanged and keep this run's rows as recovery material. + Write-LogHost " Agent 365: the catalog file is NOT written or overwritten this run because the listing did not complete; any existing target is left unchanged." -ForegroundColor Red + $a365RecoveryPath = Save-Agent365RecoveryCsv -Rows $rows.ToArray() + return @{ + CsvPath = $null + Rows = @() + Listed = $listed.Count + Emitted = $a365Emitted + DetailFailed = $a365DetailFailed + FailedDependency = $a365FailedDependency + RowBuildFailed = $a365RowBuildFailed + SkippedNoId = $a365SkippedNoId + Reconciled = $a365Reconciled + ListComplete = $false + RecoveryPath = $a365RecoveryPath + } + } + $csvPath = Export-Agent365Csv -Rows $rows.ToArray() - return @{ CsvPath = $csvPath; Rows = $rows.ToArray() } + return @{ + CsvPath = $csvPath + Rows = $rows.ToArray() + Listed = $listed.Count + Emitted = $a365Emitted + DetailFailed = $a365DetailFailed + FailedDependency = $a365FailedDependency + RowBuildFailed = $a365RowBuildFailed + SkippedNoId = $a365SkippedNoId + Reconciled = $a365Reconciled + ListComplete = $true + RecoveryPath = $null + } } # ============================================================================= @@ -18540,7 +30813,7 @@ function Test-PurviewAuditCapability { Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Gray try { - Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue Write-Host " Disconnected successfully" -ForegroundColor Green } catch { @@ -18604,136 +30877,285 @@ function Test-PurviewAuditCapability { } # ============================================== -# DUAL-MODE GROUP EXPANSION FUNCTION +# USER-SCOPE RESOLUTION FUNCTION # ============================================== -# Expand distribution/security groups to individual user principal names +# Resolve explicit UserIds and group memberships into one fail-closed target-user union -function Expand-GroupToUsers { +function Resolve-PaxUserScope { <# .SYNOPSIS - Expands a distribution or security group to individual user principal names. - + Resolves explicit UserIds and group memberships into a single deduplicated user scope. + .DESCRIPTION - Retrieves members of a group using either EOM cmdlets or Graph API. - - EOM Mode: - - Uses Get-DistributionGroupMember cmdlet - - Accepts group display name or email address - - Returns PrimarySmtpAddress of members - - Graph API Mode: - - Uses Get-MgGroupMember cmdlet - - Requires group ObjectId (auto-resolved from display name) - - Returns userPrincipalName of user members - - .PARAMETER GroupIdentity - The group identifier. Can be: - - Display name (e.g., "Executive Leadership") - - Email address (e.g., "exec-team@contoso.com") - - ObjectId/GUID (Graph mode only) - + Treats -UserIds and -GroupNames as symmetrical user-scope inputs and returns one + case-insensitive deduplicated union of explicit user principal names and the members + of every requested group. Group membership is read through injectable seams so the + function runs under unit test without a live directory: Graph mode reads group identity + and transitive user membership through the authenticated Invoke-MgGraphRequest REST seam; + Exchange Online mode reads distribution-group membership through Get-DistributionGroupMember. + + The resolution fails closed. When any requested group cannot be resolved, matches more than + one directory object, has no user members, or errors, the whole result is marked Failed so a + caller never degrades a requested scope to an unfiltered query. On success with any requested + scope the returned FinalTargetUsers is always non-empty. + + .PARAMETER UserIds + Explicit user principal names to include in the scope. + + .PARAMETER GroupNames + Group display names, mail addresses, or object ids whose user members join the scope. + .PARAMETER UseEOMMode - If true, use EOM cmdlets. If false, use Graph API. - + When true, read group membership through Exchange Online (direct members only); otherwise + read transitive user membership through Graph REST. + + .PARAMETER GraphRequest + Seam that issues a Graph REST GET for a relative uri and returns the parsed response. + + .PARAMETER EomGroupMemberOp + Seam that returns the members of an Exchange Online distribution group. + + .PARAMETER RosterValidator + Optional seam that reports whether an explicit user principal name exists in the directory. + Unmatched explicit users are reported but still included in the scope. When omitted, every + explicit user is treated as matched. + .OUTPUTS - Array of user principal names (email addresses) + A single object describing the requested inputs, per-group resolution, the FinalTargetUsers + union, an Outcome of 'Succeeded' or 'Failed', and a FailureStage when Failed. #> - + param( - [Parameter(Mandatory = $true)] - [string]$GroupIdentity, - - [Parameter(Mandatory = $false)] - [bool]$UseEOMMode = $false + [string[]]$UserIds = @(), + [string[]]$GroupNames = @(), + [bool]$UseEOMMode = $false, + [scriptblock]$GraphRequest = { param($Uri) Invoke-MgGraphRequest -Method GET -Uri $Uri -OutputType PSObject -ErrorAction Stop }, + [scriptblock]$EomGroupMemberOp = { param($GroupIdentity) Get-DistributionGroupMember -Identity $GroupIdentity -ErrorAction Stop }, + [scriptblock]$RosterValidator = $null ) - - $members = @() - - if ($UseEOMMode) { - # ======================================== - # EOM MODE: Get-DistributionGroupMember - # ======================================== - - try { - Write-LogHost " Processing group (EOM): '$GroupIdentity'" -ForegroundColor Gray - - # Get-DistributionGroupMember works with display name or email - $groupMembers = Get-DistributionGroupMember -Identity $GroupIdentity -ErrorAction Stop - - $members = $groupMembers | Select-Object -ExpandProperty PrimarySmtpAddress - - Write-LogHost " Expanded: $($members.Count) member(s)" -ForegroundColor DarkGray + + # Clean and case-insensitively deduplicate explicit UserIds, preserving first-seen casing. + $seenUsers = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $requestedUserIds = New-Object System.Collections.Generic.List[string] + foreach ($u in @($UserIds)) { + if ($null -eq $u) { continue } + $t = ([string]$u).Trim() + if ($t.Length -eq 0) { continue } + if ($seenUsers.Add($t)) { [void]$requestedUserIds.Add($t) } + } + + # Clean and case-insensitively deduplicate group names, preserving first-seen casing. + $seenGroups = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $requestedGroups = New-Object System.Collections.Generic.List[string] + foreach ($g in @($GroupNames)) { + if ($null -eq $g) { continue } + $t = ([string]$g).Trim() + if ($t.Length -eq 0) { continue } + if ($seenGroups.Add($t)) { [void]$requestedGroups.Add($t) } + } + + # Split explicit users into matched/unmatched when a roster validator is supplied; unmatched + # users are still included in the effective scope. + $matchedUsers = New-Object System.Collections.Generic.List[string] + $unmatchedUsers = New-Object System.Collections.Generic.List[string] + foreach ($u in $requestedUserIds) { + if ($RosterValidator) { + $exists = $false + try { $exists = [bool](& $RosterValidator $u) } catch { $exists = $false } + if ($exists) { [void]$matchedUsers.Add($u) } else { [void]$unmatchedUsers.Add($u) } } - catch { - Write-LogHost " Warning: Failed to expand group '$GroupIdentity': $($_.Exception.Message)" -ForegroundColor Yellow - Write-LogHost " Possible causes:" -ForegroundColor Yellow - Write-LogHost " • Group does not exist or name is misspelled" -ForegroundColor Gray - Write-LogHost " • Insufficient permissions (need Organization Management or similar)" -ForegroundColor Gray - Write-LogHost " • Group is not a distribution/mail-enabled group" -ForegroundColor Gray + else { + [void]$matchedUsers.Add($u) } } - else { - # ======================================== - # GRAPH API MODE: Get-MgGroupMember - # ======================================== - + + # Seed the effective union with the explicit UserIds (matched and unmatched both count as + # requested scope; an unmatched user simply matches zero rows downstream). + $finalSeen = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $finalUsers = New-Object System.Collections.Generic.List[string] + foreach ($u in $requestedUserIds) { + if ($finalSeen.Add($u)) { [void]$finalUsers.Add($u) } + } + + $resolvedGroups = New-Object System.Collections.Generic.List[object] + $failedGroups = New-Object System.Collections.Generic.List[string] + $ambiguousGroups = New-Object System.Collections.Generic.List[string] + $zeroMemberGroups = New-Object System.Collections.Generic.List[string] + $unauthorizedGroups = New-Object System.Collections.Generic.List[string] + $transportErrorGroups = New-Object System.Collections.Generic.List[string] + $resolutionErrorGroups = New-Object System.Collections.Generic.List[string] + $directMembers = New-Object System.Collections.Generic.List[string] + $transitiveMembers = New-Object System.Collections.Generic.List[string] + + $outcome = 'Succeeded' + $failureStage = $null + $guidPattern = '^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$' + + foreach ($g in $requestedGroups) { try { - Write-LogHost " Processing group (Graph API): '$GroupIdentity'" -ForegroundColor Gray - - # Determine if we have an ObjectId (GUID) or display name - $groupId = $null - if ($GroupIdentity -match '^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$') { - # Looks like a GUID, use directly - $groupId = $GroupIdentity - } - else { - # Display name or email - need to resolve to ObjectId - Write-LogHost " Resolving group ID from display name..." -ForegroundColor DarkGray - - # Try searching by display name first - $groupSearch = Get-MgGroup -Filter "displayName eq '$GroupIdentity'" -ErrorAction SilentlyContinue - - if (-not $groupSearch) { - # Try by mail/mailNickname - $groupSearch = Get-MgGroup -Filter "mail eq '$GroupIdentity'" -ErrorAction SilentlyContinue + if ($UseEOMMode) { + # Exchange Online: direct members only (nested groups are not expanded). + $raw = @(& $EomGroupMemberOp $g) + $eomMembers = New-Object System.Collections.Generic.List[string] + foreach ($m in $raw) { + $smtp = if ($m -is [string]) { $m } elseif ($null -ne $m.PrimarySmtpAddress) { [string]$m.PrimarySmtpAddress } else { $null } + if ($smtp) { $st = $smtp.Trim(); if ($st.Length -gt 0) { [void]$eomMembers.Add($st) } } + } + if ($eomMembers.Count -eq 0) { + [void]$zeroMemberGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupZeroMembers' } + continue } - - if ($groupSearch) { - $groupId = $groupSearch.Id - Write-LogHost " Resolved to ObjectId: $groupId" -ForegroundColor DarkGray + foreach ($m in $eomMembers) { + [void]$directMembers.Add($m); [void]$transitiveMembers.Add($m) + if ($finalSeen.Add($m)) { [void]$finalUsers.Add($m) } } - else { - throw "Unable to find group with identifier: $GroupIdentity" + [void]$resolvedGroups.Add([pscustomobject]@{ Requested = $g; Id = $null; DisplayName = $g; DirectMembers = $eomMembers.Count; TransitiveMembers = $eomMembers.Count; MembershipScope = 'DirectOnly (Exchange Online; nested groups not expanded)' }) + continue + } + + # Graph: resolve the group identity, then read transitive user membership. + $groupId = $null; $groupDisplay = $null + if ($g -match $guidPattern) { + $resp = & $GraphRequest ("/v1.0/groups/{0}?`$select=id,displayName" -f $g) + if ($resp -and $resp.id) { $groupId = [string]$resp.id; $groupDisplay = [string]$resp.displayName } + else { [void]$failedGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupNotFound' }; continue } + } + else { + $escaped = ($g -replace "'", "''") + $grpHits = @() + $r1 = & $GraphRequest ("/v1.0/groups?`$filter=displayName eq '{0}'&`$select=id,displayName" -f $escaped) + if ($r1 -and $r1.value) { $grpHits += @($r1.value) } + if ($grpHits.Count -eq 0) { + $r2 = & $GraphRequest ("/v1.0/groups?`$filter=mail eq '{0}' or mailNickname eq '{0}'&`$select=id,displayName" -f $escaped) + if ($r2 -and $r2.value) { $grpHits += @($r2.value) } + } + if ($grpHits.Count -eq 0) { [void]$failedGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupNotFound' }; continue } + if ($grpHits.Count -gt 1) { [void]$ambiguousGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupAmbiguous' }; continue } + $groupId = [string]$grpHits[0].id; $groupDisplay = [string]$grpHits[0].displayName + } + + # Transitive user members (paginated); nested-group users are included. + $transSet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $grpTrans = New-Object System.Collections.Generic.List[string] + $uri = ("/v1.0/groups/{0}/transitiveMembers/microsoft.graph.user?`$select=userPrincipalName" -f $groupId) + while ($uri) { + $page = & $GraphRequest $uri + if ($page -and $page.value) { + foreach ($m in @($page.value)) { + $upn = if ($null -ne $m.userPrincipalName) { ([string]$m.userPrincipalName).Trim() } else { $null } + if ($upn) { if ($transSet.Add($upn)) { [void]$grpTrans.Add($upn) } } + } } + $uri = if ($page -and $page.'@odata.nextLink') { [string]$page.'@odata.nextLink' } else { $null } } - - # Get group members (users only) - $groupMembers = Get-MgGroupMember -GroupId $groupId -All -ErrorAction Stop - - # Filter to users only and extract UPN - foreach ($member in $groupMembers) { - # Check if member is a user (not a nested group or service principal) - if ($member.AdditionalProperties.'@odata.type' -eq '#microsoft.graph.user') { - # Get full user object to retrieve userPrincipalName - $user = Get-MgUser -UserId $member.Id -ErrorAction SilentlyContinue - if ($user -and $user.UserPrincipalName) { - $members += $user.UserPrincipalName + if ($grpTrans.Count -eq 0) { [void]$zeroMemberGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupZeroMembers' }; continue } + + # Direct user members (paginated) for the direct/transitive split. + $dirSet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $grpDirectCount = 0 + $duri = ("/v1.0/groups/{0}/members/microsoft.graph.user?`$select=userPrincipalName" -f $groupId) + while ($duri) { + $dpage = & $GraphRequest $duri + if ($dpage -and $dpage.value) { + foreach ($m in @($dpage.value)) { + $upn = if ($null -ne $m.userPrincipalName) { ([string]$m.userPrincipalName).Trim() } else { $null } + if ($upn) { if ($dirSet.Add($upn)) { $grpDirectCount++; [void]$directMembers.Add($upn) } } } } + $duri = if ($dpage -and $dpage.'@odata.nextLink') { [string]$dpage.'@odata.nextLink' } else { $null } } - - Write-LogHost " Expanded: $($members.Count) user member(s)" -ForegroundColor DarkGray + + foreach ($upn in $grpTrans) { + [void]$transitiveMembers.Add($upn) + if ($finalSeen.Add($upn)) { [void]$finalUsers.Add($upn) } + } + [void]$resolvedGroups.Add([pscustomobject]@{ Requested = $g; Id = $groupId; DisplayName = $groupDisplay; DirectMembers = $grpDirectCount; TransitiveMembers = $grpTrans.Count; MembershipScope = 'Transitive' }) } catch { - Write-LogHost " Warning: Failed to expand group '$GroupIdentity': $($_.Exception.Message)" -ForegroundColor Yellow - Write-LogHost " Possible causes:" -ForegroundColor Yellow - Write-LogHost " • Group does not exist or identifier is invalid" -ForegroundColor Gray - Write-LogHost " • Insufficient permissions (need GroupMember.Read.All - or higher: Group.Read.All / Directory.Read.All)" -ForegroundColor Gray - Write-LogHost " • Network connectivity issues with Graph API" -ForegroundColor Gray + $ex = $_.Exception + $sc = $null + try { + if ($ex.Response -and $ex.Response.StatusCode) { $sc = [int]$ex.Response.StatusCode } + elseif ($ex.PSObject.Properties['StatusCode'] -and $null -ne $ex.StatusCode) { $sc = [int]$ex.StatusCode } + } catch { $sc = $null } + $emsg = [string]$ex.Message + if ($sc -eq 401 -or $sc -eq 403 -or $emsg -match '(?i)Authorization_RequestDenied|Forbidden|Unauthorized|Access is denied|Insufficient privileges|\b40[13]\b') { + [void]$unauthorizedGroups.Add($g); $stg = 'GroupAuthorizationError' + } + elseif (($null -ne $sc -and ($sc -ge 500 -or $sc -eq 408 -or $sc -eq 429)) -or $emsg -match '(?i)timed out|timeout|connection|network|remote server|temporarily unavailable|ServiceUnavailable|Gateway|throttl') { + [void]$transportErrorGroups.Add($g); $stg = 'GroupTransportError' + } + else { + [void]$resolutionErrorGroups.Add($g); $stg = 'GroupResolutionError' + } + $outcome = 'Failed'; if (-not $failureStage) { $failureStage = $stg } + continue } } - - return $members + + if ($outcome -eq 'Succeeded') { $failureStage = $null } + + return [pscustomobject]@{ + RequestedUserIds = $requestedUserIds.ToArray() + MatchedExplicitUserIds = $matchedUsers.ToArray() + UnmatchedExplicitUserIds = $unmatchedUsers.ToArray() + RequestedGroups = $requestedGroups.ToArray() + ResolvedGroups = $resolvedGroups.ToArray() + FailedGroups = $failedGroups.ToArray() + AmbiguousGroups = $ambiguousGroups.ToArray() + ZeroMemberGroups = $zeroMemberGroups.ToArray() + UnauthorizedGroups = $unauthorizedGroups.ToArray() + TransportErrorGroups = $transportErrorGroups.ToArray() + ResolutionErrorGroups = $resolutionErrorGroups.ToArray() + ResolvedDirectMembers = $directMembers.ToArray() + ResolvedTransitiveMembers = $transitiveMembers.ToArray() + FinalTargetUsers = $finalUsers.ToArray() + Outcome = $outcome + FailureStage = $failureStage + } +} + +# ============================================== +# DUAL-MODE GROUP EXPANSION FUNCTION +# ============================================== +# Expand distribution/security groups to individual user principal names + +function Expand-GroupToUsers { + <# + .SYNOPSIS + Expands a distribution or security group to individual user principal names. + + .DESCRIPTION + Delegates to Resolve-PaxUserScope for a single group and returns its resolved user + principal names. Graph mode reads group membership through the authenticated + Invoke-MgGraphRequest REST seam; Exchange Online mode reads distribution-group + membership through Get-DistributionGroupMember. A group that cannot be resolved, is + ambiguous, or has no user members throws so the caller never receives a silent empty + set that would widen the effective query scope. + + .PARAMETER GroupIdentity + The group identifier: display name, mail address, or object id. + + .PARAMETER UseEOMMode + When true, read membership through Exchange Online; otherwise through Graph REST. + + .OUTPUTS + Array of user principal names. + #> + + param( + [Parameter(Mandatory = $true)] + [string]$GroupIdentity, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + $resolved = Resolve-PaxUserScope -GroupNames @($GroupIdentity) -UseEOMMode $UseEOMMode + if ($resolved.Outcome -ne 'Succeeded') { + throw "Group '$GroupIdentity' could not be resolved to any user members (stage: $($resolved.FailureStage)); refusing to return an empty membership set." + } + return @($resolved.FinalTargetUsers) } # ============================================== @@ -19079,7 +31501,7 @@ function Disconnect-PurviewAudit { if ($context) { Write-LogHost "Disconnecting from Microsoft Graph..." -ForegroundColor Gray - Disconnect-MgGraph -ErrorAction Stop | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction Stop Write-LogHost " Disconnected from Microsoft Graph" -ForegroundColor Green } else { @@ -20391,6 +32813,9 @@ elseif ($script:CheckpointEnabled) { Rollup = $Rollup.IsPresent RollupPlusRaw = $RollupPlusRaw.IsPresent Dashboard = $Dashboard + OutputPathDefenderUsage = $OutputPathDefenderUsage + AppendDefenderUsage = $AppendDefenderUsage + DisableAISIDDeltaCache = $DisableAISIDDeltaCache.IsPresent # Auth (no secrets) Auth = $Auth TenantId = $TenantId @@ -20403,6 +32828,10 @@ elseif ($script:CheckpointEnabled) { UseEOM = $UseEOM.IsPresent AutoCompleteness = $AutoCompleteness.IsPresent IncludeTelemetry = $IncludeTelemetry.IsPresent + VerifyPartitionStability = $VerifyPartitionStability.IsPresent + # Hybrid directory enrichment source (path/URL only; no supplemental data VALUES are ever + # persisted). Restored on -Resume so the original supplemental input is reused. + UserInfoSupplement = $UserInfoSupplement } Initialize-CheckpointForNewRun -OutputPath $OutputPath -BaseOutputFileName $baseFileName -RunTimestamp $global:ScriptRunTimestamp -StartDate (script:Parse-DateSafe $StartDate) -EndDate (script:Parse-DateSafe $EndDate) -AllParameters $allParams | Out-Null } @@ -20656,26 +33085,24 @@ else { if ($GroupNames -and $GroupNames.Count -gt 0) { Write-LogHost "" -ForegroundColor White Write-LogHost " Group expansion (REQUIRED - because -GroupNames is set):" -ForegroundColor White - Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/members)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/transitiveMembers)" -f $auditTag) -ForegroundColor Yellow } else { Write-LogHost "" -ForegroundColor White Write-LogHost " Group expansion (only required with -GroupNames):" -ForegroundColor White - Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/members)" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/transitiveMembers)" -f $auditTag) -ForegroundColor DarkGray } # Remote output - SharePoint (audit context: same Graph token reused for SP drives API) if ($script:RemoteOutputMode -eq 'SharePoint') { Write-LogHost "" -ForegroundColor White Write-LogHost " Remote output - SharePoint (REQUIRED - because -OutputPath resolves to a SharePoint URL):" -ForegroundColor White - Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive via /sites + /drives)" -f $auditTag) -ForegroundColor Yellow - Write-LogHost (" {0} Files.ReadWrite.All (PUT/createUploadSession to /drives/{{id}}/items)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive; create folder; upload; download)" -f $auditTag) -ForegroundColor Yellow Write-LogHost " Destination user/identity ALSO needs at least Member access on the" -ForegroundColor Gray Write-LogHost " target SharePoint site (granted via SharePoint, not Graph)." -ForegroundColor Gray } else { Write-LogHost "" -ForegroundColor White Write-LogHost " Remote output - SharePoint (only required when an -OutputPath* value is a SharePoint URL):" -ForegroundColor White - Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive via /sites + /drives)" -f $auditTag) -ForegroundColor DarkGray - Write-LogHost (" {0} Files.ReadWrite.All (PUT/createUploadSession to /drives/{{id}}/items)" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive; create folder; upload; download)" -f $auditTag) -ForegroundColor DarkGray } # Remote output - Fabric / OneLake (separate token: storage.azure.com audience via Az.Accounts; @@ -20740,6 +33167,20 @@ else { } #> + # AISID dashboard (Defender advanced hunting) - reuses THIS Graph token via + # /security/runHuntingQuery. ThreatHunting.Read.All follows the same delegated-vs- + # application split as the audit scopes: delegated modes consent it at sign-in; + # app-only modes need it admin-consented as an APPLICATION permission. + if ($Dashboard -eq 'AISID') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " AISID dashboard (REQUIRED - because -Dashboard AISID is set):" -ForegroundColor White + Write-LogHost (" {0} ThreatHunting.Read.All (Defender advanced hunting via /security/runHuntingQuery)" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " AISID dashboard (only required with -Dashboard AISID):" -ForegroundColor White + Write-LogHost (" {0} ThreatHunting.Read.All (Defender advanced hunting via /security/runHuntingQuery)" -f $auditTag) -ForegroundColor DarkGray + } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green } Write-LogHost "" @@ -20750,14 +33191,14 @@ Write-LogHost "" # permissions banner and the rollup banner so users can see at a glance where each # customer-visible artifact (CSV/XLSX/log) will land. $anyDestBound = $false -foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { +foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if ($script:DestIsBound.ContainsKey($k) -and $script:DestIsBound[$k]) { $anyDestBound = $true; break } } # Bound -Append* values are destination bindings too — they tell the # run where the canonical artifact lives. Treat them as such so the banner doesn't # go silent on a pure -AppendFile/-AppendUserInfo/-AppendAgent365Info run with no # -OutputPath* in scope. -if (-not $anyDestBound -and ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info)) { +if (-not $anyDestBound -and ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info -or $AppendDefenderUsage)) { $anyDestBound = $true } if ($script:RemoteOutputMode -ne 'None' -or $anyDestBound) { @@ -20778,6 +33219,7 @@ if ($script:RemoteOutputMode -ne 'None' -or $anyDestBound) { @{ Key = 'Purview' ; Label = 'Purview audit '; ScopeCondition = $true ; AppendVar = $AppendFile ; AppendName = 'AppendFile' }, @{ Key = 'UserInfo' ; Label = 'EntraUsers CSV '; ScopeCondition = ($IncludeUserInfo -or $OnlyUserInfo) ; AppendVar = $AppendUserInfo ; AppendName = 'AppendUserInfo' }, @{ Key = 'Agent365Info' ; Label = 'Agent 365 CSV '; ScopeCondition = ($IncludeAgent365Info -or $OnlyAgent365Info) ; AppendVar = $AppendAgent365Info ; AppendName = 'AppendAgent365Info' }, + @{ Key = 'DefenderUsage'; Label = 'AISID CSVs '; ScopeCondition = ($Dashboard -eq 'AISID') ; AppendVar = $AppendDefenderUsage ; AppendName = 'AppendDefenderUsage' }, @{ Key = 'Log' ; Label = 'Run log '; ScopeCondition = $true ; AppendVar = $null ; AppendName = $null } ) # Always emit the per-data-type rows when the banner shows at all @@ -20871,6 +33313,22 @@ if ($script:RemoteOutputMode -ne 'None' -or $anyDestBound) { elseif ($script:DestRaw.ContainsKey('Agent365Info')) { $script:DestRaw['Agent365Info'] } else { '(inherits -OutputPath)' } } + elseif ($row.Key -eq 'DefenderUsage') { + # AISID dashboard: twelve fixed-name CSVs land together in ONE folder; show the + # resolved folder (honoring -OutputPathDefenderUsage / -AppendDefenderUsage, else + # co-located with -OutputPath) with a count hint rather than a single leaf. + $_duDtB = script:Resolve-DataTypePaths -DataType 'DefenderUsage' + if ($_duDtB -and $_duDtB.IsBound -and $_duDtB.Tier -eq 'Local') { + (Get-DisplayPath -LocalPath $_duDtB.EffectiveDir -Directory).TrimEnd('/','\') + '/ (12 dashboard CSVs)' + } + elseif ($_duDtB -and $_duDtB.IsBound) { + $_duDtB.EffectiveDir.TrimEnd('/') + '/ (12 dashboard CSVs)' + } + elseif ($OutputFile) { + (Get-DisplayPath -LocalPath (Split-Path $OutputFile -Parent) -Directory).TrimEnd('/','\') + '/ (12 dashboard CSVs)' + } + else { '(inherits -OutputPath) (12 dashboard CSVs)' } + } elseif ($row.Key -eq 'Log') { # Surface the resolved log destination URL (or final local path), # NOT the local _PARTIAL scratch path. $script:LogFile is the @@ -20921,7 +33379,7 @@ if ($script:RemoteOutputMode -ne 'None' -or $anyDestBound) { # forward into the union with In_Latest_Append=FALSE. Gated on any -Append* # switch being bound so the legend only appears when at least one merge tally # will actually be emitted. -if ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info) { +if ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info -or $AppendDefenderUsage) { Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan Write-LogHost " APPEND/MERGE TALLY LEGEND" -ForegroundColor Cyan Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan @@ -20968,13 +33426,13 @@ if ($Rollup -or $RollupPlusRaw) { } } $processorLabel = switch ($script:RollupProcessorMode) { - 'CopilotInteraction' { "Purview_CopilotInteraction_Processor v$($Script:EMBEDDED_PROCESSOR_COPILOT_VERSION) (--profile $($script:RollupDashboardProfile); inputs: Purview CSV + Entra users CSV)" } + 'CopilotInteraction' { "Purview_CopilotInteraction_Processor v$($Script:EMBEDDED_PROCESSOR_COPILOT_VERSION) ($(if ($script:RollupDashboard -eq 'AIBV') { 'ValueLens profile' } else { '--profile aio' }); inputs: Purview CSV + Entra users CSV)" } 'M365Bundle' { "Purview_M365_Usage_Bundle_Explosion_Processor v$($Script:EMBEDDED_PROCESSOR_M365_VERSION) (input: Purview CSV)" } default { 'unresolved' } } $rollupTargetDashboard = switch ($script:RollupProcessorMode) { 'CopilotInteraction' { - if ($script:RollupDashboard -eq 'AIBV') { 'AI Business Value (Analytics-Hub)' } + if ($script:RollupDashboard -eq 'AIBV') { 'ValueLens (Analytics-Hub)' } else { 'AI-in-One (Analytics-Hub)' } } 'M365Bundle' { 'M365 Usage Analytics (Analytics-Hub)' } @@ -21786,6 +34244,17 @@ else { $paramSnapshot['IncludeTelemetry'] = $IncludeTelemetry.IsPresent } + # AISID (Defender / AI Solutions Intelligence Dashboard) parameters — surfaced only for + # -Dashboard AISID runs so operators can audit the resolved AISID output folder plus the + # append/delta-cache switches that drove the run. The resolved folder comes from the same + # Resolve-DataTypePaths 'DefenderUsage' lookup used by the destinations banner and the + # metrics JSON, so every audit surface agrees on one value. + if ($Dashboard -eq 'AISID') { + $paramSnapshot['DefenderUsageOutput'] = (script:Get-AISIDResolvedOutputDir) + $paramSnapshot['AppendDefenderUsage'] = $(if ($AppendDefenderUsage) { $AppendDefenderUsage } else { '' }) + $paramSnapshot['DisableAISIDDeltaCache'] = $DisableAISIDDeltaCache.IsPresent + } + # Crossover (both modes) $paramSnapshot['MaxConcurrency'] = $MaxConcurrency $paramSnapshot['MaxMemoryMB'] = $(if ($script:ResolvedMaxMemoryMB -eq 0) { 'Off' } else { "$($script:ResolvedMaxMemoryMB)MB" + $(if ($MaxMemoryMB -eq -1) { ' (auto)' } else { '' }) }) @@ -22069,6 +34538,216 @@ function Export-DataTableToExcel { Send-SQLDataToExcel -DataTable $dataTable -Path $Path -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' } +function ConvertFrom-PaxCsvRecordText { + <# + .SYNOPSIS + Parses one complete RFC-4180 CSV record into an array of field strings. + .DESCRIPTION + Handles quoted fields, escaped double-quotes ("") and embedded delimiters. + The record text may contain embedded newlines inside quoted fields (see + Read-PaxCsvRecordText). Bounded to a single record. + #> + param([Parameter(Mandatory = $true)][string]$Text) + $fields = [System.Collections.Generic.List[string]]::new() + $sb = [System.Text.StringBuilder]::new() + $inQuotes = $false + for ($i = 0; $i -lt $Text.Length; $i++) { + $c = $Text[$i] + if ($c -eq '"') { + if ($inQuotes -and $i + 1 -lt $Text.Length -and $Text[$i + 1] -eq '"') { [void]$sb.Append('"'); $i++ } + else { $inQuotes = -not $inQuotes } + } elseif ($c -eq ',' -and -not $inQuotes) { + [void]$fields.Add($sb.ToString()); [void]$sb.Clear() + } else { + [void]$sb.Append($c) + } + } + [void]$fields.Add($sb.ToString()) + return ,$fields.ToArray() +} + +function Read-PaxCsvRecordText { + <# + .SYNOPSIS + Reads one complete CSV record from a StreamReader. + .DESCRIPTION + Joins continuation physical lines when a quoted field contains an embedded + newline (RFC-4180), returning the complete record text. Returns $null at end + of stream. Only one record is ever held in memory. + #> + param([Parameter(Mandatory = $true)][System.IO.StreamReader]$Reader) + $line = $Reader.ReadLine() + if ($null -eq $line) { return $null } + $buffer = [System.Text.StringBuilder]::new() + [void]$buffer.Append($line) + $quoteCount = 0 + foreach ($ch in $line.ToCharArray()) { if ($ch -eq '"') { $quoteCount++ } } + while (($quoteCount % 2) -ne 0) { + $next = $Reader.ReadLine() + if ($null -eq $next) { break } + [void]$buffer.Append("`n") + [void]$buffer.Append($next) + foreach ($ch in $next.ToCharArray()) { if ($ch -eq '"') { $quoteCount++ } } + } + return $buffer.ToString() +} + +function Get-PaxCsvHeader { + <# + .SYNOPSIS + Reads only the header record of a CSV as a string[] of column names. + Bounded — reads a single record. + #> + param([Parameter(Mandatory = $true)][string]$CsvPath) + $reader = $null + try { + $reader = New-Object System.IO.StreamReader($CsvPath, [System.Text.Encoding]::UTF8) + $h = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $h) { return @() } + return ,(ConvertFrom-PaxCsvRecordText -Text $h) + } + finally { + if ($reader) { $reader.Dispose() } + } +} + +function Write-PaxCsvToWorksheet { + <# + .SYNOPSIS + Streams a CSV file into an Excel worksheet in bounded batches. + .DESCRIPTION + Reads the CSV record-by-record and writes fixed-size batches of rows via + Send-SQLDataToExcel (the first batch with -Force to create the sheet + bold + frozen header, subsequent batches with -Append). At most one BatchSize slab + of rows is ever materialized, so peak memory is independent of file size. + All cells are written as text (-NoNumberConversion '*'), identical to the + prior whole-file DataTable path. AutoSize is applied on the first (header + + bounded sample) write only. Returns the number of DATA rows written. + .PARAMETER Append + Append rows to an existing worksheet (no new sheet / header is created). + #> + param( + [Parameter(Mandatory = $true)][string]$CsvPath, + [Parameter(Mandatory = $true)][string]$WorkbookPath, + [Parameter(Mandatory = $true)][string]$WorksheetName, + [int]$BatchSize = 5000, + [switch]$Append + ) + $reader = $null + $rowsWritten = 0 + $batch = $null + try { + $reader = New-Object System.IO.StreamReader($CsvPath, [System.Text.Encoding]::UTF8) + $headerText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $headerText) { throw "Workbook source CSV has no header row: $CsvPath" } + $columns = ConvertFrom-PaxCsvRecordText -Text $headerText + $colCount = $columns.Count + $appendMode = [bool]$Append + $firstWrite = $true + + $batch = New-Object System.Data.DataTable + foreach ($c in $columns) { [void]$batch.Columns.Add($c, [string]) } + + while ($true) { + $recText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $recText) { break } + $fields = ConvertFrom-PaxCsvRecordText -Text $recText + $nr = $batch.NewRow() + $lim = [Math]::Min($colCount, $fields.Count) + for ($j = 0; $j -lt $lim; $j++) { + $v = $fields[$j] + $nr[$j] = if ([string]::IsNullOrEmpty($v)) { [DBNull]::Value } else { $v } + } + [void]$batch.Rows.Add($nr) + $rowsWritten++ + + if ($batch.Rows.Count -ge $BatchSize) { + if ($firstWrite -and -not $appendMode) { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + } else { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Append -FreezeTopRow -BoldTopRow -NoNumberConversion '*' + } + $firstWrite = $false + $batch.Rows.Clear() + } + } + + if ($batch.Rows.Count -gt 0) { + if ($firstWrite -and -not $appendMode) { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + } else { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Append -FreezeTopRow -BoldTopRow -NoNumberConversion '*' + } + } elseif ($firstWrite -and -not $appendMode) { + # Header-only source: create the sheet with just the (bold, frozen) header. + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + } + } + finally { + if ($batch) { $batch.Dispose() } + if ($reader) { $reader.Dispose() } + } + return $rowsWritten +} + +function Split-PaxCsvByColumn { + <# + .SYNOPSIS + Partitions a CSV into one temp CSV per distinct value of a column. + .DESCRIPTION + Streams the source CSV once, writing each record verbatim into a per-group + temp CSV (each carrying the original header). First-appearance group order is + preserved. Peak memory is bounded to one open writer per distinct group value + (the number of distinct activity types, not the row count). Returns an ordered + array of [pscustomobject]@{ Value; Path; RowCount }. + #> + param( + [Parameter(Mandatory = $true)][string]$CsvPath, + [Parameter(Mandatory = $true)][string]$ColumnName, + [Parameter(Mandatory = $true)][string]$WorkDir + ) + $reader = $null + $writers = @{} + $paths = @{} + $counts = @{} + $order = [System.Collections.Generic.List[string]]::new() + try { + $reader = New-Object System.IO.StreamReader($CsvPath, [System.Text.Encoding]::UTF8) + $headerText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $headerText) { return ,@() } + $columns = ConvertFrom-PaxCsvRecordText -Text $headerText + $colIdx = [Array]::IndexOf($columns, $ColumnName) + if ($colIdx -lt 0) { throw "Column '$ColumnName' not found in CSV header: $CsvPath" } + $idx = 0 + while ($true) { + $recText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $recText) { break } + $fields = ConvertFrom-PaxCsvRecordText -Text $recText + $val = if ($colIdx -lt $fields.Count) { $fields[$colIdx] } else { '' } + if (-not $writers.ContainsKey($val)) { + $idx++ + $safe = ($val -replace '[^A-Za-z0-9_.-]', '_') + if ([string]::IsNullOrEmpty($safe)) { $safe = 'blank' } + $fp = Join-Path $WorkDir ("part_{0:D4}_{1}.csv" -f $idx, $safe) + $w = New-Object System.IO.StreamWriter($fp, $false, (New-Object System.Text.UTF8Encoding($false))) + $w.WriteLine($headerText) + $writers[$val] = $w + $paths[$val] = $fp + $counts[$val] = 0 + [void]$order.Add($val) + } + $writers[$val].WriteLine($recText) + $counts[$val] = $counts[$val] + 1 + } + } + finally { + foreach ($w in $writers.Values) { try { $w.Flush(); $w.Dispose() } catch { } } + if ($reader) { $reader.Dispose() } + } + $result = foreach ($v in $order) { [pscustomobject]@{ Value = $v; Path = $paths[$v]; RowCount = $counts[$v] } } + return ,@($result) +} + function ConvertTo-UniqueString { param([object]$items, [char]$Sep = ';') if ($null -eq $items) { return $null } @@ -22135,35 +34814,72 @@ try { function Invoke-ReplayInlineExport { param( - [Parameter(Mandatory)] [System.Collections.IEnumerable]$Logs + [Parameter(Mandatory)] [string]$SpoolPath ) Write-LogHost "Replay inline export starting..." -ForegroundColor Magenta + try { Write-PaxMemoryObservation -Stage 'ReplayExport' -Force } catch {} $exportTemp = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_export_" + [guid]::NewGuid().ToString() + ".tmp") # Fixed 153-column M code schema (matches live explosion output exactly) $columnOrder = $PurviewExplodedHeader - Open-CsvWriter -Path $exportTemp -Columns $columnOrder $total = 0 $idx = 0 $errCount = 0 - $errLimit = 25 - foreach ($log in $Logs) { - $idx++ - if ($idx % 5000 -eq 0) { Write-LogHost ("Replay inline progress: {0} records" -f $idx) -ForegroundColor DarkGray } - try { - $records = Convert-ToPurviewExplodedRecords -Record $log -Deep:$ExplodeDeep -PromptFilterValue $PromptFilter - $recordsArr = To-RecordArray $records - if ($recordsArr.Count -gt 0) { - $total += $recordsArr.Count - $emitSet = $recordsArr | ForEach-Object { $_ | Select-Object -Property $columnOrder } - $rowsOut = @($emitSet) - if ($rowsOut.Count -gt 0) { Write-CsvRows -Rows $rowsOut -Columns $columnOrder } + $spoolReader = $null + try { + Open-CsvWriter -Path $exportTemp -Columns $columnOrder + # Stream the re-iterable JSON-lines spool; rebuild the normalized replay record + # per line (identical to the prior in-memory build). The complete replay set is + # never held in memory. + $spoolReader = [System.IO.StreamReader]::new($SpoolPath, ([System.Text.UTF8Encoding]::new($false))) + while ($true) { + $spoolLine = $spoolReader.ReadLine() + if ($null -eq $spoolLine) { break } + if ([string]::IsNullOrWhiteSpace($spoolLine)) { continue } + $idx++ + if ($idx % 5000 -eq 0) { Write-LogHost ("Replay inline progress: {0} records" -f $idx) -ForegroundColor DarkGray; try { Write-PaxMemoryObservation -Stage 'ReplayExport' -RowsProcessed ([long]$idx) } catch {} } + try { + $row = $spoolLine | ConvertFrom-Json + $creation = if ($row.CreationDate) { $d = script:Parse-DateSafe $row.CreationDate; if ($d) { $d } else { Get-Date } } else { Get-Date } + $identity = if ($row.Id) { $row.Id } elseif ($row.RecordId) { $row.RecordId } else { [guid]::NewGuid().ToString() } + $log = [pscustomobject]@{ + RecordType = $(try { [int]$row.RecordType } catch { 0 }) + CreationDate = $creation + UserIds = @(if ($row.UserIds) { $row.UserIds } elseif ($row.UserId) { $row.UserId } else { $null }) + Operations = if ($row.Operations) { $row.Operations } elseif ($row.Operation) { $row.Operation } else { $null } + ResultStatus = $(try { $row.ResultStatus } catch { '' }) + ResultCount = 0 + Identity = $identity + IsValid = $true + ObjectState = '' + AuditData = $row.AuditData + Operation = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } + UserId = if ($row.UserId) { $row.UserId } elseif ($row.UserIds) { $row.UserIds } else { $null } + } + $records = Convert-ToPurviewExplodedRecords -Record $log -Deep:$ExplodeDeep -PromptFilterValue $PromptFilter + $recordsArr = To-RecordArray $records + if ($recordsArr.Count -gt 0) { + $total += $recordsArr.Count + $emitSet = $recordsArr | ForEach-Object { $_ | Select-Object -Property $columnOrder } + $rowsOut = @($emitSet) + if ($rowsOut.Count -gt 0) { Write-CsvRows -Rows $rowsOut -Columns $columnOrder } + } + } catch { + $errCount++ } - } catch { - $errCount++ } } - try { Close-CsvWriter } catch {} - try { Move-Item -Force -Path $exportTemp -Destination $OutputFile } catch {} + finally { + if ($spoolReader) { $spoolReader.Dispose() } + try { Close-CsvWriter } catch {} + } + # Fail-closed publish: terminate on failure so the caller preserves the spool. + try { + Move-Item -Force -Path $exportTemp -Destination $OutputFile -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $exportTemp) { Remove-Item -LiteralPath $exportTemp -Force -ErrorAction SilentlyContinue } + throw + } try { $script:metrics.TotalStructuredRows = $total } catch {} Write-LogHost ("Replay inline export complete: {0} rows" -f $total) -ForegroundColor Green @@ -22192,6 +34908,76 @@ function Invoke-ReplayInlineExport { Write-LogHost "" } +# ───────────────────────────────────────────────────────────────────────────── +# Explosion-mode bounded input spool (Blocker 1) +# Streams the (deduped, trimmed) normalized records to a re-iterable JSON-lines +# disk spool so the explosion pass never holds the complete normalized record set +# — nor the exploded rows — in memory. Mirrors the accepted RAW replay spool +# pattern (Invoke-ReplayInlineExport): the same normalized record shape is written +# per line and reconstructed on read, and each pass opens a fresh bounded reader. +# ───────────────────────────────────────────────────────────────────────────── +function Write-PaxNormalizedSpool { + param( + [Parameter(Mandatory)] $Records, + [Parameter(Mandatory)] [string]$SpoolPath + ) + $enc = [System.Text.UTF8Encoding]::new($false) + $sw = [System.IO.StreamWriter]::new($SpoolPath, $false, $enc) + $count = 0 + try { + foreach ($log in $Records) { + if ($null -eq $log) { continue } + $creationOut = $null + try { + if ($log.PSObject.Properties['CreationDate'] -and $log.CreationDate) { + $creationOut = ([datetime]$log.CreationDate).ToString('o', [System.Globalization.CultureInfo]::InvariantCulture) + } + } catch { try { $creationOut = [string]$log.CreationDate } catch { $creationOut = $null } } + $obj = [ordered]@{ + Id = $(try { if ($log.PSObject.Properties['Identity'] -and $log.Identity) { $log.Identity } elseif ($log.PSObject.Properties['RecordId'] -and $log.RecordId) { $log.RecordId } elseif ($log.PSObject.Properties['Id']) { $log.Id } else { $null } } catch { $null }) + RecordId = $(try { if ($log.PSObject.Properties['RecordId']) { $log.RecordId } elseif ($log.PSObject.Properties['Identity']) { $log.Identity } else { $null } } catch { $null }) + CreationDate = $creationOut + RecordType = $(try { if ($log.PSObject.Properties['RecordType']) { $log.RecordType } else { $null } } catch { $null }) + Operations = $(try { if ($log.PSObject.Properties['Operations']) { $log.Operations } else { $null } } catch { $null }) + Operation = $(try { if ($log.PSObject.Properties['Operation']) { $log.Operation } else { $null } } catch { $null }) + UserIds = $(try { if ($log.PSObject.Properties['UserIds']) { $log.UserIds } else { $null } } catch { $null }) + UserId = $(try { if ($log.PSObject.Properties['UserId']) { $log.UserId } else { $null } } catch { $null }) + ResultStatus = $(try { if ($log.PSObject.Properties['ResultStatus']) { $log.ResultStatus } else { $null } } catch { $null }) + AuditData = $(try { if ($log.PSObject.Properties['AuditData']) { $log.AuditData } else { $null } } catch { $null }) + } + $sw.WriteLine(($obj | ConvertTo-Json -Compress -Depth 20)) + $count++ + } + } + finally { + $sw.Flush(); $sw.Dispose() + } + return $count +} + +function ConvertFrom-PaxNormalizedSpoolLine { + param([Parameter(Mandatory)] [AllowEmptyString()] [string]$Line) + if ([string]::IsNullOrWhiteSpace($Line)) { return $null } + $row = $Line | ConvertFrom-Json + $creation = if ($row.CreationDate) { $d = script:Parse-DateSafe $row.CreationDate; if ($d) { $d } else { Get-Date } } else { Get-Date } + $identity = if ($row.Id) { $row.Id } elseif ($row.RecordId) { $row.RecordId } else { [guid]::NewGuid().ToString() } + return [pscustomobject]@{ + RecordType = $(try { [int]$row.RecordType } catch { 0 }) + CreationDate = $creation + UserIds = @(if ($row.UserIds) { $row.UserIds } elseif ($row.UserId) { $row.UserId } else { $null }) + Operations = if ($row.Operations) { $row.Operations } elseif ($row.Operation) { $row.Operation } else { $null } + ResultStatus = $(try { $row.ResultStatus } catch { '' }) + ResultCount = 0 + Identity = $identity + IsValid = $true + ObjectState = '' + AuditData = $row.AuditData + Operation = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } + UserId = if ($row.UserId) { $row.UserId } elseif ($row.UserIds) { $row.UserIds } else { $null } + RecordId = if ($row.RecordId) { $row.RecordId } elseif ($row.Id) { $row.Id } else { $identity } + } +} + function Get-SafeProperty { param($obj, [string]$name) try { if ($null -ne $obj -and $obj.PSObject.Properties[$name]) { return $obj.($name) } } catch {}; return $null } # --- Purview Exploded Schema (153 columns — matches M code #"Changed Type" step exactly) --- @@ -22368,8 +35154,139 @@ function Test-EntraUsersSchema { } } +# ═══════════════════════════════════════════════════════════════════════════════ +# HYBRID DIRECTORY SUPPLEMENT (-UserInfoSupplement) — shared input transport + parse/validate + merge +# The three functions below implement the customer-facing hybrid enrichment mode. They are script-scoped +# and pure (no global side effects beyond their return values) so the focused test suite can AST-extract +# and exercise the SHIPPED code paths directly. +# ═══════════════════════════════════════════════════════════════════════════════ + +function script:Get-PaxReservedDirectoryColumns { + # Authoritative reserved-column set that a supplemental column must never collide with + # (case-insensitive). Derived from the live Entra users contract ($EntraUsersHeader) UNION the + # PAX/Rollup-owned identity, licensing, append-provenance, and hierarchy columns produced downstream + # by the Users dimension. Deriving from $EntraUsersHeader keeps the Entra half in sync with the schema. + $reserved = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($c in $EntraUsersHeader) { [void]$reserved.Add([string]$c) } + foreach ($c in @('PersonId','UserKey','PersonId_Normalized','License Status','TotalEmployees', + 'Date_Added','Latest_Append_Date','In_Latest_Append','Manager_UserKey','OrgLevel','HierarchyPath', + 'TopOfChain_UserKey','IsManager','DirectReports','TotalReports')) { [void]$reserved.Add($c) } + for ($lvl = 0; $lvl -le 14; $lvl++) { [void]$reserved.Add("Level${lvl}_UserKey"); [void]$reserved.Add("Level${lvl}_Name") } + # Unary comma prevents PowerShell from unrolling the HashSet into an array on return (which would + # silently make .Contains() a case-SENSITIVE array match). Callers receive the case-insensitive set. + return ,$reserved +} + +function script:Resolve-DirectoryCsvInput { + # Shared, hardened classifier + stager for a customer-provided directory CSV INPUT (used by BOTH + # -UserInfoFile and -UserInfoSupplement). Classifies the input value's OWN tier independent of the + # output tier, requires an actual .csv file leaf (folder URLs, Fabric roots, and Fabric Tables URLs are + # rejected; a Fabric input must live under /Files/.../*.csv), and for remote tiers stages the file to + # run-specific local scratch via the matching Get-RemoteFile-* using the input URL's own parent + leaf. + # Staged inputs are never registered for upload. Returns @{ Tier; LocalPath; Leaf; Parent; Staged }. + # Throws (or Get-PathTier hard-stops) on any classification/fetch failure — callers do NOT fall back. + param( + [Parameter(Mandatory)][string]$Value, + [Parameter(Mandatory)][string]$SwitchName, + [Parameter(Mandatory)][string]$ScratchDir, + [Parameter(Mandatory)][string]$ScratchPrefix + ) + $raw = $Value.Trim() + $tier = script:Get-PathTier -Value $raw -SwitchName $SwitchName -AllowFabricFilesOnly + $leaf = ($raw -split '[\\/]' | Where-Object { $_ }) | Select-Object -Last 1 + if ([string]::IsNullOrWhiteSpace($leaf) -or ($leaf -notmatch '(?i)\.csv$')) { + Write-Host ("ERROR: -{0} must reference a .csv file (not a folder). Provided: {1}" -f $SwitchName, $raw) -ForegroundColor Red + throw ("-{0} must reference a .csv file: {1}" -f $SwitchName, $raw) + } + if ($tier -eq 'Local') { + return @{ Tier = 'Local'; LocalPath = $raw; Leaf = $leaf; Parent = (Split-Path -Path $raw -Parent); Staged = $false } + } + $parent = $raw.Substring(0, $raw.Length - $leaf.Length).TrimEnd('/', '\') + $localPath = Join-Path $ScratchDir ('{0}{1}' -f $ScratchPrefix, $leaf) + if ($tier -eq 'SharePoint') { + Get-RemoteFile-SharePoint -RelativeName $leaf -DestinationPath $localPath -ParentOverride $parent -ErrorAction Stop + } + else { + Get-RemoteFile-OneLake -RelativeName $leaf -DestinationPath $localPath -ParentOverride $parent -ErrorAction Stop + } + return @{ Tier = $tier; LocalPath = $localPath; Leaf = $leaf; Parent = $parent; Staged = $true } +} + +function script:Import-PaxSupplementCsv { + # Parse + validate a supplemental directory CSV for -UserInfoSupplement. Structured CSV parsing only + # (Import-Csv), never string splitting. Enforces: readable CSV header; >=1 data row; exactly one + # 'UserPrincipalName' header (case-insensitive; NO aliases); no case-insensitive duplicate headers; + # non-blank UPN on every row; unique normalized UPNs; and no supplemental (passthrough) column colliding + # with a reserved Entra/PAX-owned column. The join key is removed from the passthrough set (it is never + # copied into output). Returns @{ Rows; UpnColumn; PassthroughColumns; Lookup } where Lookup maps a + # normalized UPN (Trim().ToLowerInvariant()) to the ORIGINAL supplemental row. Throws a clear, + # count/list-bearing message on any violation. + param([Parameter(Mandatory)][string]$Path) + $rows = @(Import-Csv -LiteralPath $Path -ErrorAction Stop) + if ($rows.Count -eq 0) { + $firstLine = Get-Content -LiteralPath $Path -TotalCount 1 -ErrorAction SilentlyContinue + if ([string]::IsNullOrWhiteSpace($firstLine)) { throw "-UserInfoSupplement has no header row (not a CSV): $Path" } + throw "-UserInfoSupplement contains a header but zero data rows: $Path" + } + $headers = @($rows[0].PSObject.Properties.Name) + $seen = @{}; $dupHeaders = @() + foreach ($h in $headers) { $k = ([string]$h).Trim().ToLowerInvariant(); if ($seen.ContainsKey($k)) { $dupHeaders += $h } else { $seen[$k] = $h } } + if ($dupHeaders.Count -gt 0) { throw ("-UserInfoSupplement has duplicate column header(s) (case-insensitive): {0}" -f (($dupHeaders | Select-Object -Unique) -join ', ')) } + $upnCols = @($headers | Where-Object { ([string]$_).Trim().ToLowerInvariant() -eq 'userprincipalname' }) + if ($upnCols.Count -eq 0) { throw ("-UserInfoSupplement is missing the required 'UserPrincipalName' column. Found: {0}" -f ($headers -join ', ')) } + if ($upnCols.Count -gt 1) { throw ("-UserInfoSupplement has more than one 'UserPrincipalName' column; exactly one is required.") } + $upnCol = $upnCols[0] + $passthrough = @($headers | Where-Object { $_ -ne $upnCol }) + $reserved = script:Get-PaxReservedDirectoryColumns + $collisions = @($passthrough | Where-Object { $reserved.Contains([string]$_) }) + if ($collisions.Count -gt 0) { throw ("-UserInfoSupplement column(s) collide with reserved Entra/PAX-owned columns (supplemental columns are additive-only and cannot override PAX data): {0}" -f ($collisions -join ', ')) } + $lookup = [System.Collections.Generic.Dictionary[string,object]]::new([System.StringComparer]::OrdinalIgnoreCase) + $blankCount = 0; $dupUpns = [System.Collections.Generic.List[string]]::new() + foreach ($row in $rows) { + $rawUpn = [string]$row.$upnCol + if ([string]::IsNullOrWhiteSpace($rawUpn)) { $blankCount++; continue } + $norm = $rawUpn.Trim().ToLowerInvariant() + if ($lookup.ContainsKey($norm)) { $dupUpns.Add($rawUpn.Trim()) } else { $lookup[$norm] = $row } + } + if ($blankCount -gt 0) { throw ("-UserInfoSupplement has {0} row(s) with a blank UserPrincipalName; every supplemental row must carry a UPN." -f $blankCount) } + if ($dupUpns.Count -gt 0) { throw ("-UserInfoSupplement has duplicate UserPrincipalName value(s) (normalized, case-insensitive): {0}" -f (($dupUpns | Select-Object -Unique) -join ', ')) } + return @{ Rows = $rows; UpnColumn = $upnCol; PassthroughColumns = $passthrough; Lookup = $lookup } +} + +function script:Merge-PaxEntraSupplement { + # Entra-LEFT join: append the supplemental passthrough columns to each Entra row, preserving Entra row + # order and supplemental column order. A matching Entra user receives the exact supplemental value; a + # non-matching Entra user receives '' for every supplemental column. Supplemental-only users are NEVER + # added (Entra stays authoritative); the supplemental UPN is NEVER copied into output. Entra rows are + # enriched in place (each supplemental column appended AFTER all existing properties). Returns + # @{ Rows; Matched; UnmatchedSupplemental; EntraWithout; ColumnsAdded }. + param( + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$EntraRows, + [Parameter(Mandatory)][string[]]$PassthroughColumns, + [Parameter(Mandatory)]$Lookup, + [Parameter(Mandatory)][string]$UpnColumn + ) + $matchedKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $entraWithout = 0 + foreach ($er in $EntraRows) { + $eupn = [string]$er.userPrincipalName + $norm = if ($eupn) { $eupn.Trim().ToLowerInvariant() } else { '' } + $supRow = $null + if ($norm -and $Lookup.ContainsKey($norm)) { $supRow = $Lookup[$norm]; [void]$matchedKeys.Add($norm) } else { $entraWithout++ } + foreach ($pc in $PassthroughColumns) { + $val = if ($supRow) { [string]$supRow.$pc } else { '' } + $er.PSObject.Properties.Remove($pc) + $er | Add-Member -NotePropertyName $pc -NotePropertyValue $val -Force + } + } + $unmatched = [System.Collections.Generic.List[string]]::new() + foreach ($k in $Lookup.Keys) { if (-not $matchedKeys.Contains($k)) { $unmatched.Add([string]$Lookup[$k].$UpnColumn) } } + return @{ Rows = $EntraRows; Matched = $matchedKeys.Count; UnmatchedSupplemental = @($unmatched); EntraWithout = $entraWithout; ColumnsAdded = @($PassthroughColumns) } +} + # ═══════════════════════════════════════════════════════════════════════════════ # FAST ROW CREATION HELPER + # Converts a hashtable to PSCustomObject in a single operation (avoids Add-Member overhead) # Used by explosion logic to build rows efficiently via hashtable accumulation # ═══════════════════════════════════════════════════════════════════════════════ @@ -23240,15 +36157,24 @@ try { # ============================================================ # RESUME MODE VALIDATION - Ensure no conflicting parameters # ============================================================ - if ($PSBoundParameters.ContainsKey('Resume')) { + if ($ResumeSpecified) { # Resume mode is standalone - only auth-related parameters allowed $allowedWithResume = @( 'Resume', + 'RemainingArgs', 'Force', + 'UserIds', + 'GroupNames', 'Auth', 'TenantId', 'ClientId', 'ClientSecret', + 'Dashboard', + 'OutputPathDefenderUsage', + 'AppendDefenderUsage', + 'DisableAISIDDeltaCache', + 'ClearUncertainCreate', + 'ClearUncertainContract', 'ClientCertificateThumbprint', 'ClientCertificateStoreLocation', 'ClientCertificatePath', @@ -23290,8 +36216,8 @@ try { Write-Host " -TenantId Tenant ID (for AppRegistration)" -ForegroundColor Gray Write-Host " -ClientId Client ID (for AppRegistration)" -ForegroundColor Gray Write-Host " -ClientSecret Client secret (for AppRegistration)" -ForegroundColor Gray - Write-Host "" - Write-Host " Example usage:" -ForegroundColor Cyan + Write-Host " -UserIds Override the checkpoint's user scope (fail-closed)" -ForegroundColor Gray + Write-Host " -GroupNames Override the checkpoint's group scope (fail-closed)" -ForegroundColor Gray Write-Host ' .\Script.ps1 -Resume' -ForegroundColor White Write-Host ' .\Script.ps1 -Resume -Auth DeviceCode' -ForegroundColor White Write-Host ' .\Script.ps1 -Resume "C:\path\.pax_checkpoint_xxx.json" -Force' -ForegroundColor White @@ -23343,7 +36269,16 @@ try { Write-LogHost "Hydrated $restored resume artifact(s) from OneLake mirror" -ForegroundColor DarkGray } } catch { - Write-LogHost ("WARNING: Resume mirror hydration failed: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + # A hydration failure is an INTEGRITY signal, not a hint: the mirror + # exposed an authority record whose artifact set could not be proven, or + # it could not be read at all. Continuing to checkpoint discovery here + # would search a directory that either holds stale local state or holds + # nothing, and could silently resume the wrong state. Stop non-zero and + # distinguish this from the ordinary "no remote generation exists" case, + # which returns 0 and is not an error. + Write-LogHost ("ERROR: Resume mirror hydration failed - resume cannot continue: {0}" -f $_.Exception.Message) -ForegroundColor Red + Write-LogHost " No checkpoint was hydrated from the remote mirror, and no partial resume state was left behind." -ForegroundColor Gray + exit 1 } } @@ -23422,6 +36357,48 @@ try { # Note: $script:CheckpointData and $script:IsResumeMode already set by Read-Checkpoint + # Operator-only SELECTIVE recovery: when -ClearUncertainCreate is supplied it carries an + # explicit list of partition indexes to recover. Validate the selection against the + # durable uncertain-create set NOW (pre-auth): an empty, duplicate, unknown, or + # out-of-range selection is REJECTED and nothing is cleared. A valid selection is only + # STAGED here - the durable marker for a selected partition is cleared selectively and + # atomically later, at the parent-create reconcile, and ONLY after a READ-ONLY + # exact-name reconcile confirms the query is not visible (a still-visible selected + # partition is adopted, never cleared into a duplicate POST). Durable uncertain state is + # never auto-cleared by elapsed time, a collection miss, a process restart, or a failure count. + $script:ClearUncertainSelection = [System.Collections.Generic.HashSet[int]]::new() + if ($PSBoundParameters.ContainsKey('ClearUncertainCreate')) { + $clearSelection = Clear-PaxUncertainCreateState -CheckpointData $script:CheckpointData -Requested:$true -SelectedIndexes $ClearUncertainCreate -ValidateOnly + if ($clearSelection.Rejected) { + Write-LogHost "" + Write-LogHost " ERROR: -ClearUncertainCreate selection rejected - $($clearSelection.Reason). No uncertain-create state was changed." -ForegroundColor Red + $script:ResumeValidationFailed = $true + exit 1 + } + foreach ($selIdx in @($clearSelection.Selected)) { [void]$script:ClearUncertainSelection.Add([int]$selIdx) } + Write-LogHost " [RESUME] -ClearUncertainCreate: $($script:ClearUncertainSelection.Count) partition(s) selected for recovery; each is reconciled READ-ONLY first and only cleared for a fresh create if not visible by exact name." -ForegroundColor Yellow + } + # EXPLICIT-CONTRACT companion (-ClearUncertainContract): each ":" + # token names EXACTLY one durable uncertain contract, disambiguating an index that maps + # to more than one. Validated NOW (pre-auth) against the durable set: a malformed, + # unknown, multiply-matching, or duplicate token is REJECTED and nothing is cleared. A + # valid selection is only STAGED here - the durable marker is cleared selectively and + # atomically later, at the reconcile, and ONLY after a READ-ONLY exact-name reconcile + # confirms the selected contract is not visible (a still-visible contract is adopted, + # never cleared into a duplicate POST). + $script:ClearUncertainContractSelection = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + if ($PSBoundParameters.ContainsKey('ClearUncertainContract')) { + $contractSelection = Clear-PaxUncertainCreateState -CheckpointData $script:CheckpointData -Requested:$true -SelectedContracts $ClearUncertainContract -ValidateOnly + if ($contractSelection.Rejected) { + Write-LogHost "" + Write-LogHost " ERROR: -ClearUncertainContract selection rejected - $($contractSelection.Reason). No uncertain-create state was changed." -ForegroundColor Red + $script:ResumeValidationFailed = $true + exit 1 + } + foreach ($selTok in @($contractSelection.SelectedContracts)) { [void]$script:ClearUncertainContractSelection.Add([string]$selTok) } + Write-LogHost " [RESUME] -ClearUncertainContract: $($script:ClearUncertainContractSelection.Count) contract(s) selected for recovery; each is reconciled READ-ONLY first and only cleared for a fresh create if not visible by exact name." -ForegroundColor Yellow + } + # Display resume summary $completedCount = if ($checkpointData.partitions.completed) { $checkpointData.partitions.completed.Count } else { 0 } $queryCreatedCount = if ($checkpointData.partitions.queryCreated) { $checkpointData.partitions.queryCreated.Count } else { 0 } @@ -23444,6 +36421,73 @@ try { # ============================================================ Write-LogHost "Restoring parameters from checkpoint..." -ForegroundColor DarkGray $cp = $checkpointData.parameters + $script:ResumeValidationFailed = $true + # ============================================================ + # AISID resume validation. Runs BEFORE the dashboard/window + # restore below so it sets $Dashboard authoritatively and stops the rollup-dashboard + # restore from overriding an AISID resume. Reads raw checkpoint fields and + # enforces conflicts/requirements NONINTERACTIVELY (exit 1; never Read-Host). + # $checkpointData / $cp are hashtables (ConvertFrom-Json -AsHashtable). + # ============================================================ + $script:ResumeIsAISID = $false + $script:AISIDResumeForceCacheMiss = $false + $script:ResumeAISIDClass = $null + $hasAisidWindow = [bool]$checkpointData.aisidWindow + $markerRaw = if ($checkpointData.Contains('aisidResumeStateVersion')) { $checkpointData.aisidResumeStateVersion } else { $null } + if ($null -ne $markerRaw -and [string]$markerRaw -ne '1') { Write-LogHost "ERROR: this checkpoint carries an unsupported AISID resume-state version ($markerRaw); this build supports version 1 only." -ForegroundColor Red; exit 1 } + $hasMarker = ($markerRaw -eq 1) + $cpDashboard = if ($cp.Contains('dashboard')) { [string]$cp.dashboard } else { $null } + $cpRollupDash = if ($cp.Contains('rollupDashboard')) { [string]$cp.rollupDashboard } else { $null } + $cliDash = if ($PSBoundParameters.ContainsKey('Dashboard')) { [string]$Dashboard } else { $null } + $cliOut = $PSBoundParameters.ContainsKey('OutputPathDefenderUsage') + $cliApp = $PSBoundParameters.ContainsKey('AppendDefenderUsage') + if ($hasMarker -and $cpDashboard -eq 'AISID') { + $Dashboard = 'AISID'; $script:ResumeIsAISID = $true; $script:ResumeAISIDClass = 'Complete' + if ($cliDash) { Write-LogHost "ERROR: resume dashboard conflict - this checkpoint already stores its dashboard; -Dashboard cannot be supplied on resume for it. Omit -Dashboard." -ForegroundColor Red; exit 1 } + if ($cliOut -or $cliApp) { Write-LogHost "ERROR: resume DefenderUsage destination conflict - this checkpoint already stores its AISID DefenderUsage destination; do not pass -OutputPathDefenderUsage or -AppendDefenderUsage on resume." -ForegroundColor Red; exit 1 } + if ($PSBoundParameters.ContainsKey('DisableAISIDDeltaCache')) { Write-LogHost "ERROR: resume cache conflict - this checkpoint already stores the AISID delta-cache setting; do not pass -DisableAISIDDeltaCache on resume." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: restoring the AISID dashboard, DefenderUsage destination, cache state, and frozen collection window from the checkpoint." -ForegroundColor Cyan + } + elseif ($hasAisidWindow -and -not $hasMarker -and ($null -eq $cpDashboard)) { + $Dashboard = 'AISID'; $script:ResumeIsAISID = $true; $script:ResumeAISIDClass = 'Transitional'; $script:AISIDResumeForceCacheMiss = $true + if ($cliDash -and $cliDash -ne 'AISID') { Write-LogHost "ERROR: resume dashboard conflict - this AISID checkpoint can only resume as AISID; -Dashboard $cliDash is not allowed." -ForegroundColor Red; exit 1 } + if (-not ($cliOut -xor $cliApp)) { Write-LogHost "ERROR: this AISID checkpoint has no stored DefenderUsage destination; supply EXACTLY ONE of -OutputPathDefenderUsage or -AppendDefenderUsage on the resume command line (not both, not neither)." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: this AISID checkpoint has no stored DefenderUsage destination; using the destination supplied on the command line and forcing a delta-cache miss." -ForegroundColor Cyan + } + elseif (-not $hasAisidWindow -and ($null -eq $cpDashboard) -and ($cpRollupDash -eq 'AIO')) { + if (-not $cliDash) { Write-LogHost "ERROR: this older checkpoint is ambiguous - it could be an AIO run or an AISID run. Re-run with -Dashboard AIO, or -Dashboard AISID plus exactly one of -OutputPathDefenderUsage or -AppendDefenderUsage." -ForegroundColor Red; exit 1 } + elseif ($cliDash -eq 'AIO') { + $Dashboard = 'AIO'; $script:ResumeIsAISID = $false + if ($cliOut -or $cliApp) { Write-LogHost "ERROR: -OutputPathDefenderUsage and -AppendDefenderUsage are not valid with -Dashboard AIO." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: dashboard resolved to AIO from the command line." -ForegroundColor Cyan + } + elseif ($cliDash -eq 'AISID') { + $Dashboard = 'AISID'; $script:ResumeIsAISID = $true; $script:ResumeAISIDClass = 'LegacyAISID'; $script:AISIDResumeForceCacheMiss = $true + if (-not ($cliOut -xor $cliApp)) { Write-LogHost "ERROR: resuming this checkpoint as AISID requires EXACTLY ONE of -OutputPathDefenderUsage or -AppendDefenderUsage (not both, not neither)." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: dashboard resolved to AISID from the command line; the collection window is reconstructed from the checkpoint dates and creation time, and a delta-cache miss is forced." -ForegroundColor Cyan + } + else { Write-LogHost "ERROR: -Dashboard $cliDash is not a valid choice for resuming this ambiguous checkpoint (use AIO or AISID)." -ForegroundColor Red; exit 1 } + } + + # ============================================================ + # AISID AVAILABILITY GATE - restored checkpoint state + # Runs immediately after the AISID resume classification above and before any + # further parameter restoration, destination re-resolution, remote probe, + # checkpoint mutation, sign-in, collection, or publication. A checkpoint that + # restores the AISID dashboard, an AISID DefenderUsage output or append + # destination, or the AISID delta-cache setting stops here, so stored state + # cannot reach a gated AISID execution path. The checkpoint lock is released + # first so a gated resume leaves nothing behind. $cp is the checkpoint + # parameter block that the restore below reads for these same values. + # ============================================================ + $aisidResumeDashboard = ($script:ResumeIsAISID -or (([string]$Dashboard).Trim().ToUpperInvariant() -eq 'AISID') -or (($null -ne $cpDashboard) -and (([string]$cpDashboard).Trim().ToUpperInvariant() -eq 'AISID'))) + $aisidResumeOutput = ($cp.Contains('outputPathDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$cp.outputPathDefenderUsage)) + $aisidResumeAppend = ($cp.Contains('appendDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$cp.appendDefenderUsage)) + $aisidResumeCache = ($cp.Contains('disableAISIDDeltaCache') -and [bool]$cp.disableAISIDDeltaCache) + if (script:Test-AISIDEntryPointGated -DashboardIsAISID $aisidResumeDashboard -DefenderUsageOutputRequested $aisidResumeOutput -DefenderUsageAppendRequested $aisidResumeAppend -DeltaCacheSwitchRequested $aisidResumeCache) { + try { script:Release-CheckpointLock } catch { } + exit 1 + } # Restore original run timestamp so incremental files use consistent naming # This ensures all partition files (original run + resumes) share the same timestamp @@ -23471,8 +36515,8 @@ try { if ($cp.activityTypes -and $cp.activityTypes.Count -gt 0) { $ActivityTypes = $cp.activityTypes } if ($cp.recordTypes -and $cp.recordTypes.Count -gt 0) { $RecordTypes = $cp.recordTypes } if ($cp.serviceTypes -and $cp.serviceTypes.Count -gt 0) { $ServiceTypes = $cp.serviceTypes } - if ($cp.userIds -and $cp.userIds.Count -gt 0) { $UserIds = $cp.userIds } - if ($cp.groupNames -and $cp.groupNames.Count -gt 0) { $GroupNames = $cp.groupNames } + if (-not $PSBoundParameters.ContainsKey('UserIds') -and $cp.userIds -and $cp.userIds.Count -gt 0) { $UserIds = $cp.userIds } + if (-not $PSBoundParameters.ContainsKey('GroupNames') -and $cp.groupNames -and $cp.groupNames.Count -gt 0) { $GroupNames = $cp.groupNames } # Agent filtering if ($cp.agentId -and $cp.agentId.Count -gt 0) { $AgentId = $cp.agentId } @@ -23515,6 +36559,25 @@ try { # Deidentify is checkpoint-driven on resume (the resume allow-list blocks it on the # command line), so restore it here as the sole source of truth for a resumed run. if ($cp.Contains('deidentify') -and $cp.deidentify) { $Deidentify = [switch]$true; $script:PaxDeidEnabled = $true } + if ($cp.Contains('verifyPartitionStability') -and $cp.verifyPartitionStability) { $VerifyPartitionStability = [switch]$true } + + # -UserInfoSupplement (hybrid directory enrichment) is checkpoint-driven on resume so the original + # supplemental input is reused. An explicit resume override with a DIFFERENT value fails clearly; a + # matching value is accepted. Only the source path/URL is restored - no supplemental values are stored. + if ($cp.Contains('userInfoSupplement') -and $cp.userInfoSupplement) { + $cpUis = [string]$cp.userInfoSupplement + if ($PSBoundParameters.ContainsKey('UserInfoSupplement') -and -not [string]::IsNullOrWhiteSpace($UserInfoSupplement) -and ($UserInfoSupplement.Trim() -ne $cpUis.Trim())) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement was supplied on the resume command line with a value that differs from the original run." -ForegroundColor Red + Write-Host (" Original: {0}" -f $cpUis) -ForegroundColor Yellow + Write-Host (" Provided: {0}" -f $UserInfoSupplement.Trim()) -ForegroundColor Yellow + Write-Host " A resumed run reuses the original supplemental input; omit -UserInfoSupplement on resume." -ForegroundColor Yellow + Write-Host "" + exit 1 + } + $UserInfoSupplement = $cpUis + $script:UserInfoSupplementActive = $true + } # FillerLabel is checkpoint-driven on resume (the allow-list blocks it on the command # line); restore the resolved hierarchy-filler mode + literal as the sole source of truth. @@ -23531,6 +36594,13 @@ try { if ($cp.Contains('appendFile') -and $cp.appendFile) { $AppendFile = [string]$cp.appendFile } if ($cp.Contains('appendUserInfo') -and $cp.appendUserInfo) { $AppendUserInfo = [string]$cp.appendUserInfo } if ($cp.Contains('appendAgent365Info') -and $cp.appendAgent365Info) { $AppendAgent365Info = [string]$cp.appendAgent365Info } + # AISID (Defender) output set + cache switch. Restored ONLY when the checkpoint stores the full + # AISID resume state (marker present). A checkpoint without it requires an explicit destination + # recovery instead REQUIRES a dedicated -OutputPathDefenderUsage/-AppendDefenderUsage on the + # resume CLI and forces a cache miss (handled in the classification block below). + if ($cp.Contains('outputPathDefenderUsage') -and $cp.outputPathDefenderUsage) { $OutputPathDefenderUsage = [string]$cp.outputPathDefenderUsage } + if ($cp.Contains('appendDefenderUsage') -and $cp.appendDefenderUsage) { $AppendDefenderUsage = [string]$cp.appendDefenderUsage } + if ($cp.Contains('disableAISIDDeltaCache') -and $cp.disableAISIDDeltaCache) { $DisableAISIDDeltaCache = [System.Management.Automation.SwitchParameter]::new($true) } # ============================================================ # RESUME: Re-resolve destination state from RESTORED variable values. @@ -23553,6 +36623,7 @@ try { @{ Key = 'Purview' ; Name = 'OutputPath' ; Value = $OutputPath ; AllowFabricFilesOnly = $false }, @{ Key = 'UserInfo' ; Name = 'OutputPathUserInfo' ; Value = $OutputPathUserInfo ; AllowFabricFilesOnly = $false }, @{ Key = 'Agent365Info' ; Name = 'OutputPathAgent365Info' ; Value = $OutputPathAgent365Info ; AllowFabricFilesOnly = $false }, + @{ Key = 'DefenderUsage'; Name = 'OutputPathDefenderUsage' ; Value = $OutputPathDefenderUsage ; AllowFabricFilesOnly = $false }, @{ Key = 'Log' ; Name = 'OutputPathLog' ; Value = $OutputPathLog ; AllowFabricFilesOnly = $true } ) foreach ($ds in $resumeDestSwitches) { @@ -23570,7 +36641,8 @@ try { $resumeAppendSwitches = @( @{ Key = 'Purview' ; Name = 'AppendFile' ; Value = $AppendFile }, @{ Key = 'UserInfo' ; Name = 'AppendUserInfo' ; Value = $AppendUserInfo }, - @{ Key = 'Agent365Info' ; Name = 'AppendAgent365Info' ; Value = $AppendAgent365Info } + @{ Key = 'Agent365Info' ; Name = 'AppendAgent365Info' ; Value = $AppendAgent365Info }, + @{ Key = 'DefenderUsage'; Name = 'AppendDefenderUsage' ; Value = $AppendDefenderUsage } ) foreach ($as in $resumeAppendSwitches) { $hasValue = -not [string]::IsNullOrWhiteSpace($as.Value) @@ -23601,7 +36673,7 @@ try { } } if ($script:RemoteOutputMode -eq 'None') { - foreach ($k in @('UserInfo','Agent365Info')) { + foreach ($k in @('UserInfo','Agent365Info','DefenderUsage')) { if (-not $script:DestTier.ContainsKey($k)) { continue } $t = $script:DestTier[$k] if ($t -eq 'SharePoint' -or $t -eq 'Fabric') { @@ -23623,12 +36695,14 @@ try { return $u } if ($script:RemoteOutputUrl) { $script:RemoteOutputUrl = & $resumeNormalizeFolderUrl $script:RemoteOutputUrl } - foreach ($k in @('Purview','UserInfo','Agent365Info','Log')) { + foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if (-not $script:DestRaw.ContainsKey($k)) { continue } if (-not $script:DestTier.ContainsKey($k)) { continue } if ($script:DestTier[$k] -eq 'Local') { continue } $script:DestParentUrl[$k] = & $resumeNormalizeFolderUrl $script:DestRaw[$k] } + $restoredTiers = @(); foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if ($script:DestTier.ContainsKey($k)) { $restoredTiers += $script:DestTier[$k] } }; $restoredTiers = @($restoredTiers | Select-Object -Unique); if ($restoredTiers.Count -gt 1) { Write-LogHost ("ERROR: resume destination conflict - restored destinations resolve to mixed storage tiers ({0}); all destinations in one run must share one tier." -f ($restoredTiers -join ', ')) -ForegroundColor Red; exit 1 } + $restoredNorm = @{}; foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if (-not $script:DestRaw.ContainsKey($k)) { continue }; $rn = $script:DestRaw[$k].TrimEnd('/','\').ToLowerInvariant(); if ($rn -match '\.[a-z0-9]{2,5}$') { if ($restoredNorm.ContainsKey($rn)) { Write-LogHost ("ERROR: resume destination conflict - {0} and {1} resolve to the same fully qualified path: {2}" -f $restoredNorm[$rn], $k, $script:DestRaw[$k]) -ForegroundColor Red; exit 1 }; $restoredNorm[$rn] = $k } } Write-LogHost " Resume: re-resolved destination state (RemoteOutputMode=$script:RemoteOutputMode, tiers=$(($script:DestTier.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ', '))" -ForegroundColor DarkGray # Resume-side scratch-dir rewrite. Mirrors the parse-time scratch-dir rewrite @@ -23680,8 +36754,8 @@ try { # CopilotInteraction run keeps its AIO/AIBV profile instead of defaulting to AIO. # Guarded to valid values so the [ValidateSet] on $Dashboard never throws on assign. $resumeUserPassedDashboard = $PSBoundParameters.ContainsKey('Dashboard') - if (-not $resumeUserPassedDashboard -and $cp.rollupDashboard -and ([string]$cp.rollupDashboard -in @('AIO', 'AIBV', 'M365'))) { - $Dashboard = [string]$cp.rollupDashboard + if (-not $resumeUserPassedDashboard -and -not $script:ResumeIsAISID -and $cp.rollupDashboard -and ([string]$cp.rollupDashboard -in @('AIO', 'AIBV', 'M365'))) { + $Dashboard = if ([string]$cp.rollupDashboard -ieq 'AIBV') { 'ValueLens' } else { [string]$cp.rollupDashboard } Write-LogHost " Restored rollup dashboard from checkpoint: $Dashboard" -ForegroundColor DarkGray } @@ -23704,7 +36778,7 @@ try { # Honor the restored / resume-CLI dashboard so a resumed AIBV run is not # silently downgraded to the AIO default. Only AIO/AIBV reach here # (M365 implies -IncludeM365Usage -> the M365Bundle branch above). - $script:RollupDashboard = if ($Dashboard) { $Dashboard.ToUpperInvariant() } else { 'AIO' } + $script:RollupDashboard = if ($Dashboard) { if ($Dashboard.ToUpperInvariant() -eq 'VALUELENS') { 'AIBV' } else { $Dashboard.ToUpperInvariant() } } else { 'AIO' } if ($script:RollupDashboard -notin @('AIO', 'AIBV')) { $script:RollupDashboard = 'AIO' } $script:RollupDashboardProfile = $script:RollupDashboard.ToLowerInvariant() if (-not $IncludeUserInfo) { @@ -23950,6 +37024,39 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit $script:TrimStartDateUTC = if ($StartDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } $script:TrimEndDateUTC = if ($EndDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } + # AISID window restore: replay the run's frozen window on resume; never + # recompute against the resume-time clock. Newer checkpoints carry an explicit aisidWindow block; a + # checkpoint that carries two concrete dates + a parseable createdUtc (no aisidWindow) has its + # ReferenceNowUtc derived once from createdUtc. Anything else fails clearly (never the live clock). + if ($Dashboard -eq 'AISID') { + $awp = $checkpointData.aisidWindow + $rk = [System.Globalization.DateTimeStyles]::RoundtripKind + $ic = [System.Globalization.CultureInfo]::InvariantCulture + if ($awp -and $awp.referenceNowUtc -and $awp.requestedStartUtc -and $awp.requestedEndUtc) { + $script:AISIDReferenceNowUtc = [datetime]::SpecifyKind([datetime]::Parse([string]$awp.referenceNowUtc, $ic, $rk), [System.DateTimeKind]::Utc) + $reqS = [datetime]::SpecifyKind([datetime]::Parse([string]$awp.requestedStartUtc, $ic, $rk), [System.DateTimeKind]::Utc) + $reqE = [datetime]::SpecifyKind([datetime]::Parse([string]$awp.requestedEndUtc, $ic, $rk), [System.DateTimeKind]::Utc) + $script:AISIDWindow = Resolve-AISIDDateWindow -RequestedStartUtc $reqS -RequestedEndUtc $reqE -ReferenceNowUtc $script:AISIDReferenceNowUtc -RetentionDays (Get-AISIDEffectiveLookbackDays) + Write-LogHost " AISID: restored the original run's frozen collection window from checkpoint (deterministic replay)." -ForegroundColor DarkGray + } + elseif ($script:TrimStartDateUTC -and $script:TrimEndDateUTC -and $checkpointData.createdUtc) { + $refParsed = $null + try { $refParsed = [datetime]::Parse([string]$checkpointData.createdUtc, $ic, $rk) } catch { $refParsed = $null } + if (-not $refParsed) { + Write-LogHost "ERROR: Cannot resume this AISID run - the checkpoint has no explicit AISID window and its createdUtc is missing or unparseable, so the 30-day retention window cannot be anchored." -ForegroundColor Red + exit 1 + } + $script:AISIDReferenceNowUtc = [datetime]::SpecifyKind($refParsed, [System.DateTimeKind]::Utc) + $script:AISIDWindow = Resolve-AISIDDateWindow -RequestedStartUtc $script:TrimStartDateUTC -RequestedEndUtc $script:TrimEndDateUTC -ReferenceNowUtc $script:AISIDReferenceNowUtc -RetentionDays (Get-AISIDEffectiveLookbackDays) + Write-LogHost " AISID: legacy checkpoint - derived the reference instant from createdUtc for a deterministic window replay." -ForegroundColor DarkGray + } + else { + Write-LogHost "ERROR: Cannot resume this AISID run - the checkpoint lacks an explicit AISID window and does not carry two concrete dates plus a parseable createdUtc to reconstruct one." -ForegroundColor Red + exit 1 + } + } + + $script:ResumeValidationFailed = $false # Re-run MaxMemoryMB resolution after checkpoint restore (may have restored -1 for auto-detect) $script:ResolvedMaxMemoryMB = $MaxMemoryMB if ($MaxMemoryMB -eq -1) { @@ -24008,6 +37115,29 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit Write-LogHost "" } + # ==================================================================== + # Defect 2: pre-auth legacy contract-mapping resolver (resume, Graph mode). + # On resume, migrate any legacy (fingerprint-less) checkpoint entries to + # proven-unique durable fingerprints BEFORE authentication and the per-workload + # partition build. Fails closed (no auth, no mutation) on any zero/ambiguous mapping. + # ==================================================================== + if ($script:IsResumeMode -and $script:CheckpointData -and (-not $UseEOM)) { + try { + $__legacyStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + $__legacyEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + $__legacyContracts = Get-PaxResumeCurrentContractSet -StartDateObj $__legacyStart -EndDateObj $__legacyEnd -ActivityTypes $ActivityTypes -RecordTypes $RecordTypes -ServiceTypes $ServiceTypes -PartitionHours $PartitionHours -MaxPartitions $MaxPartitions -MaxConcurrency $MaxConcurrency -IncludeM365Usage:$IncludeM365Usage -UseEOM:$UseEOM -ServiceOperationMap $serviceOperationMap -RecordTypeWorkloadMap $recordTypeWorkloadMap + $__legacyResult = Invoke-PaxResumeLegacyContractMapping -CheckpointData $script:CheckpointData -CurrentContracts $__legacyContracts + if ($__legacyResult -and $__legacyResult.Status -eq 'Migrated') { + Write-LogHost ("Resume: migrated {0} legacy checkpoint entr{1} to durable per-contract fingerprints (published)." -f $__legacyResult.Migrated, $(if ($__legacyResult.Migrated -eq 1) { 'y' } else { 'ies' })) -ForegroundColor DarkCyan + } + } catch { + Write-LogHost "" -ForegroundColor Red + Write-LogHost ("RESUME BLOCKED - legacy checkpoint could not be mapped safely: {0}" -f $_.Exception.Message) -ForegroundColor Red + Write-LogHost " No authentication and no checkpoint mutation have occurred. Resolve the above and retry." -ForegroundColor Yellow + exit 1 + } + } + # Use unified authentication function Connect-PurviewAudit -AuthMethod $Auth -UseEOMMode $UseEOM @@ -24057,6 +37187,11 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit try { $appendName = [System.IO.Path]::GetFileName($AppendFile) $appendLocalPath = Join-Path $OutputPath $appendName + # Advisory: probe the remote append target's size (metadata only, no body) and + # warn if the in-memory merge could strain RAM. Fail-open - a probe error or a + # not-yet-existing (first-run) target yields no warning and does not abort. + $appendSizeWarning = Get-AppendTargetSizeWarning -SizeBytes (Get-RemoteAppendTargetSize -Tier $script:RemoteOutputMode -RelativeName $appendName) -ThresholdBytes $script:AppendTargetSizeWarnBytes -Label $appendName + if ($appendSizeWarning) { Write-LogHost $appendSizeWarning -ForegroundColor Yellow } if ($script:RemoteOutputMode -eq 'SharePoint') { Get-RemoteFile-SharePoint -RelativeName $appendName -DestinationPath $appendLocalPath -ErrorAction Stop } else { @@ -24110,36 +37245,29 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } # PAX4A-INGEST-BEGIN - # -UserInfoFile INGEST + VALIDATE + NORMALIZE (Phase 4a; UNWIRED) + # -UserInfoFile INGEST + VALIDATE + NORMALIZE # Classify the -UserInfoFile VALUE ITSELF (independent of the output tier / # $script:RemoteOutputMode). Local: read in place. SharePoint / Fabric: split the # input URL into its OWN parent + leaf and stage to local scratch via the matching # Get-RemoteFile-* with -ParentOverride = the input's own parent. Parse as CSV, # hard-stop on any fetch/format/required-field failure (NO fallback to the Entra # pull), then normalize to an Array[psobject] matching the Entra users contract. - # Result -> the NEW variable $script:UserInfoFileData; intentionally NOT wired into + # Result -> the variable $script:UserInfoFileData; not wired into # any consumer in this phase. $script:UserInfoFileData = $null if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile)) { $uiRaw = $UserInfoFile.Trim() - $uiTier = script:Get-PathTier -Value $uiRaw -SwitchName 'UserInfoFile' + $uiTier = $null $uiLocalPath = $null try { - if ($uiTier -eq 'Local') { - $uiLocalPath = $uiRaw - } - else { - $uiLeaf = ($uiRaw -split '[\\/]' | Where-Object { $_ }) | Select-Object -Last 1 - $uiParent = $uiRaw.Substring(0, $uiRaw.Length - $uiLeaf.Length).TrimEnd('/', '\') - $uiLocalPath = Join-Path $OutputPath ('.pax_userinfoseed_{0}' -f $uiLeaf) - if ($uiTier -eq 'SharePoint') { - Get-RemoteFile-SharePoint -RelativeName $uiLeaf -DestinationPath $uiLocalPath -ParentOverride $uiParent -ErrorAction Stop - } - else { - Get-RemoteFile-OneLake -RelativeName $uiLeaf -DestinationPath $uiLocalPath -ParentOverride $uiParent -ErrorAction Stop - } - Write-LogHost ("-UserInfoFile ({0}) staged to scratch: {1}" -f $uiTier, $uiLocalPath) -ForegroundColor Cyan - } + # Shared hardened classifier + stager (same strict .csv file-form validation as + # -UserInfoSupplement): rejects folder URLs, Fabric roots, and Fabric Tables URLs; a + # Fabric input must live under /Files/.../*.csv. Remote inputs stage to run scratch and + # are never registered for upload. + $uiPlan = script:Resolve-DirectoryCsvInput -Value $UserInfoFile -SwitchName 'UserInfoFile' -ScratchDir $OutputPath -ScratchPrefix '.pax_userinfoseed_' + $uiTier = $uiPlan.Tier + $uiLocalPath = $uiPlan.LocalPath + if ($uiPlan.Staged) { Write-LogHost ("-UserInfoFile ({0}) staged to scratch: {1}" -f $uiTier, $uiLocalPath) -ForegroundColor Cyan } } catch { Write-LogHost ("ERROR: -UserInfoFile could not be fetched from '{0}': {1}" -f $uiRaw, $_.Exception.Message) -ForegroundColor Red @@ -24174,7 +37302,7 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } $uiHeaderMap = @{} foreach ($h in $uiHeaders) { if ($h) { $uiHeaderMap[$h.Trim().ToLowerInvariant()] = $h } } - # Alias backfill (Phase 4b): first-match-wins resolution of documented header aliases over + # Alias backfill: first-match-wins resolution of documented header aliases over # the lower-cased $uiHeaderMap (case-insensitive; source headers are never renamed). $uiResolveCol = { param([string[]]$Aliases) foreach ($a in $Aliases) { if ($uiHeaderMap.ContainsKey($a)) { return $uiHeaderMap[$a] } } return $null } $uiUpnCol = & $uiResolveCol @('userprincipalname','upn','personid') @@ -24201,7 +37329,7 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit $uiNormalized = [System.Collections.Generic.List[object]]::new() foreach ($row in $uiRows) { $o = [ordered]@{} - # Blank-fill (Phase 4b): seed EVERY authoritative Entra column (in $EntraUsersHeader + # Blank-fill: seed EVERY authoritative Entra column (in $EntraUsersHeader # order) with '' so file rows are shape-complete vs the Entra users contract; an absent # Entra value renders as '' exactly like a real Graph /users pull. foreach ($col in $EntraUsersHeader) { $o[$col] = '' } @@ -24238,16 +37366,16 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit Write-LogHost "Fetching Entra user directory and license data..." -ForegroundColor Cyan $uiFileMode = ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile)) # Bulk license fetch runs for the live-directory path exactly as before. For the -UserInfoFile - # path it is deferred and only invoked on demand (Phase 4d license hybrid, below), so a fully + # path it is deferred and only invoked on demand, so a fully # file-supplied directory stays offline. if (-not $uiFileMode) { $script:LicenseData = Get-UserLicenseData } if ($uiFileMode) { - # Phase 4b: -UserInfoFile supplies the Entra user directory from a customer-provided CSV. + # -UserInfoFile supplies the Entra user directory from a customer-provided CSV. # Skip the Graph /users directory pull entirely and use the ingested, schema-completed rows. Write-LogHost "-UserInfoFile provided: using supplied user directory (skipping Graph /users pull)." -ForegroundColor Cyan $script:EntraUsersData = $script:UserInfoFileData # PAX4D-LICENSE-HYBRID-BEGIN - # License hybrid (Phase 4d): a file-provided HasLicense value WINS and is used as-is (no online + # License hybrid: a file-provided HasLicense value WINS and is used as-is (no online # check). Rows with a BLANK HasLicense are resolved online by UPN from the existing bulk # Get-UserLicenseData. When NO row is blank the online lookup is skipped entirely (fully offline # directory build). File-provided values are never overwritten by online data. @@ -24281,6 +37409,66 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } else { $script:EntraUsersData = Get-EntraUsersData + # PAX-UIS-MERGE-BEGIN + # Hybrid directory enrichment (-UserInfoSupplement): the live Entra directory has just been + # fetched above. Append the columns of a customer-provided supplemental CSV to each matching + # Entra row via an Entra-LEFT join on UserPrincipalName. Entra stays authoritative: every Entra + # user is preserved, supplemental-only rows are reported and excluded, and the supplemental UPN + # is a join key only (never copied into output). Runs before scope consumers, CSV writing, + # Rollup, AISID projection, and append reconciliation. + if ($script:UserInfoSupplementActive) { + $uisNoticeLines = [System.Collections.Generic.List[string]]::new() + if ($script:PaxDeidEnabled -or $Deidentify) { + $uisNoticeLines.Add(" -UserInfoSupplement + -Deidentify: the supplemental UserPrincipalName is used ONLY to match rows; it is never copied into the output. Output identity always comes from Entra.") + $uisNoticeLines.Add(" PAX de-identifies Entra-derived identity fields as usual, but passes EVERY other supplemental column and value through UNCHANGED - PAX does not inspect, mask, hash, or de-identify supplemental values.") + $uisNoticeLines.Add(" You are responsible for ensuring the supplemental values are appropriate to export.") + } + if ($Dashboard -eq 'AISID') { + $uisNoticeLines.Add(" -UserInfoSupplement + -Dashboard AISID: supplemental columns remain in the normal EntraUsers export and in the AIO/ValueLens Rollup Users dimension where applicable, but the AISID EntraUsers.csv keeps its fixed 14-column dashboard schema and does NOT receive supplemental columns.") + } + if ($uisNoticeLines.Count -gt 0) { + Write-LogHost "" + Write-LogHost "======================================================================" -ForegroundColor Yellow + Write-LogHost " HYBRID DIRECTORY SUPPLEMENT NOTICE" -ForegroundColor Yellow + foreach ($ln in $uisNoticeLines) { Write-LogHost $ln -ForegroundColor Yellow } + Write-LogHost "======================================================================" -ForegroundColor Yellow + Write-LogHost "" + } + $uisPlan = $null + try { + $uisPlan = script:Resolve-DirectoryCsvInput -Value $UserInfoSupplement -SwitchName 'UserInfoSupplement' -ScratchDir $OutputPath -ScratchPrefix '.pax_userinfosupp_' + } + catch { + Write-LogHost ("ERROR: -UserInfoSupplement could not be fetched from '{0}': {1}" -f $UserInfoSupplement.Trim(), $_.Exception.Message) -ForegroundColor Red + throw + } + if (-not (Test-Path -LiteralPath $uisPlan.LocalPath -PathType Leaf)) { + Write-LogHost ("ERROR: -UserInfoSupplement not found or unreadable: {0}" -f $UserInfoSupplement.Trim()) -ForegroundColor Red + throw "-UserInfoSupplement not found or unreadable: $($UserInfoSupplement.Trim())" + } + # Parse + validate (throws a clear message on any violation). A staged remote input is + # intentionally PRESERVED on a failure for diagnosis; it is removed only after success. + $uisParsed = script:Import-PaxSupplementCsv -Path $uisPlan.LocalPath + $uisMerge = script:Merge-PaxEntraSupplement -EntraRows @($script:EntraUsersData) -PassthroughColumns $uisParsed.PassthroughColumns -Lookup $uisParsed.Lookup -UpnColumn $uisParsed.UpnColumn + $script:EntraUsersData = $uisMerge.Rows + Write-LogHost ("-UserInfoSupplement ({0}): {1} supplemental row(s) read; {2} column(s) appended [{3}]." -f $uisPlan.Tier, @($uisParsed.Rows).Count, @($uisMerge.ColumnsAdded).Count, (@($uisMerge.ColumnsAdded) -join ', ')) -ForegroundColor Green + Write-LogHost (" Entra users examined: {0}; matched supplemental rows: {1}; Entra users without supplemental values: {2}." -f @($script:EntraUsersData).Count, $uisMerge.Matched, $uisMerge.EntraWithout) -ForegroundColor DarkCyan + if (@($uisMerge.UnmatchedSupplemental).Count -gt 0) { + Write-LogHost (" {0} supplemental row(s) did not match any Entra user and were EXCLUDED (Entra remains authoritative; no synthetic directory users were created):" -f @($uisMerge.UnmatchedSupplemental).Count) -ForegroundColor Yellow + foreach ($uUpn in @($uisMerge.UnmatchedSupplemental)) { Write-LogHost (" excluded (unmatched supplemental UPN): {0}" -f $uUpn) -ForegroundColor Yellow } + } + try { + $script:metrics.SupplementTier = $uisPlan.Tier + $script:metrics.SupplementRowsRead = @($uisParsed.Rows).Count + $script:metrics.SupplementColumnsAdded = @($uisMerge.ColumnsAdded).Count + $script:metrics.SupplementMatched = $uisMerge.Matched + $script:metrics.SupplementUnmatchedExcluded = @($uisMerge.UnmatchedSupplemental).Count + } catch {} + if ($uisPlan.Staged -and (Test-Path -LiteralPath $uisPlan.LocalPath -PathType Leaf)) { + try { Remove-Item -LiteralPath $uisPlan.LocalPath -Force -ErrorAction SilentlyContinue } catch {} + } + } + # PAX-UIS-MERGE-END } } elseif ($IncludeUserInfo -and $UseEOM) { @@ -24290,37 +37478,67 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } } + # When -OnlyUserInfo is combined with a user/group scope, narrow the exported Entra + # directory to that scope. The scope resolves as one fail-closed union; a requested + # scope that cannot be resolved stops the run so the export never silently widens to + # the whole tenant. Zero matched rows is a truthful empty scoped export, never all users. + if ($OnlyUserInfo -and ($UserIds -or $GroupNames)) { + $dirRosterValidator = $null + if ($script:EntraUsersData -and @($script:EntraUsersData).Count -gt 0) { + $dirRosterUpns = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($ru in @($script:EntraUsersData)) { $rupn = [string]$ru.userPrincipalName; if ($rupn) { [void]$dirRosterUpns.Add($rupn.Trim()) } } + $dirRosterValidator = { param($u) $dirRosterUpns.Contains(([string]$u).Trim()) }.GetNewClosure() + } + $dirScope = Resolve-PaxUserScope -UserIds $UserIds -GroupNames $GroupNames -UseEOMMode $UseEOM -RosterValidator $dirRosterValidator + $script:metrics.FilteringUserIds = @($dirScope.RequestedUserIds).Count + $script:metrics.FilteringGroupNames = @($dirScope.RequestedGroups).Count + $script:metrics.ScopeResolvedGroups = @($dirScope.ResolvedGroups).Count + $script:metrics.ScopeFailedGroups = @($dirScope.FailedGroups).Count + @($dirScope.AmbiguousGroups).Count + @($dirScope.ZeroMemberGroups).Count + @($dirScope.UnauthorizedGroups).Count + @($dirScope.TransportErrorGroups).Count + @($dirScope.ResolutionErrorGroups).Count + $script:metrics.ScopeExpandedMembers = @($dirScope.ResolvedTransitiveMembers).Count + $script:metrics.ScopeFinalTargetUsers = @($dirScope.FinalTargetUsers).Count + Write-LogHost ""; Write-LogHost "User/Group Scope Resolution (directory export):" -ForegroundColor Cyan + Write-LogHost (" Requested UserIds: {0}" -f @($dirScope.RequestedUserIds).Count) -ForegroundColor DarkCyan + Write-LogHost (" Requested GroupNames: {0}" -f @($dirScope.RequestedGroups).Count) -ForegroundColor DarkCyan + if (@($dirScope.UnmatchedExplicitUserIds).Count -gt 0) { + Write-LogHost (" UserIds not found in directory: {0}" -f (@($dirScope.UnmatchedExplicitUserIds) -join ', ')) -ForegroundColor Yellow + } + foreach ($rg in @($dirScope.ResolvedGroups)) { + Write-LogHost (" Group '{0}' -> {1} direct / {2} transitive user(s) [{3}]" -f $rg.DisplayName, $rg.DirectMembers, $rg.TransitiveMembers, $rg.MembershipScope) -ForegroundColor DarkCyan + } + if ($dirScope.Outcome -ne 'Succeeded') { + Write-LogHost "" -ForegroundColor Red + Write-LogHost "ERROR: User/group scope could not be fully resolved; refusing to export an unscoped directory." -ForegroundColor Red + if (@($dirScope.FailedGroups).Count -gt 0) { Write-LogHost (" Groups not found: {0}" -f (@($dirScope.FailedGroups) -join ', ')) -ForegroundColor Yellow } + if (@($dirScope.AmbiguousGroups).Count -gt 0) { Write-LogHost (" Groups matching more than one directory object: {0}" -f (@($dirScope.AmbiguousGroups) -join ', ')) -ForegroundColor Yellow } + if (@($dirScope.ZeroMemberGroups).Count -gt 0) { Write-LogHost (" Groups with no user members: {0}" -f (@($dirScope.ZeroMemberGroups) -join ', ')) -ForegroundColor Yellow } + if (@($dirScope.UnauthorizedGroups).Count -gt 0) { Write-LogHost (" Groups not accessible (authorization denied): {0}" -f (@($dirScope.UnauthorizedGroups) -join ', ')) -ForegroundColor Yellow } + if (@($dirScope.TransportErrorGroups).Count -gt 0) { Write-LogHost (" Groups not resolved (transient/transport error): {0}" -f (@($dirScope.TransportErrorGroups) -join ', ')) -ForegroundColor Yellow } + if (@($dirScope.ResolutionErrorGroups).Count -gt 0) { Write-LogHost (" Groups not resolved (unexpected error): {0}" -f (@($dirScope.ResolutionErrorGroups) -join ', ')) -ForegroundColor Yellow } + Write-LogHost (" Failure stage: {0}" -f $dirScope.FailureStage) -ForegroundColor Yellow + Write-LogHost "" -ForegroundColor Red + exit 1 + } + $dirScopeSet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($fu in @($dirScope.FinalTargetUsers)) { if ($fu) { [void]$dirScopeSet.Add(([string]$fu).Trim()) } } + $dirBefore = @($script:EntraUsersData).Count + $script:EntraUsersData = @(@($script:EntraUsersData) | Where-Object { $_ -and $dirScopeSet.Contains(([string]$_.userPrincipalName).Trim()) }) + $dirAfter = @($script:EntraUsersData).Count + $script:metrics.DirectoryRowsBeforeScope = $dirBefore + $script:metrics.DirectoryRowsAfterScope = $dirAfter + $script:metrics.DirectoryRowsExcludedByScope = $dirBefore - $dirAfter + Write-LogHost (" Directory rows scoped: {0} -> {1} ({2} excluded)" -f $dirBefore, $dirAfter, ($dirBefore - $dirAfter)) -ForegroundColor Green; Write-LogHost "" + } + # Skip all audit log queries when only exporting user data or only exporting Agent 365 data if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { $allLogs = New-Object System.Collections.ArrayList if ($RAWInputCSV) { Write-LogHost "Replay mode enabled: ingesting raw Purview CSV '$RAWInputCSV' (no Graph/EOM connections)" -ForegroundColor Yellow if (-not (Test-Path $RAWInputCSV)) { Write-LogHost "Replay file not found: $RAWInputCSV" -ForegroundColor Red; exit 1 } - $csvData = Import-Csv -Path $RAWInputCSV - Write-LogHost ("Replay rows ingested: {0}" -f $csvData.Count) -ForegroundColor DarkGray - try { - $sampleRow = $csvData | Select-Object -First 1 - $identity = if ($sampleRow.Id) { $sampleRow.Id } elseif ($sampleRow.RecordId) { $sampleRow.RecordId } else { [guid]::NewGuid().ToString() } - $rec = [pscustomobject]@{ - RecordType = $(try { [int]$sampleRow.RecordType } catch { 0 }) - CreationDate = $(if ($sampleRow.CreationDate) { $d = script:Parse-DateSafe $sampleRow.CreationDate; if ($d) { $d } else { Get-Date } } else { Get-Date }) - UserIds = @(if ($sampleRow.UserIds) { $sampleRow.UserIds } elseif ($sampleRow.UserId) { $sampleRow.UserId } else { $null }) - Operations = if ($sampleRow.Operations) { $sampleRow.Operations } elseif ($sampleRow.Operation) { $sampleRow.Operation } else { $null } - ResultStatus = $(try { $sampleRow.ResultStatus } catch { '' }) - ResultCount = 0 - Identity = $identity - IsValid = $true - ObjectState = '' - AuditData = $sampleRow.AuditData - Operation = if ($sampleRow.Operation) { $sampleRow.Operation } elseif ($sampleRow.Operations) { $sampleRow.Operations } else { $null } - UserId = if ($sampleRow.UserId) { $sampleRow.UserId } elseif ($sampleRow.UserIds) { $sampleRow.UserIds } else { $null } - } - $sampleOut = Convert-ToPurviewExplodedRecords -Record $rec -SkipMetrics - # sample row count preview removed (verbosity reduction) - # sample columns preview removed (verbosity reduction) - } catch { - Write-LogHost ("Replay sample conversion failed: {0}" -f $_.Exception.Message) -ForegroundColor DarkYellow - } + # Stream the replay CSV (RFC-4180) instead of loading it whole. Filters are + # applied per row as it is read; the raw-input array is never materialized and + # the previously-dead $filteredRows collection is removed. The inline exporter + # below consumes $allLogs one record at a time. $applyDateFilter = ($PSBoundParameters.ContainsKey('StartDate') -or $PSBoundParameters.ContainsKey('EndDate')) $applyActivityFilter = ($PSBoundParameters.ContainsKey('ActivityTypes') -and $ActivityTypes -and $ActivityTypes.Count -gt 0) $startFilter = $null; $endFilter = $null @@ -24330,47 +37548,83 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } $activitySet = $null if ($applyActivityFilter) { $activitySet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase); foreach ($a in $ActivityTypes) { if ($a) { [void]$activitySet.Add($a) } } } - $filteredRows = New-Object System.Collections.Generic.List[object] - foreach ($row in $csvData) { - $keep = $true - $creationRaw = $row.CreationDate - $creation = if ($creationRaw) { script:Parse-DateSafe $creationRaw } else { $null } - if ($applyDateFilter -and $creation) { - if ($startFilter -and $creation -lt $startFilter) { $keep = $false } - if ($endFilter -and $creation -ge $endFilter) { $keep = $false } - } - if ($keep -and $applyActivityFilter) { - $op = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } - if (-not $op -or -not $activitySet.Contains([string]$op)) { $keep = $false } - } - if (-not $keep) { continue } - $auditData = $row.AuditData - $identity = if ($row.Id) { $row.Id } elseif ($row.RecordId) { $row.RecordId } else { [guid]::NewGuid().ToString() } - $rec = [pscustomobject]@{ - RecordType = $(try { [int]$row.RecordType } catch { 0 }) - CreationDate = $(if ($creation) { $creation } else { Get-Date }) - UserIds = @(if ($row.UserIds) { $row.UserIds } elseif ($row.UserId) { $row.UserId } else { $null }) - Operations = if ($row.Operations) { $row.Operations } elseif ($row.Operation) { $row.Operation } else { $null } - ResultStatus = $(try { $row.ResultStatus } catch { '' }) - ResultCount = 0 - Identity = $identity - IsValid = $true - ObjectState = '' - AuditData = $auditData - Operation = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } - UserId = if ($row.UserId) { $row.UserId } elseif ($row.UserIds) { $row.UserIds } else { $null } - } - [void]$filteredRows.Add($row) - [void]$allLogs.Add($rec) - } - $ingested = $allLogs.Count + # Accepted replay records are spilled to a re-iterable disk spool (JSON-lines) + # instead of an in-memory ArrayList, so the complete replay set is never held in + # memory. The inline exporter streams the spool; the fallback pass reopens it. The + # spool is deleted on success and preserved on a failed export for recovery. + $replaySpoolPath = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_replay_spool_${global:ScriptRunTimestamp}_" + [guid]::NewGuid().ToString('N') + ".jsonl") + $replayReader = [System.IO.StreamReader]::new((Resolve-Path -LiteralPath $RAWInputCSV).Path, $true) + $replaySpoolWriter = [System.IO.StreamWriter]::new($replaySpoolPath, $false, ([System.Text.UTF8Encoding]::new($false))) + $replayTotalRead = 0 + $replayAccepted = 0 + $replaySampleDone = $false + try { + $replayHeader = script:Read-PaxCsvRecord -Reader $replayReader + if ($null -ne $replayHeader) { + $replayCols = @($replayHeader | ForEach-Object { [string]$_ }) + $replayColCount = $replayCols.Length + while ($true) { + $rf = script:Read-PaxCsvRecord -Reader $replayReader + if ($null -eq $rf) { break } + $replayTotalRead++ + $rowMap = [ordered]@{} + for ($ci = 0; $ci -lt $replayColCount; $ci++) { $rowMap[$replayCols[$ci]] = if ($ci -lt $rf.Length) { [string]$rf[$ci] } else { '' } } + $row = [pscustomobject]$rowMap + if (-not $replaySampleDone) { + $replaySampleDone = $true + try { + $identity = if ($row.Id) { $row.Id } elseif ($row.RecordId) { $row.RecordId } else { [guid]::NewGuid().ToString() } + $srec = [pscustomobject]@{ + RecordType = $(try { [int]$row.RecordType } catch { 0 }) + CreationDate = $(if ($row.CreationDate) { $d = script:Parse-DateSafe $row.CreationDate; if ($d) { $d } else { Get-Date } } else { Get-Date }) + UserIds = @(if ($row.UserIds) { $row.UserIds } elseif ($row.UserId) { $row.UserId } else { $null }) + Operations = if ($row.Operations) { $row.Operations } elseif ($row.Operation) { $row.Operation } else { $null } + ResultStatus = $(try { $row.ResultStatus } catch { '' }) + ResultCount = 0 + Identity = $identity + IsValid = $true + ObjectState = '' + AuditData = $row.AuditData + Operation = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } + UserId = if ($row.UserId) { $row.UserId } elseif ($row.UserIds) { $row.UserIds } else { $null } + } + $sampleOut = Convert-ToPurviewExplodedRecords -Record $srec -SkipMetrics + } catch { + Write-LogHost ("Replay sample conversion failed: {0}" -f $_.Exception.Message) -ForegroundColor DarkYellow + } + } + $keep = $true + $creationRaw = $row.CreationDate + $creation = if ($creationRaw) { script:Parse-DateSafe $creationRaw } else { $null } + if ($applyDateFilter -and $creation) { + if ($startFilter -and $creation -lt $startFilter) { $keep = $false } + if ($endFilter -and $creation -ge $endFilter) { $keep = $false } + } + if ($keep -and $applyActivityFilter) { + $op = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } + if (-not $op -or -not $activitySet.Contains([string]$op)) { $keep = $false } + } + if (-not $keep) { continue } + # Spill the accepted raw row to the disk spool; the normalized replay + # record is rebuilt from it (identically) at export time. + $replaySpoolWriter.WriteLine(($row | ConvertTo-Json -Compress -Depth 5)) + $replayAccepted++ + if (($replayAccepted % 5000) -eq 0) { try { Write-PaxMemoryObservation -Stage 'ReplaySpool' -RowsProcessed ([long]$replayAccepted) } catch {} } + } + } + } + finally { try { $replaySpoolWriter.Flush(); $replaySpoolWriter.Dispose() } catch {}; $replayReader.Dispose() } + Write-LogHost ("Replay rows ingested: {0}" -f $replayTotalRead) -ForegroundColor DarkGray + $ingested = $replayAccepted Write-LogHost ("Replay rows after filters: {0}" -f $ingested) -ForegroundColor DarkGray - # Inline export for replay: execute immediately, bypass downstream pipeline + # Inline export for replay: stream from the re-iterable spool (no in-memory set). try { - Invoke-ReplayInlineExport -Logs $allLogs + Invoke-ReplayInlineExport -SpoolPath $replaySpoolPath + try { if (Test-Path -LiteralPath $replaySpoolPath) { Remove-Item -LiteralPath $replaySpoolPath -Force -ErrorAction SilentlyContinue } } catch {} return } catch { + Write-LogHost ("Replay inline export failed: {0}. Spool preserved for recovery at: {1}" -f $_.Exception.Message, $replaySpoolPath) -ForegroundColor Yellow } $queryPlan = @(); $sequentialGroups = 0; $parallelDecision = @{ Enabled = $false; Reason = 'Replay'; AutoEligible = $false }; $parallelOverallEnabled = $false $script:metrics.TotalRecordsFetched = $ingested @@ -24384,20 +37638,92 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit $script:targetUsers = @() if ($UserIds -or $GroupNames) { - Write-LogHost ""; Write-LogHost "User/Group Filtering Enabled:" -ForegroundColor Cyan - if ($UserIds) { $script:targetUsers += $UserIds; Write-LogHost " Individual users: $($UserIds.Count)" -ForegroundColor DarkCyan } - if ($GroupNames) { - Write-LogHost " Expanding groups to individual users..." -ForegroundColor DarkCyan - foreach ($group in $GroupNames) { - $members = Expand-GroupToUsers -GroupIdentity $group -UseEOMMode $UseEOM - if ($members.Count -gt 0) { - $script:targetUsers += $members + # Resolve the effective user scope as a single fail-closed union of explicit + # UserIds and group members. A requested scope that cannot be fully resolved + # stops the run so a query never widens to the whole tenant. + $rosterValidator = $null + if ($script:EntraUsersData -and @($script:EntraUsersData).Count -gt 0) { + $rosterUpns = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($ru in @($script:EntraUsersData)) { $rupn = [string]$ru.userPrincipalName; if ($rupn) { [void]$rosterUpns.Add($rupn.Trim()) } } + $rosterValidator = { param($u) $rosterUpns.Contains(([string]$u).Trim()) }.GetNewClosure() + } + $resolvedScope = Resolve-PaxUserScope -UserIds $UserIds -GroupNames $GroupNames -UseEOMMode $UseEOM -RosterValidator $rosterValidator + $script:metrics.FilteringUserIds = @($resolvedScope.RequestedUserIds).Count + $script:metrics.FilteringGroupNames = @($resolvedScope.RequestedGroups).Count + $script:metrics.ScopeResolvedGroups = @($resolvedScope.ResolvedGroups).Count + $script:metrics.ScopeFailedGroups = @($resolvedScope.FailedGroups).Count + @($resolvedScope.AmbiguousGroups).Count + @($resolvedScope.ZeroMemberGroups).Count + @($resolvedScope.UnauthorizedGroups).Count + @($resolvedScope.TransportErrorGroups).Count + @($resolvedScope.ResolutionErrorGroups).Count + $script:metrics.ScopeExpandedMembers = @($resolvedScope.ResolvedTransitiveMembers).Count + $script:metrics.ScopeFinalTargetUsers = @($resolvedScope.FinalTargetUsers).Count + Write-LogHost ""; Write-LogHost "User/Group Scope Resolution:" -ForegroundColor Cyan + Write-LogHost (" Requested UserIds: {0}" -f @($resolvedScope.RequestedUserIds).Count) -ForegroundColor DarkCyan + Write-LogHost (" Requested GroupNames: {0}" -f @($resolvedScope.RequestedGroups).Count) -ForegroundColor DarkCyan + if (@($resolvedScope.UnmatchedExplicitUserIds).Count -gt 0) { + if ($Dashboard -eq 'AISID') { + # F3 fail-closed: under -Dashboard AISID the EntraUsers dimension is REQUIRED, so every + # explicitly requested target UPN MUST resolve in the authenticated Entra directory. An + # unmatched explicit target (e.g. the wrong tenant/account was signed in) makes the scoped + # dashboard invalid, so the run stops BEFORE any audit query or publication — no Purview, + # Rollup, or AISID customer data is produced. Directory membership is checked against the real + # roster (never inferred from the UPN suffix, no tenant/domain assumed). The established AISID + # gap flag drives the finally-block exit precedence to a nonzero result (exit 40); a bare + # 'exit 1' alone would be overwritten by that recompute, so the flag is what makes it truthful. + $script:AISIDHadGaps = $true + Write-LogHost "" -ForegroundColor Red + Write-LogHost ("ERROR: -Dashboard AISID requires every requested user to exist in the authenticated Entra directory; not found: {0}" -f (@($resolvedScope.UnmatchedExplicitUserIds) -join ', ')) -ForegroundColor Red + Write-LogHost " The AISID EntraUsers dimension cannot be scoped to a user who is absent from the signed-in tenant's directory." -ForegroundColor Yellow + Write-LogHost " Sign in to the tenant that contains the requested user, or request a user that exists in the current directory, then retry." -ForegroundColor Cyan + Write-LogHost " No audit query was run and no output was produced." -ForegroundColor Yellow + Write-LogHost "" -ForegroundColor Red + exit 1 + } + else { + Write-LogHost (" UserIds not found in directory (still included in scope): {0}" -f (@($resolvedScope.UnmatchedExplicitUserIds) -join ', ')) -ForegroundColor Yellow } } + foreach ($rg in @($resolvedScope.ResolvedGroups)) { + Write-LogHost (" Group '{0}' -> {1} direct / {2} transitive user(s) [{3}]" -f $rg.DisplayName, $rg.DirectMembers, $rg.TransitiveMembers, $rg.MembershipScope) -ForegroundColor DarkCyan + } + if ($resolvedScope.Outcome -ne 'Succeeded') { + Write-LogHost "" -ForegroundColor Red + Write-LogHost "ERROR: User/group scope could not be fully resolved; refusing to run an unfiltered audit query." -ForegroundColor Red + if (@($resolvedScope.FailedGroups).Count -gt 0) { Write-LogHost (" Groups not found: {0}" -f (@($resolvedScope.FailedGroups) -join ', ')) -ForegroundColor Yellow } + if (@($resolvedScope.AmbiguousGroups).Count -gt 0) { Write-LogHost (" Groups matching more than one directory object: {0}" -f (@($resolvedScope.AmbiguousGroups) -join ', ')) -ForegroundColor Yellow } + if (@($resolvedScope.ZeroMemberGroups).Count -gt 0) { Write-LogHost (" Groups with no user members: {0}" -f (@($resolvedScope.ZeroMemberGroups) -join ', ')) -ForegroundColor Yellow } + if (@($resolvedScope.UnauthorizedGroups).Count -gt 0) { Write-LogHost (" Groups not accessible (authorization denied): {0}" -f (@($resolvedScope.UnauthorizedGroups) -join ', ')) -ForegroundColor Yellow } + if (@($resolvedScope.TransportErrorGroups).Count -gt 0) { Write-LogHost (" Groups not resolved (transient/transport error): {0}" -f (@($resolvedScope.TransportErrorGroups) -join ', ')) -ForegroundColor Yellow } + if (@($resolvedScope.ResolutionErrorGroups).Count -gt 0) { Write-LogHost (" Groups not resolved (unexpected error): {0}" -f (@($resolvedScope.ResolutionErrorGroups) -join ', ')) -ForegroundColor Yellow } + Write-LogHost (" Failure stage: {0}" -f $resolvedScope.FailureStage) -ForegroundColor Yellow + Write-LogHost " Verify the group name(s) and that the sign-in has GroupMember.Read.All, then retry." -ForegroundColor Cyan + Write-LogHost "" -ForegroundColor Red + exit 1 + } + $script:targetUsers = @($resolvedScope.FinalTargetUsers) + if ($script:targetUsers.Count -eq 0) { + Write-LogHost "" -ForegroundColor Red + Write-LogHost "ERROR: A user/group scope was requested but resolved to zero users; refusing to run an unfiltered audit query." -ForegroundColor Red + Write-LogHost "" -ForegroundColor Red + exit 1 + } + Write-LogHost (" Effective target users (deduplicated union): {0}" -f $script:targetUsers.Count) -ForegroundColor Green; Write-LogHost "" + # Apply that same resolved scope to the directory roster so a scoped run publishes + # only in-scope users to the Entra users CSV, the workbook Entra tab, the Rollup + # Users input, and any append or upload built from them. The scope is resolved once + # per run above and applied once here, so group membership is never expanded twice + # and a requested scope cannot widen: an unresolvable scope or an empty union has + # already stopped the run. -OnlyUserInfo scopes its roster from its own resolution + # and is excluded here. UPN matching is case-insensitive and trimmed. + if ($IncludeUserInfo -and -not $OnlyUserInfo -and $script:EntraUsersData -and @($script:EntraUsersData).Count -gt 0) { + $normScopeSet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($fu in @($resolvedScope.FinalTargetUsers)) { if ($fu) { [void]$normScopeSet.Add(([string]$fu).Trim()) } } + $normBefore = @($script:EntraUsersData).Count + $script:EntraUsersData = @(@($script:EntraUsersData) | Where-Object { $_ -and $normScopeSet.Contains(([string]$_.userPrincipalName).Trim()) }) + $normAfter = @($script:EntraUsersData).Count + $script:metrics.DirectoryRowsBeforeScope = $normBefore + $script:metrics.DirectoryRowsAfterScope = $normAfter + $script:metrics.DirectoryRowsExcludedByScope = $normBefore - $normAfter + Write-LogHost (" Directory rows scoped: {0} -> {1} ({2} excluded)" -f $normBefore, $normAfter, ($normBefore - $normAfter)) -ForegroundColor Green; Write-LogHost "" + } } - $script:targetUsers = $script:targetUsers | Select-Object -Unique - Write-LogHost " Total target users after deduplication: $($script:targetUsers.Count)" -ForegroundColor Green; Write-LogHost "" - } $startDateObj = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) $endDateObj = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) @@ -24459,7 +37785,8 @@ $(if (-not $logFileExisted) { "=== Portable Audit eXporter (PAX) - Purview Audit } } if ($RAWInputCSV) { - Invoke-ReplayInlineExport -Logs $allLogs + Invoke-ReplayInlineExport -SpoolPath $replaySpoolPath + try { if (Test-Path -LiteralPath $replaySpoolPath) { Remove-Item -LiteralPath $replaySpoolPath -Force -ErrorAction SilentlyContinue } } catch {} # Skip the rest of the pipeline; replay handled return } @@ -25282,6 +38609,47 @@ Write-LogHost "" # Output mode display with format-specific defaults # Replace partitions with only those needing work (fetch-only + create-and-fetch) $partitions = @() $partitions += $partitionCategories.ToFetchOnly + # Durable uncertain-create resume reconcile (READ-ONLY, never a POST here). Each + # uncertain partition is resolved by exact fingerprinted name against the live + # collection: exactly one match is adopted + checkpointed and joins fetch-only; + # multiple fail closed; none stays uncertain. Unresolved / fail-closed partitions + # are deliberately NOT added to $partitions, so no worker is launched for an + # unresolved uncertain create (worker-launch gating). + if ($partitionCategories.ToReconcileOnly.Count -gt 0) { + Write-LogHost " [RESUME] $($partitionCategories.ToReconcileOnly.Count) partition(s) have an unproven prior create - reconciling READ-ONLY (no POST)" -ForegroundColor Yellow + $reconcileOutcome = Invoke-PaxResumeUncertainReconcile -Partitions $partitionCategories.ToReconcileOnly -Activities $activities -Log { param($m) Write-Log $m } + if (@($reconcileOutcome.AdoptedFetchOnly).Count -gt 0) { + Write-LogHost " [RESUME] Adopted $(@($reconcileOutcome.AdoptedFetchOnly).Count) prior create(s) by exact name - fetching data only (no POST)" -ForegroundColor Green + $partitions += $reconcileOutcome.AdoptedFetchOnly + } + # Operator-selected recovery: a NOT-VISIBLE (Unresolved) uncertain partition the + # operator explicitly chose with -ClearUncertainCreate has its durable marker + # cleared SELECTIVELY + atomically (single durable write, bucket-exclusive) BEFORE + # any fresh create, then is routed to a fresh create-and-fetch this run. A still- + # visible partition never reaches here (it was adopted above, or fails closed on + # multiple matches), so a visible query is never cleared into a duplicate POST. + $operatorCleared = 0 + foreach ($uncertainPart in @($reconcileOutcome.Unresolved)) { + # Derive this partition's contract fingerprint so an explicit + # -ClearUncertainContract ":" selection can match EXACTLY + # this contract even when another contract shares the index. On any derive + # failure the fingerprint is $null and only an unambiguous index selection applies. + $__ucFp = $null + try { $__ucFp = New-GraphAuditQueryFingerprint -PStart $uncertainPart.PStart -PEnd $uncertainPart.PEnd -Activities $uncertainPart.Activities -RecordTypes $uncertainPart.RecordTypes -ServiceFilter $uncertainPart.ServiceFilter } catch { $__ucFp = $null } + if (Resolve-PaxOperatorClearedUncertain -Index ([int]$uncertainPart.Index) -Fingerprint $__ucFp) { + if ($script:partitionStatus.ContainsKey([int]$uncertainPart.Index)) { $script:partitionStatus[[int]$uncertainPart.Index].UncertainCreate = $false } + $partitions += $uncertainPart + $operatorCleared++ + } + } + if ($operatorCleared -gt 0) { + Write-LogHost " [RESUME] -ClearUncertainCreate: cleared $operatorCleared not-visible partition(s) for a fresh create this run (durable state persisted before any create)." -ForegroundColor Yellow + } + $heldCount = @($reconcileOutcome.Unresolved).Count + @($reconcileOutcome.FailedClosed).Count - $operatorCleared + if ($heldCount -gt 0) { + Write-LogHost " [RESUME] $heldCount partition(s) remain uncertain (no proven create) - NOT launching a worker; re-run with -ClearUncertainCreate to force a fresh create for a specific partition" -ForegroundColor Yellow + } + } $partitions += $partitionCategories.ToCreateAndFetch if ($partitions.Count -eq 0) { @@ -25412,6 +38780,27 @@ Write-LogHost "" # Output mode display with format-specific defaults Status = 'NotStarted' # NotStarted, Sent, Complete, Failed, Subdivided, JobCreated LastError = $null RecordCount = 0 + LastFailureSignature = $null + ConsecutiveIdenticalFailures = 0 + IsTerminalFailure = $false + LastFailureStage = $null + LastFailureReason = $null + # Structured per-query lifecycle state (additive, optional on + # legacy checkpoint load). Mirrors New-GraphAuditLifecycleState so the + # parent entry and its checkpoint representation carry the same fields. + QueryApiVersion = $null + LifecycleOperation = $null + FailureCategory = 'None' + AlternateVersionChecked = $false + AlternateVersionResult = $null + QueryConfirmedAbsent = $false + ReplacementAttempted = $false + ReplacementQueryId = $null + LastLifecycleHttpStatus = $null + LifecycleRetryCount = 0 + ZeroRecordComplete = $false + UncertainCreate = $false + UncertainCreateInfo = $null } # v6 shape-contract guard: catches a future PR that drops a required # partitionStatus field. Cheap (runs once per partition at init). @@ -25431,10 +38820,269 @@ Write-LogHost "" # Output mode display with format-specific defaults # Track which partitions have had jobs created for them (for dynamic subdivision support) $script:partitionsWithJobs = New-Object System.Collections.Generic.HashSet[int] - + + # Run-scoped set of durable-uncertain partitions that a same-process READ-ONLY + # reconcile could NOT resolve (fail-closed on multiple, unresolved on none). Held + # partitions are never re-POSTed and are excluded from the outer-loop pending + # selector so the create loop terminates instead of re-dispatching an unproven + # create forever - mirroring the IsTerminalFailure exclusion. + $script:uncertainReconcileHeld = [System.Collections.Generic.HashSet[int]]::new() + + #region PAX-1E query-poll observability (additive; no timeout / no query-age ceiling) + # --------------------------------------------------------------------------------- + # Shared, parent-owned, thread-safe observation channel. Each active partition + # worker publishes an immutable snapshot into this dictionary after every + # SUCCESSFUL server status poll (and updates error/host-gap fields on failures). + # The parent monitor loops read this dictionary DIRECTLY - they never parse + # repeated display text and never depend on Receive-Job timing to know the last + # successful server poll. Snapshots are replaced atomically (never mutated in + # place) so the parent can never observe a half-updated object. + # NOTE: the worker runs in an isolated ThreadJob runspace and does NOT inherit + # these script-scoped helpers; it mirrors the SAME snapshot field set inline. + # tests/Purview.QueryPollObservability.Tests.ps1 proves (a) this pure builder's + # field logic and (b) that the worker's inline mirror carries an identical field + # set, so the tested logic is the shipped logic. + if (-not $script:QueryObservations) { + $script:QueryObservations = [System.Collections.Concurrent.ConcurrentDictionary[int, object]]::new() + } else { + $script:QueryObservations.Clear() + } + # Cumulative host-suspension time excluded from network-outage accounting across + # all partitions this run (reported by the parent heartbeat only when nonzero). + $script:QueryHostGapSecondsTotal = 0.0 + # Parent monitor-loop timing (UTC of the last status emission) used to detect an + # extended parent-loop / log-delivery delay. + $script:PaxLastStatusEmitUtc = $null + # Parent-owned token-refresh state used by the authoritative heartbeat. Never + # holds token material - only lifecycle state + timing. + if (-not $script:PaxTokenRefreshState) { + $script:PaxTokenRefreshState = [pscustomobject]@{ + State = 'Idle' # Idle | Refreshing | Succeeded | Failed + LastReason = $null + LastAuthMode = $null + StartedUtc = $null + LastSuccessUtc = $null + LastElapsedSeconds = $null + LastError = $null # category/short reason only - never token/credential text + } + } + + # Pure builder for one immutable partition observation snapshot. All time inputs + # are UTC instants. QueryAgeSeconds is $null when the submission instant is not + # genuinely known (reused QueryId with no checkpoint-provided creation time) - + # never fabricated. + function script:New-PaxQueryObservation { + param( + [Parameter(Mandatory)][int]$PartitionIndex, + [int]$AttemptNumber = 1, + [string]$ServerStatus = $null, + [Nullable[datetime]]$QuerySubmittedUtc = $null, + [Nullable[datetime]]$FirstObservedUtc = $null, + [Nullable[datetime]]$LastSuccessfulPollUtc = $null, + [Nullable[datetime]]$LastStatusTransitionUtc = $null, + [int]$PollCount = 0, + [int]$ConsecutivePollErrors = 0, + [string]$LastPollErrorCategory = $null, + [Nullable[datetime]]$TokenRefreshInstantUtc = $null, + [double]$HostGapSecondsExcluded = 0.0, + [Nullable[datetime]]$NowUtc = $null + ) + $now = if ($NowUtc) { [datetime]$NowUtc } else { (Get-Date).ToUniversalTime() } + $queryAge = $null + if ($QuerySubmittedUtc) { $queryAge = [Math]::Round(($now - [datetime]$QuerySubmittedUtc).TotalSeconds, 1) } + $firstObsAge = $null + if ($FirstObservedUtc) { $firstObsAge = [Math]::Round(($now - [datetime]$FirstObservedUtc).TotalSeconds, 1) } + $lastPollAge = $null + if ($LastSuccessfulPollUtc) { $lastPollAge = [Math]::Round(($now - [datetime]$LastSuccessfulPollUtc).TotalSeconds, 1) } + return [pscustomobject]@{ + PartitionIndex = $PartitionIndex + AttemptNumber = $AttemptNumber + ServerStatus = $ServerStatus + QuerySubmittedUtc = $QuerySubmittedUtc + FirstObservedUtc = $FirstObservedUtc + LastSuccessfulPollUtc = $LastSuccessfulPollUtc + LastStatusTransitionUtc = $LastStatusTransitionUtc + PollCount = $PollCount + ConsecutivePollErrors = $ConsecutivePollErrors + LastPollErrorCategory = $LastPollErrorCategory + TokenRefreshInstantUtc = $TokenRefreshInstantUtc + HostGapSecondsExcluded = [Math]::Round($HostGapSecondsExcluded, 1) + QueryAgeSeconds = $queryAge + FirstObservedAgeSeconds = $firstObsAge + LastSuccessfulPollAgeSeconds = $lastPollAge + SnapshotUtc = $now + } + } + + # Map a poll failure to a NON-sensitive category. Never returns or inspects + # response bodies, tokens, URLs, or query ids - category only. + function script:Get-PaxPollErrorCategory { + param([string]$StatusCode = $null, [string]$Message = $null) + $m = ("{0} {1}" -f $StatusCode, $Message) + if ($m -match '429|Too\s*Many\s*Requests|throttl') { return 'Throttled' } + if ($m -match '401|403|Unauthorized|Forbidden|InvalidAuthenticationToken|token') { return 'Authentication' } + if ($m -match '404|Not\s*Found|QUERY-GONE|gone') { return 'Gone' } + if ($m -match '5\d{2}|timed out|timeout|connection|unable to connect|could not be resolved|temporarily unavailable|network|Bad Gateway|Service Unavailable|Gateway Timeout|Internal Server Error') { return 'Network' } + return 'Other' + } + + # Host-gap accounting. Given the sleep we INTENDED and the wall-clock actually + # consumed, return the excess seconds attributable to host suspension - but only + # when that excess is material (>= tolerance). Small scheduler jitter returns 0. + # This is REPORTING/exclusion math only; it never terminates or truncates work. + function script:Measure-PaxHostGap { + param( + [double]$IntendedSeconds, + [Parameter(Mandatory)][datetime]$BeforeUtc, + [Parameter(Mandatory)][datetime]$AfterUtc, + [double]$ToleranceSeconds = 30.0 + ) + $actual = ($AfterUtc - $BeforeUtc).TotalSeconds + $rawExcess = $actual - $IntendedSeconds + if ($rawExcess -ge $ToleranceSeconds) { return [Math]::Round($rawExcess, 1) } + return 0.0 + } + + # Shift an active network-outage start FORWARD by the excluded host-gap seconds so + # host-suspension time is not counted as network downtime. Returns the (possibly + # unchanged) start instant; never erases real time spent in a failing network call. + function script:Get-PaxOutageStartAfterHostGap { + param([Nullable[datetime]]$NetOutageStartUtc, [double]$ExcludedSeconds) + if ($null -eq $NetOutageStartUtc) { return $null } + if ($ExcludedSeconds -le 0) { return [datetime]$NetOutageStartUtc } + return ([datetime]$NetOutageStartUtc).AddSeconds($ExcludedSeconds) + } + + # Measure parent monitor-loop scheduling delay. Returns excess seconds beyond the + # expected cadence + tolerance, else 0. Cause is NOT classified here. + function script:Measure-PaxParentLoopDelay { + param( + [Nullable[datetime]]$PreviousIterationUtc, + [Parameter(Mandatory)][datetime]$CurrentUtc, + [double]$ExpectedCadenceSeconds, + [double]$ToleranceSeconds = 30.0 + ) + if ($null -eq $PreviousIterationUtc) { return 0.0 } + $gap = ($CurrentUtc - [datetime]$PreviousIterationUtc).TotalSeconds + $excess = $gap - ($ExpectedCadenceSeconds + $ToleranceSeconds) + if ($excess -gt 0) { return [Math]::Round($gap, 1) } + return 0.0 + } + + # Token-refresh lifecycle setters (Set-PaxTokenRefreshing / Resolve-PaxTokenRefresh) + # are defined EARLY (just before Refresh-GraphTokenIfNeeded) so that every refresh + # call site - including the ones that run before this parallel section - can record + # truthful state. See that definition for details. + + # Build the authoritative parent heartbeat from partition status + shared + # observations. Aggregate counts only - never per-partition rows, never any + # row/query/token/customer data. Returns a structured object plus a Text line. + function script:Format-PaxParentHeartbeat { + param( + [object[]]$PartitionStatuses = @(), + [object[]]$Observations = @(), + [object]$PageCounts = $null, + [object]$TokenRefreshState = $null, + [Nullable[datetime]]$NowUtc = $null, + [double]$ExcludedHostGapSeconds = 0.0, + [int]$ResumeCompletedOffset = 0 + ) + $now = if ($NowUtc) { [datetime]$NowUtc } else { (Get-Date).ToUniversalTime() } + $active = @($PartitionStatuses | Where-Object { $_ -and $_.Status -ne 'Subdivided' }) + $total = $active.Count + $ResumeCompletedOffset + $completed = @($active | Where-Object { $_.Status -eq 'Complete' }).Count + $ResumeCompletedOffset + $failed = @($active | Where-Object { $_.Status -eq 'Failed' }).Count + $remaining = $active.Count - @($active | Where-Object { $_.Status -eq 'Complete' }).Count - $failed + if ($remaining -lt 0) { $remaining = 0 } + # Server-state counts come from live observations (authoritative last poll). + $serverCounts = [ordered]@{ queued = 0; notStarted = 0; running = 0; other = 0 } + $oldestQueryAge = $null; $oldestFirstObsAge = $null; $stalestPollAge = $null + foreach ($o in @($Observations)) { + if (-not $o) { continue } + switch (("" + $o.ServerStatus)) { + 'queued' { $serverCounts.queued++ } + 'notStarted' { $serverCounts.notStarted++ } + 'running' { $serverCounts.running++ } + default { if ($o.ServerStatus) { $serverCounts.other++ } } + } + if ($null -ne $o.QueryAgeSeconds) { + if ($null -eq $oldestQueryAge -or $o.QueryAgeSeconds -gt $oldestQueryAge) { $oldestQueryAge = $o.QueryAgeSeconds } + } + if ($null -ne $o.FirstObservedUtc) { + $fa = [Math]::Round(($now - [datetime]$o.FirstObservedUtc).TotalSeconds, 1) + if ($null -eq $oldestFirstObsAge -or $fa -gt $oldestFirstObsAge) { $oldestFirstObsAge = $fa } + } + if ($null -ne $o.LastSuccessfulPollUtc) { + $pa = [Math]::Round(($now - [datetime]$o.LastSuccessfulPollUtc).TotalSeconds, 1) + if ($null -eq $stalestPollAge -or $pa -gt $stalestPollAge) { $stalestPollAge = $pa } + } + } + $trState = 'Idle'; $trLastSuccessAge = $null + if ($TokenRefreshState) { + $trState = "" + $TokenRefreshState.State + if ($TokenRefreshState.LastSuccessUtc) { $trLastSuccessAge = [Math]::Round(($now - [datetime]$TokenRefreshState.LastSuccessUtc).TotalSeconds, 1) } + } + $pageProgress = $null + if ($PageCounts -and $PageCounts.Count -gt 0) { + $pageProgress = ($PageCounts.GetEnumerator() | Sort-Object Key | ForEach-Object { "P$($_.Key):$([int]$_.Value)pg" }) -join ' ' + } + $fmtAge = { param($s) if ($null -eq $s) { 'unknown' } elseif ($s -ge 60) { "{0}m" -f [Math]::Round($s/60,1) } else { "{0}s" -f [Math]::Round($s,0) } } + $text = "Total Queries: $total | Completed: $completed | Remaining: $remaining | Failed: $failed | Server: queued=$($serverCounts.queued) notStarted=$($serverCounts.notStarted) running=$($serverCounts.running)" + if ($serverCounts.other -gt 0) { $text += " other=$($serverCounts.other)" } + $ageStr = if ($null -ne $oldestQueryAge) { (& $fmtAge $oldestQueryAge) } else { "first-observed " + (& $fmtAge $oldestFirstObsAge) } + $text += " | Oldest query: $ageStr | Stalest successful poll: $(& $fmtAge $stalestPollAge)" + # Token-refresh heartbeat reports only what is known. Initial authentication is not a + # refresh, so before the first successful refresh the age is reported in words rather + # than as a numeric age. A prior success stays visible after a later failed attempt. + $trStateLower = $trState.ToLower() + if ($null -eq $trLastSuccessAge) { + $trSuffix = if ($trStateLower -eq 'failed') { 'no successful refresh yet' } else { 'no refresh completed yet' } + } else { + $trSuffix = "last success $(& $fmtAge $trLastSuccessAge) ago" + } + $text += " | Token refresh: $trStateLower, $trSuffix" + if ($ExcludedHostGapSeconds -gt 0) { $text += " | Host-gap excluded: $(& $fmtAge $ExcludedHostGapSeconds)" } + return [pscustomobject]@{ + Total = $total; Completed = $completed; Remaining = $remaining; Failed = $failed + ServerQueued = $serverCounts.queued; ServerNotStarted = $serverCounts.notStarted + ServerRunning = $serverCounts.running; ServerOther = $serverCounts.other + OldestQueryAgeSeconds = $oldestQueryAge; OldestFirstObservedAgeSeconds = $oldestFirstObsAge + StalestSuccessfulPollAgeSeconds = $stalestPollAge + TokenRefreshState = $trState; LastTokenSuccessAgeSeconds = $trLastSuccessAge + PageProgress = $pageProgress; ExcludedHostGapSeconds = [Math]::Round($ExcludedHostGapSeconds, 1) + Text = $text + } + } + + # Deliver a heartbeat line and measure terminal/log persistence latency. When the + # delivering call itself takes materially longer than the expected interval, a + # NON-recursive follow-up warning is emitted via -WarnAction with the event and + # delivery instants. Cause is not classified (not called a Purview/host/folder + # failure). -WarnAction must not re-enter this helper. + function script:Invoke-PaxHeartbeatDelivery { + param( + [Parameter(Mandatory)][scriptblock]$DeliverAction, + [double]$ExpectedIntervalSeconds = 60.0, + [double]$ToleranceSeconds = 30.0, + [scriptblock]$WarnAction = $null, + [Nullable[datetime]]$EventUtc = $null, + [scriptblock]$NowProvider = $null + ) + $nowFn = if ($NowProvider) { $NowProvider } else { { (Get-Date).ToUniversalTime() } } + $eventAt = if ($EventUtc) { [datetime]$EventUtc } else { (& $nowFn) } + & $DeliverAction + $deliveredAt = (& $nowFn) + $elapsed = ($deliveredAt - $eventAt).TotalSeconds + if ($elapsed -gt ($ExpectedIntervalSeconds + $ToleranceSeconds) -and $WarnAction) { + & $WarnAction $eventAt $deliveredAt ([Math]::Round($elapsed, 1)) + } + return [Math]::Round($elapsed, 1) + } + #endregion PAX-1E query-poll observability + # Define the ThreadJob scriptblock once for reuse in both initial and retry attempts $queryJobScriptBlock = { - param($pStart, $pEnd, [array]$activity, $resultSize, $userIds, $idx, $tot, $sharedAuthState, $partition, $maxOutageMinutes, $apiVersion, $logPath, $existingQueryId, $incrementalDir, $runTimestamp, $memoryFlushEnabled) + param($pStart, $pEnd, [array]$activity, $resultSize, $userIds, $idx, $tot, $sharedAuthState, $partition, $maxOutageMinutes, $apiVersion, $logPath, $existingQueryId, $incrementalDir, $runTimestamp, $memoryFlushEnabled, $queryObservations) # Suppress web request progress bar in job runspace $ProgressPreference = 'SilentlyContinue' @@ -25629,345 +39277,28 @@ Write-LogHost "" # Output mode display with format-specific defaults } } - # SKIP CREATE if we already have a QueryId (retry after 403 fetch failure) - $skipCreate = $false - if ($queryId) { - $skipCreate = $true - Write-Output "[RETRY-FETCH] Partition $idx/$tot - Reusing existing QueryId: $queryId (skipping CREATE)" - $telemetry.QueryCreatedAt = 'reused_existing' - } - - # Convert outage tolerance to seconds BEFORE the skipCreate check so FETCH phase always has it + # The parent creates every partition query before this worker is launched and + # passes its id in through $existingQueryId ($queryId). A worker never creates + # a query. Convert the outage tolerance to seconds here so the FETCH phase + # always has it regardless of how the query id arrived. $maxNetworkOutageSeconds = $maxOutageMinutes * 60 - - # Retry loop for query creation with 429 handling (unlimited retries for throttling) - # Also handles transient network errors (502, 503, connection failures) with time-based tolerance - if (-not $skipCreate) { - try { # CREATE phase try-catch wrapper - handles CREATE-specific failures separately from FETCH errors - $createRetries = 0 - $createSuccess = $false - $networkErrorStart = $null - - # Build query body ONCE before retry loop - $queryBody = @{ - displayName = $displayName - filterStartDateTime = $pStart.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) - filterEndDateTime = $pEnd.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) - operationFilters = @($queryActivities) - } - if ($activeRecordFilters -and $activeRecordFilters.Count -gt 0) { - $queryBody.recordTypeFilters = @($activeRecordFilters) - } - if ($activeServiceFilter) { - $queryBody.serviceFilter = $activeServiceFilter - } - - $queryBodyJson = $queryBody | ConvertTo-Json -Depth 5 - Write-GraphQueryDebug -Header "Graph API Query Body for partition $idx/$tot (Operations: $($queryActivities -join ', ')):" -Operations $queryActivities -RecordFilters $activeRecordFilters -ServiceFilter $activeServiceFilter -PayloadJson $queryBodyJson - - $attemptTimestamp = Get-Date -Format 'HH:mm:ss' - $attemptMessage = "[ATTEMPT] [$attemptTimestamp] Partition $idx/$tot - Starting query creation..." - Write-Host $attemptMessage -ForegroundColor DarkGray - - while (-not $createSuccess) { - try { - $queryUri = "https://graph.microsoft.com/$apiVersion/security/auditLog/queries" - $createResponse = Invoke-RestMethod -Method POST -Uri $queryUri -Headers (Get-CurrentHeaders -ClientRequestId $clientRequestId) -Body $queryBodyJson -ErrorAction Stop - $queryId = $createResponse.id - $telemetry.QueryCreatedAt = (Get-Date).ToString('yyyy-MM-dd HH:mm:ss') - - # LOG: Query successfully sent to server (output stream only, displayed by main thread) - $sentTimestamp = Get-Date -Format 'HH:mm:ss' - $sentMessage = "[SENT] [$sentTimestamp] Partition $idx/$tot - Query sent to Purview (QueryId: $queryId)" - Write-Output $sentMessage - - $createSuccess = $true - if ($networkErrorStart) { - $outageSeconds = [Math]::Round(((Get-Date) - $networkErrorStart).TotalSeconds, 1) - Write-Output "[NETWORK] Partition $idx/$tot - Recovered after network error (${outageSeconds}s outage)" - } - $networkErrorStart = $null # Reset network error timer on success - } - catch { - # CRITICAL: Check for transient network errors FIRST (502, 503, 504) - must be prioritized - # before 429 check to ensure these errors are always caught and retried correctly - $isNetworkError = $false - $errorMessage = $_.Exception.Message - $errorSummary = "Unknown error" - - if ($_.Exception.Response) { - $statusCode = $_.Exception.Response.StatusCode.value__ - if ($statusCode -ge 500) { - $isNetworkError = $true - $errorSummary = "$statusCode Server Error" - } - } - if (-not $isNetworkError -and ($errorMessage -match '5\d{2}|Bad Gateway|Service Unavailable|Gateway Timeout|Internal Server Error')) { - $isNetworkError = $true - $errorSummary = "Server error (from message)" - } - if (-not $isNetworkError -and ($errorMessage -match 'timed out|connection|unable to connect|could not be resolved')) { - $isNetworkError = $true - $errorSummary = "Network connectivity issue" - } - - # Check for 429 throttling (AFTER network error check) - $is429Create = $false - if (-not $isNetworkError) { - if ($_.Exception.Response) { - $statusCode = $_.Exception.Response.StatusCode - if ($statusCode -eq 429 -or $statusCode -eq 'TooManyRequests' -or $statusCode.value__ -eq 429) { - $is429Create = $true - } - } - if (-not $is429Create -and ($errorMessage -match '429' -or $errorMessage -match 'Too Many Requests')) { - $is429Create = $true - } - } - - # Check for 403 Forbidden (Microsoft service-side issue) - $is403Create = $false - if (-not $isNetworkError -and -not $is429Create) { - if ($_.Exception.Response) { - $statusCode = $_.Exception.Response.StatusCode - if ($statusCode -eq 403 -or $statusCode -eq 'Forbidden' -or $statusCode.value__ -eq 403) { - $is403Create = $true - } - } - if (-not $is403Create -and ($errorMessage -match '403' -or $errorMessage -match 'Forbidden')) { - $is403Create = $true - } - } - - # Handle different error types - if ($is403Create) { - # Extract diagnostic info from 403 response (critical for Microsoft support) - $responseBody403 = $null - $requestId403 = $null - $wwwAuth403 = $null - $isPermanent403 = $false - - # PowerShell 7 error handling: ErrorDetails.Message contains response body - try { - if ($_.ErrorDetails -and $_.ErrorDetails.Message) { - $responseBody403 = $_.ErrorDetails.Message - } - } catch {} - - # Try to get headers from the response - try { - if ($_.Exception.Response) { - # Try different header access patterns for PS7 compatibility - try { - $requestId403 = $_.Exception.Response.Headers['request-id'] - } catch { - try { - $requestId403 = $_.Exception.Response.Headers.GetValues('request-id') | Select-Object -First 1 - } catch {} - } - try { - $wwwAuth403 = $_.Exception.Response.Headers['WWW-Authenticate'] - } catch { - try { - $wwwAuth403 = $_.Exception.Response.Headers.GetValues('WWW-Authenticate') | Select-Object -First 1 - } catch {} - } - - # Fallback: try to read response stream if ErrorDetails was empty - if (-not $responseBody403) { - try { - $respStream = $_.Exception.Response.GetResponseStream() - if ($respStream -and $respStream.CanRead) { - $reader = New-Object System.IO.StreamReader($respStream) - $responseBody403 = $reader.ReadToEnd() - $reader.Dispose() - } - } catch {} - } - } - } catch {} - - # Also capture the full exception message as fallback - $exceptionMessage403 = $_.Exception.Message - - # Check if this is a PERMANENT 403 (don't retry these) - if ($responseBody403 -match 'InsufficientPrivileges|Authorization_RequestDenied|AccessDenied|InvalidAuthenticationToken') { - $isPermanent403 = $true - } - # Claims challenge indicates CAE - token needs refresh, not retry - if ($wwwAuth403 -match 'claims') { - $isPermanent403 = $true # Don't retry with same token, need fresh token - } - - # Log diagnostic info for Microsoft support - try { - $diagLog = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [403-DIAG] Partition $idx/$tot`n" - $diagLog += " client-request-id: $clientRequestId`n" - $diagLog += " request-id: $requestId403`n" - $diagLog += " exception: $exceptionMessage403`n" - if ($wwwAuth403) { $diagLog += " WWW-Authenticate: $wwwAuth403`n" } - if ($responseBody403) { $diagLog += " Response body: $responseBody403`n" } - $diagLog += " Permanent error: $isPermanent403" - $diagLog | Add-Content -Path $logPath -Encoding UTF8 -ErrorAction SilentlyContinue - } catch {} - - if ($isPermanent403) { - # Permanent 403 - don't retry, fail immediately for partition retry with fresh token - Write-Output "[403-PERM] Partition $idx/$tot - PERMANENT 403 (permissions/CAE) - Failing partition | request-id: $requestId403 | client-request-id: $clientRequestId" - throw "403 Forbidden (permanent) - $responseBody403" - } - - # Transient 403 - retry with exponential backoff - $createRetries++ - $max403Retries = 3 # Limited retries since we can't refresh token inside ThreadJob - - if ($createRetries -le $max403Retries) { - # Exponential backoff: 15s, 30s, 60s - $retryAfter = [Math]::Min(15 * [Math]::Pow(2, $createRetries - 1), 60) - - # Thread-safe file logging - try { - $logMsg = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [403-CREATE] Partition $idx/$tot - Transient 403 (Attempt $createRetries/$max403Retries) - Retrying in ${retryAfter}s" - $logMsg | Add-Content -Path $logPath -Encoding UTF8 -ErrorAction SilentlyContinue - } catch {} - - Write-Output "[403-CREATE] Partition $idx/$tot - Transient 403 (Attempt $createRetries/$max403Retries) - Retrying in ${retryAfter}s | request-id: $requestId403 | client-request-id: $clientRequestId" - Start-Sleep -Seconds $retryAfter - } else { - # Max 403 retries exceeded - throw to trigger partition-level retry (which will refresh token) - try { - $logMsg = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [403-CREATE] Partition $idx/$tot - Max transient 403 retries exceeded ($max403Retries), failing partition for retry with fresh token" - $logMsg | Add-Content -Path $logPath -Encoding UTF8 -ErrorAction SilentlyContinue - } catch {} - - Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 retries exceeded - Failing partition | request-id: $requestId403 | client-request-id: $clientRequestId" - throw "403 Forbidden (transient) - max retries exceeded (partition will retry with fresh token)" - } - } - elseif ($is429Create) { - $createRetries++ - # Throttling - get retry-after value - if ($_.Exception.Response.Headers -and $_.Exception.Response.Headers['Retry-After']) { - $retryAfter = [int]$_.Exception.Response.Headers['Retry-After'] - } - else { - # Exponential backoff: 60s, 120s, 240s, then cap at 300s (5 min) - $retryAfter = [Math]::Min(60 * [Math]::Pow(2, $createRetries - 1), 300) - } - - # Send real-time throttle notification to parent process - try { - $throttleNotification = [PSCustomObject]@{ - Partition = "$idx/$tot" - Attempt = $createRetries - RetryAfter = $retryAfter - Timestamp = (Get-Date) - } - $script:throttleNotifications.Enqueue($throttleNotification) - } - catch { - # Fallback to output if synchronized collection fails - Write-Output "[THROTTLE] Partition $idx/$tot - Query creation throttled (Attempt $createRetries) - Waiting $retryAfter seconds..." - } - - Start-Sleep -Seconds $retryAfter - - # Safety cap: prevent truly infinite throttle loops - if ($createRetries -ge 20) { - throw "Query creation throttled after $createRetries attempts - aborting partition" - } - } - elseif ($isNetworkError) { - # Network error - check if we're still within the outage tolerance window - if (-not $networkErrorStart) { - $networkErrorStart = Get-Date - # Log to file only (no terminal spam) - Write-Output "[NETWORK] Partition $idx/$tot - $errorSummary - Starting retry window (max ${maxOutageMinutes}m)" - Write-Output "[NETWORK] First network error for partition $idx/$tot : $errorMessage" - } - - $elapsedOutageSeconds = ((Get-Date) - $networkErrorStart).TotalSeconds - - if ($elapsedOutageSeconds -lt $maxNetworkOutageSeconds) { - $remainingMinutes = [Math]::Ceiling(($maxNetworkOutageSeconds - $elapsedOutageSeconds) / 60) - $retryDelay = 30 + (Get-Random -Minimum 10 -Maximum 30) # 30-60s random delay - - # Suppress subsequent retry messages to terminal (first error already shown) - # Full error details to log file only for troubleshooting - $elapsedFormatted = [Math]::Round($elapsedOutageSeconds, 1) - Write-Output "[NETWORK] Retry attempt for partition $idx/$tot (${elapsedFormatted}s elapsed) : $errorMessage" - - Start-Sleep -Seconds $retryDelay - } - else { - # Network outage exceeded tolerance - $outageMinutes = [Math]::Round($elapsedOutageSeconds / 60, 1) - Write-Output "[CREATE-FAILED] Partition $idx/$tot - Network outage exceeded $maxOutageMinutes minute tolerance (${outageMinutes}m elapsed) - will retry at end of run" - throw "Network outage exceeded $maxOutageMinutes minute tolerance during query creation" - } - } - else { - # Non-retriable error - log it and re-throw - $statusCode = $null - $bodyText = $null - try { - if ($_.Exception.Response) { - $statusCode = [int]$_.Exception.Response.StatusCode.value__ - } - } catch {} - if (-not $statusCode -and $_.Exception.Response) { - try { $statusCode = [int]$_.Exception.Response.StatusCode } catch {} - } - try { - if ($_.Exception.Response) { - $respStream = $_.Exception.Response.GetResponseStream() - if ($respStream) { - $reader = New-Object System.IO.StreamReader($respStream) - $bodyText = $reader.ReadToEnd() - $reader.Dispose() - } - } - } catch {} - - # No automatic filter fallback allowed – capture diagnostics only + $telemetry.QueryCreatedAt = 'reused_existing' - $errorDetails = "StatusCode: $(if ([string]::IsNullOrEmpty($statusCode)) { $_.Exception.Response.StatusCode } else { $statusCode }), Message: $($_.Exception.Message)" - Write-Host "[CREATE-FAILED] Partition $idx/$tot - Query creation failed: $errorDetails" -ForegroundColor Red - Write-Output "[ERROR] Partition $idx/$tot - Query creation failed (will retry at end of run): $errorDetails" - if ($bodyText) { - Write-Output "[GRAPH-ERROR] Partition $idx/$tot - Response body: $bodyText" - } - throw # Re-throw non-network, non-throttle errors - } - } - } - - # Track throttle retries for summary - $telemetry.ThrottleRetriesDuringCreation = $createRetries - - if (-not $queryId) { - throw "Failed to create query" - } - - # Log query details - $debugInfo = "Query ID: $queryId | DateRange: $($pStart.ToString('yyyy-MM-dd HH:mm')) to $($pEnd.ToString('yyyy-MM-dd HH:mm')) UTC | Activities: $($queryActivities -join ', ')" - } # End of CREATE phase try block - catch { - # CREATE phase failed - re-throw with CREATE-FAILED marker so outer catch knows this is not a FETCH error - $createError = $_.Exception.Message - $createStack = $_.ScriptStackTrace - - # Log CREATE failure + # A missing query id means the parent-side creation did not produce one for + # this partition. That is a structured failure for the partition, surfaced the + # same way a create failure was, never a silent create here. + if (-not $queryId) { + $telemetry.Status = 'create_missing' try { - $logMsg = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [CREATE-FAILED] Partition $idx/$tot - Query creation failed: $createError" + $logMsg = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [CREATE-FAILED] Partition $idx/$tot - No QueryId supplied to worker (parent-side creation did not provide one)" $logMsg | Add-Content -Path $logPath -Encoding UTF8 -ErrorAction SilentlyContinue } catch {} - - # Re-throw with marker for outer catch to identify as CREATE failure - throw "[CREATE-FAILED] $createError" + throw "[CREATE-FAILED] No QueryId supplied to worker (parent-side creation did not provide one)" } - } # End of if (-not $skipCreate) block - + + # Log query details for the query the parent created. + $debugInfo = "Query ID: $queryId | DateRange: $($pStart.ToString('yyyy-MM-dd HH:mm')) to $($pEnd.ToString('yyyy-MM-dd HH:mm')) UTC | Activities: $($queryActivities -join ', ')" + # Step 2: Poll for completion (no artificial timeout — polls until Purview responds) # Microsoft guidance: 30-60s intervals for enterprise-scale parallel execution # Network outage tolerance (adaptive) using passed-in MaxNetworkOutageMinutes parameter @@ -25988,14 +39319,70 @@ Write-LogHost "" # Output mode display with format-specific defaults $pollStartTime = Get-Date $queryComplete = $false - + + # PAX-1E worker observation tracking (published atomically into the shared + # parent dictionary $queryObservations after each successful/failed poll). + $obsFirstObservedUtc = $null + $obsLastSuccessfulPollUtc = $null + $obsLastStatusTransitionUtc = $null + $obsLastServerStatus = $null + $obsConsecutivePollErrors = 0 + $obsLastErrorCategory = $null + $obsHostGapSecondsExcluded = 0.0 + $obsLastIntendedSleep = 0.0 + # Genuine PAX creation instant when THIS run created the query; $null for a + # reused/resumed QueryId (age is then reported as unknown/first-observed, never + # fabricated). + $obsQuerySubmittedUtc = if (-not $existingQueryId) { (Get-Date).ToUniversalTime() } else { $null } + # Inline snapshot builder - mirrors the script-scoped New-PaxQueryObservation + # field set EXACTLY (proven by tests/Purview.QueryPollObservability.Tests.ps1). + function New-ObsSnapshot { + param($ServerStatus, $NowUtc) + $now = if ($NowUtc) { [datetime]$NowUtc } else { (Get-Date).ToUniversalTime() } + $qAge = if ($obsQuerySubmittedUtc) { [Math]::Round(($now - $obsQuerySubmittedUtc).TotalSeconds, 1) } else { $null } + $foAge = if ($obsFirstObservedUtc) { [Math]::Round(($now - $obsFirstObservedUtc).TotalSeconds, 1) } else { $null } + $lpAge = if ($obsLastSuccessfulPollUtc) { [Math]::Round(($now - $obsLastSuccessfulPollUtc).TotalSeconds, 1) } else { $null } + $refUtc = if ($sharedAuthState -and $sharedAuthState.PSObject.Properties.Name -contains 'LastRefreshUtc' -and $sharedAuthState.LastRefreshUtc) { $sharedAuthState.LastRefreshUtc } else { $null } + return [pscustomobject]@{ + PartitionIndex = $idx + AttemptNumber = $(if ($telemetry -and $telemetry.PSObject.Properties.Name -contains 'RetryAttempt' -and $telemetry.RetryAttempt) { [int]$telemetry.RetryAttempt } else { 1 }) + ServerStatus = $ServerStatus + QuerySubmittedUtc = $obsQuerySubmittedUtc + FirstObservedUtc = $obsFirstObservedUtc + LastSuccessfulPollUtc = $obsLastSuccessfulPollUtc + LastStatusTransitionUtc = $obsLastStatusTransitionUtc + PollCount = $pollCount + ConsecutivePollErrors = $obsConsecutivePollErrors + LastPollErrorCategory = $obsLastErrorCategory + TokenRefreshInstantUtc = $refUtc + HostGapSecondsExcluded = [Math]::Round($obsHostGapSecondsExcluded, 1) + QueryAgeSeconds = $qAge + FirstObservedAgeSeconds = $foAge + LastSuccessfulPollAgeSeconds = $lpAge + SnapshotUtc = $now + } + } + # Initial wait before first poll with randomization to prevent synchronization $initialWaitSeconds = Get-Random -Minimum 30 -Maximum 60 $telemetry.InitialPollDelaySeconds = $initialWaitSeconds Start-Sleep -Seconds $initialWaitSeconds - + + $obsIterStartUtc = (Get-Date).ToUniversalTime() while (-not $queryComplete) { $pollCount++ + # PAX-1E host-gap accounting: a wall-clock gap between poll iterations that + # materially exceeds the sleep we intended indicates the host suspended this + # worker (container throttle / VM sleep). Exclude that excess from network- + # outage accounting; NEVER fail, cancel, or age-out the query on it. + $obsIterNowUtc = (Get-Date).ToUniversalTime() + $obsIterGap = ($obsIterNowUtc - $obsIterStartUtc).TotalSeconds - $obsLastIntendedSleep + if ($obsIterGap -ge 30) { + $obsHostGapSecondsExcluded += [Math]::Round($obsIterGap, 1) + if ($netOutageStart) { $netOutageStart = $netOutageStart.AddSeconds($obsIterGap) } + Write-Output ("[HOSTGAP] Partition $idx/$tot - Worker scheduling resumed after an extended host delay (~{0}s excluded from outage accounting)" -f [Math]::Round($obsIterGap, 0)) + } + $obsIterStartUtc = $obsIterNowUtc # Poll query status with 429 throttling detection try { @@ -26025,6 +39412,15 @@ Write-LogHost "" # Output mode display with format-specific defaults } } catch { + # PAX-1E: record a poll ERROR as a NON-sensitive category (never body/token/ + # url/queryId) and publish, WITHOUT disturbing the last-successful-poll instant. + $obsConsecutivePollErrors++ + $obsErrCode = '' + if ($_.Exception.Response) { try { $obsErrCode = "" + $_.Exception.Response.StatusCode } catch {} } + $obsErrMsg = "" + $_.Exception.Message + $obsErrProbe = "$obsErrCode $obsErrMsg" + $obsLastErrorCategory = if ($obsErrProbe -match '429|Too\s*Many\s*Requests|throttl') { 'Throttled' } elseif ($obsErrProbe -match '401|403|Unauthorized|Forbidden|InvalidAuthenticationToken|token') { 'Authentication' } elseif ($obsErrProbe -match '404|Not\s*Found|QUERY-GONE|gone') { 'Gone' } elseif ($obsErrProbe -match '5\d{2}|timed out|timeout|connection|unable to connect|could not be resolved|temporarily unavailable|network|Bad Gateway|Service Unavailable|Gateway Timeout|Internal Server Error') { 'Network' } else { 'Other' } + if ($queryObservations) { $queryObservations[$idx] = New-ObsSnapshot -ServerStatus $obsLastServerStatus -NowUtc ((Get-Date).ToUniversalTime()) } # Check for 429 throttling using safe detection $is429 = $false if ($_.Exception.Response) { @@ -26160,13 +39556,33 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } if ($is404Poll) { - # Query no longer exists on server - QueryId is dead, must CREATE a new one on retry + # The pinned-version query is not visible right now. The worker NEVER + # creates or replaces a query; it RETURNS a structured visibility-failure + # signal so the parent can revalidate the retained id through the shared + # decision layer (bounded same-id retry, then a read-only same-id GET on + # the alternate version) and only replace it, at most once, after the + # parent positively confirms absence. Diagnostics are preserved. try { - $logMsg = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [QUERY-GONE] Partition $idx/$tot - 404 Not Found on status poll - QueryId $queryId is dead | client-request-id: $clientRequestId" + $logMsg = "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] [QUERY-GONE] Partition $idx/$tot - 404 Not Found on status poll - QueryId $queryId not visible on $apiVersion - returning visibility signal to parent | client-request-id: $clientRequestId" $logMsg | Add-Content -Path $logPath -Encoding UTF8 -ErrorAction SilentlyContinue } catch {} - Write-Output "[QUERY-GONE] Partition $idx/$tot - 404 Not Found - Query $queryId no longer exists on server" - throw "[QUERY-GONE] 404 Not Found during status poll - QueryId $queryId is dead" + Write-Output "[QUERY-GONE] Partition $idx/$tot - 404 Not Found - Query $queryId not visible on $apiVersion (parent will revalidate/replace)" + return [PSCustomObject]@{ + QueryId = $queryId + Status = 'visibility_failure' + QueryApiVersion = $apiVersion + FailureCategory = 'QueryNotVisible' + AlternateVersionChecked = $false + QueryConfirmedAbsent = $false + ReplacementAttempted = $false + LastLifecycleHttpStatus = 404 + PartitionStart = $pStart + PartitionEnd = $pEnd + PartitionIndex = $idx + PartitionTotal = $tot + RetrievedCount = 0 + Telemetry = $telemetry + } } # Detect transient network outage vs hard failure @@ -26227,6 +39643,17 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } } + # PAX-1E: record a SUCCESSFUL server poll and publish an atomic snapshot into + # the shared parent dictionary. Status transition time changes only on an + # actual state change; last-successful-poll updates every poll. + $obsNowUtc = (Get-Date).ToUniversalTime() + if (-not $obsFirstObservedUtc) { $obsFirstObservedUtc = $obsNowUtc } + $obsLastSuccessfulPollUtc = $obsNowUtc + $obsConsecutivePollErrors = 0 + $obsCurrentStatus = "" + $statusResponse.status + if ($obsCurrentStatus -ne $obsLastServerStatus) { $obsLastStatusTransitionUtc = $obsNowUtc; $obsLastServerStatus = $obsCurrentStatus } + if ($queryObservations) { $queryObservations[$idx] = New-ObsSnapshot -ServerStatus $obsCurrentStatus -NowUtc $obsNowUtc } + # Process status response switch ($statusResponse.status) { 'succeeded' { @@ -26337,6 +39764,9 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc Start-Sleep -Seconds $waitSeconds } } + # PAX-1E: remember the sleep we intended this iteration so the next iteration's + # host-gap check can distinguish host suspension from a legitimately long poll. + $obsLastIntendedSleep = [double]$waitSeconds } # Step 3: Retrieve records with pagination @@ -26643,7 +40073,13 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $allRecords.Clear() $threadSavedToDisk = $true } catch { - Write-Output "[FLUSH-WARN] Partition $idx/$tot - Page flush failed, keeping in memory: $($_.Exception.Message)" + # Fail closed: keeping the page in memory would defeat the memory + # budget and grow the in-memory buffer without bound. Terminate this + # partition truthfully; already-flushed pages remain on disk and the + # run finishes as a terminal failure (nonzero) without continuing to + # collect the result set in memory. + Write-Output "[FLUSH-FAIL] Partition $idx/$tot - Page flush failed: $($_.Exception.Message). Failing closed (no in-memory fallback)." + throw [System.Exception]::new("Bounded page flush failed for partition $idx/$tot ($($_.Exception.Message))") } } @@ -26814,7 +40250,14 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $threadSavedToDisk = $true Write-Output "[SAVE-THREAD] Partition $($idx)/$($tot): $($allRecords.Count) records persisted by thread job" } catch { - Write-Output "[SAVE-THREAD-WARN] Partition $($idx)/$($tot): final thread persistence failed: $($_.Exception.Message)" + # Fail closed (bounded memory-flush mode): a persistence failure must NOT + # fall through to returning the full record set in memory below (that would + # reintroduce the unbounded in-memory buffer and violate the memory budget). + # Terminate this partition truthfully; already-completed incremental files and + # checkpoint state are preserved on disk, and the run finishes as a terminal + # failure (nonzero) without certifying completeness or continuing collection. + Write-Output "[SAVE-THREAD-FAIL] Partition $($idx)/$($tot): bounded persistence failed: $($_.Exception.Message). Failing closed (no in-memory fallback)." + throw [System.Exception]::new("Bounded memory-flush persistence failed for partition $($idx)/$($tot): $($_.Exception.Message)") } } elseif ($memoryFlushEnabled -and $pageFlushFilePath) { # All records were already flushed per-page; just ensure thread-saved references are set @@ -26884,12 +40327,19 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # Find partitions that need jobs created (haven't been processed yet). # Include 'Failed' status to enable retry of failed partitions — for # Failed partitions, allow retry even if they're already tracked in - # $script:partitionsWithJobs (their previous job failed). + # $script:partitionsWithJobs (their previous job failed). Partitions flagged + # IsTerminalFailure (N consecutive same-category failures) are NOT re-queued — + # this is what lets the loop finalize instead of re-dispatching forever. $pendingPartitions = @($partitions | Where-Object { $statusObj = $script:partitionStatus[$_.Index] if (-not $statusObj) { return $false } + # A durable-uncertain partition (unproven prior create, no query id) that a + # same-process read-only reconcile already failed to resolve is HELD: never + # re-POSTed and excluded here so the outer create loop can terminate, exactly + # mirroring the IsTerminalFailure exclusion below. + if ($statusObj.UncertainCreate -and -not $statusObj.QueryId -and $script:uncertainReconcileHeld.Contains($_.Index)) { return $false } if ($statusObj.Status -eq 'NotStarted' -and -not $script:partitionsWithJobs.Contains($_.Index)) { return $true } - if ($statusObj.Status -eq 'Failed') { return $true } # Allow retry of failed partitions + if ($statusObj.Status -eq 'Failed' -and -not $statusObj.IsTerminalFailure) { return $true } # retry failed, but not terminally-failed return $false }) @@ -26904,9 +40354,155 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc Write-LogHost "=== Subdivision Pass $subdivisionPass ===" -ForegroundColor Yellow Write-LogHost "Processing $($pendingPartitions.Count) new sub-partitions from previous subdivisions..." -ForegroundColor Yellow } - + + # --- Parent-side query creation pre-pass (bounded parallel) --- + # Create the audit query for every pending partition that has no query id + # yet, BEFORE any worker is launched, reusing the launch concurrency bound + # ($maxConcurrentPartitions). Creation runs in the main runspace where the + # token can be refreshed. A partition that already carries a query id (its + # status QueryId or StoredQueryId) skips creation and is wired to StoredQueryId + # so the existing launch reads it. A successful creation is checkpointed + # QueryCreated before launch; a failed creation is recorded so no worker is + # launched for it this pass. + $createFailedThisPass = [System.Collections.Generic.HashSet[int]]::new() + $needCreate = New-Object System.Collections.Generic.List[object] + $needReconcile = New-Object System.Collections.Generic.List[object] + foreach ($cp in $pendingPartitions) { + $existingId = $null + if ($script:partitionStatus.ContainsKey($cp.Index) -and $script:partitionStatus[$cp.Index].QueryId) { $existingId = [string]$script:partitionStatus[$cp.Index].QueryId } + elseif ($cp.StoredQueryId) { $existingId = [string]$cp.StoredQueryId } + if ($existingId) { + $cp | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue $existingId -Force + } elseif ($script:partitionStatus.ContainsKey($cp.Index) -and $script:partitionStatus[$cp.Index].UncertainCreate) { + # A partition that became durable-uncertain in an earlier pass of THIS run has + # an unproven prior create and no query id. It must NEVER be re-POSTed as a + # fresh create; route it away from the create list to a same-process READ-ONLY + # reconcile instead (exact fingerprinted name, never a POST). + [void]$needReconcile.Add($cp) + } else { + [void]$needCreate.Add($cp) + } + } + # Same-process READ-ONLY reconcile of durable-uncertain partitions (never a POST): + # exactly one exact-name match is adopted + checkpointed QueryCreated (uncertain + # vacated) and wired to launch this pass; multiple (fail closed) or none + # (unresolved) are held - not launched, recorded create-failed this pass, and added + # to the run-scoped held set so the pending selector excludes them and the outer + # loop can terminate. A collection miss never sets QueryConfirmedAbsent. + if ($needReconcile.Count -gt 0) { + Write-LogHost " [PARENT-RECONCILE] $($needReconcile.Count) partition(s) have an unproven prior create - reconciling READ-ONLY in-process (no POST)..." -ForegroundColor Yellow + $inlineReconcile = Invoke-PaxResumeUncertainReconcile -Partitions $needReconcile.ToArray() -Activities $activities -Log { param($m) Write-Log $m } + foreach ($adopted in @($inlineReconcile.AdoptedFetchOnly)) { + # Adopted: reconcile stored a proven real QueryId on the partition object and + # checkpointed it QueryCreated (uncertain vacated). Mirror it into the live + # status so the launch below reads a real query id and fetches data only. + $script:partitionStatus[$adopted.Index].QueryId = [string]$adopted.StoredQueryId + $script:partitionStatus[$adopted.Index].UncertainCreate = $false + Write-LogHost " [PARENT-RECONCILE] Partition $($adopted.Index)/$($adopted.Total) adopted by exact name (no POST) - fetching data only" -ForegroundColor Green + } + foreach ($held in (@($inlineReconcile.Unresolved) + @($inlineReconcile.FailedClosed))) { + # Held: no proven create. Never re-POSTed, no worker launched this pass, and + # added to the run-scoped held set so the pending selector excludes it and the + # outer create loop terminates instead of re-dispatching forever. + [void]$createFailedThisPass.Add([int]$held.Index) + [void]$script:uncertainReconcileHeld.Add([int]$held.Index) + } + $heldInline = @($inlineReconcile.Unresolved).Count + @($inlineReconcile.FailedClosed).Count + if ($heldInline -gt 0) { + Write-LogHost " [PARENT-RECONCILE] $heldInline partition(s) remain uncertain (no proven create) - held, not launched; re-run with -ClearUncertainCreate to force a fresh create for a specific partition" -ForegroundColor Yellow + } + } + if ($needCreate.Count -gt 0) { + $createBound = [Math]::Max(1, [int]$maxConcurrentPartitions) + Write-LogHost " [PARENT-CREATE] Creating $($needCreate.Count) partition query(ies) in bounded parallel (limit $createBound)..." -ForegroundColor DarkCyan + # Refresh the token before the create pre-pass so every queued and retrying + # create job reads a current token from the shared auth state. + $createRefresh = Refresh-GraphTokenIfNeeded -BufferMinutes 5 + if ($createRefresh -is [string] -and $createRefresh -eq 'Quit') { + if ($script:CheckpointEnabled) { Save-Checkpoint -Force } + Show-CheckpointExitMessage + exit 0 + } + $createMap = Invoke-PaxParentQueryCreation -Partitions $needCreate.ToArray() -ThrottleLimit $createBound -Activities $activities -MaxNetworkOutageMinutes $MaxNetworkOutageMinutes + # A token quit or refresh error during creation fails the pass closed before any + # partition is persisted: enter the graceful authentication-exit path so nothing + # is recorded as created off a quit/error decision (resume re-runs the pre-create + # reconcile, which adopts an already-created query without a duplicate POST). + if ($createMap['__AuthQuit']) { + # Persist any query the pool actually created before the quit AND mark the + # still-in-flight partitions durable-uncertain, so a known QueryId is never + # discarded and a possibly-accepted in-flight POST is reconciled read-only on + # resume. A user-driven quit stays a graceful (exit 0) checkpoint-and-resume. + Save-PaxInterruptedCreatePool -CreateMap $createMap -Requested $needCreate -Activities $activities -Phase 'initial' + Write-LogHost " [PARENT-CREATE] Authentication quit requested during query creation - completed persisted, in-flight marked uncertain, saving checkpoint and exiting (use -Resume to continue)" -ForegroundColor Yellow + if ($script:CheckpointEnabled) { Save-Checkpoint -Force } + Show-CheckpointExitMessage + exit 0 + } + if ($createMap['__RefresherError']) { + # Same durable persistence as the quit path, but a token-refresh EXCEPTION is a + # failure, not a user decision: after persisting completed + in-flight-uncertain + # it terminates NONZERO through the established fatal-exit machinery + # ($script:GenericFatal -> exit 1) so automation sees the run did not succeed. + Save-PaxInterruptedCreatePool -CreateMap $createMap -Requested $needCreate -Activities $activities -Phase 'initial' + Write-LogHost " [PARENT-CREATE] Token refresh error during query creation ($($createMap['__RefresherError'])) - completed persisted, in-flight marked uncertain, saving checkpoint and exiting NONZERO (use -Resume to continue)" -ForegroundColor Red + $script:GenericFatal = $true + if ($script:CheckpointEnabled) { Save-Checkpoint -Force } + Show-CheckpointExitMessage + exit 1 + } + foreach ($cp in $needCreate) { + $cr = $createMap[[int]$cp.Index] + if ($cr -and $cr.QueryId) { + $cp | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue ([string]$cr.QueryId) -Force + $script:partitionStatus[$cp.Index].QueryId = [string]$cr.QueryId + $script:partitionStatus[$cp.Index].QueryApiVersion = $cr.Version + $script:partitionStatus[$cp.Index].LifecycleOperation = 'Create' + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -QueryId ([string]$cr.QueryId) -State 'QueryCreated' -LifecycleState $script:partitionStatus[$cp.Index] + } + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) query created (QueryId: $($cr.QueryId))" -ForegroundColor DarkCyan + } else { + $cat = if ($cr) { [string]$cr.FailureCategory } else { 'NotCreated' } + [void]$createFailedThisPass.Add([int]$cp.Index) + if ($cat -eq 'UncertainCreateUnresolved') { + # A first POST whose acceptance could not be proven (collection miss / 5xx + # that cannot prove rejection) is NEVER re-POSTed. Persist durable + # uncertain-create state so a resume routes this partition to a READ-ONLY + # reconcile (adopt one exact match, fail closed on many, stay uncertain on + # none) instead of a duplicate create. Deliberately NOT Set-PartitionFailure: + # the partition is not marked Failed and is never requeued for a re-create. + $script:partitionStatus[$cp.Index].UncertainCreate = $true + $script:partitionStatus[$cp.Index].UncertainCreateInfo = @{ + displayName = New-GraphAuditPartitionDisplayName -PStart $cp.PStart -PEnd $cp.PEnd -Index $cp.Index -Total $cp.Total -Activities $activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + fingerprint = New-GraphAuditQueryFingerprint -PStart $cp.PStart -PEnd $cp.PEnd -Activities $activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + pStart = $cp.PStart.ToString('o') + pEnd = $cp.PEnd.ToString('o') + attempt = [int]$script:partitionStatus[$cp.Index].AttemptNumber + phase = 'initial' + } + $script:partitionStatus[$cp.Index].LastError = "Parent-side query creation uncertain - durable reconcile pending (category: $cat)" + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -State 'UncertainCreate' -LifecycleState $script:partitionStatus[$cp.Index] + } + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) create uncertain (category: $cat) - persisted for read-only reconcile, no re-POST, worker not launched this pass" -ForegroundColor Yellow + } else { + Set-PartitionFailure -Index $cp.Index -Stage 'CREATE' -Reason $cat + $script:partitionStatus[$cp.Index].LastError = "Parent-side query creation failed (category: $cat)" + $diag = "HTTP $($cr.HttpStatus) | client-request-id: $($cr.ClientRequestId) | request-id: $($cr.RequestId)" + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) creation failed (category: $cat; $diag) - worker not launched this pass" -ForegroundColor Yellow + if ($cr.ResponseBody) { Write-LogHost " Graph response: $($cr.ResponseBody)" -ForegroundColor DarkYellow } + } + } + } + } + # Create initial jobs for all PENDING partitions with backpressure foreach ($pt in $pendingPartitions) { + # Skip launching a worker for any partition whose parent-side creation + # failed this pass; it has no query id and is already marked Failed for + # a future pass. + if ($createFailedThisPass.Contains([int]$pt.Index)) { continue } # ============================================================ # PROACTIVE TOKEN REFRESH: Check before each job creation # Job launch phase can take 20-35+ minutes with 60 partitions @@ -27099,10 +40695,14 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $jobPartition = $jobMeta[$activeJob.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { $script:partitionStatus[$jobPartition.Index].QueryId = $extractedQueryId + # Record the effective creation version (creation is always beta) so + # the checkpoint carries the per-query lifecycle version. + $script:partitionStatus[$jobPartition.Index].QueryApiVersion = $script:GraphAuditApiVersion_Previous + $script:partitionStatus[$jobPartition.Index].LifecycleOperation = 'Create' # CHECKPOINT: Save QueryCreated state so we can resume data fetch if interrupted if ($script:CheckpointEnabled) { - Save-Checkpoint -PartitionIndex $jobPartition.Index -QueryId $extractedQueryId -State 'QueryCreated' + Save-Checkpoint -PartitionIndex $jobPartition.Index -QueryId $extractedQueryId -State 'QueryCreated' -LifecycleState $script:partitionStatus[$jobPartition.Index] } } } @@ -27119,7 +40719,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # CRITICAL: Mark this partition as Failed for retry after re-auth $jobPartition = $jobMeta[$activeJob.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'AUTH' -Reason '401' $script:partitionStatus[$jobPartition.Index].LastError = '401 Unauthorized - token expired' } break # IMMEDIATE EXIT - don't process more output @@ -27128,7 +40728,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # must also mark the partition as Failed for retry. $jobPartition = $jobMeta[$activeJob.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'FETCH' -Reason 'JOB_ERROR' $script:partitionStatus[$jobPartition.Index].LastError = $output Write-LogHost " [RETRY-QUEUE] Partition $($jobPartition.Index)/$($jobPartition.Total) queued for retry at end of run" -ForegroundColor Yellow } @@ -27167,7 +40767,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($output.SplitRequired -eq $true) { $jobPartition = $jobMeta[$activeJob.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'FETCH' -Reason 'STILL_RUNNING' $script:partitionStatus[$jobPartition.Index].LastError = "Query still running after $([Math]::Round($output.ElapsedMinutes,1)) min - queued for retry" Write-LogHost " [RETRY-QUEUE] Partition $($jobPartition.Index)/$($jobPartition.Total) - query still processing after $([Math]::Round($output.ElapsedMinutes,1)) min, queued for retry" -ForegroundColor Yellow } @@ -27180,7 +40780,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $currentStatus = $script:partitionStatus[$jobPartition.Index].Status if ($currentStatus -ne 'Complete') { if ([string]::IsNullOrWhiteSpace([string]$output.QueryId) -and [int]($output.RetrievedCount ?? 0) -le 0) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'CREATE' -Reason 'EMPTY_QUERYID' $script:partitionStatus[$jobPartition.Index].LastError = 'ThreadJob returned empty QueryId with zero records (query was not created/sent)' Write-LogHost " [DATA-CHECK] Partition $($jobPartition.Index) returned empty QueryId with 0 records - marked Failed for retry" -ForegroundColor Yellow [void]$script:processedJobIds.Add($activeJob.Id) @@ -27267,8 +40867,12 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($existingQueryIdForJob) { Write-LogHost "[RESUME] Partition $($pt.Index)/$($pt.Total) - Using stored QueryId: $existingQueryIdForJob" -ForegroundColor Yellow } + # The worker polls and fetches on the SAME version the query was created on. + # The parent records that effective version on the partition state; fall back + # to the run version only when a partition carries none. + $effectiveApiVersion = if ($script:partitionStatus[$pt.Index].QueryApiVersion) { $script:partitionStatus[$pt.Index].QueryApiVersion } else { $script:GraphAuditApiVersion } - $job = Start-ThreadJob -ThrottleLimit $maxConcurrentPartitions -ScriptBlock $queryJobScriptBlock -ArgumentList $pt.PStart, $pt.PEnd, $activities, $graphResultSize, $UserIds, $pt.Index, $pt.Total, $script:SharedAuthState, $pt, $MaxNetworkOutageMinutes, $script:GraphAuditApiVersion, $script:LogFile, $existingQueryIdForJob, $threadIncrementalDir, $threadRunTimestamp, $threadMemoryFlushEnabled + $job = Start-ThreadJob -ThrottleLimit $maxConcurrentPartitions -ScriptBlock $queryJobScriptBlock -ArgumentList $pt.PStart, $pt.PEnd, $activities, $graphResultSize, $script:targetUsers, $pt.Index, $pt.Total, $script:SharedAuthState, $pt, $MaxNetworkOutageMinutes, $effectiveApiVersion, $script:LogFile, $existingQueryIdForJob, $threadIncrementalDir, $threadRunTimestamp, $threadMemoryFlushEnabled, $script:QueryObservations $jobs += $job $jobMeta[$job.Id] = $pt @@ -27328,10 +40932,14 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { $script:partitionStatus[$jobPartition.Index].QueryId = $extractedQueryId Write-Verbose "[QUERYID-CAPTURED] Partition $($jobPartition.Index) QueryId=$extractedQueryId" -Verbose:$VerbosePreference + # Record the effective creation version (creation is always beta) so + # the checkpoint carries the per-query lifecycle version. + $script:partitionStatus[$jobPartition.Index].QueryApiVersion = $script:GraphAuditApiVersion_Previous + $script:partitionStatus[$jobPartition.Index].LifecycleOperation = 'Create' # CHECKPOINT: Save QueryCreated state so we can resume data fetch if interrupted if ($script:CheckpointEnabled) { - Save-Checkpoint -PartitionIndex $jobPartition.Index -QueryId $extractedQueryId -State 'QueryCreated' + Save-Checkpoint -PartitionIndex $jobPartition.Index -QueryId $extractedQueryId -State 'QueryCreated' -LifecycleState $script:partitionStatus[$jobPartition.Index] } } } @@ -27354,7 +40962,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # CRITICAL: Mark this partition as Failed for retry after re-auth $jobPartition = $jobMeta[$job.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'AUTH' -Reason '401' $script:partitionStatus[$jobPartition.Index].LastError = '401 Unauthorized - token expired' } break # IMMEDIATE EXIT @@ -27400,7 +41008,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $currentStatus = $script:partitionStatus[$jobPartition.Index].Status if ($currentStatus -ne 'Complete') { if ([string]::IsNullOrWhiteSpace([string]$output.QueryId) -and [int]($output.RetrievedCount ?? 0) -le 0) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'CREATE' -Reason 'EMPTY_QUERYID' $script:partitionStatus[$jobPartition.Index].LastError = 'ThreadJob returned empty QueryId with zero records (query was not created/sent)' Write-LogHost " [DATA-CHECK] Partition $($jobPartition.Index) returned empty QueryId with 0 records - marked Failed for retry" -ForegroundColor Yellow [void]$script:processedJobIds.Add($job.Id) @@ -27522,7 +41130,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # CRITICAL: Mark this partition as Failed for retry after re-auth $jobPartition = $jobMeta[$existingJob.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'AUTH' -Reason '401' $script:partitionStatus[$jobPartition.Index].LastError = '401 Unauthorized - token expired' } break # IMMEDIATE EXIT @@ -27572,7 +41180,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($output.SplitRequired -eq $true) { $jobPartition = $jobMeta[$existingJob.Id] if ($jobPartition -and $script:partitionStatus.ContainsKey($jobPartition.Index)) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'FETCH' -Reason 'STILL_RUNNING' $script:partitionStatus[$jobPartition.Index].LastError = "Query still running after $([Math]::Round($output.ElapsedMinutes,1)) min - queued for retry" Write-LogHost " [RETRY-QUEUE] Partition $($jobPartition.Index)/$($jobPartition.Total) - query still processing after $([Math]::Round($output.ElapsedMinutes,1)) min, queued for retry" -ForegroundColor Yellow } @@ -27585,7 +41193,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $currentStatus = $script:partitionStatus[$jobPartition.Index].Status if ($currentStatus -ne 'Complete') { if ([string]::IsNullOrWhiteSpace([string]$output.QueryId) -and [int]($output.RetrievedCount ?? 0) -le 0) { - $script:partitionStatus[$jobPartition.Index].Status = 'Failed' + Set-PartitionFailure -Index $jobPartition.Index -Stage 'CREATE' -Reason 'EMPTY_QUERYID' $script:partitionStatus[$jobPartition.Index].LastError = 'ThreadJob returned empty QueryId with zero records (query was not created/sent)' Write-LogHost " [DATA-CHECK] Partition $($jobPartition.Index) returned empty QueryId with 0 records - marked Failed for retry" -ForegroundColor Yellow [void]$script:processedJobIds.Add($existingJob.Id) @@ -27660,6 +41268,19 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } } + $parentNowUtc = (Get-Date).ToUniversalTime() + # PAX-1E: measure parent monitor-loop scheduling delay (gap between successive + # status emissions vs the configured cadence). A material excess is surfaced + # once per stall, neutrally; the cause is not classified and no query state + # changes. + $parentLoopExcess = Measure-PaxParentLoopDelay -PreviousIterationUtc $script:PaxLastStatusEmitUtc -CurrentUtc $parentNowUtc -ExpectedCadenceSeconds $StatusIntervalSeconds -ToleranceSeconds ([Math]::Max(30, $StatusIntervalSeconds)) + if ($parentLoopExcess -gt 0 -and $script:PaxLastStatusEmitUtc) { + $stalePollAge = $null + foreach ($o in @($script:QueryObservations.Values)) { if ($o -and $o.LastSuccessfulPollUtc) { $pa = ($parentNowUtc - [datetime]$o.LastSuccessfulPollUtc).TotalSeconds; if ($null -eq $stalePollAge -or $pa -gt $stalePollAge) { $stalePollAge = [Math]::Round($pa, 0) } } } + Write-LogHost ("[MONITOR] Query monitoring resumed after an extended parent-loop delay (~{0}s); server state is being refreshed (last known worker-poll age: {1})." -f [Math]::Round($parentLoopExcess, 0), $(if ($null -ne $stalePollAge) { "${stalePollAge}s" } else { 'unknown' })) -ForegroundColor Yellow + } + $script:PaxLastStatusEmitUtc = $parentNowUtc + $elapsedSinceLastUpdate = ((Get-Date) - $script:lastStatusUpdate).TotalSeconds if ($elapsedSinceLastUpdate -ge $StatusIntervalSeconds) { # Count partitions by status (EXCLUDE 'Subdivided' parent partitions from total) @@ -27677,7 +41298,22 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $pageStr = ($script:partitionPageCounts.GetEnumerator() | Sort-Object Key | ForEach-Object { "P$($_.Key):$($_.Value.ToString('N0'))pg" }) -join ' ' $statusLine += " | Pages $pageStr" } - Write-LogHost $statusLine -ForegroundColor White + # PAX-1E: authoritative heartbeat built DIRECTLY from the shared worker + # observations + partition status (never parsed from Receive-Job text). + # Delivered through a latency-measuring wrapper that emits a neutral + # follow-up warning if terminal/log persistence itself was materially + # delayed (the cause is not classified as a Purview/host/folder failure). + $paxHostGapTotal = 0.0 + foreach ($o in @($script:QueryObservations.Values)) { if ($o -and $o.HostGapSecondsExcluded) { $paxHostGapTotal += [double]$o.HostGapSecondsExcluded } } + $paxHb = Format-PaxParentHeartbeat -PartitionStatuses @($script:partitionStatus.Values) -Observations @($script:QueryObservations.Values) -PageCounts $script:partitionPageCounts -TokenRefreshState $script:PaxTokenRefreshState -NowUtc $parentNowUtc -ExcludedHostGapSeconds $paxHostGapTotal -ResumeCompletedOffset $script:resumeCompletedOffset + $paxEventUtc = (Get-Date).ToUniversalTime() + $null = Invoke-PaxHeartbeatDelivery -EventUtc $paxEventUtc -ExpectedIntervalSeconds $StatusIntervalSeconds -ToleranceSeconds ([Math]::Max(30, $StatusIntervalSeconds)) -DeliverAction { + Write-LogHost $statusLine -ForegroundColor White + Write-LogHost ("[MONITOR] " + $paxHb.Text) -ForegroundColor DarkGray + } -WarnAction { + param($evt, $del, $el) + Write-LogHost ("[MONITOR] Terminal or log delivery was delayed ~{0}s (event {1:HH:mm:ss}Z, delivered {2:HH:mm:ss}Z); query state is being refreshed." -f $el, $evt, $del) -ForegroundColor Yellow + } $script:lastStatusUpdate = Get-Date } @@ -27762,7 +41398,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($pt -and $script:partitionStatus.ContainsKey($pt.Index)) { $currentStatus = $script:partitionStatus[$pt.Index].Status if ($currentStatus -notin 'Complete', 'Failed', 'Subdivided') { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'FETCH' -Reason 'JOB_ERROR' $script:partitionStatus[$pt.Index].LastError = $jobErrors[0].Exception.Message Write-LogHost " [RETRY-QUEUE] Partition $($pt.Index)/$($pt.Total) marked as Failed for retry (from monitoring loop)" -ForegroundColor Yellow } @@ -27834,7 +41470,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # CRITICAL: Mark this partition as Failed for retry after re-auth $pt = $jobMeta[$job.Id] if ($pt -and $script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'AUTH' -Reason '401' $script:partitionStatus[$pt.Index].LastError = '401 Unauthorized - token expired' } break # IMMEDIATE EXIT @@ -27875,7 +41511,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($output.SplitRequired -eq $true) { $pt = $jobMeta[$job.Id] if ($pt -and $script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'FETCH' -Reason 'STILL_RUNNING' $script:partitionStatus[$pt.Index].LastError = "Query still running after $([Math]::Round($output.ElapsedMinutes,1)) min - queued for retry" Write-LogHost " [RETRY-QUEUE] Partition $($pt.Index)/$($pt.Total) - query still processing after $([Math]::Round($output.ElapsedMinutes,1)) min, queued for retry" -ForegroundColor Yellow } @@ -27891,7 +41527,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($currentStatus -ne 'Complete') { if ([string]::IsNullOrWhiteSpace([string]$output.QueryId) -and [int]($output.RetrievedCount ?? 0) -le 0) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'CREATE' -Reason 'EMPTY_QUERYID' $script:partitionStatus[$pt.Index].LastError = 'ThreadJob returned empty QueryId with zero records (query was not created/sent)' Write-LogHost " [DATA-CHECK] Partition $($pt.Index) returned empty QueryId with 0 records - marked Failed for retry" -ForegroundColor Yellow [void]$script:processedJobIds.Add($job.Id) @@ -28150,7 +41786,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # Mark as Failed for retry (job exceptions need explicit status update) $pt = $jobMeta[$job.Id] if ($pt -and $script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECEIVE' -Reason 'RECEIVE_ERROR' $script:partitionStatus[$pt.Index].LastError = $_.Exception.Message Write-LogHost " [RETRY-QUEUE] Partition $($pt.Index)/$($pt.Total) marked as Failed for retry" -ForegroundColor Yellow } @@ -28164,6 +41800,27 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($null -ne $res) { # CHECK FOR SUBDIVISION SIGNAL: Job returned needs_subdivision status if ($res.Status -eq 'needs_subdivision') { + # Defensive runaway guard: if this lineage has already been subdivided + # $script:MaxSubdivisionLevel times and STILL reports the record limit, + # stop subdividing (do NOT multiply further). Mark the window with a + # DISTINCT 'SubdivisionLimit' state — not silently dropped, and not + # conflated with the terminal create/fetch failure class. -Resume can + # retry it later once the condition clears. + $currentSubLevel = if ($pt.SubdivisionLevel) { [int]$pt.SubdivisionLevel } else { 0 } + if ($currentSubLevel -ge $script:MaxSubdivisionLevel) { + $winText = "$($pt.PStart.ToString('yyyy-MM-dd HH:mm')) to $($pt.PEnd.ToString('yyyy-MM-dd HH:mm')) UTC" + Write-LogHost "" -ForegroundColor Red + Write-LogHost "[SUBDIVISION-LIMIT] Partition $($pt.Index)/$($pt.Total) reached the maximum subdivision depth ($script:MaxSubdivisionLevel) and still exceeds the record limit." -ForegroundColor Red + Write-LogHost " Window: $winText" -ForegroundColor Red + Write-LogHost " This window was NOT collected this run. Re-run with -Resume to retry it once the condition clears, or narrow the run's date range for this window." -ForegroundColor Yellow + if ($script:partitionStatus.ContainsKey($pt.Index)) { + $script:partitionStatus[$pt.Index].Status = 'SubdivisionLimit' + $script:partitionStatus[$pt.Index].RecordCount = 0 + $script:partitionStatus[$pt.Index].LastError = "Reached maximum subdivision depth ($script:MaxSubdivisionLevel) with the window still over the record limit" + } + [void]$script:processedJobIds.Add($job.Id) + continue + } Write-LogHost "[SUBDIVISION] Partition $($pt.Index)/$($pt.Total) requires subdivision - Preview count: $($res.PreviewCount)" -ForegroundColor Yellow # Calculate subdivision time windows @@ -28288,6 +41945,15 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc RecordCount = $existingStatus.RecordCount ParentPartition = $existingStatus.ParentPartition SubdivisionReason = $existingStatus.SubdivisionReason + LastFailureSignature = $existingStatus.LastFailureSignature + ConsecutiveIdenticalFailures = $existingStatus.ConsecutiveIdenticalFailures + IsTerminalFailure = $existingStatus.IsTerminalFailure + LastFailureStage = $existingStatus.LastFailureStage + LastFailureReason = $existingStatus.LastFailureReason + # Carry durable uncertain-create state forward so a subdivided + # partition never loses its no-re-POST guarantee. + UncertainCreate = $existingStatus.UncertainCreate + UncertainCreateInfo = $existingStatus.UncertainCreateInfo } } else { # New sub-partition - initialize fresh @@ -28300,6 +41966,13 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc LastError = $null RecordCount = 0 ParentPartition = $pt.Index + LastFailureSignature = $null + ConsecutiveIdenticalFailures = 0 + IsTerminalFailure = $false + LastFailureStage = $null + LastFailureReason = $null + UncertainCreate = $false + UncertainCreateInfo = $null } } } @@ -28341,6 +42014,149 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # Set auth failure flag to trigger token refresh before retry $script:AuthFailureDetected = $true } + elseif ($res.Status -eq 'visibility_failure') { + # STRUCTURED VISIBILITY-FAILURE RECOVERY (the worker never creates or + # replaces). The worker observed a 404 on the pinned version and RETURNED a + # structured signal instead of throwing. The parent revalidates the retained + # id through the shared decision layer (bounded same-id eventual-consistency + # retry and then a read-only same-id GET on the alternate version). Only after + # positively-confirmed absence does the parent attempt at-most-one replacement, + # persisted to the checkpoint BEFORE the POST so a crash or resume can never + # double-create. A recovered query is re-queued with its retained id and no + # POST; records are decided independently and never marked complete-with-zero. + $retainedId = if ($res.QueryId) { [string]$res.QueryId } elseif ($script:partitionStatus.ContainsKey($pt.Index)) { [string]$script:partitionStatus[$pt.Index].QueryId } else { $null } + $entry = if ($script:partitionStatus.ContainsKey($pt.Index)) { $script:partitionStatus[$pt.Index] } else { $null } + if ($entry) { + Set-GraphAuditLifecycleFields -Entry $entry + $entry.LastLifecycleHttpStatus = 404 + } + $revalidated = $null + if (-not [string]::IsNullOrWhiteSpace($retainedId)) { + $revalidated = Get-GraphAuditQueryStatus -QueryId $retainedId + } + if ($null -ne $revalidated) { + # Still visible (recovered on the pinned or the alternate version). Retain + # the id and re-queue WITHOUT any POST; this is not counted as a failure. + Write-LogHost "[VISIBILITY-RECOVER] Partition $($pt.Index)/$($pt.Total) - retained QueryId $retainedId is still visible (status '$($revalidated.Status)') - re-queued with the same id, no replacement" -ForegroundColor Cyan + if ($entry) { + $entry.QueryId = $retainedId + $entry.Status = 'NotStarted' + $entry.LastError = 'Visibility 404 recovered on revalidation - retained query re-queued' + $entry.AlternateVersionChecked = $true + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $pt.Index -QueryId $retainedId -State 'QueryCreated' -LifecycleState $entry + } + } + } + elseif ([string]::IsNullOrWhiteSpace($retainedId)) { + # No retained id to revalidate or replace. Re-queue through the normal + # create path (a null QueryId there triggers a duplicate-safe create). + Write-LogHost "[VISIBILITY-ABSENT] Partition $($pt.Index)/$($pt.Total) - visibility failure with no retained QueryId - re-queued for a duplicate-safe create" -ForegroundColor Yellow + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'QUERY-GONE' + } + else { + # Positively confirmed unavailable on both versions. + $alreadyAttempted = ($entry -and $entry.ContainsKey('ReplacementAttempted') -and $entry.ReplacementAttempted) + if ($alreadyAttempted) { + # Durable at-most-once guard: a replacement was already persisted. Never + # create a second one; mark Failed (bounded by the terminal guard), never + # complete-with-zero (which would silently drop the window). + Write-LogHost "[VISIBILITY-ABSENT] Partition $($pt.Index)/$($pt.Total) - retained QueryId $retainedId confirmed absent and a replacement was already attempted - not creating a second replacement" -ForegroundColor Yellow + if ($entry) { $entry.LastError = 'Query confirmed absent; replacement already attempted (at-most-once) - not replaced again' } + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'QUERY-GONE' + } + else { + # Persisting confirmed-absence + ReplacementAttempted is delegated to the + # resolver via -PersistReplacementIntent, so it happens ONLY after the + # resolver's own exact-name miss + confirmed absence, immediately before the + # sole replacement POST - never before an Adopt/FailClosed/StillVisible + # outcome that performs no POST. + $__persistIntent = { + param($__st) + if ($entry) { + $entry.QueryConfirmedAbsent = $true + $entry.ReplacementAttempted = $true + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $pt.Index -QueryId $retainedId -State 'QueryCreated' -LifecycleState $entry + } + } + } + # Duplicate-safe, at-most-one beta replacement after confirmed absence. + $__rState = New-GraphAuditLifecycleState + $__displayName = New-GraphAuditPartitionDisplayName -PStart $pt.PStart -PEnd $pt.PEnd -Index $pt.Index -Total $pt.Total -Activities $activities -RecordTypes $pt.RecordTypes -ServiceFilter $pt.ServiceFilter + $__listReplace = { + $__items = New-Object System.Collections.Generic.List[object] + $__lu = Get-GraphAuditApiUri -Path 'queries' + while ($__lu) { + $__page = Invoke-MgGraphRequest -Method GET -Uri $__lu -ErrorAction Stop + if ($__page -and $__page.value) { + foreach ($__q in $__page.value) { $__items.Add(@{ id = $__q.id; displayName = $__q.displayName }) } + } + $__lu = $null + if ($__page) { $__lu = $__page.'@odata.nextLink' } + } + return $__items.ToArray() + } + $__checkReplace = { + param($Version, $Qid) + try { + $__u = "https://graph.microsoft.com/$Version/security/auditLog/queries/$Qid" + $null = Invoke-MgGraphRequest -Method GET -Uri $__u -ErrorAction Stop + return @{ HttpStatus = 200 } + } + catch { + $__sc = $null + if ($_.Exception.Response) { + try { $__sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $__sc) { try { $__sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + if ($null -eq $__sc -and ($_.Exception.Message -match '404|Not Found')) { $__sc = 404 } + return @{ HttpStatus = $__sc } + } + } + $__createReplace = { + param($Version, $Name) + try { + $__body = New-GraphAuditQueryRequest -PStart $pt.PStart -PEnd $pt.PEnd -Activities $activities -RecordTypes $pt.RecordTypes -ServiceFilter $pt.ServiceFilter + $__body.displayName = $Name + $__u = "https://graph.microsoft.com/$Version/security/auditLog/queries" + $__resp = Invoke-MgGraphRequest -Method POST -Uri $__u -Body $__body -ErrorAction Stop + $__id = $null + if ($__resp) { $__id = $__resp.id } + return @{ HttpStatus = 201; QueryId = $__id; Ambiguous = $false } + } + catch { + $__sc = $null + if ($_.Exception.Response) { + try { $__sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $__sc) { try { $__sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + return @{ HttpStatus = $__sc; QueryId = $null; Ambiguous = ($__sc -eq 409) } + } + } + $__repl = Resolve-GraphAuditDuplicateSafeQuery -DisplayName $__displayName -ExistingQueryId $retainedId -State $__rState -ListQueries $__listReplace -CheckItem $__checkReplace -CreateQuery $__createReplace -PersistReplacementIntent $__persistIntent + if ($__repl -and $__repl.QueryId) { + Write-LogHost "[VISIBILITY-REPLACE] Partition $($pt.Index)/$($pt.Total) - confirmed-absent query replaced (Action=$($__repl.Action); Reason=$($__repl.Reason)); new QueryId $($__repl.QueryId) - re-queued" -ForegroundColor Yellow + if ($entry) { + $entry.QueryId = [string]$__repl.QueryId + $entry.ReplacementQueryId = [string]$__repl.QueryId + $entry.Status = 'NotStarted' + $entry.LastError = 'Confirmed-absent query replaced (at-most-once) - re-queued with new id' + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $pt.Index -QueryId ([string]$__repl.QueryId) -State 'QueryCreated' -LifecycleState $entry + } + } + } + else { + Write-LogHost "[VISIBILITY-REPLACE] Partition $($pt.Index)/$($pt.Total) - replacement not created (Action=$($__repl.Action); Reason=$($__repl.Reason)) - marked Failed for bounded retry" -ForegroundColor Yellow + if ($entry) { $entry.LastError = "Confirmed absent; replacement not created (Action=$($__repl.Action); Reason=$($__repl.Reason))" } + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'QUERY-GONE' + } + } + } + if ($res.Telemetry) { $script:telemetryData += $res.Telemetry } + [void]$script:processedJobIds.Add($job.Id) + } else { # NORMAL COMPLETION: Not a subdivision - process as completed query # Track whether data was actually added to collection @@ -28357,15 +42173,25 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # Update partition status tracking - mark Complete if query succeeded (even with 0 records) if ($script:partitionStatus.ContainsKey($pt.Index)) { - $queryName = "PAX_Query_$($pt.PStart.ToString('yyyyMMdd_HHmm'))-$($pt.PEnd.ToString('yyyyMMdd_HHmm'))_Part$($pt.Index)/$($pt.Total)" + $queryName = New-GraphAuditPartitionDisplayName -PStart $pt.PStart -PEnd $pt.PEnd -Index $pt.Index -Total $pt.Total -Activities $activities -RecordTypes $pt.RecordTypes -ServiceFilter $pt.ServiceFilter $script:partitionStatus[$pt.Index].QueryName = $queryName $script:partitionStatus[$pt.Index].Status = 'Complete' $script:partitionStatus[$pt.Index].QueryId = $res.QueryId $script:partitionStatus[$pt.Index].RecordCount = $res.RetrievedCount + # Thread the structured lifecycle fields into the completion checkpoint. A + # genuine zero-record completion is an explicit, first-class terminal state + # (ZeroRecordComplete) - distinct from a dropped or never-run window - so a + # resume treats it as done rather than re-querying. Legacy-safe: fields are + # added only if missing and the stored QueryId is never overwritten here. + $__completeEntry = $script:partitionStatus[$pt.Index] + Set-GraphAuditLifecycleFields -Entry $__completeEntry | Out-Null + if ($res.QueryApiVersion) { $__completeEntry.QueryApiVersion = [string]$res.QueryApiVersion } + $__completeEntry.ZeroRecordComplete = ([int]$res.RetrievedCount -eq 0) + # CHECKPOINT: Save Completed state - this partition's data is now fully fetched if ($script:CheckpointEnabled) { - Save-Checkpoint -PartitionIndex $pt.Index -QueryId $res.QueryId -State 'Completed' + Save-Checkpoint -PartitionIndex $pt.Index -QueryId $res.QueryId -State 'Completed' -LifecycleState $__completeEntry } # INCREMENTAL SAVE: Write partition records to disk immediately (prevents data loss on auth failure) @@ -28450,7 +42276,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } if ($pt -and $script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'JOB_FAILED' $script:partitionStatus[$pt.Index].LastError = 'ThreadJob failed before completion' } @@ -28484,7 +42310,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # Mark partition as Failed for retry if ($pt -and $script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'ERROR_STREAM' $script:partitionStatus[$pt.Index].LastError = $errorMsg Write-LogHost " [RETRY-QUEUE] Partition $($pt.Index)/$($pt.Total) marked as Failed for retry due to error stream" -ForegroundColor Yellow } @@ -28536,7 +42362,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($jobErrors.Count -gt 0 -and $script:partitionStatus[$pt.Index].Status -notin @('Complete', 'Subdivided')) { # Job completed WITH errors - mark as Failed for retry - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'ERROR_STREAM' $script:partitionStatus[$pt.Index].LastError = $jobErrors[0].ToString() # QueryId preserved — query may still be valid on Purview for retry resume Write-LogHost " [ERROR-CHECK] Partition $($pt.Index) job completed with errors - marked Failed for retry" -ForegroundColor Yellow @@ -28565,7 +42391,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } } else { # No JSONL file — job finished but never saved data (likely 504/auth failure swallowed internally) - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'NO_DATA' $script:partitionStatus[$pt.Index].LastError = 'Job completed without saving data (no JSONL file found) - likely network/auth failure swallowed by retry logic' Write-LogHost " [DATA-CHECK] Partition $($pt.Index) job completed but NO data saved to disk - marked Failed for retry" -ForegroundColor Yellow } @@ -28574,7 +42400,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $queryIdMissing = [string]::IsNullOrWhiteSpace([string]$script:partitionStatus[$pt.Index].QueryId) $recordCountZero = ([int]($script:partitionStatus[$pt.Index].RecordCount ?? 0) -le 0) if ($queryIdMissing -and $recordCountZero) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'INVALID_COMPLETE' $script:partitionStatus[$pt.Index].LastError = 'Partition marked Complete but has empty QueryId and zero records' Write-LogHost " [DATA-CHECK] Partition $($pt.Index) had invalid completion state (empty QueryId + 0 records) - marked Failed for retry" -ForegroundColor Yellow } @@ -28587,7 +42413,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc foreach ($failedJob in $failedJobs) { $pt = $jobMeta[$failedJob.Id] if ($script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'JOB_FAILED' try { $errorOutput = Receive-Job -Job $failedJob -ErrorAction SilentlyContinue 2>&1 if ($errorOutput) { @@ -28616,8 +42442,9 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $partitionsToRetry = @() foreach ($idx in $script:partitionStatus.Keys) { $status = $script:partitionStatus[$idx] - # Only retry actual failures, not queries that completed with 0 records - if ($status.Status -in @('Failed', 'NotStarted')) { + # Only retry actual failures, not queries that completed with 0 records, + # and never retry terminally-failed partitions. + if ($status.Status -in @('Failed', 'NotStarted') -and -not $status.IsTerminalFailure) { if ($status.AttemptNumber -lt $maxAttempts) { $partitionsToRetry += $status.Partition } @@ -28716,8 +42543,110 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $retryJobMeta = @{} $retryMaxConcurrency = [Math]::Min($maxConcurrentPartitions, 3) Write-LogHost " [RETRY] Using reduced retry concurrency: $retryMaxConcurrency (initial phase: $maxConcurrentPartitions)" -ForegroundColor DarkCyan - + + # --- Parent-side query creation pre-pass (bounded parallel, retry phase) --- + # A retry partition that never obtained a query id has nothing to reuse. + # Create it here in bounded parallel under the reduced retry bound so its + # status QueryId is populated before the retry worker launches. A creation + # failure is recorded so no worker is launched for that partition this pass. + $retryCreateFailed = [System.Collections.Generic.HashSet[int]]::new() + $retryNeedCreate = New-Object System.Collections.Generic.List[object] + foreach ($cp in $partitionsToRetry) { + # Never re-create a durable uncertain-create partition: a possibly-accepted + # first POST is resolved READ-ONLY on resume, never by a second POST here. + if ($script:partitionStatus.ContainsKey($cp.Index) -and $script:partitionStatus[$cp.Index].UncertainCreate) { continue } + if (-not ($script:partitionStatus.ContainsKey($cp.Index) -and $script:partitionStatus[$cp.Index].QueryId)) { + [void]$retryNeedCreate.Add($cp) + } + } + if ($retryNeedCreate.Count -gt 0) { + $retryCreateBound = [Math]::Max(1, [int]$retryMaxConcurrency) + Write-LogHost " [PARENT-CREATE] Creating $($retryNeedCreate.Count) retry partition query(ies) in bounded parallel (limit $retryCreateBound)..." -ForegroundColor DarkCyan + # Refresh the token before the retry create pre-pass so every queued and + # retrying create job reads a current token from the shared auth state. + $retryCreateRefresh = Refresh-GraphTokenIfNeeded -BufferMinutes 5 + if ($retryCreateRefresh -is [string] -and $retryCreateRefresh -eq 'Quit') { + if ($script:CheckpointEnabled) { Save-Checkpoint -Force } + Show-CheckpointExitMessage + exit 0 + } + $retryCreateMap = Invoke-PaxParentQueryCreation -Partitions $retryNeedCreate.ToArray() -ThrottleLimit $retryCreateBound -Activities $activities -MaxNetworkOutageMinutes $MaxNetworkOutageMinutes + # A token quit or refresh error during creation fails the pass closed before + # any partition is persisted: enter the graceful authentication-exit path so + # nothing is recorded as created off a quit/error decision (resume re-runs the + # pre-create reconcile, which adopts an already-created query without a + # duplicate POST). + if ($retryCreateMap['__AuthQuit']) { + # Persist any query the pool actually created before the quit AND mark the + # still-in-flight partitions durable-uncertain, so a known QueryId is never + # discarded and a possibly-accepted in-flight POST is reconciled read-only on + # resume. A user-driven quit stays a graceful (exit 0) checkpoint-and-resume. + Save-PaxInterruptedCreatePool -CreateMap $retryCreateMap -Requested $retryNeedCreate -Activities $activities -Phase 'retry' + Write-LogHost " [PARENT-CREATE] Authentication quit requested during query creation - completed persisted, in-flight marked uncertain, saving checkpoint and exiting (use -Resume to continue)" -ForegroundColor Yellow + if ($script:CheckpointEnabled) { Save-Checkpoint -Force } + Show-CheckpointExitMessage + exit 0 + } + if ($retryCreateMap['__RefresherError']) { + # Same durable persistence as the quit path, but a token-refresh EXCEPTION is a + # failure, not a user decision: after persisting completed + in-flight-uncertain + # it terminates NONZERO through the established fatal-exit machinery + # ($script:GenericFatal -> exit 1) so automation sees the run did not succeed. + Save-PaxInterruptedCreatePool -CreateMap $retryCreateMap -Requested $retryNeedCreate -Activities $activities -Phase 'retry' + Write-LogHost " [PARENT-CREATE] Token refresh error during query creation ($($retryCreateMap['__RefresherError'])) - completed persisted, in-flight marked uncertain, saving checkpoint and exiting NONZERO (use -Resume to continue)" -ForegroundColor Red + $script:GenericFatal = $true + if ($script:CheckpointEnabled) { Save-Checkpoint -Force } + Show-CheckpointExitMessage + exit 1 + } + foreach ($cp in $retryNeedCreate) { + $cr = $retryCreateMap[[int]$cp.Index] + if ($cr -and $cr.QueryId) { + $cp | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue ([string]$cr.QueryId) -Force + $script:partitionStatus[$cp.Index].QueryId = [string]$cr.QueryId + $script:partitionStatus[$cp.Index].QueryApiVersion = $cr.Version + $script:partitionStatus[$cp.Index].LifecycleOperation = 'Create' + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -QueryId ([string]$cr.QueryId) -State 'QueryCreated' -LifecycleState $script:partitionStatus[$cp.Index] + } + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) query created (QueryId: $($cr.QueryId))" -ForegroundColor DarkCyan + } else { + $cat = if ($cr) { [string]$cr.FailureCategory } else { 'NotCreated' } + [void]$retryCreateFailed.Add([int]$cp.Index) + if ($cat -eq 'UncertainCreateUnresolved') { + # A first POST whose acceptance could not be proven is NEVER re-POSTed. + # Persist durable uncertain-create state so a resume routes this + # partition to a READ-ONLY reconcile instead of a duplicate create. + # Deliberately NOT Set-PartitionFailure: not marked Failed, never requeued. + $script:partitionStatus[$cp.Index].UncertainCreate = $true + $script:partitionStatus[$cp.Index].UncertainCreateInfo = @{ + displayName = New-GraphAuditPartitionDisplayName -PStart $cp.PStart -PEnd $cp.PEnd -Index $cp.Index -Total $cp.Total -Activities $activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + fingerprint = New-GraphAuditQueryFingerprint -PStart $cp.PStart -PEnd $cp.PEnd -Activities $activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + pStart = $cp.PStart.ToString('o') + pEnd = $cp.PEnd.ToString('o') + attempt = [int]$script:partitionStatus[$cp.Index].AttemptNumber + phase = 'retry' + } + $script:partitionStatus[$cp.Index].LastError = "Parent-side query creation uncertain - durable reconcile pending (category: $cat)" + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -State 'UncertainCreate' -LifecycleState $script:partitionStatus[$cp.Index] + } + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) create uncertain (category: $cat) - persisted for read-only reconcile, no re-POST, retry worker not launched this pass" -ForegroundColor Yellow + } else { + Set-PartitionFailure -Index $cp.Index -Stage 'CREATE' -Reason $cat + $script:partitionStatus[$cp.Index].LastError = "Parent-side query creation failed (category: $cat)" + $diag = "HTTP $($cr.HttpStatus) | client-request-id: $($cr.ClientRequestId) | request-id: $($cr.RequestId)" + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) creation failed (category: $cat; $diag) - retry worker not launched this pass" -ForegroundColor Yellow + if ($cr.ResponseBody) { Write-LogHost " Graph response: $($cr.ResponseBody)" -ForegroundColor DarkYellow } + } + } + } + } + foreach ($pt in $partitionsToRetry) { + # Skip launching a retry worker for any partition whose parent-side + # creation failed this pass; it has no query id to reuse. + if ($retryCreateFailed.Contains([int]$pt.Index)) { continue } $script:partitionStatus[$pt.Index].AttemptNumber++ $script:partitionStatus[$pt.Index].Status = 'NotStarted' # Reset for retry @@ -28725,20 +42654,27 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # Re-create the job using the same scriptblock # CRITICAL: Use $graphResultSize (0 for Graph API = unlimited) instead of $ResultSize (10000) - # Pass existing QueryId if available (for retry after 403 fetch failure) + # Pass existing QueryId if available (retry/resume of an already-created query). $existingQueryId = $script:partitionStatus[$pt.Index].QueryId - # Clear dead QueryId if the partition failed due to 404 (query vanished server-side). - # A dead QueryId would cause infinite retry loops polling a non-existent query. - $lastErr = $script:partitionStatus[$pt.Index].LastError - if ($existingQueryId -and $lastErr -and ($lastErr -match 'QUERY-GONE|404.*Not Found|404.*status poll')) { - Write-LogHost " [QUERY-GONE] Clearing dead QueryId $existingQueryId for Partition $($pt.Index)/$($pt.Total) - will CREATE fresh query" -ForegroundColor Yellow - $script:partitionStatus[$pt.Index].QueryId = $null - $existingQueryId = $null - } + # Structured retry control: NEVER clear an existing QueryId based on error + # TEXT. A single 404 (or any diagnostic LastError string) is not proof the + # query is absent - portal-completed queries were observed returning 404 on + # one API version while remaining fully retrievable on the pinned version, + # and eventual consistency can briefly 404 a freshly-created query. Reusing + # the QueryId lets the pinned proven-lifecycle version re-poll and fetch the + # already-completed server-side query instead of creating a duplicate. The + # parent pre-pass creates any missing query before this retry launch, so a + # partition that reaches launch always carries a QueryId to reuse. Partitions + # that stay unretrievable are bounded by the consecutive-failure terminal guard + # ($script:MaxConsecutiveIdenticalFailures), not by clearing the ID, so no + # duplicate replacement query is ever created while the original ID remains. + # LastError is diagnostic-only and no longer controls retry. if ($existingQueryId) { - Write-LogHost " [REUSE] Reusing existing QueryId: $existingQueryId" -ForegroundColor Cyan + Write-LogHost " [REUSE] Reusing existing QueryId: $existingQueryId (retry reuses the created query; no duplicate is created)" -ForegroundColor Cyan } - $job = Start-ThreadJob -ThrottleLimit $retryMaxConcurrency -ScriptBlock $queryJobScriptBlock -ArgumentList $pt.PStart, $pt.PEnd, $activities, $graphResultSize, $UserIds, $pt.Index, $pt.Total, $script:SharedAuthState, $pt, $MaxNetworkOutageMinutes, $script:GraphAuditApiVersion, $script:LogFile, $existingQueryId, $threadIncrementalDir, $threadRunTimestamp, $threadMemoryFlushEnabled + # The worker polls and fetches on the SAME version the query was created on. + $effectiveApiVersion = if ($script:partitionStatus[$pt.Index].QueryApiVersion) { $script:partitionStatus[$pt.Index].QueryApiVersion } else { $script:GraphAuditApiVersion } + $job = Start-ThreadJob -ThrottleLimit $retryMaxConcurrency -ScriptBlock $queryJobScriptBlock -ArgumentList $pt.PStart, $pt.PEnd, $activities, $graphResultSize, $script:targetUsers, $pt.Index, $pt.Total, $script:SharedAuthState, $pt, $MaxNetworkOutageMinutes, $effectiveApiVersion, $script:LogFile, $existingQueryId, $threadIncrementalDir, $threadRunTimestamp, $threadMemoryFlushEnabled, $script:QueryObservations $retryJobs += $job $retryJobMeta[$job.Id] = $pt } @@ -28779,7 +42715,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $errorText = ($job.Error | Out-String).Trim() Write-LogHost "[ERROR-STREAM] Retry Partition $($pt.Index)/$($pt.Total) - Job completed but had error: $errorText" -ForegroundColor Yellow if ($script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RECONCILE' -Reason 'ERROR_STREAM' $script:partitionStatus[$pt.Index].LastError = $errorText Write-LogHost " [RETRY-QUEUE] Partition $($pt.Index)/$($pt.Total) marked as Failed for next retry pass due to error stream" -ForegroundColor Yellow } @@ -28802,14 +42738,14 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($null -ne $res -and $script:partitionStatus.ContainsKey($pt.Index)) { if ([string]::IsNullOrWhiteSpace([string]$res.QueryId) -and [int]($res.RetrievedCount ?? 0) -le 0) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'CREATE' -Reason 'EMPTY_QUERYID' $script:partitionStatus[$pt.Index].LastError = 'Retry ThreadJob returned empty QueryId with zero records (query was not created/sent)' Write-LogHost " [DATA-CHECK] Retry Partition $($pt.Index) returned empty QueryId with 0 records - marked Failed for next retry" -ForegroundColor Yellow continue } $script:partitionStatus[$pt.Index].QueryId = $res.QueryId - $script:partitionStatus[$pt.Index].QueryName = "PAX_Query_$($pt.PStart.ToString('yyyyMMdd_HHmm'))-$($pt.PEnd.ToString('yyyyMMdd_HHmm'))_Part$($pt.Index)/$($pt.Total)" + $script:partitionStatus[$pt.Index].QueryName = New-GraphAuditPartitionDisplayName -PStart $pt.PStart -PEnd $pt.PEnd -Index $pt.Index -Total $pt.Total -Activities $activities -RecordTypes $pt.RecordTypes -ServiceFilter $pt.ServiceFilter $script:partitionStatus[$pt.Index].RecordCount = $res.RetrievedCount $script:partitionStatus[$pt.Index].Status = 'Complete' @@ -28849,42 +42785,47 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } catch { Write-LogHost " ✗ Retry failed for Partition $($pt.Index)/$($pt.Total): $($_.Exception.Message)" -ForegroundColor Red if ($script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + $isAuthFailure = Test-TransientAuthSignal -Text $_.Exception.Message + Set-PartitionFailure -Index $pt.Index -Stage $(if ($isAuthFailure) { 'AUTH' } else { 'RETRY' }) -Reason $(if ($isAuthFailure) { '401' } else { 'EXCEPTION' }) $script:partitionStatus[$pt.Index].LastError = $_.Exception.Message - - # 401/Unauthorized failures don't count against retry limit (auth issue, not partition issue) - if ($_.Exception.Message -match '401|Unauthorized') { + + # Genuine transient-auth failures don't count against retry limit (auth issue, not partition issue) + if ($isAuthFailure) { $script:partitionStatus[$pt.Index].AttemptNumber-- - Write-LogHost " [AUTH] 401 error detected - retry attempt not counted (auth issue, not partition issue)" -ForegroundColor Yellow + Write-LogHost " [AUTH] Transient auth failure detected - retry attempt not counted (auth issue, not partition issue)" -ForegroundColor Yellow } } } } elseif ($job.State -eq 'Failed') { Write-LogHost " ✗ Retry job failed for Partition $($pt.Index)/$($pt.Total)" -ForegroundColor Red if ($script:partitionStatus.ContainsKey($pt.Index)) { - $script:partitionStatus[$pt.Index].Status = 'Failed' + # Read the job's error output FIRST so the failure can be classified + # (auth vs generic) before it is recorded and counted. + $lastErrorText = $null try { $errorOutput = Receive-Job -Job $job -ErrorAction SilentlyContinue 2>&1 - if ($errorOutput) { - $lastErrorText = ($errorOutput | Out-String).Trim() - $script:partitionStatus[$pt.Index].LastError = $lastErrorText - - # Log the actual error for diagnostics (useful level) - Write-LogHost " Error details: $lastErrorText" -ForegroundColor DarkYellow - - # 401/Unauthorized failures don't count against retry limit (auth issue, not partition issue) - if ($lastErrorText -match '401|Unauthorized') { - $script:partitionStatus[$pt.Index].AttemptNumber-- - Write-LogHost " [AUTH] 401 error detected - retry attempt not counted (auth issue, not partition issue)" -ForegroundColor Yellow - } - } + if ($errorOutput) { $lastErrorText = ($errorOutput | Out-String).Trim() } } catch {} + $isAuthFailure = Test-TransientAuthSignal -Text $lastErrorText + Set-PartitionFailure -Index $pt.Index -Stage $(if ($isAuthFailure) { 'AUTH' } else { 'RECONCILE' }) -Reason $(if ($isAuthFailure) { '401' } else { 'JOB_FAILED' }) + if ($lastErrorText) { + $script:partitionStatus[$pt.Index].LastError = $lastErrorText + # Log the actual error for diagnostics (useful level) + Write-LogHost " Error details: $lastErrorText" -ForegroundColor DarkYellow + } + # Genuine transient-auth failures don't count against retry limit (auth issue, not partition issue) + if ($isAuthFailure) { + $script:partitionStatus[$pt.Index].AttemptNumber-- + Write-LogHost " [AUTH] Transient auth failure detected - retry attempt not counted (auth issue, not partition issue)" -ForegroundColor Yellow + } } } } # Cleanup retry jobs - Remove-Job -Job $retryJobs -Force -ErrorAction SilentlyContinue | Out-Null + if ($retryJobs.Count -gt 0) { + Remove-Job -Job $retryJobs -Force -ErrorAction SilentlyContinue | Out-Null + } } # FINAL SUMMARY AND RECONCILIATION @@ -28896,7 +42837,22 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $totalPartitions = $script:partitionStatus.Count $completedPartitions = @($script:partitionStatus.Values | Where-Object { $_.Status -eq 'Complete' }) $sentButIncomplete = @($script:partitionStatus.Values | Where-Object { $_.Status -eq 'Sent' -and $_.QueryId }) - $neverSent = @($script:partitionStatus.Values | Where-Object { $_.Status -in @('Failed', 'NotStarted') -and -not $_.QueryId }) + $terminallyFailed = @($script:partitionStatus.Values | Where-Object { $_.IsTerminalFailure }) + $subdivisionLimited = @($script:partitionStatus.Values | Where-Object { $_.Status -eq 'SubdivisionLimit' }) + $neverSent = @($script:partitionStatus.Values | Where-Object { $_.Status -in @('Failed', 'NotStarted') -and -not $_.QueryId -and -not $_.IsTerminalFailure }) + # Record "completed with gaps" signals for the process exit code (read at script end). + # Set-only (monotonic) so a later summary pass cannot clear a gap already seen. + if ($terminallyFailed.Count -gt 0) { $script:HadTerminalFailures = $true } + if ($subdivisionLimited.Count -gt 0) { $script:HadSubdivisionLimit = $true } + # Bounded partition-stability verification (opt-in via -VerifyPartitionStability). A partition + # certified Complete can still be a STRICT SUBSET of the true result (observed live: SharePoint + # partition 26 = 13,350 vs 15,465, 2,115 RecordIds silently lost). Invoke-PartitionStabilityVerification + # runs ONE independent bounded repeat fetch per completed partition, populating .RecordIds (first + # fetch, from persisted output) and .StabilityRepeatIds (the single repeat query), then + # Test-AuditPartitionConsistency treats any RecordId-set difference (missing OR extra) as a gap + # (sets HadTerminalFailures -> exit 40): never silently certified, never blind-accept-larger, never + # unioned, never count-only. Default OFF => no repeat query, no RecordId retention => byte-identical. + script:Invoke-PartitionStabilityVerification -CompletedPartitions $completedPartitions -Enabled:$VerifyPartitionStability -TargetUsers $script:targetUsers Write-LogHost " Total Partitions: $totalPartitions" -ForegroundColor White Write-LogHost " Sent and Complete: $($completedPartitions.Count)" -ForegroundColor Green @@ -28917,7 +42873,30 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc Write-LogHost " - Partition $($pt.Index)/$($pt.Total): QueryName=$($status.QueryName)$errorMsg" -ForegroundColor Red } } - + + if ($subdivisionLimited.Count -gt 0) { + Write-LogHost " [!] Subdivision-Limit (window exceeded max subdivision depth $script:MaxSubdivisionLevel — NOT a create/fetch failure): $($subdivisionLimited.Count)" -ForegroundColor Red + Write-LogHost " These windows were not collected this run; re-run with -Resume to retry them once the underlying condition clears." -ForegroundColor Yellow + foreach ($status in $subdivisionLimited) { + $pt = $status.Partition + $win = "$($pt.PStart.ToString('yyyy-MM-dd HH:mm')) to $($pt.PEnd.ToString('yyyy-MM-dd HH:mm')) UTC" + Write-LogHost " - Partition $($pt.Index)/$($pt.Total): $win" -ForegroundColor Red + } + } + + if ($terminallyFailed.Count -gt 0) { + Write-LogHost " ✗ Terminally Failed (stopped after $script:MaxConsecutiveIdenticalFailures consecutive same-category failures): $($terminallyFailed.Count)" -ForegroundColor Red + Write-LogHost " Not retried further this run; re-run with -Resume to retry them once the underlying issue clears." -ForegroundColor Yellow + foreach ($status in $terminallyFailed) { + $pt = $status.Partition + $label = if ($status.LastFailureStage -and $status.LastFailureReason) { "$($status.LastFailureStage)/$($status.LastFailureReason)" } else { 'UNKNOWN' } + $win = "$($pt.PStart.ToString('yyyy-MM-dd HH:mm')) to $($pt.PEnd.ToString('yyyy-MM-dd HH:mm')) UTC" + $qid = if ($status.QueryId) { " | QueryId=$($status.QueryId)" } else { "" } + Write-LogHost " - Partition $($pt.Index)/$($pt.Total): $label x$($status.ConsecutiveIdenticalFailures) | $win$qid" -ForegroundColor Red + if ($status.LastError) { Write-LogHost " Last error: $($status.LastError)" -ForegroundColor DarkYellow } + } + } + # Check for any missing/skipped partitions (exclude intentionally skipped partitions from resume mode) $expectedPartitions = 1..$totalPartitions $attemptedPartitions = @($script:partitionStatus.Keys) @@ -29013,7 +42992,9 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } # Cleanup - Remove-Job -Job $jobs -Force -ErrorAction SilentlyContinue | Out-Null + if ($jobs.Count -gt 0) { + Remove-Job -Job $jobs -Force -ErrorAction SilentlyContinue | Out-Null + } # Normalize progress if needed if ($script:progressState.Query.Current -lt $script:progressState.Query.Total -and $script:progressState.Query.Total -le 200) { @@ -29211,7 +43192,7 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc Save-Checkpoint -PartitionIndex $pt.Index -State 'Completed' -RecordCount ($logArray.Count) } } catch { - $script:partitionStatus[$pt.Index].Status = 'Failed' + Set-PartitionFailure -Index $pt.Index -Stage 'RETRY' -Reason 'EXCEPTION' $script:partitionStatus[$pt.Index].LastError = $_.Exception.Message Write-LogHost " [FAILED] Partition $($pt.Index): $($_.Exception.Message)" -ForegroundColor Red } @@ -29716,7 +43697,9 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } # For OnlyUserInfo mode, skip the early return - we don't need audit logs, just Entra data # For streaming merge mode, $allLogs is intentionally empty - data will be streamed from JSONL files - if ($allLogs.Count -eq 0 -and -not $OnlyUserInfo -and -not $script:UseStreamingMergeForExport) { + # -Dashboard AISID still emits the header-only Purview CSV here but then CONTINUES + # to the same downstream finalization instead of returning (Test-PaxZeroRecordContinuesAfterHeaderOnly). + if (script:Test-PaxZeroRecordHeaderOnlyPath -AllLogsCount $allLogs.Count -OnlyUserInfo ([bool]$OnlyUserInfo) -UseStreamingMergeForExport ([bool]$script:UseStreamingMergeForExport)) { Write-LogHost ""; Write-LogHost "No audit logs found in the specified date range for the selected activity types." -ForegroundColor Yellow # Warn if token refresh occurred during monitoring — 0 records may indicate auth-related data loss if ($script:SharedAuthState.RefreshCount -gt 0) { @@ -29804,15 +43787,28 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc # that normally writes these lines. $script:metrics.TotalStructuredRows = 0 $script:metrics.EffectiveChunkSize = 0 - Set-ProgressPhase -Phase 'Complete' -Status 'No data' - Complete-Progress - Write-LogHost "" - Write-LogHost "Pipeline Summary:" -ForegroundColor Cyan - Write-LogHost " Retained: 0 records" -ForegroundColor White - Write-LogHost " Exported: 0 rows" -ForegroundColor White - Write-LogHost ("Log file: {0}" -f (Get-DisplayPath -LocalPath $LogFile)) -ForegroundColor White - $script:ScriptCompleted = $true - return + if (script:Test-PaxZeroRecordContinuesAfterHeaderOnly -Dashboard ([string]$Dashboard)) { + # A zero-record AISID run CONTINUES to the common downstream finalization + # (Rollup empty Fact + Users dimension -> normal Entra publication -> AISID export of all twelve + # CSVs incl. EntraUsers.csv -> completion summary + output roster -> truthful exit). The + # header-only Purview CSV emitted above is the Rollup's input; the explosion is skipped (no + # records to explode) via $script:ZeroRecordContinueToPostProcessing, and the "0 records" + # pipeline summary + ScriptCompleted are deferred to the normal end-of-run tail so the AISID + # completion/roster are reported truthfully. Zero Purview records are a valid empty result. + $script:ZeroRecordContinueToPostProcessing = $true + Write-LogHost "No Purview records: continuing to Rollup + Entra + AISID export (zero records is a valid empty result for -Dashboard AISID)..." -ForegroundColor Cyan + } + else { + Set-ProgressPhase -Phase 'Complete' -Status 'No data' + Complete-Progress + Write-LogHost "" + Write-LogHost "Pipeline Summary:" -ForegroundColor Cyan + Write-LogHost " Retained: 0 records" -ForegroundColor White + Write-LogHost " Exported: 0 rows" -ForegroundColor White + Write-LogHost ("Log file: {0}" -f (Get-DisplayPath -LocalPath $LogFile)) -ForegroundColor White + $script:ScriptCompleted = $true + return + } } # Determine explosion mode: @@ -29876,6 +43872,24 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $useParallelExplosion = $script:IsPS7 -and ($effectiveRecordCount -gt 500) -and ($actualExplosionThreads -gt 1) $parallelBatchSize = 1000 # Records per parallel batch $parallelThrottleLimit = $actualExplosionThreads + + # ── Explosion-mode bounded input spool (Blocker 1) ────────────────────────── + # Capture the input count, then (in explosion mode) stream the deduped/trimmed + # normalized records to a re-iterable JSON-lines disk spool and release the + # in-memory collection so the explosion pass holds neither the complete + # normalized set nor the exploded rows in memory. Serial sample-based schema + # discovery streams from the spool. Mirrors the accepted RAW replay spool. + $explosionInputCount = $allLogs.Count + $script:ExplosionInputSpoolPath = $null + if ($fullExplode -and -not $script:UseStreamingMergeForExport) { + $script:ExplosionInputSpoolPath = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_explode_spool_" + [guid]::NewGuid().ToString() + ".jsonl") + $null = Write-PaxNormalizedSpool -Records $allLogs -SpoolPath $script:ExplosionInputSpoolPath + try { Write-PaxMemoryObservation -Stage 'ExplosionSpool' -RowsProcessed ([long]$explosionInputCount) -Force } catch {} + $allLogs = New-Object System.Collections.ArrayList # release complete normalized set before explosion + [System.GC]::Collect() + $useParallelExplosion = $false # bounded serial sample-schema path streams from the spool + Write-LogHost ("Explosion input spooled to disk ({0:N0} records); streaming bounded explosion..." -f $explosionInputCount) -ForegroundColor DarkCyan + } if ($useParallelExplosion) { Write-LogHost "Parallel processing: ENABLED (PS7+ detected, $parallelThrottleLimit threads)" -ForegroundColor Green @@ -29899,8 +43913,8 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $schemaFrozen = $false; $schemaSampleRows = New-Object System.Collections.Generic.List[object]; $postFreezeNewColumns = 0; $lateIgnoredColumns = New-Object System.Collections.Generic.HashSet[string]; $columnOrder = $null; $buffer = New-Object System.Collections.Generic.List[object]; $exportTemp = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_export_" + [guid]::NewGuid().ToString() + ".tmp"); $csvWriter = $false # Ensure TotalRecordsFetched reflects allLogs count (may not have been set in all code paths) - if ($script:metrics.TotalRecordsFetched -eq 0 -and $allLogs.Count -gt 0) { - $script:metrics.TotalRecordsFetched = $allLogs.Count + if ($script:metrics.TotalRecordsFetched -eq 0 -and $explosionInputCount -gt 0) { + $script:metrics.TotalRecordsFetched = $explosionInputCount } # ═══════════════════════════════════════════════════════════════════════════════ @@ -30331,6 +44345,10 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc } else { $skipToPostProcessing = $false } + # A zero-record AISID run emitted the header-only Purview CSV above and must skip the + # explosion (there are no records to explode) yet still reach post-processing (Rollup -> Entra -> + # AISID export -> completion/roster). Route it to post-processing using the already-written CSV. + if ($script:ZeroRecordContinueToPostProcessing) { $skipToPostProcessing = $true } # ═══════════════════════════════════════════════════════════════════════════════ # PARALLEL MODE: Skip serial loop entirely - do parallel schema discovery + processing @@ -30347,8 +44365,60 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc $explosionProgressInterval = 20000 $processedRecordCount = 0 $lastProgressUpdate = 0 - - foreach ($log in $allLogs) { + + # ── Full-schema discovery prepass (Blocker 2) ────────────────────────────── + # Scan the COMPLETE spool once, exploding each record to derive every column the + # selected mode can emit — including columns that first appear only in late + # records — WITHOUT retaining any row values or exploded row objects. Only column + # NAMES accumulate (bounded by schema width, not row count). The complete header is + # frozen and written BEFORE any output row, so no late column is ever missed. The + # per-record explosion below then streams straight to the post-freeze write path. + if ($script:ExplosionInputSpoolPath) { + $columnOrder = New-Object System.Collections.Generic.List[string] + if ($ExplodeArrays -or $ExplodeDeep -or $ForcedRawInputCsvExplosion) { + if ($IncludeM365Usage -and $RAWInputCSV) { + foreach ($c in (Get-M365UsageWideHeader -RawCsvPath $RAWInputCSV -BaseHeader $M365UsageBaseHeader)) { [void]$columnOrder.Add($c) } + } else { + foreach ($c in $PurviewExplodedHeader) { [void]$columnOrder.Add($c) } + } + if ($ExplodeDeep -and $script:DeepExtraColumns -and $script:DeepExtraColumns.Count -gt 0) { + foreach ($c in $script:DeepExtraColumns) { if (-not $columnOrder.Contains($c)) { [void]$columnOrder.Add($c) } } + } + } + $schemaScanSeen = 0 + foreach ($schemaLine in [System.IO.File]::ReadLines($script:ExplosionInputSpoolPath)) { + $slog = ConvertFrom-PaxNormalizedSpoolLine -Line $schemaLine + if ($null -eq $slog) { continue } + $srecords = if ($effectiveExplode) { Convert-ToPurviewExplodedRecords -Record $slog -Deep:$ExplodeDeep -PartialExplode:$partialExplode -PromptFilterValue $PromptFilter -SkipMetrics } else { Convert-ToStructuredRecord -Record $slog -EnableExplosion:$false } + foreach ($sr in (To-RecordArray $srecords)) { + if ($null -eq $sr) { continue } + foreach ($pn in $sr.PSObject.Properties.Name) { if (-not $columnOrder.Contains($pn)) { [void]$columnOrder.Add($pn) } } + } + $schemaScanSeen++ + if (($schemaScanSeen % 20000) -eq 0) { Write-LogHost ("Schema scan: {0:N0} records" -f $schemaScanSeen) -ForegroundColor DarkGray; try { Write-PaxMemoryObservation -Stage 'ExplosionSchemaScan' -RowsProcessed ([long]$schemaScanSeen) } catch {} } + } + try { Write-PaxMemoryObservation -Stage 'ExplosionSchemaScan' -RowsProcessed ([long]$schemaScanSeen) -Force } catch {} + # Only pre-freeze when the scan produced a schema (array/deep always seed a base header; + # standard mode with a non-empty spool discovers columns). An empty result falls through to + # the established small-dataset finalization below. + if ($columnOrder.Count -gt 0) { + $effectiveChunkSize = $StreamingChunkSize + $colCount = $columnOrder.Count + if ($colCount -gt 1000) { $effectiveChunkSize = [int][Math]::Min($effectiveChunkSize, 1000) } + elseif ($colCount -gt 750) { $effectiveChunkSize = [int][Math]::Min($effectiveChunkSize, 1500) } + elseif ($colCount -gt 500) { $effectiveChunkSize = [int][Math]::Min($effectiveChunkSize, 2500) } + elseif ($colCount -gt 250) { $effectiveChunkSize = [int][Math]::Min($effectiveChunkSize, 4000) } + else { if ($colCount -le 60 -and $StreamingChunkSize -lt 15000) { $autoBoost = [int][Math]::Min(15000, [Math]::Max($StreamingChunkSize * 3, 8000)); $effectiveChunkSize = $autoBoost } } + $script:metrics.EffectiveChunkSize = $effectiveChunkSize + if (-not $csvWriter) { Open-CsvWriter -Path $exportTemp -Columns $columnOrder; $csvWriter = $true } + $schemaFrozen = $true + Write-LogHost "Schema frozen with $($columnOrder.Count) columns after a full-spool scan of $($schemaScanSeen) records (bounded prepass — complete schema, no late-column loss)" -ForegroundColor DarkCyan + } + } + + foreach ($spoolLine in $(if ($script:ExplosionInputSpoolPath) { [System.IO.File]::ReadLines($script:ExplosionInputSpoolPath) } else { @() })) { + $log = ConvertFrom-PaxNormalizedSpoolLine -Line $spoolLine + if ($null -eq $log) { continue } $processedRecordCount++ # NOTE: Retrieved counts are tracked during partition retrieval (lines 11052, 11431, 11784) @@ -30358,13 +44428,13 @@ Write-Output "[403-MAX] Partition $idx/$tot - Max transient 403 poll retries exc if ($processedRecordCount -ge ($lastProgressUpdate + $explosionProgressInterval)) { $elapsed = (Get-Date) - $te0 $rate = [int]($processedRecordCount / $elapsed.TotalSeconds) - $pct = [int](($processedRecordCount / $allLogs.Count) * 100) + $pct = [int](($processedRecordCount / $explosionInputCount) * 100) $elapsedStr = if ($elapsed.TotalMinutes -ge 1) { "{0}m {1}s" -f [int]$elapsed.TotalMinutes, $elapsed.Seconds } else { "{0}s" -f [int]$elapsed.TotalSeconds } - Write-LogHost ("[{0}] Processing: {1:N0} / {2:N0} records ({3}%) | Elapsed: {4} | Rate: {5:N0} rec/sec" -f (Get-Date -Format "HH:mm:ss"), $processedRecordCount, $allLogs.Count, $pct, $elapsedStr, $rate) -ForegroundColor DarkCyan + Write-LogHost ("[{0}] Processing: {1:N0} / {2:N0} records ({3}%) | Elapsed: {4} | Rate: {5:N0} rec/sec" -f (Get-Date -Format "HH:mm:ss"), $processedRecordCount, $explosionInputCount, $pct, $elapsedStr, $rate) -ForegroundColor DarkCyan $lastProgressUpdate = $processedRecordCount } $records = if ($effectiveExplode) { Convert-ToPurviewExplodedRecords -Record $log -Deep:$ExplodeDeep -PartialExplode:$partialExplode -PromptFilterValue $PromptFilter } else { Convert-ToStructuredRecord -Record $log -EnableExplosion:$false } @@ -30940,13 +45010,15 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread if ($csvWriter) { try { Close-CsvWriter } catch {} } # Replay fallback: if no structured rows but we have raw logs, emit compact rows (non-exploded) - if ($structuredDataCount -eq 0 -and $allLogs.Count -gt 0 -and $RAWInputCSV) { + if ($structuredDataCount -eq 0 -and $explosionInputCount -gt 0 -and $RAWInputCSV) { Write-LogHost "Replay fallback: emitting compact non-exploded rows" -ForegroundColor Yellow try { $columnOrder = @('RecordId','CreationDate','RecordType','Operation','UserId','AuditData','AssociatedAdminUnits','AssociatedAdminUnitsNames') if (-not $csvWriter) { Open-CsvWriter -Path $exportTemp -Columns $columnOrder; $csvWriter = $true } $fallbackCount = 0 - foreach ($log in $allLogs) { + foreach ($spoolLine in $(if ($script:ExplosionInputSpoolPath) { [System.IO.File]::ReadLines($script:ExplosionInputSpoolPath) } else { @() })) { + $log = ConvertFrom-PaxNormalizedSpoolLine -Line $spoolLine + if ($null -eq $log) { continue } try { $rows = Convert-ToStructuredRecord -Record $log -EnableExplosion:$false $rowsArr = To-RecordArray $rows @@ -30987,7 +45059,7 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # Parallel mode has its own completion summary, skip this to avoid duplicate/confusing output # Also skip when fast path or streaming merge already handled export (skipToPostProcessing=true) if (-not $useParallelExplosion -and -not $skipToPostProcessing) { - if ($allLogs.Count -ge $explosionProgressInterval) { + if ($explosionInputCount -ge $explosionProgressInterval) { $elapsed = (Get-Date) - $te0 $rate = [int]($processedRecordCount / $elapsed.TotalSeconds) $elapsedStr = if ($elapsed.TotalMinutes -ge 1) { @@ -30995,16 +45067,16 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } else { "{0}s" -f [int]$elapsed.TotalSeconds } - Write-LogHost ("[{0}] Processing: {1:N0} / {2:N0} records (100%) | Elapsed: {3} | Rate: {4:N0} rec/sec" -f (Get-Date -Format "HH:mm:ss"), $processedRecordCount, $allLogs.Count, $elapsedStr, $rate) -ForegroundColor DarkCyan + Write-LogHost ("[{0}] Processing: {1:N0} / {2:N0} records (100%) | Elapsed: {3} | Rate: {4:N0} rec/sec" -f (Get-Date -Format "HH:mm:ss"), $processedRecordCount, $explosionInputCount, $elapsedStr, $rate) -ForegroundColor DarkCyan } $te1 = Get-Date; try { $script:metrics.ExplosionMs += [int]($te1 - $te0).TotalMilliseconds } catch {} - Write-LogHost "Standard processing (streamed) complete: $($allLogs.Count) input -> $structuredDataCount output" -ForegroundColor Cyan + Write-LogHost "Standard processing (streamed) complete: $($explosionInputCount) input -> $structuredDataCount output" -ForegroundColor Cyan } # Explain record count changes (filtering vs explosion) - if ($structuredDataCount -lt $allLogs.Count) { - $recordsFiltered = $allLogs.Count - $structuredDataCount + if ($structuredDataCount -lt $explosionInputCount) { + $recordsFiltered = $explosionInputCount - $structuredDataCount Write-LogHost "" Write-LogHost " ℹ Record count decreased: $recordsFiltered parent record(s) were filtered out" -ForegroundColor Yellow Write-LogHost " Note: Filtering happens DURING explosion (before array expansion)" -ForegroundColor DarkYellow @@ -31041,7 +45113,7 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } } # Use stored count for streaming merge (allLogs was cleared), otherwise use allLogs.Count - $inputRecordCount = if ($script:OriginalInputRecordCount) { $script:OriginalInputRecordCount } else { $allLogs.Count } + $inputRecordCount = if ($script:OriginalInputRecordCount) { $script:OriginalInputRecordCount } else { $explosionInputCount } if ($structuredDataCount -eq $inputRecordCount -and ($ExplodeArrays -or $ExplodeDeep)) { Write-LogHost "" @@ -31176,6 +45248,22 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } } # End of skipToPostProcessing guard + # Explosion input spool cleanup (Blocker 1): delete on successful publication; + # preserve for recovery on failure. A failed publish leaves the output missing + # (or the AppendFile branch exits before here), so the spool is retained. + if ($script:ExplosionInputSpoolPath -and (Test-Path -LiteralPath $script:ExplosionInputSpoolPath)) { + $explosionPublished = $false + try { if ($OutputFile -and (Test-Path -LiteralPath $OutputFile)) { $explosionPublished = $true } } catch {} + if (-not $explosionPublished) { try { if ($script:CsvOutputFile -and (Test-Path -LiteralPath $script:CsvOutputFile)) { $explosionPublished = $true } } catch {} } + if ($explosionPublished) { + Remove-Item -LiteralPath $script:ExplosionInputSpoolPath -Force -ErrorAction SilentlyContinue + try { Write-PaxMemoryObservation -Stage 'ExplosionExportPublished' -Force } catch {} + $script:ExplosionInputSpoolPath = $null + } else { + Write-LogHost ("Explosion input spool preserved for recovery: {0}" -f $script:ExplosionInputSpoolPath) -ForegroundColor DarkYellow + } + } + $script:progressState.Export.Total = 1; $script:progressState.Export.Current = 1; Update-Progress -Status 'Export complete (stream)'; Set-ProgressPhase -Phase 'Complete' -Status 'Done'; Complete-Progress # --- Dynamic Downgrade: If combined CSV contains data for only one activity type, rename to single-activity convention --- @@ -31369,6 +45457,13 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $header = ($script:EntraUsersData | Select-Object -First 1 | Get-Member -MemberType NoteProperty | Select-Object -ExpandProperty Name) if ($header) { ($null | Select-Object $header) | Export-Csv -Path $entraFile -NoTypeInformation -Encoding UTF8 } } + # Carry forward the AUTHORITATIVE local path just emitted for the raw Entra users CSV so the + # CopilotInteraction rollup post-processor consumes THIS exact file (honoring a separate + # -OutputPathUserInfo destination, the effective basename, and any pristine-raw '_raw' suffix) + # instead of reconstructing it under the Purview output directory. Under -Rollup/-RollupPlusRaw + + # CopilotInteraction the raw persists to rollup time; the rollup consumer re-validates existence, + # so a later -AppendUserInfo in-band cleanup path is safe. + $script:RollupEntraLocalPath = $entraFile # Report the raw path only when it persists past this step. Outside # -AppendUserInfo the raw IS the customer's output; under -AppendUserInfo # with a deferred merge (-Rollup / -RollupPlusRaw + CopilotInteraction) @@ -31442,33 +45537,75 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread Write-LogHost "Reading combined CSV: $(Get-DisplayPath -LocalPath $OutputFile)" -ForegroundColor Gray try { - # Read combined CSV - $allRecords = Import-Csv -Path $OutputFile -ErrorAction Stop - Write-LogHost "Loaded $($allRecords.Count) records from combined CSV" -ForegroundColor Gray - - # Group by Operation field - $groupedRecords = $allRecords | Group-Object -Property Operation - Write-LogHost "Found $($groupedRecords.Count) activity types" -ForegroundColor Gray - - # Write separate CSV files - $outputDir = Split-Path $OutputFile -Parent - $createdFiles = @() - - foreach ($group in $groupedRecords) { - $activityType = $group.Name - if ([string]::IsNullOrWhiteSpace($activityType)) { $activityType = "Unknown" } - - # Sanitize filename (remove invalid characters) - $safeActivityName = $activityType -replace '[\\/:*?"<>|]', '_' - # Base name already contains full prefix+timestamp, just prepend activity type - $fileName = "Purview_Audit_UsageActivity_${safeActivityName}_${global:ScriptRunTimestamp}.csv" - $filePath = Join-Path $outputDir $fileName # Export to separate CSV - # Atomic temp+rename write per activity-type group. - Save-CsvAtomic -InputObject $group.Group -Path $filePath -NoTypeInformation -Encoding UTF8 - $createdFiles += $filePath - - Write-LogHost " • $activityType → $fileName ($($group.Count) records)" -ForegroundColor DarkCyan + # Bounded split: stream the combined CSV once (RFC-4180) and route each row to + # its activity-type writer, holding at most a fixed cap of writers open (LRU + # close + reopen in append mode beyond the cap). Input row order is preserved + # within each activity file; the whole export is never materialized and no + # activity group is collected. The combined CSV is deleted only after every + # split writer closes successfully. + Write-LogHost "Streaming combined CSV (bounded per-activity writers)" -ForegroundColor Gray + $outputDir = Split-Path $OutputFile -Parent + $createdFiles = @() + $splitWriterCap = 64 + $u8Split = [System.Text.UTF8Encoding]::new($false) + $splitReader = [System.IO.StreamReader]::new((Resolve-Path -LiteralPath $OutputFile).Path, $true) + $splitWriters = @{} # activityType -> StreamWriter + $splitLru = New-Object System.Collections.Generic.List[string] # LRU order of open writers + $splitPath = @{} # activityType -> file path + $splitSeen = New-Object System.Collections.Generic.List[string] # first-seen order + $splitCounts = @{} + $splitTotal = 0 + try { + $splitHeader = script:Read-PaxCsvRecord -Reader $splitReader + if ($null -eq $splitHeader) { throw "combined CSV '$OutputFile' has no header row." } + $opIdx = -1 + for ($i = 0; $i -lt $splitHeader.Length; $i++) { if ([string]::Equals([string]$splitHeader[$i], 'Operation', [System.StringComparison]::OrdinalIgnoreCase)) { $opIdx = $i; break } } + $getSplitWriter = { + param($type) + if ($splitWriters.ContainsKey($type)) { + [void]$splitLru.Remove($type); $splitLru.Add($type) + return $splitWriters[$type] + } + $fp = $splitPath[$type] + $existed = Test-Path -LiteralPath $fp + if ($splitWriters.Count -ge $splitWriterCap) { + $lru = $splitLru[0]; [void]$splitLru.RemoveAt(0) + $splitWriters[$lru].Flush(); $splitWriters[$lru].Dispose(); $splitWriters.Remove($lru) + } + $sw = [System.IO.StreamWriter]::new($fp, $existed, $u8Split) # append if reopened after LRU close + if (-not $existed) { script:Write-PaxCsvRecord -Writer $sw -Fields $splitHeader } + $splitWriters[$type] = $sw; $splitLru.Add($type) + return $sw + } + while ($true) { + $fields = script:Read-PaxCsvRecord -Reader $splitReader + if ($null -eq $fields) { break } + $op = if ($opIdx -ge 0 -and $opIdx -lt $fields.Length) { [string]$fields[$opIdx] } else { '' } + $activityType = if ([string]::IsNullOrWhiteSpace($op)) { 'Unknown' } else { $op } + if (-not $splitPath.ContainsKey($activityType)) { + $safeActivityName = $activityType -replace '[\\/:*?"<>|]', '_' + $fileName = "Purview_Audit_UsageActivity_${safeActivityName}_${global:ScriptRunTimestamp}.csv" + $splitPath[$activityType] = Join-Path $outputDir $fileName + $splitSeen.Add($activityType) + $splitCounts[$activityType] = 0 + } + $w = & $getSplitWriter $activityType + script:Write-PaxCsvRecord -Writer $w -Fields $fields + $splitCounts[$activityType] = $splitCounts[$activityType] + 1 + $splitTotal++ + } } + finally { + foreach ($k in @($splitWriters.Keys)) { try { $splitWriters[$k].Flush(); $splitWriters[$k].Dispose() } catch {} } + $splitReader.Dispose() + } + Write-LogHost "Found $($splitSeen.Count) activity types across $splitTotal records" -ForegroundColor Gray + foreach ($t in $splitSeen) { + $fp = $splitPath[$t] + $createdFiles += $fp + Write-LogHost (" • {0} → {1} ({2} records)" -f $t, (Split-Path -Leaf $fp), $splitCounts[$t]) -ForegroundColor DarkCyan + } + # Emit EntraUsers listing inline with the other split-file bullets so the visual # split-summary group is contiguous BEFORE the "Removed combined CSV" closer. @@ -31648,17 +45785,12 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $tabName = "CombinedUsageActivity" if ($AppendFile -and $script:ExistingExcelSheets -contains $tabName) { - # Append mode: Need to validate headers (requires loading CSV) - Write-LogHost "Reading CSV data for header validation: $csvFilePath" -ForegroundColor Gray - $csvData = Import-Csv -Path $csvFilePath -ErrorAction Stop - $totalRows = $csvData.Count - Write-LogHost "Loaded $totalRows rows from CSV" -ForegroundColor Gray - - # Validate headers match + # Append mode: validate headers (bounded — reads only the CSV header row) Write-LogHost "Validating headers for tab: $tabName" -ForegroundColor Gray $existingWorkbook = Import-Excel -Path $excelFilePath -WorksheetName $tabName -StartRow 1 -EndRow 1 -NoHeader $existingHeaders = $existingWorkbook[0].PSObject.Properties.Value | Where-Object { $_ } - $newHeaders = $csvData[0].PSObject.Properties.Name + $newHeaders = Get-PaxCsvHeader -CsvPath $csvFilePath + $totalRows = 0 $headerMismatch = $false if ($existingHeaders.Count -ne $newHeaders.Count) { @@ -31677,20 +45809,16 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $timestampedTabName = "${tabName}_$excelTimestamp" Write-LogHost "WARNING: Header mismatch detected for tab '$tabName'" -ForegroundColor Yellow Write-LogHost "Creating timestamped duplicate tab: $timestampedTabName" -ForegroundColor Yellow - $dataTable = $csvData | ConvertTo-DataTable - Send-SQLDataToExcel -DataTable $dataTable -Path $excelFilePath -WorkSheetName $timestampedTabName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + $totalRows = Write-PaxCsvToWorksheet -CsvPath $csvFilePath -WorkbookPath $excelFilePath -WorksheetName $timestampedTabName } else { # Append to existing tab Write-LogHost "Appending to existing tab: $tabName" -ForegroundColor Gray - $dataTable = $csvData | ConvertTo-DataTable - Send-SQLDataToExcel -DataTable $dataTable -Path $excelFilePath -WorkSheetName $tabName -Append -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + $totalRows = Write-PaxCsvToWorksheet -CsvPath $csvFilePath -WorkbookPath $excelFilePath -WorksheetName $tabName -Append } } else { # Create new tab or new workbook Write-LogHost "Creating tab: $tabName" -ForegroundColor Gray - $dataTable = Import-CsvToDataTable -Path $csvFilePath - $totalRows = $dataTable.Rows.Count - Send-SQLDataToExcel -DataTable $dataTable -Path $excelFilePath -WorkSheetName $tabName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + $totalRows = Write-PaxCsvToWorksheet -CsvPath $csvFilePath -WorkbookPath $excelFilePath -WorksheetName $tabName # Append EntraUsers tab if requested if ($IncludeUserInfo -and $script:EntraUsersData) { $entraTab = 'EntraUsers_MAClicensing' @@ -31704,13 +45832,14 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread Write-LogHost ("{0}: {1}" -f $xlsxVerb1, (Get-DisplayPath -LocalPath $excelFilePath)) -ForegroundColor Green Write-LogHost " Tab: $tabName | Rows: $totalRows" -ForegroundColor White } else { # --- Multi-tab Mode: One tab per activity type --- - # Multi-tab mode requires loading CSV for grouping - Write-LogHost "Reading CSV data for multi-tab grouping: $csvFilePath" -ForegroundColor Gray - $csvData = Import-Csv -Path $csvFilePath -ErrorAction Stop - Write-LogHost "Loaded $($csvData.Count) rows from CSV" -ForegroundColor Gray - - # Group CSV data by Operation column - $groupedData = $csvData | Group-Object -Property Operation + # Partition the CSV by Operation with a single streaming pass (one + # bounded per-activity temp file each), so the full row set is never + # held in memory for grouping. + Write-LogHost "Partitioning CSV by activity type: $csvFilePath" -ForegroundColor Gray + $multiTabWorkDir = Join-Path ([System.IO.Path]::GetDirectoryName($excelFilePath)) (".paxwb_" + [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $multiTabWorkDir -Force | Out-Null + $groupedData = Split-PaxCsvByColumn -CsvPath $csvFilePath -ColumnName 'Operation' -WorkDir $multiTabWorkDir + Write-LogHost "Partitioned into $($groupedData.Count) activity group(s)" -ForegroundColor Gray # Calculate total tab count (activity types + EntraUsers if present) $activityTabCount = $groupedData.Count @@ -31724,16 +45853,15 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $tabsCreated = @() foreach ($group in $groupedData) { - $activityType = $group.Name - $activityData = $group.Group - $activityRows = $activityData.Count + $activityType = $group.Value + $activityRows = $group.RowCount if ($AppendFile -and $script:ExistingExcelSheets -contains $activityType) { - # Validate headers match + # Validate headers match (bounded — reads only the group header row) Write-LogHost "Validating headers for tab: $activityType" -ForegroundColor Gray $existingWorkbook = Import-Excel -Path $excelFilePath -WorksheetName $activityType -StartRow 1 -EndRow 1 -NoHeader $existingHeaders = $existingWorkbook[0].PSObject.Properties.Value | Where-Object { $_ } - $newHeaders = $activityData[0].PSObject.Properties.Name + $newHeaders = Get-PaxCsvHeader -CsvPath $group.Path $headerMismatch = $false if ($existingHeaders.Count -ne $newHeaders.Count) { @@ -31752,21 +45880,18 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $timestampedTabName = "${activityType}_$excelTimestamp" Write-LogHost "WARNING: Header mismatch detected for tab '$activityType'" -ForegroundColor Yellow Write-LogHost "Creating timestamped duplicate tab: $timestampedTabName" -ForegroundColor Yellow - $activityDataTable = $activityData | ConvertTo-DataTable - Send-SQLDataToExcel -DataTable $activityDataTable -Path $excelFilePath -WorkSheetName $timestampedTabName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + [void](Write-PaxCsvToWorksheet -CsvPath $group.Path -WorkbookPath $excelFilePath -WorksheetName $timestampedTabName) $tabsCreated += "$timestampedTabName ($activityRows rows)" } else { # Append to existing tab Write-LogHost "Appending to existing tab: $activityType" -ForegroundColor Gray - $activityDataTable = $activityData | ConvertTo-DataTable - Send-SQLDataToExcel -DataTable $activityDataTable -Path $excelFilePath -WorkSheetName $activityType -Append -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + [void](Write-PaxCsvToWorksheet -CsvPath $group.Path -WorkbookPath $excelFilePath -WorksheetName $activityType -Append) $tabsCreated += "$activityType ($activityRows rows appended)" } } else { # Create new tab Write-LogHost "Creating tab: $activityType ($activityRows rows)" -ForegroundColor Gray - $activityDataTable = $activityData | ConvertTo-DataTable - Send-SQLDataToExcel -DataTable $activityDataTable -Path $excelFilePath -WorkSheetName $activityType -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + [void](Write-PaxCsvToWorksheet -CsvPath $group.Path -WorkbookPath $excelFilePath -WorksheetName $activityType) $tabsCreated += "$activityType ($activityRows rows)" } @@ -31785,6 +45910,9 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $xlsxVerb2 = if ($AppendFile) { 'Excel workbook updated (append-merge)' } else { 'Excel workbook created' } Write-LogHost ("{0}: {1}" -f $xlsxVerb2, (Get-DisplayPath -LocalPath $excelFilePath)) -ForegroundColor Green Write-LogHost " Tabs: $($tabsCreated -join ', ')" -ForegroundColor White + if ($multiTabWorkDir -and (Test-Path -LiteralPath $multiTabWorkDir)) { + Remove-Item -LiteralPath $multiTabWorkDir -Recurse -Force -ErrorAction SilentlyContinue + } } # Delete temporary CSV file (with retry for file lock issues) if ($AppendFile -and $script:AppendFileTempCsv) { @@ -31948,12 +46076,10 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread Write-LogHost " - ExcludeAgents filter: $excludeCount" -ForegroundColor DarkGray } if ($UserIds) { - $userCount = if ($script:metrics.FilteringUserIds -gt 0) { $script:metrics.FilteringUserIds } else { 0 } - Write-LogHost " - UserIds filter: $userCount" -ForegroundColor DarkGray + Write-LogHost " - UserIds in scope: $($script:metrics.FilteringUserIds)" -ForegroundColor DarkGray } if ($GroupNames) { - $groupCount = if ($script:metrics.FilteringGroupNames -gt 0) { $script:metrics.FilteringGroupNames } else { 0 } - Write-LogHost " - GroupNames filter: $groupCount" -ForegroundColor DarkGray + Write-LogHost " - GroupNames in scope: $($script:metrics.FilteringGroupNames) group(s), $($script:metrics.ScopeExpandedMembers) member(s)" -ForegroundColor DarkGray } # Calculate explained filtering count @@ -31964,8 +46090,8 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread if ($AgentId) { $explainedFiltering += $script:metrics.FilteringAgentFiltered } if ($AgentsOnly) { $explainedFiltering += $script:metrics.FilteringAgentFiltered } if ($ExcludeAgents) { $explainedFiltering += $script:metrics.FilteringExcludeAgents } - if ($UserIds) { $explainedFiltering += $script:metrics.FilteringUserIds } - if ($GroupNames) { $explainedFiltering += $script:metrics.FilteringGroupNames } + # UserIds/GroupNames now carry requested-scope counts (not per-record exclusions), + # so they are reported separately above and excluded from the record-exclusion tally. # Show unspecified reason if filtered count doesn't match explained reasons if ($explainedFiltering -eq 0 -or $actFiltered -gt $explainedFiltering) { @@ -32340,9 +46466,11 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread Write-LogHost "Rollup: using Python $($pyResolved.Version) at '$pyDisplay'" -ForegroundColor Gray # 2. Best-effort orjson install (processors fall back to stdlib json on import failure). + Write-PaxRollupReaderMode [void](Install-OrjsonIfMissing -PythonExe $pyResolved.Path -LauncherArgs $pyResolved.Args) # 3. Build processor-specific argument vector. + Write-PaxRollupStage -Stage 'args-build' -Status 'entry' $rollupArgs = $null $rollupRawCsvList = New-Object System.Collections.Generic.List[string] # Internal-input deletion list: files consumed only as join inputs by the @@ -32394,7 +46522,17 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread if (-not (Test-Path -LiteralPath $rollupPurviewCsv)) { throw "Rollup: expected Purview CSV not found at '$rollupPurviewCsv'." } - $rollupEntraCsv = Join-Path $rollupOutputDir (& $script:GetEffectiveEntraBasename) + # Resolve the Entra users CSV to the AUTHORITATIVE local path actually emitted this run (honors a + # separate -OutputPathUserInfo destination, the effective basename, and any pristine-raw '_raw' + # suffix). Only fall back to the Purview output directory when no emitted local path was captured + # (e.g. a flat layout where UserInfo and Purview resolve to the same folder, or a remote run whose + # local staging equals the rollup dir). NEVER infer the customer UserInfo destination from the + # rollup/Purview output location. + $rollupEntraCsv = if ($script:RollupEntraLocalPath -and (Test-Path -LiteralPath $script:RollupEntraLocalPath -PathType Leaf)) { + $script:RollupEntraLocalPath + } else { + Join-Path $rollupOutputDir (& $script:GetEffectiveEntraBasename) + } if (-not (Test-Path -LiteralPath $rollupEntraCsv)) { throw "Rollup: required Entra users CSV not found at '$rollupEntraCsv'. -IncludeUserInfo must produce a CSV (not a workbook tab)." } @@ -32402,7 +46540,7 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread if ($rollupEntraInfo.Length -le 0) { throw "Rollup: Entra users CSV at '$rollupEntraCsv' is empty (0 bytes)." } - # Dashboard profile (aio|aibv) selects the embedded v4.0.0 output schema; + # Dashboard profile (aio|aibv) selects the embedded output schema; # resolved in the rollup-mode decision (default aio). --combined-entra is # passed explicitly because PAX always emits a single combined # users+licensing Entra CSV (the processor's 2-input path); the separate @@ -32427,15 +46565,62 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $rollupRetainedExtraList.Add($rollupEntraCsv) } - # -AppendUserInfo: seed the embedded processor with the target's existing - # {PersonId_Normalized -> UserKey} map so retained users keep their stable - # integer keys across runs. Failure to prepare the seed is non-fatal — the - # processor will mint fresh keys and the post-Python Merge-UsersCsv call - # will still produce a correct union. - if ($AppendUserInfo) { + # Append publication state. Both append targets describe one set of people, so + # each union is built to a candidate path and nothing replaces an existing + # target until the joint referential-integrity gate has passed. + $script:PaxAuditOnlyUsersCsv = $null + $script:PaxAuditOnlyUsersWorkFile = $null + $script:PaxUserKeyContinuityFailed = $false + $script:PaxUsersCandidatePath = $null + $script:PaxUsersCandidateFinal = $null + $script:PaxUsersTransientCsv = $null + $script:PaxFactCandidatePath = $null + $script:PaxFactCandidateFinal = $null + $script:PaxFactTransientCsv = $null + + # -AppendUserInfo: seed the embedded processor with the continuity map taken + # from the existing targets so retained users keep their stable integer keys + # across runs. Continuity is derived from BOTH targets, because a user who + # only ever appears in activity data has no directory row and would otherwise + # have no key reserved at all. + if ($AppendUserInfo -or $AppendFile) { + # Continuity is prepared for whichever append target(s) the run requested. + $ukSeedLabel = if ($AppendUserInfo) { '-AppendUserInfo' } else { '-AppendFile' } + $ukSeedTarget = if ($AppendUserInfo) { $AppendUserInfo } else { $AppendFile } + $ukSeedRawKey = if ($AppendUserInfo) { 'UserInfo' } else { 'Purview' } $rollupUserInfoSeedJson = Join-Path $rollupIncDir "PAX_AppendUserInfo_seed_${global:ScriptRunTimestamp}.json" + $rollupUserInfoAuditOnlyCsv = Join-Path $rollupIncDir "PAX_AppendUserInfo_auditonly_${global:ScriptRunTimestamp}.csv" + Write-PaxRollupStage -Stage 'users-seed' -Status 'entry' try { - $rollupUserInfoSeedCount = ConvertTo-UsersSeedMap -TargetUsersCsv $AppendUserInfo -OutJsonPath $rollupUserInfoSeedJson + # Observability-only heartbeat for a seed preparation that can run a long time. + # The stage name and EVERY counter key are literal at this call site, so the + # counter-name channel is closed by inspection: no key, row, field, file name, + # path, URL, package identifier, user principal name, tenant or account + # identifier, or credential can reach a record. Only AGGREGATE numeric values + # are read from the snapshot, and any failure here is swallowed so a logging + # problem can never alter seed output, counts, ordering, or the run result. + $rollupUserInfoSeedProgress = { + param($Snapshot) + try { + if ($null -eq $Snapshot) { return } + Write-PaxRollupStage -Stage 'users-seed' -Status 'heartbeat' -Counters @{ + rowsRead = [int64]$Snapshot.RowsRead + rowsWritten = [int64]$Snapshot.RowsWritten + runCount = [int64]$Snapshot.RunCount + mergePassCount = [int64]$Snapshot.MergePassCount + maxOpenReaders = [int64]$Snapshot.MaxOpenReaders + temporaryBytes = [int64]$Snapshot.TempBytesWritten + elapsedSeconds = [double]$Snapshot.ElapsedSeconds + } + } + catch { } + } + $rollupUserInfoContinuity = ConvertTo-PaxUserKeyContinuity -TargetUsersCsv $AppendUserInfo -TargetFactCsv $AppendFile -OutJsonPath $rollupUserInfoSeedJson -OutAuditOnlyCsvPath $rollupUserInfoAuditOnlyCsv -ProgressCallback $rollupUserInfoSeedProgress -ProgressIntervalSeconds $StatusIntervalSeconds + $rollupUserInfoSeedCount = [int]$rollupUserInfoContinuity.SeededKeys + $script:PaxAuditOnlyUsersCsv = $rollupUserInfoAuditOnlyCsv + if ($rollupUserInfoContinuity.AuditOnlyIdentities -gt 0 -and $AppendUserInfo) { + Write-LogHost ("Rollup: -AppendUserInfo: {0:N0} user(s) appear only in prior activity; their existing keys are reserved and each receives a minimal Users row." -f $rollupUserInfoContinuity.AuditOnlyIdentities) -ForegroundColor DarkCyan + } # Differentiate seed outcomes so zero-count is not # silently glossed as "seeded 0" (which reads like a success line). # In remote-output mode $AppendUserInfo has already been redirected to a @@ -32444,21 +46629,29 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # scratch leaf) that does not exist on the destination. Surface the # canonical customer-supplied URL from $script:AppendRaw['UserInfo'] # when available. - $auiDisplayPath = if ($script:AppendRaw.ContainsKey('UserInfo') -and $script:AppendRaw['UserInfo']) { $script:AppendRaw['UserInfo'] } else { Get-DisplayPath -LocalPath $AppendUserInfo } + $auiDisplayPath = if ($script:AppendRaw.ContainsKey($ukSeedRawKey) -and $script:AppendRaw[$ukSeedRawKey]) { $script:AppendRaw[$ukSeedRawKey] } else { Get-DisplayPath -LocalPath $ukSeedTarget } if ($rollupUserInfoSeedCount -gt 0) { - Write-LogHost ("Rollup: -AppendUserInfo: seeded {0:N0} retained UserKey(s)" -f $rollupUserInfoSeedCount) -ForegroundColor DarkCyan + Write-LogHost ("Rollup: {0}: seeded {1:N0} retained UserKey(s)" -f $ukSeedLabel, $rollupUserInfoSeedCount) -ForegroundColor DarkCyan Write-LogHost (" from: {0}" -f $auiDisplayPath) -ForegroundColor DarkCyan - } elseif (-not (Test-Path -LiteralPath $AppendUserInfo -PathType Leaf)) { - Write-LogHost ("Rollup: -AppendUserInfo: target not found; no UserKeys to seed (new keys will be minted).") -ForegroundColor Yellow + } elseif (-not (Test-Path -LiteralPath $ukSeedTarget -PathType Leaf)) { + Write-LogHost ("Rollup: {0}: target not found; no UserKeys to seed (new keys will be minted)." -f $ukSeedLabel) -ForegroundColor Yellow Write-LogHost (" from: {0}" -f $auiDisplayPath) -ForegroundColor Yellow } else { - Write-LogHost ("Rollup: -AppendUserInfo: target exists but contains no retainable UserKeys (header-only or unrecognized schema); new keys will be minted.") -ForegroundColor Yellow + Write-LogHost ("Rollup: {0}: target exists but contains no retainable UserKeys (header-only or unrecognized schema); new keys will be minted." -f $ukSeedLabel) -ForegroundColor Yellow Write-LogHost (" from: {0}" -f $auiDisplayPath) -ForegroundColor Yellow } $rollupArgs += @('--seed-userkey-map', $rollupUserInfoSeedJson) + Write-PaxRollupStage -Stage 'users-seed' -Status 'complete' -Counters @{ seededKeys = [int64]$rollupUserInfoSeedCount } } catch { - Write-LogHost ("Rollup: -AppendUserInfo seed preparation failed: {0}. Continuing without retained UserKeys; new keys will be minted." -f $_.Exception.Message) -ForegroundColor Yellow + # Fail closed. Minting fresh keys over a target that already has its own is + # exactly how one key ends up shared between two people, so neither append + # target is touched this run and the run reports completed with gaps. + $script:PaxUserKeyContinuityFailed = $true + $script:HadTerminalFailures = $true + Write-LogHost ("Rollup: {0}: user key continuity could not be established: {1}" -f $ukSeedLabel, $_.Exception.Message) -ForegroundColor Red + Write-LogHost "Rollup: -> Neither append target is updated this run (completed with gaps, exit 40); this run's output is preserved for review." -ForegroundColor Red + Write-PaxRollupStage -Stage 'users-seed' -Status 'failed' } } @@ -32471,10 +46664,57 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # surrogate values. if ($AppendFile) { if (Test-Path -LiteralPath $AppendFile -PathType Leaf) { + # Advisory: for a genuinely-local append target (remote targets are probed + # before download) warn if it is large enough that the in-memory merge could + # strain RAM. Fail-open, non-blocking. + if (-not $script:AppendRemoteOriginal) { + $localAppendSizeWarning = Get-AppendTargetSizeWarning -SizeBytes (Get-RemoteAppendTargetSize -Tier 'Local' -Path $AppendFile) -ThresholdBytes $script:AppendTargetSizeWarnBytes -Label (Get-DisplayPath -LocalPath $AppendFile) + if ($localAppendSizeWarning) { Write-LogHost $localAppendSizeWarning -ForegroundColor Yellow } + } $rollupMidSeedJson = Join-Path $rollupIncDir "PAX_AppendFile_midseed_${global:ScriptRunTimestamp}.json" $rollupThreadSeedJson = Join-Path $rollupIncDir "PAX_AppendFile_threadseed_${global:ScriptRunTimestamp}.json" + Write-PaxRollupStage -Stage 'fact-seed' -Status 'entry' try { - $rollupFactSeed = ConvertTo-FactSeedMaps -TargetFactCsv $AppendFile -OutMidJsonPath $rollupMidSeedJson -OutThreadJsonPath $rollupThreadSeedJson + # Observability-only heartbeat for a seed preparation that can run a long time. + # The stage name and EVERY counter key are literal at this call site, so the + # counter-name channel is closed by inspection: no key, row, field, file name, + # path, URL, package identifier, user principal name, tenant or account + # identifier, or credential can reach a record. Only AGGREGATE numeric values + # are read from the snapshot, and any failure here is swallowed so a logging + # problem can never alter seed output, counts, ordering, or the run result. + $rollupFactSeedProgress = { + param($Snapshot) + try { + if ($null -eq $Snapshot) { return } + Write-PaxRollupStage -Stage 'fact-seed' -Status 'heartbeat' -Counters @{ + rowsRead = [int64]$Snapshot.RowsRead + rowsWritten = [int64]$Snapshot.RowsWritten + runCount = [int64]$Snapshot.RunCount + mergePassCount = [int64]$Snapshot.MergePassCount + maxOpenReaders = [int64]$Snapshot.MaxOpenReaders + temporaryBytes = [int64]$Snapshot.TempBytesWritten + elapsedSeconds = [double]$Snapshot.ElapsedSeconds + } + } + catch { } + } + # Prepare both seeds through the embedded disk-backed builder first: it + # folds a large target far faster while keeping memory bounded. An + # explicit compatibility outcome means the target holds a key form the + # builder is not proven to serialize identically, so the accepted + # implementation runs instead and produces the same seeds. Any other + # accelerator outcome abandons seed preparation rather than silently + # changing continuity. + $rollupFactSeedAccel = Invoke-PaxFactSeedAccelerator -TargetFactCsv $AppendFile -OutMidJsonPath $rollupMidSeedJson -OutThreadJsonPath $rollupThreadSeedJson -PythonExe $pyResolved.Path -LauncherArgs $pyResolved.Args -ScriptDir $rollupIncDir -HeartbeatIntervalSeconds $StatusIntervalSeconds + if ($rollupFactSeedAccel.Status -eq 'Success') { + $rollupFactSeed = [PSCustomObject]@{ MidCount = $rollupFactSeedAccel.MidCount; ThreadCount = $rollupFactSeedAccel.ThreadCount } + } + elseif ($rollupFactSeedAccel.Status -eq 'CompatibilityFallbackRequired') { + $rollupFactSeed = ConvertTo-FactSeedMaps -TargetFactCsv $AppendFile -OutMidJsonPath $rollupMidSeedJson -OutThreadJsonPath $rollupThreadSeedJson -ProgressCallback $rollupFactSeedProgress -ProgressIntervalSeconds $StatusIntervalSeconds + } + else { + throw [System.InvalidOperationException]::new(("continuity seed preparation did not complete ({0})" -f $rollupFactSeedAccel.Reason)) + } # Differentiate populated vs empty target. Use the canonical customer-supplied # URL from $script:AppendRaw['Purview'] when available so remote-output runs # don't surface a scratch-leaf-derived URL if the scratch file's leaf ever @@ -32489,9 +46729,19 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } $rollupArgs += @('--seed-mid-map', $rollupMidSeedJson) $rollupArgs += @('--seed-thread-map', $rollupThreadSeedJson) + Write-PaxRollupStage -Stage 'fact-seed' -Status 'complete' -Counters @{ midKeys = [int64]$rollupFactSeed.MidCount; threadKeys = [int64]$rollupFactSeed.ThreadCount } } catch { - Write-LogHost ("Rollup: -AppendFile seed preparation failed: {0}. Continuing without seeded surrogates." -f $_.Exception.Message) -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'fact-seed' -Status 'failed' + # Fail closed: the continuity seeds are required for correct cross-run + # Message_Id / ThreadKey surrogate continuity. Continuing without them + # would let the embedded processor mint fresh surrogates and silently + # change continuity, so a seed-preparation failure aborts the rollup. + # The existing append target and any prior seed artifacts are left + # untouched, and the embedded processor is never invoked without seeds. + $seedErr = $_ + Write-LogHost ("Rollup: -AppendFile continuity seed preparation failed: {0}" -f $seedErr.Exception.Message) -ForegroundColor Red + throw [System.InvalidOperationException]::new(("Rollup aborted: -AppendFile continuity seed preparation failed; refusing to run the rollup without continuity seeds so Message_Id and thread continuity are never silently changed. Underlying error: {0}" -f $seedErr.Exception.Message), $seedErr.Exception) } } else { @@ -32507,18 +46757,144 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # (further below) reads the already-hashed merged rollup, so it must NOT re-hash. if ($script:PaxDeidEnabled) { $rollupArgs += '--deidentify' } + Write-PaxRollupStage -Stage 'args-build' -Status 'complete' -Counters @{ argumentCount = [int64]@($rollupArgs).Count } + # 4. Invoke embedded processor (writes temp .py into .pax_incremental, deletes on exit). + Write-PaxRollupStage -Stage 'processor' -Status 'entry' $rollupExit = Invoke-EmbeddedProcessor ` -ProcessorMode $script:RollupProcessorMode ` -PythonExe $pyResolved.Path ` -LauncherArgs $pyResolved.Args ` -ProcessorArgs $rollupArgs ` -IncrementalDir $rollupIncDir + $__rollupStageStatus = if ($rollupExit -eq 0) { 'complete' } else { 'failed' } + Write-PaxRollupStage -Stage 'processor' -Status $__rollupStageStatus -Counters @{ exitCode = [int64]$rollupExit } if ($rollupExit -eq 0) { $rollupSuccess = $true Write-LogHost "Rollup: post-processor completed successfully (exit 0)." -ForegroundColor Green + # D3: a "successful" CopilotInteraction processor MUST have produced BOTH required deliverables — + # the rolled-up Fact (_Interactions.csv) and Users dim (_Users.csv). + # Verify HERE, before any -AppendFile/-AppendUserInfo merge can consume or delete them. A missing + # deliverable is a rollup-processor failure (consumed as a requested-output failure under + # -Dashboard AISID); for non-AISID best-effort modes the flag is set but never read, so exit + # semantics there are unchanged. + if ($script:RollupProcessorMode -eq 'CopilotInteraction') { + $__d3FactCsv = Join-Path $rollupOutputDir ("{0}_Interactions.csv" -f [System.IO.Path]::GetFileNameWithoutExtension($rollupPurviewCsv)) + $__d3UsersCsv = Join-Path $rollupOutputDir ("{0}_Users.csv" -f [System.IO.Path]::GetFileNameWithoutExtension($rollupEntraCsv)) + if (-not (Test-Path -LiteralPath $__d3FactCsv -PathType Leaf)) { + $script:RollupProcessorFailed = $true + Write-LogHost ("Rollup: expected Fact CSV not produced at '{0}' despite processor exit 0 (requested-output failure)." -f (Get-DisplayPath -LocalPath $__d3FactCsv)) -ForegroundColor Red + } + if (-not (Test-Path -LiteralPath $__d3UsersCsv -PathType Leaf)) { + $script:RollupProcessorFailed = $true + Write-LogHost ("Rollup: expected Users dim CSV not produced at '{0}' despite processor exit 0 (requested-output failure)." -f (Get-DisplayPath -LocalPath $__d3UsersCsv)) -ForegroundColor Red + } + } + + # ============================================================ + # AISID DASHBOARD EXPORT (only under -Dashboard AISID) + # Placed HERE — immediately after the CopilotInteraction rollup Fact CSV + # (_Interactions.csv) is written and BEFORE the -AppendFile + # merge/delete below can consume it — so Get-AISIDCopilotUsage's source is + # guaranteed present regardless of -AppendFile. This is the single, flag- + # independent point where the Fact exists and no deletion has yet occurred. + # Destination reuses the DefenderUsage path registry (consuming + # -OutputPathDefenderUsage / -AppendDefenderUsage, else co-locating with the + # Purview output folder). Failure is isolated inside Invoke-AISIDPilotExport. + # This is what makes -Dashboard AISID customer-functional: the run reaches + # here under -Dashboard AISID and produces the + # twelve dashboard CSVs with per-table failure isolation and, under -Deidentify, + # deidentification applied before merge. + # ============================================================ + if ($Dashboard -eq 'AISID') { + # Canonical per-run AISID folder — the twelve fixed-name CSVs publish into + # 'AISID_' beneath the DefenderUsage destination on EVERY tier + # (Local / SharePoint / Fabric). One resolver drives the write dir, history-read parent, + # completion check, upload target ($script:AISIDRemoteRunFolderUrl), roster registration, + # and the run-specific scratch cleanup. The twelve filenames inside are never renamed. + $aisidRun = script:Resolve-AISIDRunFolder + $aisidRunLeaf = $aisidRun.RunLeaf + $aisidWriteDir = $aisidRun.WriteDir + $script:AISIDRunFolder = $aisidRun + $script:AISIDRemoteRunFolderUrl = $aisidRun.RemoteRunFolderUrl + # Blocker-3: the AISID history-read source is SEPARATE from this run's fresh output folder. + # Prior history is read ONLY from an explicitly supplied -AppendDefenderUsage prior bundle + # (local folder or remote parent URL), never from Purview and never from the new timestamped + # child. A plain -OutputPathDefenderUsage run reads NO prior history. The reconciled result + # is written into the AISID_ child; the prior bundle is only read and left untouched. + $aisidHist = script:Resolve-AISIDHistorySource + $aisidHistoryDir = [string]$aisidHist.LocalDir + $aisidHistoryParent = [string]$aisidHist.RemoteParentUrl + + # Collision-safe folder creation (items #3/#4/#5 + defects 1/2/3): any pre-existing EXACT run + # folder is a collision — FAIL CLOSED (skip publication, mark an AISID gap -> exit 40, log the + # exact folder, and NEVER delete or modify the existing folder). In remote mode EXCLUSIVE + # ownership of the EXACT remote AISID_ child folder is acquired by CREATING it atomically + # (SharePoint POST /children conflictBehavior=fail; Fabric PUT ?resource=directory + # If-None-Match:*): the create is authoritative, so a 409/conflict is a collision even if the + # earlier read-only probe reported Absent, and any inaccessible lookup/create also fails closed. + # The local publish/staging folder is acquired via an ATOMIC ownership marker (blocker-5, + # FileMode.CreateNew in the parent) so two concurrent same-second runs cannot both proceed. + $aisidBundleOk = $true + $script:AISIDOwnershipMarker = '' + $script:AISIDRemoteOwnershipAcquired = $false + $__aisidCreateTarget = if ($script:RemoteOutputMode -ne 'None') { $aisidRun.RemoteScratchStaging } else { $aisidRun.LocalRunFolder } + if ($aisidBundleOk -and $script:RemoteOutputMode -ne 'None' -and $aisidRun.RemoteRunFolderUrl) { + $__remoteOwn = script:Acquire-AISIDRemoteRunFolderOwnership -Url $aisidRun.RemoteRunFolderUrl -Mode $script:RemoteOutputMode + if ($__remoteOwn.Acquired) { + $script:AISIDRemoteOwnershipAcquired = $true + Write-LogHost ("[AISID] Remote run folder ownership acquired ({0}): {1}" -f $__remoteOwn.Mode, $aisidRun.RemoteRunFolderUrl) -ForegroundColor DarkGray + } + else { + Write-LogHost ("[AISID] Remote run folder ownership NOT acquired ({0}: {1}) for {2}; refusing to upload (collision -> AISID gap). Existing remote content is left untouched." -f $__remoteOwn.State, $__remoteOwn.Reason, $aisidRun.RemoteRunFolderUrl) -ForegroundColor Yellow + $script:AISIDHadGaps = $true; $script:AnyUploadFailed = $true; $aisidBundleOk = $false + } + } + if ($aisidBundleOk) { + $__aisidOwn = script:New-AISIDRunFolderOwnership -RunFolder $__aisidCreateTarget + if ($__aisidOwn.Acquired) { $script:AISIDOwnershipMarker = [string]$__aisidOwn.MarkerPath } + else { + Write-LogHost ("[AISID] Run bundle folder ownership not acquired for {0} ({1}); refusing to reuse it (collision -> AISID gap). The existing folder is left untouched." -f $__aisidCreateTarget, $__aisidOwn.Reason) -ForegroundColor Yellow + $script:AISIDHadGaps = $true; $aisidBundleOk = $false + } + } + + if ($aisidBundleOk) { + $aisidResult = Invoke-AISIDPilotExport -DestinationDir $aisidWriteDir -ParentOverride $aisidHistoryParent -HistoryDir $aisidHistoryDir -DisableDeltaCache:$DisableAISIDDeltaCache -RawPurviewCsv $rollupPurviewCsv + # A2 truthful completion: evaluate completeness FIRST (per-table state contract + twelve-file + # check), then mark completed ONLY if every table reached a terminal state. Any gap sets + # AISIDHadGaps -> exit 40 ("completed with gaps"); a failure is never a factual empty. + $script:AISIDCompletionStatus = Get-AISIDCompletionStatus -Result $aisidResult -DestinationDir $aisidWriteDir + if ($script:AISIDCompletionStatus.HadGaps) { $script:AISIDHadGaps = $true } + if ($script:AISIDCompletionStatus.AllTablesTerminal) { $script:AISIDExportCompleted = $true } + # Register the twelve fixed-name leaves + destination so the LOCAL roster lists all twelve + # and the REMOTE upload sweep ships each exactly once to the per-run AISID_ child URL + # ($script:AISIDRemoteRunFolderUrl). The '.aisid_cache' delta-cache is never registered. + $script:AISIDDestDir = $aisidWriteDir + if (-not $script:AISIDUploadLeafs) { $script:AISIDUploadLeafs = New-Object System.Collections.Generic.List[string] } + foreach ($aisidSchema in $script:AISIDTableSchemas.Values) { + $aisidLeaf = $aisidSchema.FileName + if ($aisidLeaf -and -not $script:AISIDUploadLeafs.Contains($aisidLeaf)) { [void]$script:AISIDUploadLeafs.Add($aisidLeaf) } + } + # Scratch/cache lifecycle (item #4): a successful run leaves NO cache or scratch. The + # run-specific scratch root holds the OffHours cache and (remote) the staged twelve files. + # In remote mode the staged files must survive until the end-of-run upload sweep, so the + # scratch delete is DEFERRED to the sweep (via $script:AISIDScratchPendingCleanup). In + # local mode only the cache lives there, so delete now and verify — a residual scratch + # is a completed-with-gaps condition (exit 40). + if (-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') { + $__aisidScratch = script:Get-AISIDScratchRoot + if (-not (script:Remove-AISIDScratchRoot -Root $__aisidScratch)) { + Write-LogHost ("[AISID] Internal scratch could not be fully removed: {0}; flagging an AISID gap (exit 40)." -f $__aisidScratch) -ForegroundColor Yellow + $script:AISIDHadGaps = $true + } + } + else { $script:AISIDScratchPendingCleanup = script:Get-AISIDScratchRoot } + } + } + # -RollupPlusRaw + -Deidentify: the embedded processor has consumed the # IDENTIFIED raw (and produced a deidentified rollup via --deidentify); now # scrub the RETAINED raw file(s) in place so the kept raw is anonymized too. @@ -32546,32 +46922,83 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $_rollupAuiMergeOut = if ($script:AppendIsRemote.ContainsKey('UserInfo') -and $script:AppendIsRemote['UserInfo'] -and $script:AppendRaw.ContainsKey('UserInfo') -and $script:AppendRaw['UserInfo']) { Join-Path $rollupOutputDir ([System.IO.Path]::GetFileName($script:AppendRaw['UserInfo'])) } else { $AppendUserInfo } - $rollupUsersMergeStats = Merge-UsersCsv -TargetUsersCsv $AppendUserInfo -CurrentUsersCsv $rollupUsersCsv -OutputPath $_rollupAuiMergeOut - Write-LogHost ("Rollup: -AppendUserInfo merge: Retained={0:N0} New={1:N0} Departed={2:N0} Union={3:N0}" -f $rollupUsersMergeStats.Retained, $rollupUsersMergeStats.New, $rollupUsersMergeStats.Departed, $rollupUsersMergeStats.Union) -ForegroundColor Green - $_rollupAuiDisplay = if ($script:AppendRaw.ContainsKey('UserInfo') -and $script:AppendRaw['UserInfo']) { $script:AppendRaw['UserInfo'] } else { $AppendUserInfo } - Write-LogHost ("Entra Users File: appended to {0}" -f $_rollupAuiDisplay) -ForegroundColor Gray - # Delete the transient rollup-Python Users CSV: its content is now - # fully represented in $AppendUserInfo (pristine-raw separation), so - # leaving it on disk would cause it to be picked up by the run-timestamp - # upload sweep as a duplicate of the merged target. - try { - if ((Test-Path -LiteralPath $rollupUsersCsv) -and ([System.IO.Path]::GetFullPath($rollupUsersCsv) -ne [System.IO.Path]::GetFullPath($AppendUserInfo))) { - Remove-Item -LiteralPath $rollupUsersCsv -Force -ErrorAction Stop + if ($script:PaxUserKeyContinuityFailed) { + throw 'user key continuity was not established for this run' + } + # Every user in the activity data must resolve to a Users row, so a user + # who appears only in activity and has no directory row this run receives + # a minimal row carrying nothing but the identity and the key already + # reserved for it. A real directory row always wins. + $_rollupUsersForMerge = $rollupUsersCsv + if ($script:PaxAuditOnlyUsersCsv -and (Test-Path -LiteralPath $script:PaxAuditOnlyUsersCsv -PathType Leaf)) { + $_rollupUsersWithAudit = $rollupUsersCsv + '.withaudit' + # Only a run that also appends the activity target can resolve keys + # minted for a person seen for the first time in this run. + $_rollupCurrentFactForUsers = $null + if ($AppendFile) { + $_rollupCurrentFactProbe = Join-Path $rollupOutputDir ("{0}_Interactions.csv" -f ([System.IO.Path]::GetFileNameWithoutExtension($rollupPurviewCsv))) + if (Test-Path -LiteralPath $_rollupCurrentFactProbe -PathType Leaf) { $_rollupCurrentFactForUsers = $_rollupCurrentFactProbe } + } + $_rollupPlaceholderCount = Add-PaxAuditOnlyUserRows -CurrentUsersCsv $rollupUsersCsv -AuditOnlyCsv $script:PaxAuditOnlyUsersCsv -OutputPath $_rollupUsersWithAudit -CandidateFactCsv $_rollupCurrentFactForUsers + $_rollupUsersForMerge = $_rollupUsersWithAudit + $script:PaxAuditOnlyUsersWorkFile = $_rollupUsersWithAudit + if ($_rollupPlaceholderCount -gt 0) { + Write-LogHost ("Rollup: -AppendUserInfo: {0:N0} minimal Users row(s) added so every user in the activity data resolves to a Users row." -f $_rollupPlaceholderCount) -ForegroundColor DarkCyan } } - catch { - Write-LogHost ("Rollup: -AppendUserInfo: transient Users CSV cleanup failed (non-fatal): {0}" -f $_.Exception.Message) -ForegroundColor DarkYellow + Write-PaxRollupStage -Stage 'users-merge' -Status 'entry' + # Observability-only heartbeat for a union that can run a long time. The + # stage name and EVERY counter key are literal at this call site, so the + # counter-name channel is closed by inspection: no key, row, field, file + # name, path, URL, user principal name, tenant or account identifier, or + # credential can reach a record. Only AGGREGATE numeric values are read + # from the snapshot, and any failure here is swallowed so a logging + # problem can never alter merge output, ordering, counts, or the result. + $rollupUsersMergeProgress = { + param($Snapshot) + try { + if ($null -eq $Snapshot) { return } + Write-PaxRollupStage -Stage 'users-merge' -Status 'heartbeat' -Counters @{ + rowsRead = [int64]$Snapshot.RowsRead + rowsWritten = [int64]$Snapshot.RowsWritten + runCount = [int64]$Snapshot.RunCount + mergePassCount = [int64]$Snapshot.MergePassCount + maxOpenReaders = [int64]$Snapshot.MaxOpenReaders + temporaryBytes = [int64]$Snapshot.TempBytesWritten + elapsedSeconds = [double]$Snapshot.ElapsedSeconds + } + } + catch { } } + # The union goes to a candidate path; the existing target is replaced only + # after the joint referential-integrity gate passes. + $_rollupAuiCandidate = $_rollupAuiMergeOut + '.paxcandidate' + $rollupUsersMergeStats = Merge-UsersCsv -TargetUsersCsv $AppendUserInfo -CurrentUsersCsv $_rollupUsersForMerge -OutputPath $_rollupAuiCandidate -ProgressCallback $rollupUsersMergeProgress -ProgressIntervalSeconds $StatusIntervalSeconds + Write-PaxRollupStage -Stage 'users-merge' -Status 'complete' -Counters @{ retained = [int64]$rollupUsersMergeStats.Retained; added = [int64]$rollupUsersMergeStats.New; departed = [int64]$rollupUsersMergeStats.Departed; union = [int64]$rollupUsersMergeStats.Union } + Write-LogHost ("Rollup: -AppendUserInfo merge: Retained={0:N0} New={1:N0} Departed={2:N0} Union={3:N0}" -f $rollupUsersMergeStats.Retained, $rollupUsersMergeStats.New, $rollupUsersMergeStats.Departed, $rollupUsersMergeStats.Union) -ForegroundColor Green + # The transient rollup-Python Users CSV is the recovery artifact until the + # gate publishes; it is removed there, not here. + $script:PaxUsersCandidatePath = $_rollupAuiCandidate + $script:PaxUsersCandidateFinal = $_rollupAuiMergeOut + $script:PaxUsersTransientCsv = $rollupUsersCsv } else { Write-LogHost ("Rollup: -AppendUserInfo: expected Users CSV not found at '{0}'; skipping merge." -f $rollupUsersCsv) -ForegroundColor Yellow } } catch { - # Non-fatal: the un-merged current-run Users CSV is still on disk - # (we just failed to fold it into the -AppendUserInfo target). Surface - # both facts so the customer knows (a) the target was NOT updated - # this run, and (b) where to find the new-run output to merge by hand. + # The un-merged current-run Users CSV is still on disk (we just failed to + # fold it into the -AppendUserInfo target). Surface both facts so the + # customer knows (a) the target was NOT updated this run, and (b) where to + # find the new-run output. Because the two targets are published together, + # this also stops the Fact target from being replaced on its own. + $script:PaxUserKeyContinuityFailed = $true + $script:HadTerminalFailures = $true + if ($script:PaxUsersCandidatePath -and (Test-Path -LiteralPath $script:PaxUsersCandidatePath -PathType Leaf)) { + try { Remove-Item -LiteralPath $script:PaxUsersCandidatePath -Force -ErrorAction SilentlyContinue } catch { } + } + $script:PaxUsersCandidatePath = $null + Write-PaxRollupStage -Stage 'users-merge' -Status 'failed' Write-LogHost ("Rollup: -AppendUserInfo merge FAILED: {0}" -f $_.Exception.Message) -ForegroundColor Yellow Write-LogHost ("Rollup: -> Target NOT updated this run: {0}" -f (Get-DisplayPath -LocalPath $AppendUserInfo)) -ForegroundColor Yellow Write-LogHost ("Rollup: -> New-run Users CSV preserved at: {0}" -f (Get-DisplayPath -LocalPath $rollupUsersCsv)) -ForegroundColor Yellow @@ -32586,9 +47013,8 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # customer-facing Users dimension and must be uploaded (plain overwrite) to # the UserInfo destination. When -OutputPathUserInfo supplies a file-form leaf, # the rolled-up leaf ('_Users.csv') does NOT carry the current - # run timestamp, so the remote upload sweep's timestamp wildcard misses it - # (previously, '-AppendFile + -OutputPathUserInfo + -Rollup' produced only - # 2 artifacts, dropping the Entra Users dim). Register the leaf explicitly — + # run timestamp, so the remote upload sweep's timestamp wildcard misses it. + # Register the leaf explicitly — # exactly like the M365 sidecar leaves — so the upload sweep includes it; # per-data-type routing (Get-DataTypeForOutputFile -> 'UserInfo') then lands # it at -OutputPathUserInfo. No union/merge, no In_Latest_Append provenance. @@ -32675,39 +47101,160 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread Write-LogHost "Rollup: -> To re-baseline: use this new file as your -AppendFile target going forward (it carries Message_Id_Raw and the full grain-composite key); subsequent appends reconcile on real grain + message identity." -ForegroundColor Yellow } elseif (Test-Path -LiteralPath $rollupFactCsv) { - $rollupFactMergeStats = Merge-FactCsv -TargetFactCsv $AppendFile -CurrentFactCsv $rollupFactCsv -KeyColumn 'Message_Id_Raw' -CompositeKeyColumn $rollupAfCompositeKey -OutputPath $AppendFile - Write-LogHost ("Rollup: -AppendFile merge: Retained={0:N0} New={1:N0} Departed={2:N0} Union={3:N0}" -f $rollupFactMergeStats.Retained, $rollupFactMergeStats.New, $rollupFactMergeStats.Departed, $rollupFactMergeStats.Union) -ForegroundColor Green - $_rollupAfDisplay = if ($script:AppendRaw.ContainsKey('Purview') -and $script:AppendRaw['Purview']) { $script:AppendRaw['Purview'] } else { $AppendFile } - Write-LogHost ("Appended to: {0}" -f $_rollupAfDisplay) -ForegroundColor White - # Delete the transient rollup-Python Fact CSV: its content is now - # fully represented in $AppendFile (pristine-raw separation), so - # leaving it on disk would cause it to be picked up by the run-timestamp - # upload sweep as a duplicate of the merged target. - try { - if ((Test-Path -LiteralPath $rollupFactCsv) -and ([System.IO.Path]::GetFullPath($rollupFactCsv) -ne [System.IO.Path]::GetFullPath($AppendFile))) { - Remove-Item -LiteralPath $rollupFactCsv -Force -ErrorAction Stop - } + # Announce the local bounded reconciliation before it starts so a + # large field-scale append shows what is happening and, importantly, + # that nothing has been sent anywhere yet — this whole step is a + # disk-bounded local merge that runs before any remote upload. + $rollupAfTargetBytes = try { [int64](Get-Item -LiteralPath $AppendFile).Length } catch { [int64]0 } + $rollupAfCurrentBytes = try { [int64](Get-Item -LiteralPath $rollupFactCsv).Length } catch { [int64]0 } + Write-LogHost ("Rollup: -AppendFile: local bounded reconciliation has begun (memory budget 64 MiB, fan-in 8; scratch under '{0}'). Existing target {1:N0} bytes + this run {2:N0} bytes. Remote upload has not started." -f $env:TEMP, $rollupAfTargetBytes, $rollupAfCurrentBytes) -ForegroundColor Cyan + # Surface external-sort progress from inside the reconciliation. The + # callback is interval-throttled by Invoke-PaxExternalSort itself + # (ProgressIntervalSeconds), so it prints at most one line per status + # interval per sort phase plus the forced first-of-stage markers. + $rollupFactMergeProgress = { + param($snapshot) + try { + Write-LogHost ("Rollup: -AppendFile reconciliation [{0}]: rowsRead={1:N0} rowsWritten={2:N0} runs={3:N0} elapsed={4:N0}s" -f $snapshot.Stage, [int64]$snapshot.RowsRead, [int64]$snapshot.RowsWritten, [int64]$snapshot.RunCount, [double]$snapshot.ElapsedSeconds) -ForegroundColor DarkGray + } catch { } } - catch { - Write-LogHost ("Rollup: -AppendFile: transient Fact CSV cleanup failed (non-fatal): {0}" -f $_.Exception.Message) -ForegroundColor DarkYellow + if ($script:PaxUserKeyContinuityFailed) { + throw 'user key continuity was not established for this run' } + # The union goes to a candidate path; the existing target is replaced only + # after the joint referential-integrity gate passes. + $_rollupAfCandidate = $AppendFile + '.paxcandidate' + $rollupFactMergeStats = Merge-FactCsv -TargetFactCsv $AppendFile -CurrentFactCsv $rollupFactCsv -KeyColumn 'Message_Id_Raw' -CompositeKeyColumn $rollupAfCompositeKey -OutputPath $_rollupAfCandidate -ProgressCallback $rollupFactMergeProgress -ProgressIntervalSeconds $StatusIntervalSeconds + Write-LogHost ("Rollup: -AppendFile merge: Retained={0:N0} New={1:N0} Departed={2:N0} Union={3:N0}" -f $rollupFactMergeStats.Retained, $rollupFactMergeStats.New, $rollupFactMergeStats.Departed, $rollupFactMergeStats.Union) -ForegroundColor Green + # The transient rollup-Python Fact CSV is the recovery artifact until the + # gate publishes; it is removed there, not here. + $script:PaxFactCandidatePath = $_rollupAfCandidate + $script:PaxFactCandidateFinal = $AppendFile + $script:PaxFactTransientCsv = $rollupFactCsv } else { Write-LogHost ("Rollup: -AppendFile: expected Fact CSV not found at '{0}'; skipping merge." -f $rollupFactCsv) -ForegroundColor Yellow } } catch { - # Non-fatal: the un-merged current-run Fact CSV is still on disk - # (we just failed to fold it into the -AppendFile target). Surface - # both facts so the customer knows (a) the target was NOT updated - # this run, and (b) where to find the new-run output to merge by hand. - Write-LogHost ("Rollup: -AppendFile merge FAILED: {0}" -f $_.Exception.Message) -ForegroundColor Yellow - Write-LogHost ("Rollup: -> Target NOT updated this run: {0}" -f (Get-DisplayPath -LocalPath $AppendFile)) -ForegroundColor Yellow - Write-LogHost ("Rollup: -> New-run Fact CSV preserved at: {0}" -f (Get-DisplayPath -LocalPath $rollupFactCsv)) -ForegroundColor Yellow + # A REQUESTED output failed: the -AppendFile reconciliation threw, so the + # target was NOT updated this run. This is a completed-with-gaps condition + # (exit 40), NOT a best-effort skip. Flag HadTerminalFailures so the process + # exit is truthful, and flag FactAppendFailed so the end-of-run cleanup + # PRESERVES the recovery artifacts (unmerged current-run Fact + remote scratch + # + Fabric resume mirror) instead of deleting them as on a clean run. It is NOT + # an upload failure, so AnyUploadFailed is left untouched (a local merge fault + # must never be reported as a remote upload fault). The un-merged current-run + # Fact CSV stays on disk as the recovery artifact; register its leaf so the + # end-of-run upload sweep + local listing EXCLUDE it — it must never be published + # as the completed -AppendFile target, nor as a second normal Fact output. + $script:HadTerminalFailures = $true + $script:FactAppendFailed = $true + if ($script:PaxFactCandidatePath -and (Test-Path -LiteralPath $script:PaxFactCandidatePath -PathType Leaf)) { + try { Remove-Item -LiteralPath $script:PaxFactCandidatePath -Force -ErrorAction SilentlyContinue } catch { } + } + $script:PaxFactCandidatePath = $null + if ($rollupFactCsv -and (Test-Path -LiteralPath $rollupFactCsv -PathType Leaf)) { + if (-not $script:FactAppendRecoveryLeafs) { $script:FactAppendRecoveryLeafs = New-Object System.Collections.Generic.List[string] } + $rollupAfRecoveryLeaf = [System.IO.Path]::GetFileName($rollupFactCsv) + if ($rollupAfRecoveryLeaf -and -not $script:FactAppendRecoveryLeafs.Contains($rollupAfRecoveryLeaf)) { [void]$script:FactAppendRecoveryLeafs.Add($rollupAfRecoveryLeaf) } + } + Write-LogHost ("Rollup: -AppendFile merge FAILED: {0}" -f $_.Exception.Message) -ForegroundColor Red + Write-LogHost "Rollup: -> Requested append output NOT completed this run (completed with gaps, exit 40)." -ForegroundColor Red + Write-LogHost ("Rollup: -> Target left UNCHANGED this run: {0}" -f (Get-DisplayPath -LocalPath $AppendFile)) -ForegroundColor Yellow + Write-LogHost ("Rollup: -> Recovery Fact CSV preserved (excluded from upload): {0}" -f (Get-DisplayPath -LocalPath $rollupFactCsv)) -ForegroundColor Yellow Write-LogHost "Rollup: -> Manual union-merge required to apply this run to the target." -ForegroundColor Yellow } } + # Referential-integrity gate. The candidate Fact and the candidate Users + # describe one set of people, so they are validated TOGETHER and published + # together. If they disagree, NEITHER replaces its existing target: both + # targets stay byte-for-byte as they were, this run's output is preserved for + # review, and the run reports completed with gaps. + if ($script:RollupProcessorMode -eq 'CopilotInteraction' -and ($script:PaxUsersCandidatePath -or $script:PaxFactCandidatePath)) { + $_paxGateOk = $false + try { + $_paxGateUsers = if ($script:PaxUsersCandidatePath) { $script:PaxUsersCandidatePath } + elseif ($AppendUserInfo -and (Test-Path -LiteralPath $AppendUserInfo -PathType Leaf)) { $AppendUserInfo } + else { $null } + $_paxGateFact = if ($script:PaxFactCandidatePath) { $script:PaxFactCandidatePath } else { $null } + if ($_paxGateUsers -or $_paxGateFact) { + Write-PaxRollupStage -Stage 'append-integrity' -Status 'entry' + $_paxGate = Test-PaxUserKeyReferentialIntegrity -CandidateUsersCsv $_paxGateUsers -CandidateFactCsv $_paxGateFact + if ($_paxGate.Passed) { + Write-PaxRollupStage -Stage 'append-integrity' -Status 'complete' -Counters @{ usersRows = [int64]$_paxGate.UsersRows; factRows = [int64]$_paxGate.FactRows; conflictingUserKeys = [int64]$_paxGate.ConflictingUserKeys; splitIdentities = [int64]$_paxGate.SplitIdentities; orphanFactKeys = [int64]$_paxGate.OrphanFactKeys; mismatchedFactRows = [int64]$_paxGate.MismatchedFactRows } + $_paxGateOk = $true + } + else { + Write-PaxRollupStage -Stage 'append-integrity' -Status 'failed' -Counters @{ conflictingUserKeys = [int64]$_paxGate.ConflictingUserKeys; splitIdentities = [int64]$_paxGate.SplitIdentities; orphanFactKeys = [int64]$_paxGate.OrphanFactKeys; mismatchedFactRows = [int64]$_paxGate.MismatchedFactRows } + } + } + else { + # Neither candidate is present, so there is nothing to validate. + $_paxGateOk = $true + } + } + catch { + $_paxGateOk = $false + Write-PaxRollupStage -Stage 'append-integrity' -Status 'failed' + Write-LogHost ("Rollup: append integrity check failed: {0}" -f $_.Exception.Message) -ForegroundColor Red + } + + if ($_paxGateOk) { + try { + if ($script:PaxUsersCandidatePath) { + Move-Item -LiteralPath $script:PaxUsersCandidatePath -Destination $script:PaxUsersCandidateFinal -Force -ErrorAction Stop + $script:PaxUsersCandidatePath = $null + $_rollupAuiDisplay = if ($script:AppendRaw.ContainsKey('UserInfo') -and $script:AppendRaw['UserInfo']) { $script:AppendRaw['UserInfo'] } else { $AppendUserInfo } + Write-LogHost ("Entra Users File: appended to {0}" -f $_rollupAuiDisplay) -ForegroundColor Gray + } + if ($script:PaxFactCandidatePath) { + Move-Item -LiteralPath $script:PaxFactCandidatePath -Destination $script:PaxFactCandidateFinal -Force -ErrorAction Stop + $script:PaxFactCandidatePath = $null + $_rollupAfDisplay = if ($script:AppendRaw.ContainsKey('Purview') -and $script:AppendRaw['Purview']) { $script:AppendRaw['Purview'] } else { $AppendFile } + Write-LogHost ("Appended to: {0}" -f $_rollupAfDisplay) -ForegroundColor White + } + } + catch { + $_paxGateOk = $false + Write-LogHost ("Rollup: append publication failed: {0}" -f $_.Exception.Message) -ForegroundColor Red + } + } + + if ($_paxGateOk) { + # Both targets now carry this run, so the transient current-run inputs are + # duplicates and are removed. + foreach ($_paxTransient in @($script:PaxUsersTransientCsv, $script:PaxFactTransientCsv, $script:PaxAuditOnlyUsersWorkFile)) { + if ($_paxTransient -and (Test-Path -LiteralPath $_paxTransient -PathType Leaf)) { + try { Remove-Item -LiteralPath $_paxTransient -Force -ErrorAction Stop } + catch { Write-LogHost ("Rollup: transient append input cleanup failed (non-fatal): {0}" -f $_.Exception.Message) -ForegroundColor DarkYellow } + } + } + } + else { + # Publish NEITHER candidate. Both existing targets are left exactly as they + # were and this run's output is kept, so nothing is lost. + $script:HadTerminalFailures = $true + $script:FactAppendFailed = $true + foreach ($_paxCandidate in @($script:PaxUsersCandidatePath, $script:PaxFactCandidatePath, $script:PaxAuditOnlyUsersWorkFile)) { + if ($_paxCandidate -and (Test-Path -LiteralPath $_paxCandidate -PathType Leaf)) { + try { Remove-Item -LiteralPath $_paxCandidate -Force -ErrorAction SilentlyContinue } catch { } + } + } + if ($script:PaxFactTransientCsv -and (Test-Path -LiteralPath $script:PaxFactTransientCsv -PathType Leaf)) { + if (-not $script:FactAppendRecoveryLeafs) { $script:FactAppendRecoveryLeafs = New-Object System.Collections.Generic.List[string] } + $_paxRecoveryLeaf = [System.IO.Path]::GetFileName($script:PaxFactTransientCsv) + if ($_paxRecoveryLeaf -and -not $script:FactAppendRecoveryLeafs.Contains($_paxRecoveryLeaf)) { [void]$script:FactAppendRecoveryLeafs.Add($_paxRecoveryLeaf) } + } + Write-LogHost "Rollup: append NOT completed this run: the Fact and Users output did not agree on user identity keys." -ForegroundColor Red + Write-LogHost "Rollup: -> Both existing targets are unchanged (completed with gaps, exit 40)." -ForegroundColor Red + Write-LogHost "Rollup: -> This run's output is preserved for review; re-baseline to apply it." -ForegroundColor Yellow + } + $script:PaxUsersCandidatePath = $null + $script:PaxFactCandidatePath = $null + } + # -AppendFile (M365Bundle mode): merge the current run's rollup into # the customer's target rollup, then regenerate the UserStats and # SessionCohort sidecars from the merged content and union-merge @@ -32721,7 +47268,7 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # $AppendFile (column-tolerant — legacy 9/10-col targets are # padded with empty agent columns + IsAgentInteraction=FALSE) # and writes the union back to $AppendFile. - # 2b. Merge-M365SessionStatsCsv (v2.6.0+) unions the current + # 2b. Merge-M365SessionStatsCsv unions the current # SessionStats with the anchored '_SessionStats.csv' # target (creating it on first run) using additive counter # semantics keyed on (UserId, CreationDate, AppHost). @@ -32904,12 +47451,14 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } else { Write-LogHost "Rollup: post-processor returned non-zero exit code $rollupExit. Raw CSV(s) preserved." -ForegroundColor Red + $script:RollupProcessorFailed = $true } } catch { Write-LogHost "Rollup: post-processor failed: $($_.Exception.Message)" -ForegroundColor Red Write-Log "Rollup: $($_.ScriptStackTrace)" $rollupSuccess = $false + $script:RollupProcessorFailed = $true } # 5. Retention: -Rollup deletes raw CSV(s) on success; -RollupPlusRaw always keeps them; @@ -33030,6 +47579,12 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $null = $listingDirs.Add($_streamDt.EffectiveDir) } } + # AISID dashboard destination (DefenderUsage): its twelve fixed-name leaves are + # customer-visible outputs. When that folder is local (co-located with -OutputPath + # or a local -OutputPathDefenderUsage), include it so the roster lists all twelve. + if ($script:AISIDDestDir -and (Test-Path -LiteralPath $script:AISIDDestDir -PathType Container)) { + $null = $listingDirs.Add($script:AISIDDestDir) + } if ($listingDirs.Count -gt 0) { # Mirror the remote upload sweep's append-leaf collection (~30 lines below) # so the local listing surfaces the merged Append* targets even though @@ -33061,9 +47616,11 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread if (-not (Test-Path $_sweepDir)) { continue } $listingCandidates += Get-ChildItem -Path $_sweepDir -File -ErrorAction SilentlyContinue | Where-Object { - ($_.Name -like "*${global:ScriptRunTimestamp}*" -or $listingAppendLeafs.Contains($_.Name)) ` + ($_.Name -like "*${global:ScriptRunTimestamp}*" -or $listingAppendLeafs.Contains($_.Name) -or ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Contains($_.Name))) ` -and $_.Name -notlike '.pax_*' ` - -and $_.Name -notlike '*_PARTIAL.*' + -and $_.Name -notlike '*_PARTIAL.*' ` + -and (-not ($script:FactAppendRecoveryLeafs -and $script:FactAppendRecoveryLeafs.Contains($_.Name))) ` + -and (-not ($script:Agent365RecoveryLeafs -and $script:Agent365RecoveryLeafs.Contains($_.Name))) } } # Dedupe by full path (a stream override directory could theoretically @@ -33162,11 +47719,33 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } $uploadCandidates = Get-ChildItem -Path $uploadDir -File -ErrorAction SilentlyContinue | Where-Object { - ($_.Name -notlike '.pax_*') -and ( + ($_.Name -notlike '.pax_*') -and + (-not ($script:FactAppendRecoveryLeafs -and $script:FactAppendRecoveryLeafs.Contains($_.Name))) -and (-not ($script:Agent365RecoveryLeafs -and $script:Agent365RecoveryLeafs.Contains($_.Name))) -and ( ($_.Name -like "*${global:ScriptRunTimestamp}*") -or ($appendLeafs.Count -gt 0 -and $appendLeafs.Contains($_.Name)) ) } + # Remote AISID artifacts are staged under the run-specific OS-temp staging + # folder ($script:AISIDRunFolder.RemoteScratchStaging), NOT under $uploadDir; the general sweep + # above therefore never sees them. Discover the twelve fixed-name AISID leaves AUTHORITATIVELY from + # staging (Get-AISIDUploadPlan) and union them in, taking each file's path from staging so a + # same-named file in $uploadDir can never impersonate an AISID artifact. Any general-dir entry that + # collides with an AISID leaf name is dropped first. A missing staged leaf is an AISID gap that + # preserves scratch (exit 40). Per-leaf upload success is tracked in $script:AISIDUploadSucceeded. + # The AISID upload plan is executed ONLY when EXCLUSIVE remote ownership of the + # AISID_ child folder was acquired this run ($script:AISIDRemoteOwnershipAcquired); without it + # no AISID file is uploaded (fail closed, scratch preserved). + $script:AISIDUploadSucceeded = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + if ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Count -gt 0 -and $script:AISIDRunFolder -and $script:AISIDRemoteOwnershipAcquired) { + $aisidLeafSet = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($lf in $script:AISIDUploadLeafs) { if ($lf) { [void]$aisidLeafSet.Add($lf) } } + $aisidPlan = script:Get-AISIDUploadPlan -StagingDir $script:AISIDRunFolder.RemoteScratchStaging -ExpectedLeafs @($script:AISIDUploadLeafs) + if (-not $aisidPlan.Complete) { + $script:AISIDHadGaps = $true; $script:AnyUploadFailed = $true + foreach ($ml in $aisidPlan.MissingLeafs) { Write-LogHost ("[AISID] Expected staged upload artifact missing from staging: {0} (AISID gap; scratch preserved)." -f $ml) -ForegroundColor Yellow } + } + $uploadCandidates = @(@($uploadCandidates) | Where-Object { -not $aisidLeafSet.Contains($_.Name) }) + @($aisidPlan.Files) + } if ($uploadCandidates -and $uploadCandidates.Count -gt 0) { # Detect Fabric Tables/ Delta destination. The trailing segment is either @@ -33233,6 +47812,8 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread $originTypeByLeaf = @{} if ($script:Agent365UploadLeafs) { foreach ($lf in $script:Agent365UploadLeafs) { if ($lf) { $originTypeByLeaf[$lf] = 'Agent365Info' } } } if ($script:RollupUsersDimLeafs) { foreach ($lf in $script:RollupUsersDimLeafs) { if ($lf) { $originTypeByLeaf[$lf] = 'UserInfo' } } } + # AISID dashboard leaves are DefenderUsage-destined (fixed names, no run-timestamp). + if ($script:AISIDUploadLeafs) { foreach ($lf in $script:AISIDUploadLeafs) { if ($lf) { $originTypeByLeaf[$lf] = 'DefenderUsage' } } } foreach ($originKey in @('UserInfo','Agent365Info')) { if ($script:AppendIsBound.ContainsKey($originKey) -and $script:AppendIsBound[$originKey] -and $script:AppendRaw.ContainsKey($originKey)) { $originAppendLeaf = [System.IO.Path]::GetFileName($script:AppendRaw[$originKey]) @@ -33264,7 +47845,13 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread # Fall back to DestRaw if the normalized dict has no entry (e.g. local tier). $dtParentUrl = if ($script:DestParentUrl.ContainsKey($dtKey)) { $script:DestParentUrl[$dtKey] } else { $script:DestRaw[$dtKey] } } - if ($fabricDeltaMode -and $deltaReady -and $isCsv) { + # AISID dashboard leaves publish into the per-run + # 'AISID_' child folder on the DefenderUsage destination for EVERY + # remote tier (SharePoint / Fabric). Override the flat DefenderUsage parent with the + # resolved run-folder URL so uploads land under AISID_, never flat. Uses the + # established transport (this only changes the target URL; the upload path is unchanged). + $dtParentUrl = script:Resolve-AISIDUploadParentUrl -Leaf $uploadFile.Name -DefaultParentUrl $dtParentUrl + if ($fabricDeltaMode -and $deltaReady -and $isCsv -and -not ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Contains($uploadFile.Name))) { # Derive stable table name: strip "_YYYYMMDD_HHMMSS" suffix when present. $tableName = [System.IO.Path]::GetFileNameWithoutExtension($uploadFile.Name) $tableName = ($tableName -replace '_\d{8}_\d{6}$', '') @@ -33305,12 +47892,27 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread else { try { Invoke-OutputUpload -LocalPath $uploadFile.FullName -ParentOverride $dtParentUrl + if ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Contains($uploadFile.Name)) { [void]$script:AISIDUploadSucceeded.Add($uploadFile.Name) } } catch { Write-LogHost ("WARNING: Upload failed for '{0}': {1}" -f $uploadFile.Name, (Get-GraphErrorDetail -ErrorRecord $_)) -ForegroundColor Yellow $script:AnyUploadFailed = $true } } } + # The deferred AISID scratch delete is allowed ONLY after all twelve fixed-name + # leaves uploaded successfully to the timestamped child URL. A missing, failed, or unattempted + # AISID upload is a gap (exit 40) that PRESERVES the staging scratch for recovery — never delete + # staged AISID files just because the general sweep recorded no failure. + if ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Count -gt 0) { + $aisidExpected = @($script:AISIDUploadLeafs | Where-Object { $_ } | Select-Object -Unique) + $aisidConfirmed = @($aisidExpected | Where-Object { $script:AISIDUploadSucceeded.Contains($_) }) + if ($aisidConfirmed.Count -ne $aisidExpected.Count) { + foreach ($ml in @($aisidExpected | Where-Object { -not $script:AISIDUploadSucceeded.Contains($_) })) { + Write-LogHost ("[AISID] Remote upload not confirmed for artifact: {0} (AISID gap; scratch preserved)." -f $ml) -ForegroundColor Yellow + } + $script:AISIDHadGaps = $true; $script:AnyUploadFailed = $true + } + } } } } @@ -33363,6 +47965,17 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread Write-LogHost "Note: Could not remove rollup seed JSONs: $($_.Exception.Message)" -ForegroundColor DarkGray } } + # Reap the run-scoped audit-only continuity list emitted alongside the seed JSONs. + $thisRunContinuityPattern = "PAX_Append*_${global:ScriptRunTimestamp}.csv" + $thisRunContinuityFiles = Get-ChildItem -Path $incrementalDir -Filter $thisRunContinuityPattern -ErrorAction SilentlyContinue + if ($thisRunContinuityFiles -and $thisRunContinuityFiles.Count -gt 0) { + try { + $thisRunContinuityFiles | Remove-Item -Force -ErrorAction Stop + Write-LogHost "Rollup continuity lists cleaned up ($($thisRunContinuityFiles.Count) files from this run)" -ForegroundColor DarkGray + } catch { + Write-LogHost "Note: Could not remove rollup continuity lists: $($_.Exception.Message)" -ForegroundColor DarkGray + } + } # Also remove the directory if it's now empty $remaining = Get-ChildItem -Path $incrementalDir -ErrorAction SilentlyContinue if (-not $remaining -or $remaining.Count -eq 0) { @@ -33387,6 +48000,9 @@ function Profile-AuditData { param([object]$AuditData) } # No-op stub for thread } } } + # All completion work finished without a late exception; mark a clean end-of-run tail so the + # AISID collection-window summary is emitted only for a genuinely completed run. + $script:AISIDCleanTail = $true } catch { # Handle Ctrl+C (PipelineStoppedException) @@ -33399,14 +48015,23 @@ catch { if ($msg -eq '__PAX_EARLY_EXIT__' -or $script:EarlyExit) { # Graceful early exit path (e.g., header-only CSV) Write-LogHost "Early exit executed: $script:EarlyExit" -ForegroundColor DarkGray + } elseif ($script:CtrlCPressed) { + # User-initiated cancel (Ctrl+C): handled by the graceful path, not a fatal abort. + Write-LogHost "Run cancelled by user (Ctrl+C)." -ForegroundColor DarkGray } else { + # 1D-a: a genuine, unexpected terminating error aborted the run. Record it as a generic + # fatal so the finally exit selector reports exit 1 (Fatal / aborted) instead of falling + # through to a false exit 0, and so the durable recovery artifacts are preserved. + $script:GenericFatal = $true Write-LogHost "Script failed: $msg" -ForegroundColor Red Write-LogHost $_.ScriptStackTrace -ForegroundColor Red } } finally { # Check if script was interrupted (didn't complete normally and not an early exit) - if (-not $script:ScriptCompleted -and -not $script:EarlyExit -and -not $script:CtrlCPressed) { + # 1D-a: exclude a recorded generic fatal so a genuine mid-run crash is NOT mis-inferred + # as a user Ctrl+C (which would suppress the fatal exit code and mislabel the log). + if (-not $script:ScriptCompleted -and -not $script:EarlyExit -and -not $script:CtrlCPressed -and -not $script:ResumeValidationFailed -and -not $script:GenericFatal -and -not $script:AISIDGateBlocked) { # Script was interrupted - likely Ctrl+C that wasn't caught by PipelineStoppedException $script:CtrlCPressed = $true } @@ -33416,7 +48041,7 @@ finally { # Agent365 phase, output summary), reap this run's incremental JSONLs and checkpoint here. # Gated on the SAME success criteria as the _PARTIAL log rename: script completed AND not # Ctrl+C AND not early-exit. Idempotent — no-op if the success path already cleaned up. - if ($script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and $global:ScriptRunTimestamp) { + if ($script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and -not $script:GenericFatal -and $global:ScriptRunTimestamp) { try { $cleanupBaseDir = if ($OutputFile) { Split-Path $OutputFile -Parent } elseif ($OutputPath) { $OutputPath } else { $null } if ($cleanupBaseDir -and (Test-Path $cleanupBaseDir)) { @@ -33475,7 +48100,7 @@ finally { # Disconnect from Microsoft Graph - ALWAYS attempt disconnect Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Cyan try { - Disconnect-MgGraph -ErrorAction Stop | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction Stop Write-Host " Microsoft Graph disconnected" -ForegroundColor Green } catch { @@ -33526,6 +48151,22 @@ finally { try { if ($script:metrics -and $script:metrics.StartTime) { $elapsed = $endUtc - $script:metrics.StartTime; $totalHours = [math]::Floor($elapsed.TotalHours); $remainder = $elapsed - [TimeSpan]::FromHours($totalHours); $elapsedFormatted = ("{0}:{1:00}:{2:00}.{3:000}" -f $totalHours, $remainder.Minutes, $remainder.Seconds, $remainder.Milliseconds); Write-Log ("Total elapsed time: {0} (hours:minutes:seconds.milliseconds)" -f $elapsedFormatted) } } catch {} $script:summaryWritten = $true } + + # AISID collection-window summary. Emitted here in the end-of-run tail - after + # export/append/merge/artifact/upload-sweep work and BEFORE the final metrics + log re-upload below + # - so the remote (SharePoint/Fabric) log copy captures it. Completed FULL/PARTIAL AISID runs only; + # a NONE or one-sided run exits before setting $script:AISIDExportCompleted and never reaches here. + # D3: under -Dashboard AISID a REQUESTED-output failure — the rollup processor failing (throw, nonzero + # exit, or a missing required Fact/Users deliverable) or an AISID export that never reached terminal + # completion — is a completed-with-gaps condition. Force the gap so the run cannot exit 0 and the clean + # AISID completion summary below is suppressed. Guarded to genuine end-of-run states (not Ctrl+C / early + # graceful exit); this only raises the exit to 40, so any higher-precedence failure code still wins. + if ($Dashboard -eq 'AISID' -and -not $script:AISIDGateBlocked -and -not $script:EarlyExit -and -not $script:CtrlCPressed -and ($script:RollupProcessorFailed -or -not $script:AISIDExportCompleted)) { + $script:AISIDHadGaps = $true + } + if ($Dashboard -eq 'AISID' -and $script:AISIDExportCompleted -and -not $script:AISIDHadGaps -and $script:AISIDCleanTail -and $script:AISIDWindow -and -not $script:EarlyExit -and -not $script:CtrlCPressed -and -not $script:AnyUploadFailed) { + Write-AISIDWindowSummary + } # NOTE: Graph / EOM disconnect is deliberately deferred until AFTER the metrics + # log re-upload below. Disconnect-MgGraph clears the in-memory Graph session state, @@ -33546,7 +48187,8 @@ finally { $outputDir = Split-Path $OutputFile -Parent $metricsPath = Join-Path $outputDir "${baseName}_metrics_$global:ScriptRunTimestamp.json" } - $emitObj = [ordered]@{ version = $ScriptVersion; timestampUtc = (Get-Date).ToUniversalTime().ToString('o'); parameters = $paramSnapshot; metrics = $script:metrics } + $aisidMetricsFields = script:Get-AISIDSnapshotFields -Dashboard $Dashboard -ResolvedOutputDir (script:Get-AISIDResolvedOutputDir) -AppendDefenderUsage $AppendDefenderUsage -DisableAISIDDeltaCache $DisableAISIDDeltaCache.IsPresent -ExportCompleted ([bool]$script:AISIDExportCompleted) + $emitObj = [ordered]@{ version = $ScriptVersion; timestampUtc = (Get-Date).ToUniversalTime().ToString('o'); parameters = $paramSnapshot; aisid = $aisidMetricsFields; metrics = $script:metrics } ($emitObj | ConvertTo-Json -Depth 6) | Out-File -FilePath $metricsPath -Encoding UTF8 Write-LogHost "Metrics JSON emitted: $metricsPath" -ForegroundColor DarkCyan } catch { @@ -33558,6 +48200,18 @@ finally { # (after explosion, before this finally block) using timestamp-based file matching. # This finally block only handles abnormal termination scenarios. + # Terminal-result logging (pre-remote-log-upload). Emit the process exit result plus the + # recovery/scratch-preservation location to the run log BEFORE the log is uploaded below, + # so the remote (SharePoint/Fabric) log copy records the terminal outcome and where local + # run files can be recovered. The authoritative exit code is re-derived from the same + # precedence at the true end of the finally block; if the finalization upload below itself + # fails, AnyUploadFailed can escalate a clean 0 to 40, reflected in the local log only. + $prelimExit = 0; if ($script:AISIDGateBlocked) { $prelimExit = 1 } elseif ($script:ResumeValidationFailed -or $script:GenericFatal) { $prelimExit = 1 } elseif ($script:circuitBreakerOpen) { $prelimExit = 20 } elseif (($script:Hit10KLimit -or $script:Hit1MLimit) -and -not $AutoCompleteness) { $prelimExit = 10 } elseif ($script:EntraUsersFetchFailed) { $prelimExit = 30 } elseif ($script:HadTerminalFailures -or $script:HadSubdivisionLimit -or $script:AISIDHadGaps -or $script:Agent365HadGaps -or $script:AnyUploadFailed) { $prelimExit = 40 } + $prelimExitMeaning = switch ($prelimExit) { 1 {'fatal error'} 20 {'stopped - repeated failures'} 10 {'row-limit reached'} 30 {'user directory fetch failed'} 40 {'completed with gaps'} default {'success'} } + Write-Log ("Terminal exit result: {0} ({1})" -f $prelimExit, $prelimExitMeaning) + if ($script:RemoteScratchDir -and ($script:AnyUploadFailed -or $script:FactAppendFailed -or $script:GenericFatal)) { Write-Log ("Recovery location - local run files preserved for retry: {0}" -f $script:RemoteScratchDir) } + Write-Log "Note: lines written after this point (Microsoft Graph disconnect confirmation and any late upload-failure escalation) appear in the local log only, not in the uploaded remote copy." + # Remote-output: final upload of metrics JSON + log file (best-effort; runs on every # exit path including error/Ctrl+C so operators can see the run output remotely). # Skip entirely when the destination probe never succeeded (e.g. pre-flight failure): @@ -33620,7 +48274,7 @@ finally { # Graph API mode: Disconnect from Microsoft Graph Write-LogHost "Disconnecting from Microsoft Graph..." -ForegroundColor Gray try { - Disconnect-MgGraph -ErrorAction Stop | Out-Null + script:Invoke-PaxGraphDisconnect -ErrorAction Stop Write-LogHost " Microsoft Graph disconnected" -ForegroundColor Green } catch { @@ -33646,8 +48300,10 @@ finally { # Remote-output: scratch directory cleanup. Only on a clean, successful run — preserve # scratch on Ctrl+C / early-exit / failure so the operator can inspect locally if running - # attended, or so a re-run from the same container can resume from local state. - if ($script:RemoteScratchDir -and $script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed) { + # attended, or so a re-run from the same container can resume from local state. A requested + # -AppendFile reconciliation failure (FactAppendFailed) is also a requested-output failure + # whose recovery Fact lives under this scratch dir in remote mode, so it likewise preserves. + if ($script:RemoteScratchDir -and $script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and -not $script:FactAppendFailed -and -not $script:GenericFatal) { try { if (Test-Path -LiteralPath $script:RemoteScratchDir) { Remove-Item -LiteralPath $script:RemoteScratchDir -Recurse -Force -ErrorAction Stop @@ -33660,15 +48316,62 @@ finally { elseif ($script:RemoteScratchDir -and $script:AnyUploadFailed -and (Test-Path -LiteralPath $script:RemoteScratchDir)) { Write-LogHost ("Local run files preserved: one or more uploads did not complete successfully. Location: {0}" -f $script:RemoteScratchDir) -ForegroundColor Yellow } + elseif ($script:RemoteScratchDir -and $script:FactAppendFailed -and (Test-Path -LiteralPath $script:RemoteScratchDir)) { + Write-LogHost ("Local run files preserved: the requested -AppendFile reconciliation did not complete (target left unchanged; recovery Fact retained). Location: {0}" -f $script:RemoteScratchDir) -ForegroundColor Yellow + } + elseif ($script:RemoteScratchDir -and $script:GenericFatal -and (Test-Path -LiteralPath $script:RemoteScratchDir)) { + Write-LogHost ("Local run files preserved: the run aborted on an unexpected fatal error (exit 1). Location: {0}" -f $script:RemoteScratchDir) -ForegroundColor Yellow + } + + # AISID run-specific scratch (OffHours cache + remote staging): a successful AISID run leaves NO cache + # or scratch directory anywhere. In remote mode the staged twelve files must survive until the upload + # sweep above, so the delete was DEFERRED here. On a clean, successful run remove the whole scratch + # root and verify — a residual scratch is a completed-with-gaps condition (exit 40). On any non-clean + # exit the scratch is preserved for recovery and the log identifies it. + if ($script:AISIDScratchPendingCleanup) { + if ($script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and -not $script:GenericFatal) { + if (-not (script:Remove-AISIDScratchRoot -Root $script:AISIDScratchPendingCleanup)) { + Write-LogHost ("[AISID] Internal scratch could not be fully removed after upload: {0}; flagging an AISID gap (exit 40)." -f $script:AISIDScratchPendingCleanup) -ForegroundColor Yellow + $script:AISIDHadGaps = $true + } + else { $script:AISIDScratchPendingCleanup = $null } + } + elseif (Test-Path -LiteralPath $script:AISIDScratchPendingCleanup) { + Write-LogHost ("[AISID] Internal scratch preserved for recovery (run did not complete cleanly): {0}" -f $script:AISIDScratchPendingCleanup) -ForegroundColor Yellow + } + } + + # Blocker-5: the local AISID ownership marker (atomic run-folder lock) is removed ONLY after a clean, + # successful, gap-free run. On any non-clean exit or gap it is preserved so a re-run detects the prior + # incomplete attempt and fails closed (a re-run reuses the folder only via verified resume). Another + # run's marker is never touched. In remote mode the marker lives under the scratch root and is removed + # with it above; this handles the local-tier marker in the customer output folder. + if ($script:AISIDOwnershipMarker -and (Test-Path -LiteralPath $script:AISIDOwnershipMarker)) { + if ($script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and -not $script:AISIDHadGaps -and -not $script:GenericFatal) { + # Defect-3 fail-closed: a clean exit 0 is allowed ONLY when the marker is absent. Attempt + # deletion, then VERIFY absence; if it remains, flag an AISID gap (exit 40) and log the exact + # marker path — deletion failures are never swallowed. + try { Remove-Item -LiteralPath $script:AISIDOwnershipMarker -Force -ErrorAction Stop } catch {} + if (Test-Path -LiteralPath $script:AISIDOwnershipMarker) { + Write-LogHost ("[AISID] Ownership marker could not be removed after a clean run: {0}; flagging an AISID gap (exit 40)." -f $script:AISIDOwnershipMarker) -ForegroundColor Yellow + $script:AISIDHadGaps = $true + } + } + else { + Write-LogHost ("[AISID] Ownership marker preserved for recovery (run did not complete cleanly): {0}" -f $script:AISIDOwnershipMarker) -ForegroundColor Yellow + } + } # Fabric tier: drop the durable resume mirror on a clean, successful run so the # Lakehouse Files/.pax_resume/ namespace stays clean. Preserved on any non-clean - # exit so a subsequent container can resume from it. - if ($script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and $global:ScriptRunTimestamp) { + # exit so a subsequent container can resume from it — including a requested + # -AppendFile reconciliation failure (FactAppendFailed), which is a completed-with-gaps + # outcome, not a clean run. + if ($script:ScriptCompleted -and -not $script:CtrlCPressed -and -not $script:EarlyExit -and -not $script:AnyUploadFailed -and -not $script:FactAppendFailed -and -not $script:GenericFatal -and $global:ScriptRunTimestamp) { try { Remove-FabricResumeMirror -RunTimestamp $global:ScriptRunTimestamp } catch {} } - $exitCode = 0; if ($script:circuitBreakerOpen) { $exitCode = 20 } elseif (($script:Hit10KLimit -or $script:Hit1MLimit) -and -not $AutoCompleteness) { $exitCode = 10 } elseif ($script:EntraUsersFetchFailed) { $exitCode = 30 } + $exitCode = 0; if ($script:AISIDGateBlocked) { $exitCode = 1 } elseif ($script:ResumeValidationFailed -or $script:GenericFatal) { $exitCode = 1 } elseif ($script:circuitBreakerOpen) { $exitCode = 20 } elseif (($script:Hit10KLimit -or $script:Hit1MLimit) -and -not $AutoCompleteness) { $exitCode = 10 } elseif ($script:EntraUsersFetchFailed) { $exitCode = 30 } elseif ($script:HadTerminalFailures -or $script:HadSubdivisionLimit -or $script:AISIDHadGaps -or $script:Agent365HadGaps -or $script:AnyUploadFailed) { $exitCode = 40 } Write-LogHost "Exit code: $exitCode" -ForegroundColor DarkGray exit $exitCode } diff --git a/README.md b/README.md index 8b8440d..1e839b6 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@

PAX PowerShell Script  |  PAX Cookbook  |  PAX Cookbook Mini-Kitchen

-**⬇️ Download the script:** [`PAX_Purview_Audit_Log_Processor_v1.11.14.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.14/PAX_Purview_Audit_Log_Processor_v1.11.14.ps1)  |  Release Date: July 6, 2026 +**⬇️ Download the script:** [`PAX_Purview_Audit_Log_Processor_v1.11.15.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.15/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1)  |  Release Date: July 31, 2026 **📖 Script Resources:** [Latest Documentation](https://github.com/microsoft/PAX/blob/release/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md) | [Latest Release Notes](https://github.com/microsoft/PAX/blob/release/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md) diff --git a/fabric_resources/CompatibilityMatrix.md b/fabric_resources/CompatibilityMatrix.md index 39389ed..17b2c6e 100644 --- a/fabric_resources/CompatibilityMatrix.md +++ b/fabric_resources/CompatibilityMatrix.md @@ -87,13 +87,13 @@ Three trailing columns are appended at merge time to **any appended file**: | `Latest_Append_Date` | `YYYY-MM-DD` | Latest run timestamp that touched the file. Same value on every row; updated each append. | | `In_Latest_Append` | `TRUE` / `FALSE` | Whether the row appeared in the latest run's audit window or membership snapshot. `FALSE` for retained-but-departed rows. | -The CopilotInteraction rollup Fact CSV additionally carries stable identity columns (always present, not just at append time) so per-run integer surrogates remain stable across appends: the two trailing raw keys **`Message_Id_Raw`** (raw audit `Messages[].Id` GUID) and **`ThreadId_Raw`** (raw `CopilotEventData.ThreadId` GUID), plus a stable normalized user-identity column — **`User_Id_Normalized`** on the AIO fact (additive; parity with the **`Audit_UserId_Normalized`** column the AIBV fact already carries). Merge keys: the rollup Fact CSV dedups on a **composite key** — the profile grain (every grain column, with per-run integer surrogates replaced by their stable forms: `UserKey` → `User_Id_Normalized` on AIO / `Audit_UserId_Normalized` on AIBV, and `ThreadId` → `ThreadId_Raw`) **plus `Message_Id_Raw`** — because one message can fan out to several rows (one per distinct grain, e.g. per accessed resource) that must each reconcile independently rather than collapse to one. The non-rollup raw audit CSV keys on `RecordId`, the EntraUsers CSV on `PersonId_Normalized`, and the Agent 365 CSV on `AgentId`. +The CopilotInteraction rollup Fact CSV additionally carries stable identity columns (always present, not just at append time) so per-run integer surrogates remain stable across appends: the two trailing raw keys **`Message_Id_Raw`** (raw audit `Messages[].Id` GUID) and **`ThreadId_Raw`** (raw `CopilotEventData.ThreadId` GUID), plus a stable normalized user-identity column — **`User_Id_Normalized`** on the AIO fact (additive; parity with the **`Audit_UserId_Normalized`** column the ValueLens fact already carries). Merge keys: the rollup Fact CSV dedups on a **composite key** — the profile grain (every grain column, with per-run integer surrogates replaced by their stable forms: `UserKey` → `User_Id_Normalized` on AIO / `Audit_UserId_Normalized` on ValueLens, and `ThreadId` → `ThreadId_Raw`) **plus `Message_Id_Raw`** — because one message can fan out to several rows (one per distinct grain, e.g. per accessed resource) that must each reconcile independently rather than collapse to one. The non-rollup raw audit CSV keys on `RecordId`, the EntraUsers CSV on `PersonId_Normalized`, and the Agent 365 CSV on `AgentId`. **Rollup Fact CSV one-time re-baseline (parallel to the Users schema-compatibility probe below).** When an existing rollup Fact CSV `-AppendFile` target predates these keys — i.e. it is missing `Message_Id_Raw` or any composite-key column (for example an older AIO file written before `User_Id_Normalized` existed) — PAX does **not** silently overwrite or mis-merge it. It leaves the old target untouched, writes the current run to a new timestamped file, and prints re-baseline guidance. The practical effect is a one-time re-baseline with no data lost. The 0-row overwrite refusal and union-shrink guards remain in force alongside it. **Deidentification (`-Deidentify`) and merge keys.** Under `-Deidentify`, identity values are replaced with irreversible deterministic tokens before any write. The documented merge keys remain usable: `PersonId_Normalized` and the fact composite key's user component (`User_Id_Normalized` / `Audit_UserId_Normalized`) are tokenized but deterministic (so EntraUsers/Users joins, distinct counts, and cross-run fact reconciliation are preserved across runs), while `Message_Id_Raw` / `ThreadId_Raw` are message/thread GUIDs that are **not** tokenized. A run's deidentify state is detected from the target's identity columns; appending a `-Deidentify` run into a non-deidentified target (or the reverse) is hard-rejected at pre-flight to prevent token/raw identity mixing. `-Deidentify` is persisted in the checkpoint and restored on `-Resume`. -**Org / manager-hierarchy columns and Delta schema evolution.** AIO / AIBV rollups add org/manager-hierarchy columns to the Users output (`OrgLevel`, `Manager_UserKey`, `TopOfChain_UserKey`, `HierarchyPath`, `IsManager`, `DirectReports`, `TotalReports`, and `Level0`…`Level14` name/key pairs; the M365 dashboard has none). On Fabric, the Users append runs a schema-compatibility probe that **tolerates added columns but rejects missing ones**: appending a current (hierarchy-bearing) run into a Delta table created by an older PAX simply adds the new columns, but appending older output that lacks the hierarchy columns into a table that already has them fails the compatibility check. Keep a given Fabric Users table on a single PAX version line, or recreate it, to avoid mixed-schema appends. The hierarchy needs no new Graph scope (manager data comes from the existing `$expand=manager` under `User.Read.All`). +**Org / manager-hierarchy columns and Delta schema evolution.** AIO / ValueLens rollups add org/manager-hierarchy columns to the Users output (`OrgLevel`, `Manager_UserKey`, `TopOfChain_UserKey`, `HierarchyPath`, `IsManager`, `DirectReports`, `TotalReports`, and `Level0`…`Level14` name/key pairs; the M365 dashboard has none). On Fabric, the Users append runs a schema-compatibility probe that **tolerates added columns but rejects missing ones**: appending a current (hierarchy-bearing) run into a Delta table created by an older PAX simply adds the new columns, but appending older output that lacks the hierarchy columns into a table that already has them fails the compatibility check. Keep a given Fabric Users table on a single PAX version line, or recreate it, to avoid mixed-schema appends. The hierarchy needs no new Graph scope (manager data comes from the existing `$expand=manager` under `User.Read.All`). **M365 rollup append anchoring (`-IncludeM365Usage` + `-Rollup` / `-RollupPlusRaw`).** The embedded M365 Bundle Explosion Processor produces a 4-file bundle in the destination folder/Lakehouse: `_Rollup.csv`, `_UserStats.csv`, `_SessionCohort.csv`, `_SessionStats.csv`. Only `_Rollup.csv` is the merge anchor; the merge is PowerShell-driven (`Merge-M365RollupCsv` unions the Rollup, then `--rebuild-sidecars-from-rollup` regenerates the sidecars). UserStats and SessionCohort are RECOMPUTED over the merged Rollup on every run; SessionStats is union-merged in place via `Merge-M365SessionStatsCsv` (additive counter semantics keyed on `UserId`, `CreationDate`, `AppHost`). All three sidecars are anchored off the `-AppendFile` leaf stem and overwrite their derived destination URLs in-place. `-AppendFile` MUST point to a `_Rollup.csv` leaf; sidecar leaves (`_UserStats.csv`, `_SessionCohort.csv`, `_SessionStats.csv`) and CopilotInteraction (`_Interactions.csv`) / event-level (`_Exploded.csv`) leaves are rejected at pre-flight. Renamed AppendFile is fine as long as the leaf still ends in `_Rollup.csv` (e.g. `MyM365Rollup_Rollup.csv` produces sidecars `MyM365Rollup_UserStats.csv`, `MyM365Rollup_SessionCohort.csv`, `MyM365Rollup_SessionStats.csv`); the validator also accepts a timestamped `_Rollup_.csv` anchor. Prior sidecars with different leaf names are not auto-deleted. On Fabric `Tables/`, all 4 leaves become 4 stable Delta tables under the same schema (table name = leaf stem), refreshed via `Convert-CsvToDelta -Mode overwrite` on every run. @@ -123,7 +123,14 @@ major exceeds 2. Operators should never edit checkpoints by hand. | `parallelism` | **1** | Two replicas against the same OutputPath would corrupt the checkpoint. The script holds an exclusive lock but operators should not rely on it as a primary defence. | | `PAX_NONINTERACTIVE` | **1** | The image already sets this. Removing it re-enables Read-Host prompts that will silently hang on stdin in ACA. | | Image user | non-root (`pax`, UID 10001) | Enforced by the Dockerfile. | -| `OutputPath` | Local path **or** URL (tier inferred from the path form) | Customer-visible output can be Local / SharePoint / OneLake. Checkpoint + `_PARTIAL` files always live on local container scratch internally and are mirrored to the remote tier by `Mirror-ResumeArtifactsToOneLake` so resume can re-open them. | +| `OutputPath` | Local path **or** URL (tier inferred from the path form) | Customer-visible output can be Local / SharePoint / OneLake. Working checkpoint and `_PARTIAL` files are created on local container scratch; Fabric runs automatically maintain a durable mirror at `Files/.pax_resume//` so a fresh host can restore and resume. | + +
+v1.11.15 hosted resume validation + +PAX v1.11.15 corrects exact-byte transmission for checkpoint and `_PARTIAL` artifacts, and unexpected fatal top-level errors return exit code `1`. Before broad production use, run an explicit dated hosted validation and confirm both the process exit result and resume restoration from the Fabric durable mirror. + +
## 7. Environment variables PAX honours @@ -139,8 +146,7 @@ major exceeds 2. Operators should never edit checkpoints by hand. - `-Auth ManagedIdentity` with `-IncludeAgent365Info` / `-OnlyAgent365Info` **without** the Agent 365 application permissions granted — the app-only Agent 365 phase returns a runtime **403** until the identity holds `CopilotPackages.Read.All` + `Application.Read.All` (grant via `Grant-PAXPermissions.ps1 -IncludeAgent365`; opt-in / pre-GA). It is **not** rejected at startup. - Two simultaneous PAX runs against the same `OutputPath` — guarded by an exclusive checkpoint lock; operators should still set ACA `parallelism = 1`. -- Remote (URL) `OutputPath` for checkpoint storage — checkpoints MUST be on the local - filesystem so resume can re-open them safely. +- Fabric-hosted resume uses local working checkpoint and `_PARTIAL` files plus an automatic durable mirror at `Files/.pax_resume//`; do not manually relocate or edit these artifacts. - UNC paths (`\\server\share\…`) on any destination switch (`-OutputPath`, `-OutputPathUserInfo`, `-OutputPathAgent365Info`, `-OutputPathLog`, `-AppendFile`, `-AppendUserInfo`, `-AppendAgent365Info`) — rejected at parameter validation. - Mixed-tier destinations in a single run — every supplied `-OutputPath*` / `-Append*` (URL form) must resolve to the same tier (Local / SharePoint / Fabric). - `-OutputPathLog` resolving to a Fabric `Tables/*` URL — logs are not tabular; only Fabric `Files/…` is accepted. diff --git a/fabric_resources/Deploy/README.md b/fabric_resources/Deploy/README.md index 44cb06f..0c03b93 100644 --- a/fabric_resources/Deploy/README.md +++ b/fabric_resources/Deploy/README.md @@ -60,9 +60,9 @@ The container image itself is **auth-agnostic** — it just bakes in PowerShell ## Deploy -> **Dashboard selection.** In `-ScriptArgs`, `-Rollup` targets the **AI-in-One (AIO)** dashboard by default. Add `'-Dashboard','AIBV'` for the **AI Business Value** dashboard, or `'-Dashboard','M365'` (equivalently `'-IncludeM365Usage'`) for **M365 Usage Analytics**. AIO and AIBV are produced from the same CopilotInteraction + Entra/MAC licensing data — no other args change. +> **Dashboard selection.** In `-ScriptArgs`, `-Rollup` targets the **AI-in-One (AIO)** dashboard by default. Add `'-Dashboard','ValueLens'` for the **ValueLens** dashboard, or `'-Dashboard','M365'` (equivalently `'-IncludeM365Usage'`) for **M365 Usage Analytics**. AIO and ValueLens use the same CopilotInteraction + Entra/MAC licensing data, output schema, and append behavior; ValueLens is a name-only change, and existing checkpoints resume without customer intervention. -> **Anonymization & hierarchy.** Add `'-Deidentify'` to `-ScriptArgs` to write anonymized output to the destination — every identity is replaced with an irreversible token on the host before upload (off by default; works under managed identity and app registration alike; no extra Graph permission). AIO / AIBV rollups automatically include org/manager-hierarchy columns in the Users output; `'-FillerLabel','Fixed','-FillerLabelText',''` only controls how empty deeper org levels are labelled (the M365 dashboard has no hierarchy). +> **Anonymization & hierarchy.** Add `'-Deidentify'` to `-ScriptArgs` to write anonymized output to the destination — every identity is replaced with an irreversible token on the host before upload (off by default; works under managed identity and app registration alike; no extra Graph permission). AIO / ValueLens rollups automatically include org/manager-hierarchy columns in the Users output; `'-FillerLabel','Fixed','-FillerLabelText',''` only controls how empty deeper org levels are labelled (the M365 dashboard has no hierarchy). ### SharePoint destination, daily 06:00 UTC @@ -118,6 +118,9 @@ The two examples above route every output to the same location (Purview audit, E | Purview audit (raw / rollup / event-level) | `-OutputPath` | `-AppendFile` | | EntraUsers / MAC licensing | `-OutputPathUserInfo` | `-AppendUserInfo` (auto-enables `-IncludeUserInfo`) | | Agent 365 catalog | `-OutputPathAgent365Info` | `-AppendAgent365Info` (auto-enables `-IncludeAgent365Info`) | + | Run log | `-OutputPathLog` | _(n/a)_ | Rules enforced at parameter validation: every supplied URL must resolve to the same tier (mixed Local/SP/Fabric is rejected); UNC paths are rejected on every destination switch; exactly one of each stream's pair must be bound. @@ -155,7 +158,7 @@ Rules enforced at parameter validation: every supplied URL must resolve to the s Every appended file (rollup Fact CSV under `-AppendFile`, raw audit CSV under non-rollup `-AppendFile`, EntraUsers CSV under `-AppendUserInfo`) gains three trailing columns at merge time: `Date_Added`, `Latest_Append_Date`, `In_Latest_Append`. Rows that departed from the current run's audit window are retained in the union with `In_Latest_Append = FALSE` so historical fact-table joins continue to resolve. The CopilotInteraction rollup Fact CSV additionally carries two stable identity columns (`Message_Id_Raw`, `ThreadId_Raw`) so per-run integer surrogates remain stable across appends. Under `-AppendUserInfo`, the raw Entra membership snapshot the audit phase writes is kept **pristine** — the union lands at the `-AppendUserInfo` target, the raw file keeps its natural timestamped name (or gets a `_raw` suffix on the rare path-and-leaf collision). -**Under `-Deidentify`:** `PersonId_Normalized` is tokenized but deterministic (still a stable join key), while `Message_Id_Raw` / `ThreadId_Raw` keep their real GUID values; appending a deidentified run into a non-deidentified target (or vice-versa) is hard-rejected. **Org / manager-hierarchy columns** are added to the rollup Users output on AIO / AIBV runs — the Fabric Users append adds new columns but rejects appends that are *missing* existing columns, so keep a given Users Delta table on one PAX version line (or recreate it) to avoid mixed-schema appends. +**Under `-Deidentify`:** `PersonId_Normalized` is tokenized but deterministic (still a stable join key), while `Message_Id_Raw` / `ThreadId_Raw` keep their real GUID values; appending a deidentified run into a non-deidentified target (or vice-versa) is hard-rejected. **Org / manager-hierarchy columns** are added to the rollup Users output on AIO / ValueLens runs — the Fabric Users append adds new columns but rejects appends that are *missing* existing columns, so keep a given Users Delta table on one PAX version line (or recreate it) to avoid mixed-schema appends. ## Alternative: app registration instead of managed identity @@ -327,6 +330,9 @@ Or open the file share in Azure Storage Explorer / Azure portal. The failed cont ## Notes +- **Replica safety:** Keep ACA job `parallelism` at `1`, and do not run concurrent replicas against the same output target. +- **Validation sequence:** Run an explicit one-day UTC window first, for example `-StartDate '2026-07-01' -EndDate '2026-07-02'`; then run an explicit historical range such as `-StartDate '2026-06-01' -EndDate '2026-07-01'`; then append the next explicit range using the matching `-Append*` switches. Keep the PAX version and `-Deidentify` state consistent across the seed and append runs. +- **Hosted resume and exit behavior:** Fabric mirrors durable resume artifacts to `Files/.pax_resume//`. PAX v1.11.15 corrects exact-byte transmission for checkpoint and `_PARTIAL` artifacts, and unexpected fatal top-level errors return exit code `1`. Observe the hosted process exit and resume behavior before broad production use. - **Agent 365 enrichment under `-Auth ManagedIdentity` (app-only, opt-in / pre-GA).** PAX runs `-IncludeAgent365Info` / `-OnlyAgent365Info` app-only under `-Auth ManagedIdentity` (and `-Auth AppRegistration`) with no interactive sign-in, provided the identity holds the application permissions `CopilotPackages.Read.All` + `Application.Read.All` (admin-consented). Grant them with `../Prereqs/Grant-PAXPermissions.ps1 -IncludeAgent365` — they are not granted by default. A missing app-role, unlicensed tenant, or absent program enrollment surfaces as a runtime 403 (the rest of the run completes normally). This app-only path is **pre-GA** — validate it before relying on it in production. (Delegated Agent 365 instead relies on the signed-in user's AI Administrator / Global Administrator directory role, e.g. on the local-run path.) - **Fabric `-OutputPath` shapes the script accepts:** - `https://.onelake.dfs.fabric.microsoft.com//.Lakehouse` — main Delta tables go under the Lakehouse's default `Tables/` area. diff --git a/fabric_resources/LocalRun/README.md b/fabric_resources/LocalRun/README.md index 223ed8b..904dc57 100644 --- a/fabric_resources/LocalRun/README.md +++ b/fabric_resources/LocalRun/README.md @@ -98,14 +98,25 @@ This is the fastest possible Fabric-from-laptop on-ramp. It uses `-Auth WebLogin (No `-StartDate` / `-EndDate` => script defaults to the last 30 days, UTC. To pin a window, pass e.g. `-StartDate '2026-04-14' -EndDate '2026-05-14'`.) You will get a browser prompt for the Graph scopes and (separately) for Az.Accounts to mint the OneLake storage token. After both succeed the script writes the audit Delta table(s) under `Tables/dbo/` and the run log under `Files/pax_logs/`. - > **Dashboard selection.** `-Rollup` targets the **AI-in-One (AIO)** dashboard by default. Add `-Dashboard AIBV` for the **AI Business Value** dashboard, or `-Dashboard M365` (equivalently `-IncludeM365Usage`) for **M365 Usage Analytics**. AIO and AIBV are produced from the same CopilotInteraction + Entra/MAC licensing data — no other switches change. + > **Dashboard selection.** `-Rollup` targets the **AI-in-One (AIO)** dashboard by default. Add `-Dashboard ValueLens` for the **ValueLens** dashboard, or `-Dashboard M365` (equivalently `-IncludeM365Usage`) for **M365 Usage Analytics**. AIO and ValueLens use the same CopilotInteraction + Entra/MAC licensing data, output schema, and append behavior; ValueLens is a name-only change, and existing checkpoints resume without customer intervention. > **Anonymized output (`-Deidentify`).** Add `-Deidentify` to replace every identity (including the identity fields inside the raw `AuditData` JSON) with an irreversible, deterministic token **before** anything is written to the lakehouse — useful when the Fabric data will be shared more broadly. Off by default, works under any auth mode, and needs no extra Graph scope. See the main PAX documentation for the full field list. - > **Org / manager hierarchy.** AIO / AIBV rollups automatically add org/manager-hierarchy columns to the Users output (level, manager, full management chain, direct/total report counts) from the Entra manager data PAX already collects — ready for parent-child org views in Power BI. `-FillerLabel` (`Self` / `RepeatManager` / `Fixed`, with `-FillerLabelText ""` for the literal) only controls how empty deeper level columns are labelled. The M365 dashboard has no hierarchy. + > **Org / manager hierarchy.** AIO / ValueLens rollups automatically add org/manager-hierarchy columns to the Users output (level, manager, full management chain, direct/total report counts) from the Entra manager data PAX already collects — ready for parent-child org views in Power BI. `-FillerLabel` (`Self` / `RepeatManager` / `Fixed`, with `-FillerLabelText ""` for the literal) only controls how empty deeper level columns are labelled. The M365 dashboard has no hierarchy. That is the full minimum path. The rest of this README covers the variants you are likely to want next. +
+Validation and scale-up sequence + +1. Validate a single UTC day first with explicit dates, for example `-StartDate '2026-07-01' -EndDate '2026-07-02'`. +2. Run an explicit historical range next, for example `-StartDate '2026-06-01' -EndDate '2026-07-01'`. +3. Then append the next explicit range by replacing every `-OutputPath*` switch used in validation with its matching `-Append*` switch. Keep the PAX version and `-Deidentify` state consistent across the seed and append runs. + +Omitting both dates selects the last 30 UTC days. Fabric mirrors durable resume artifacts to `Files/.pax_resume//`. PAX v1.11.15 corrects exact-byte transmission for checkpoint and `_PARTIAL` artifacts, and unexpected fatal top-level errors return exit code `1`. Observe the hosted process exit and resume behavior before broad production use. + +
+ --- ## Variant: scheduled run on the same host @@ -224,7 +235,7 @@ pwsh -File .\PAX_Purview_Audit_Log_Processor_v.ps1 ` `-AppendFile` is supported under `-Rollup` and `-RollupPlusRaw` on all three tiers. The same example works against SharePoint or local paths by changing the URL / path form — no other switches change. > **Two append cautions when writing to Fabric Delta tables.** -> - **Hierarchy / schema drift.** AIO / AIBV rollups add org/manager-hierarchy columns to the Users output. The Fabric Users append tolerates *added* columns but rejects *missing* ones — so appending current output into an older Users Delta table just adds the columns, but appending older (pre-hierarchy) output into a table that already has them fails the schema check. Keep a Users table on one PAX version line or recreate it. +> - **Hierarchy / schema drift.** AIO / ValueLens rollups add org/manager-hierarchy columns to the Users output. The Fabric Users append tolerates *added* columns but rejects *missing* ones — so appending current output into an older Users Delta table just adds the columns, but appending older (pre-hierarchy) output into a table that already has them fails the schema check. Keep a Users table on one PAX version line or recreate it. > - **Deidentify consistency.** Appending a `-Deidentify` run into a non-deidentified table (or the reverse) is hard-rejected at pre-flight — use the same `-Deidentify` choice for every run that targets a given file/table. ### M365 rollup append anchoring (`-IncludeM365Usage` + `-Rollup` / `-RollupPlusRaw`) @@ -264,7 +275,7 @@ Every appended file gains three trailing columns at merge time: | `Latest_Append_Date` | `YYYY-MM-DD` | Latest run that touched the file (same on every row). | | `In_Latest_Append` | `TRUE` / `FALSE` | Whether the row appeared in this run's audit window / membership snapshot. `FALSE` for rows retained from prior runs that no longer surface today. | -The CopilotInteraction rollup Fact CSV additionally carries stable identity columns: the raw keys **`Message_Id_Raw`** and **`ThreadId_Raw`**, plus a normalized user column (**`User_Id_Normalized`** on the AIO fact; **`Audit_UserId_Normalized`** on AIBV). These keep the per-run integer surrogates (`Message_Id`, `ThreadId`, `UserKey`) stable across appends — the cross-run append merge dedups on the full grain plus `Message_Id_Raw`, so multi-month threads remain a single thread and fan-out rows (many per message) reconcile independently in downstream models. +The CopilotInteraction rollup Fact CSV additionally carries stable identity columns: the raw keys **`Message_Id_Raw`** and **`ThreadId_Raw`**, plus a normalized user column (**`User_Id_Normalized`** in AIO output; **`Audit_UserId_Normalized`** in ValueLens output). These keep the per-run integer surrogates (`Message_Id`, `ThreadId`, `UserKey`) stable across appends — the cross-run append merge dedups on the full grain plus `Message_Id_Raw`, so multi-month threads remain a single thread and fan-out rows (many per message) reconcile independently in downstream models. ### Pristine raw EntraUsers under `-AppendUserInfo` diff --git a/fabric_resources/README.md b/fabric_resources/README.md index f64fb50..15fa7e9 100644 --- a/fabric_resources/README.md +++ b/fabric_resources/README.md @@ -69,13 +69,16 @@ PAX routes each data type to its own destination through a symmetric `-OutputPat - `https://…onelake.dfs.fabric.microsoft.com/…` URL → Fabric tier. - UNC paths (`\\server\share\…`) are rejected on every destination switch. -The seven switches and their pairings: +The nine switches and their pairings: | Data destination switch | Paired append switch | Purpose | |---|---|---| | `-OutputPath` | `-AppendFile` | Purview audit output (raw, rollup, or event-level) | | `-OutputPathUserInfo` | `-AppendUserInfo` | EntraUsers / MAC licensing CSV | | `-OutputPathAgent365Info` | `-AppendAgent365Info` | Agent 365 catalog CSV | + | `-OutputPathLog` | _(n/a)_ | Run log | Operating rules: @@ -84,10 +87,24 @@ Operating rules: - **Pair XOR.** For each stream in scope, exactly one of its `-OutputPath*` / `-Append*` pair must be supplied — both bound or neither bound is rejected. - **Append surface.** `-AppendFile` works under `-Rollup` and `-RollupPlusRaw` on all three tiers. Per-dimension `-AppendUserInfo` and `-AppendAgent365Info` perform a union merge on the EntraUsers and Agent 365 catalog respectively, keyed on `PersonId_Normalized` and `AgentId`. - **Deidentification (`-Deidentify`).** Optional anonymization that replaces every identifying value (including the identity fields inside the raw `AuditData` JSON) with an irreversible, deterministic token, applied on the host **before** any write or upload — so anonymized data is what lands in the Fabric lakehouse / SharePoint folder / local path. Off by default. It is dashboard- and tier-agnostic (works under every `-Auth` mode and on all three tiers, including Fabric Delta tables) and needs no extra Graph scope. Merge keys stay usable: `PersonId_Normalized` is tokenized but deterministic, so EntraUsers/Users joins and distinct counts are preserved; `Message_Id_Raw` / `ThreadId_Raw` are message/thread GUIDs and are **not** tokenized. A hard-stop guard rejects mixing a `-Deidentify` run into a non-deidentified append target (and vice-versa). `-Deidentify` is persisted in the checkpoint and restored on `-Resume`. -- **Org / manager hierarchy (AIO / AIBV rollup).** When `-Rollup` / `-RollupPlusRaw` produces **AI-in-One** or **AI Business Value** input, the rolled-up Users output automatically gains org/manager-hierarchy columns (`OrgLevel`, `Manager_UserKey`, `TopOfChain_UserKey`, `HierarchyPath`, `IsManager`, `DirectReports`, `TotalReports`, and `Level0`…`Level14` name/key pairs) derived from the Entra manager data PAX already collects via `$expand=manager` (no new Graph scope). On Fabric these land in the rollup Users Delta table; the structural columns are built on a stable internal key, so they are byte-identical with or without `-Deidentify`. The M365 dashboard has no hierarchy. `-FillerLabel` (`Self` / `RepeatManager` / `Fixed`, with `-FillerLabelText` for the literal) only controls how empty deeper level columns are labelled. +- **Org / manager hierarchy (AIO / ValueLens rollup).** When `-Rollup` / `-RollupPlusRaw` produces **AI-in-One** or **ValueLens** input, the rolled-up Users output automatically gains org/manager-hierarchy columns (`OrgLevel`, `Manager_UserKey`, `TopOfChain_UserKey`, `HierarchyPath`, `IsManager`, `DirectReports`, `TotalReports`, and `Level0`…`Level14` name/key pairs) derived from the Entra manager data PAX already collects via `$expand=manager` (no new Graph scope). On Fabric these land in the rollup Users Delta table; the structural columns are built on a stable internal key, so they are byte-identical with or without `-Deidentify`. The M365 dashboard has no hierarchy. `-FillerLabel` (`Self` / `RepeatManager` / `Fixed`, with `-FillerLabelText` for the literal) only controls how empty deeper level columns are labelled. - **M365 rollup append.** Under `-IncludeM365Usage` + `-Rollup` / `-RollupPlusRaw`, the embedded M365 Bundle processor emits four files into the same destination — `_Rollup.csv`, `_UserStats.csv`, `_SessionCohort.csv`, and `_SessionStats.csv`. `-AppendFile` MUST point to the `_Rollup.csv` leaf; the three sidecars are recomputed/merged each run, anchored off the same leaf stem, and overwrite their derived destination URLs in-place (`_SessionStats.csv` is union-merged in place with additive counter semantics). Sidecar leaves (`_UserStats.csv`, `_SessionCohort.csv`, `_SessionStats.csv`) and CopilotInteraction (`_Interactions.csv`) / event-level (`_Exploded.csv`) leaves are rejected at pre-flight. - **Provenance columns.** Every appended file (Fact CSV, raw `-AppendFile` audit CSV, EntraUsers CSV) gains three trailing columns at merge time: `Date_Added`, `Latest_Append_Date`, `In_Latest_Append`. Rows that departed from the latest audit window are retained in the union with `In_Latest_Append = FALSE` so historical fact-table joins continue to resolve. The CopilotInteraction rollup Fact CSV additionally carries two stable raw-identity columns: `Message_Id_Raw` and `ThreadId_Raw`. (Under `-Deidentify` these two keep their real GUID values; only `PersonId_Normalized` is tokenized — deterministically, so it stays a stable join key.) - **Pristine raw EntraUsers under `-AppendUserInfo`.** The raw Entra membership snapshot written by the audit phase is never overwritten by the append-merge. The union lands at the `-AppendUserInfo` target; the raw file keeps its natural timestamped leaf (or gets a `_raw` suffix on the rare path/leaf collision). + + +
+Hosted operational validation and scale-up + +1. Validate a single UTC day first with explicit dates, for example `-StartDate '2026-07-01' -EndDate '2026-07-02'`. +2. Run an explicit historical range next, for example `-StartDate '2026-06-01' -EndDate '2026-07-01'`. +3. Then append the next explicit range by replacing every `-OutputPath*` switch used in validation with its matching `-Append*` switch. Keep the PAX version and `-Deidentify` state consistent across the seed and append runs. + +Always provide dates for this validation sequence; omitting both dates selects the last 30 UTC days. Fabric mirrors durable resume artifacts to `Files/.pax_resume//`. PAX v1.11.15 corrects exact-byte transmission for checkpoint and `_PARTIAL` artifacts, and unexpected fatal top-level errors return exit code `1`. Observe the hosted process exit and resume behavior before broad production use. + +
When any `-OutputPath*` resolves to a Fabric OneLake URL, customer-visible outputs are written as **Delta tables** under the Lakehouse `Tables/` namespace (Schemas-mode Lakehouse `Tables/` is the current Fabric default; `dbo` is typical). Operational artifacts (run log, metrics JSON) land under `Files/`. The Python `deltalake` package is auto-installed on first use; offline hosts can pre-install it (see `LocalRun/README.md` and `Dockerfile/PAX.Dockerfile`). diff --git a/release_documentation/.gitkeep b/release_documentation/.gitkeep index 725eac7..49b1f17 100644 --- a/release_documentation/.gitkeep +++ b/release_documentation/.gitkeep @@ -1 +1 @@ -# Last updated: 2026-07-06 (Purview v1.11.14, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.36) \ No newline at end of file +# Last updated: 2026-07-31 (Purview v1.11.15, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.37) \ No newline at end of file diff --git a/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md b/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md index 2ad7318..29c1b3a 100644 --- a/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md +++ b/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md @@ -1,8 +1,8 @@ # Portable Audit eXporter (PAX) -
Purview Audit Log Processor -> **📥 Quick Start:** Download the script → [`PAX_Purview_Audit_Log_Processor_v1.11.14.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.14/PAX_Purview_Audit_Log_Processor_v1.11.14.ps1) +> **📥 Quick Start:** Download the script → [`PAX_Purview_Audit_Log_Processor_v1.11.15.ps1`](https://github.com/microsoft/PAX/releases/download/purview-v1.11.15/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1) > -> **📅 Script v1.11.14 Release Date:** July 6, 2026 +> **📅 Script v1.11.15 Release Date:** July 31, 2026 > > **📋 Release Notes:** See what's new → [v1.11.x Release Notes](https://github.com/microsoft/PAX/blob/release/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md) | [All Release Notes](https://github.com/microsoft/PAX/tree/release/release_notes/Purview_Audit_Log_Processor) > @@ -10,7 +10,7 @@ > > **📚 Documentation Archive:** [All Documentation](https://github.com/microsoft/PAX/tree/release/release_documentation/Purview_Audit_Log_Processor) -**Documentation Version:** v1.11.x (Current Script Version: v1.11.14) +**Documentation Version:** v1.11.x (Current Script Version: v1.11.15) **Audience:** IT admins, security/compliance analysts, BI/data teams **Runtime:** PowerShell 7+ (required for default Graph API mode); PowerShell 5.1 supported only with `-UseEOM` **License:** MIT @@ -96,7 +96,7 @@ The **Portable Audit eXporter (PAX)** is an enterprise-grade PowerShell script t - Retrieves audit events from Microsoft 365 Unified Audit Log via **Graph API (default)** or **EOM mode** (`-UseEOM`) - **Graph API filter passthrough:** Optional `-RecordTypes` / `-ServiceTypes` switches target documented Purview workloads (SharePoint, OneDrive, and future additions) so non-Copilot office app activity returns alongside Copilot operations - **Microsoft 365 usage data (`-IncludeM365Usage`):** Curated cross-workload activity bundle spanning Outlook, Teams, SharePoint, OneDrive, Word, Excel, PowerPoint, OneNote, Forms, Stream, Planner, and PowerApps — captured in the same Graph audit run alongside Copilot telemetry for ROI and behavior-change analysis -- **Multi-dashboard Power BI rollup (`-Rollup` / `-RollupPlusRaw` + `-Dashboard`):** Produce ready-to-load input for the Copilot Analytics Lab Power BI dashboards — **AI-in-One (AIO)**, the new **AI Business Value (AIBV)**, or **M365 Usage Analytics** — directly from a single PAX run via an embedded post-processor. The `-Dashboard` selector (default `AIO`) chooses the target dashboard; AIO and AIBV are produced from the same CopilotInteraction + Entra/MAC licensing data +- **Multi-dashboard Power BI rollup (`-Rollup` / `-RollupPlusRaw` + `-Dashboard`):** Produce ready-to-load input for the Copilot Analytics Lab Power BI dashboards — **AI-in-One (AIO)**, **ValueLens**, or **M365 Usage Analytics** — directly from a single PAX run via an embedded post-processor. The `-Dashboard` selector (default `AIO`) chooses the target dashboard; AIO and ValueLens are produced from the same CopilotInteraction + Entra/MAC licensing data - Exports to structured CSV - Includes enriched usage & ROI fields (tokens, models, latency, acceptance metrics) - Implements adaptive time slicing to navigate service limits intelligently @@ -111,7 +111,7 @@ The **Portable Audit eXporter (PAX)** is an enterprise-grade PowerShell script t - **Microsoft Agent 365 catalog (`-IncludeAgent365Info` / `-OnlyAgent365Info`):** Export a point-in-time inventory of the agents registered in your tenant as a separate `Agent365_.csv` — alongside the audit export or on its own — for tenants licensed for Microsoft Agent 365 (requires an interactive AI Administrator / Global Administrator sign-in); see [Microsoft Agent 365 Parameters](#microsoft-agent-365-parameters) - **EOM mode (`-UseEOM`):** Supports group expansion via `-GroupNames` (uses `Get-DistributionGroupMember`) and 10K-per-query limit detection - **Optional anonymized output (`-Deidentify`):** A single switch replaces every identity in the output with irreversible, format-preserving tokens, so anonymized data can be shared for reporting without exposing who did what — while keeping the analytical fields and the relationships between records intact. OFF by default (output is unchanged when it is not used); see [Deidentification (Anonymized Output)](#deidentification-anonymized-output) -- **Built-in org / manager hierarchy (Power BI rollup):** When producing input for the AI-in-One or AI Business Value dashboards, the rolled-up Users output automatically includes each person's place in the org chart — their level, manager, full management chain, and team-size counts — derived from the Entra manager data PAX already collects, ready for org-based Power BI views; see [Rollup Post-Processor (Power BI)](#rollup-post-processor-power-bi) +- **Built-in org / manager hierarchy (Power BI rollup):** When producing input for the AI-in-One or ValueLens dashboards, the rolled-up Users output automatically includes each person's place in the org chart — their level, manager, full management chain, and team-size counts — derived from the Entra manager data PAX already collects, ready for org-based Power BI views; see [Rollup Post-Processor (Power BI)](#rollup-post-processor-power-bi) @@ -144,7 +144,7 @@ The **Portable Audit eXporter (PAX)** is an enterprise-grade PowerShell script t - **Entra ID Enrichment + M365 Copilot Licensing (Graph API Mode Only):** Enrich audit data with Entra user attributes and M365 Copilot (MAC) license information via `-IncludeUserInfo` (default mode, not compatible with `-UseEOM`) - **User-Only Export (Graph API Mode Only):** Export only Entra ID user data and M365 Copilot licensing without audit records via `-OnlyUserInfo` (requires `-IncludeUserInfo`, not compatible with `-UseEOM`) - **Microsoft Agent 365 Catalog (Graph API Mode Only):** Inventory the tenant's registered agents (name, publisher, developer, and package metadata) via `-IncludeAgent365Info` (alongside audit) or `-OnlyAgent365Info` (catalog only); requires a Microsoft Agent 365 license and either an interactive AI Administrator / Global Administrator sign-in or app-only authentication (AppRegistration certificate/secret or managed identity) with the required application permission admin-consented. See [Microsoft Agent 365 Parameters](#microsoft-agent-365-parameters) -- **Org / Manager Hierarchy (Power BI Rollup):** The AI-in-One and AI Business Value rollups add org/manager-hierarchy columns to the Users output — each person's level, manager, full management chain, and direct/total report counts — derived from the Entra manager data PAX already collects; ready for parent-child hierarchies, leaderboards, and team rollups in Power BI. See [Rollup Post-Processor (Power BI)](#rollup-post-processor-power-bi) +- **Org / Manager Hierarchy (Power BI Rollup):** The AI-in-One and ValueLens rollups add org/manager-hierarchy columns to the Users output — each person's level, manager, full management chain, and direct/total report counts — derived from the Entra manager data PAX already collects; ready for parent-child hierarchies, leaderboards, and team rollups in Power BI. See [Rollup Post-Processor (Power BI)](#rollup-post-processor-power-bi) - **Deidentification (`-Deidentify`):** Optionally anonymize every identifying value in the output with irreversible, format-preserving tokens for anonymous-style reporting, while preserving relationships and analytical fields. OFF by default. See [Deidentification (Anonymized Output)](#deidentification-anonymized-output) - **Streaming Export:** Memory-efficient chunked data writing for large datasets - **UTF-8 Encoding:** Consistent UTF-8 (no BOM) output for CSV files @@ -212,7 +212,7 @@ The **Portable Audit eXporter (PAX)** is an enterprise-grade PowerShell script t - Identify power users and underutilized licenses - Calculate ROI metrics based on time saved and acceptance rates - Analyze Word, Excel, PowerPoint, and OneNote document activity by pairing `-ActivityTypes` (e.g., `FileAccessed`, `FilePreviewed`) with `-RecordTypes`/`-ServiceTypes` to capture SharePoint and OneDrive workloads alongside Copilot usage -- Roll Copilot adoption up by team, department, or management chain — and build leader leaderboards and drill-down org views — using the org/manager-hierarchy columns the AI-in-One / AI Business Value rollup adds to its Users output (see [Rollup Post-Processor (Power BI)](#rollup-post-processor-power-bi)) +- Roll Copilot adoption up by team, department, or management chain — and build leader leaderboards and drill-down org views — using the org/manager-hierarchy columns the AI-in-One / ValueLens rollup adds to its Users output (see [Rollup Post-Processor (Power BI)](#rollup-post-processor-power-bi)) @@ -298,7 +298,7 @@ The **Portable Audit eXporter (PAX)** is an enterprise-grade PowerShell script t | **Unified Audit Logging** | Enabled in tenant | Verify in Microsoft Purview compliance portal | | **Graph API Permissions** | See [Permission Details](#permission-details) below | Required for Graph API mode (default). Consented during interactive sign-in or pre-configured for app registrations. | | **Audit Role** | Purview Audit Reader (or higher) | Required only for EOM mode (`-UseEOM`) and the Purview UI. Not required for Graph API mode (default), regardless of authentication method. | -| **SharePoint write access** *(only when `-OutputPath` is a SharePoint URL)* | Edit/Contribute on the destination library + folder, plus the Graph delegated or application permissions `Sites.ReadWrite.All` and `Files.ReadWrite.All` | See [Sending Output to SharePoint](#sending-output-to-sharepoint) for the full setup. | +| **SharePoint write access** *(only when `-OutputPath` is a SharePoint URL)* | Edit/Contribute on the destination library + folder, plus the Graph delegated or application permission `Sites.ReadWrite.All` (`Files.ReadWrite.All` is an accepted alternative but is not requested by the tool) | See [Sending Output to SharePoint](#sending-output-to-sharepoint) for the full setup. | | **Microsoft Fabric / OneLake access** *(only when `-OutputPath` is a Fabric lakehouse URL)* | All three layers are required: **(1)** the Azure role `Storage Blob Data Contributor` on the OneLake storage scope, **(2)** the **Contributor** (or higher) role on the Fabric workspace in the Fabric portal, **(3)** the tenant setting allowing service principals / Entra IDs to access Fabric APIs must be enabled by your Fabric admin | See [Sending Output to Microsoft Fabric (OneLake)](#sending-output-to-microsoft-fabric-onelake) for the full setup, and the `fabric_resources` folder in the repo root for detailed container/runbook material. | | **Az.Accounts PowerShell module** *(only when `-OutputPath` is a Fabric lakehouse URL)* | Used to obtain the OneLake storage token | Install manually if missing: `Install-Module Az.Accounts -Scope CurrentUser`. PAX surfaces a clear error at pre-flight if the module is not present. (Already present on Azure Cloud Shell and the PAX-on-ACA container image.) | | **Network Access** | Microsoft 365 endpoints | Ensure firewall allows connections to Microsoft Graph and Exchange Online endpoints. When `-OutputPath` is a SharePoint URL, also allow `*.sharepoint.com`. When `-OutputPath` is a Fabric lakehouse URL, also allow `onelake.dfs.fabric.microsoft.com`. | @@ -326,7 +326,7 @@ Graph API mode requests scopes conditionally based on the switches you pass. The | **Graph: Application.Read.All** | Publisher / developer name resolution for Agent 365 | `-IncludeAgent365Info` or `-OnlyAgent365Info` | ✅ Yes | ✅ Yes (interactive) | — N/A | | **Entra role: AI Administrator OR Global Administrator** | Required by the Agent 365 catalog API (delegated only) | `-IncludeAgent365Info` or `-OnlyAgent365Info` | ✅ Yes | ✅ Yes | — N/A | | **Graph: Sites.ReadWrite.All** | Resolve the SharePoint site/library/folder and upload output files | `-OutputPath` is a SharePoint URL | ✅ Yes | ✅ Yes | — N/A | -| **Graph: Files.ReadWrite.All** | Create, replace, and resume uploads of output files in the SharePoint folder | `-OutputPath` is a SharePoint URL | ✅ Yes | ✅ Yes | — N/A | +| **Graph: Files.ReadWrite.All** *(optional — not requested)* | Accepted alternative to `Sites.ReadWrite.All` for creating, replacing, and resuming uploads of output files; the tool does not request it | Optional — `Sites.ReadWrite.All` alone authorizes all SharePoint output | Optional | Optional | — N/A | | **Azure role: Storage Blob Data Contributor** | Write PAX output into the OneLake `Tables/` namespace (Delta tables) and `Files/` namespace (operational artifacts) of the destination Lakehouse | `-OutputPath` is a Fabric lakehouse URL | ✅ Required on the signed-in user / managed identity | ✅ Required on the service principal | — N/A | | **Fabric portal role: Contributor** (or higher) on the workspace | Allow the identity to see and write into the lakehouse via Fabric APIs | `-OutputPath` is a Fabric lakehouse URL | ✅ Required | ✅ Required | — N/A | | **Fabric tenant setting: "Service principals can use Fabric APIs"** | Enables Entra service principals / managed identities to call Fabric/OneLake | `-OutputPath` is a Fabric lakehouse URL (only when using `-Auth AppRegistration` or `-Auth ManagedIdentity`) | — | ✅ Must be enabled by a Fabric admin | — N/A | @@ -355,7 +355,7 @@ The **Purview Audit Reader** role is only required for EOM mode (`-UseEOM`) and - Not applicable in EOM mode (`-UseEOM`) **Sending output directly to SharePoint (Optional Feature - Graph API Mode Only):** -- Requires the Graph permissions **Sites.ReadWrite.All** and **Files.ReadWrite.All**, requested only when `-OutputPath` is a SharePoint URL +- Requires the Graph permission **Sites.ReadWrite.All**, requested only when `-OutputPath` is a SharePoint URL (**Files.ReadWrite.All** is an accepted alternative but is not requested by the tool) - The signed-in account (or, for unattended runs, the service principal / managed identity) must additionally have **Edit** or **Contribute** permission on the destination SharePoint library and folder - See [Sending Output to SharePoint](#sending-output-to-sharepoint) for the full walkthrough, including how to get a valid URL and what kinds of links cannot be used - Not applicable in EOM mode (`-UseEOM`) @@ -391,6 +391,8 @@ The **Purview Audit Reader** role is only required for EOM mode (`-UseEOM`) and **PowerShell 5.1 (legacy):** Supported ONLY with `-UseEOM` (serial Exchange Online Management mode). Default Graph API mode and parallel query features require PowerShell 7+. +**Language mode:** PAX requires `FullLanguage` mode. If WDAC, AppLocker, or another endpoint policy places PowerShell in `ConstrainedLanguage` mode, PAX stops before initialization and directs you to contact your IT or security team for an approved environment or to have PAX allowlisted. `ExecutionPolicy Bypass` does not change the PowerShell language mode. Check the current mode with `$ExecutionContext.SessionState.LanguageMode`. + @@ -406,7 +408,7 @@ The **Purview Audit Reader** role is only required for EOM mode (`-UseEOM`) and ### Download the Script -- **Script:** [PAX_Purview_Audit_Log_Processor_v1.11.14.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.14/PAX_Purview_Audit_Log_Processor_v1.11.14.ps1) +- **Script:** [PAX_Purview_Audit_Log_Processor_v1.11.15.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.15/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1) - **Release Notes:** [v1.11.x](https://github.com/microsoft/PAX/blob/release/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md) Save the downloaded script to a working directory (e.g., `C:\Scripts\PAX\`). @@ -1220,6 +1222,38 @@ All audit-related parameters are incompatible and will trigger validation errors --- +#### `-UserInfoSupplement` (string) + +**Purpose:** **Enrich** (rather than replace) the live Entra directory. PAX fetches the live Entra `/users` directory normally, then appends the columns of a **CSV you provide** to each matching Entra row — an **Entra-left join** keyed on `UserPrincipalName`. Use this when you want to add your own columns (e.g., business unit, cost-center label, region) to the authoritative live directory without replacing it. +**Default:** Not set (no supplemental enrichment) +**Accepts:** A **local `.csv`**, a **SharePoint** `.csv` file URL, or a **Microsoft Fabric / OneLake** `/Files/.../*.csv` URL (folder URLs, Fabric roots, and Fabric `Tables` URLs are rejected). Remote inputs are staged to local scratch, parsed, and removed after use; they are never uploaded. +**Use When:** + +- You want to add curated attributes to the *live* directory (not substitute for it) +- You need supplemental columns to flow into the EntraUsers CSV and the AIO/ValueLens Rollup Users dimension + +**Example:** `-IncludeUserInfo -UserInfoSupplement "C:\Data\dept_tags.csv"` + +**Notes:** + +- **Matching:** exactly one `UserPrincipalName` column is required (the only required column), matched **case-insensitively and whitespace-trimmed**. It is a **join key only and is never copied into the output** — the output identity always comes from Entra. +- **Every Entra user is preserved.** A non-matching Entra user receives blank values in the supplemental columns. +- **Unmatched supplemental rows are reported and excluded** — they never become synthetic directory users, and excluding them does not mark the run "completed with gaps". +- **Additive-only, no transformation:** supplemental column names, order, blank values, and values are preserved exactly. A column name that collides (case-insensitively) with any Entra/PAX-owned directory, licensing, append-provenance, or hierarchy column is a **fatal input error**, as are a blank/duplicate `UserPrincipalName` and a duplicate header — so PAX-owned data can never be overwritten and data-quality errors are never silently hidden. +- **Downstream:** supplemental columns appear in the normal EntraUsers CSV and the AIO/ValueLens Rollup Users dimension. +- **De-identification:** with `-Deidentify`, PAX de-identifies Entra-derived identity fields as usual but passes **every supplemental column and value through unchanged** — PAX does not inspect, mask, hash, or de-identify supplemental values. A prominent notice is shown before processing and cannot be suppressed; you are responsible for ensuring the supplemental values are appropriate to export. +- **Modes:** auto-enables `-IncludeUserInfo`; compatible with `-UserIds`, `-GroupNames`, `-OnlyUserInfo`, AIO/ValueLens, `-AppendUserInfo`, and `-Deidentify`; mutually exclusive with `-UserInfoFile`, `-UseEOM`, and `-RAWInputCSV`. +- **Difference from `-UserInfoFile`:** `-UserInfoFile` *replaces* the directory with your CSV; `-UserInfoSupplement` *enriches* the live directory and keeps Entra authoritative. + +SharePoint / Fabric examples: + +```powershell +-IncludeUserInfo -UserInfoSupplement "https://contoso.sharepoint.com/sites/HR/Shared Documents/dept_tags.csv" -OutputPath "C:\PAX\out" +-IncludeUserInfo -UserInfoSupplement "https://onelake.dfs.fabric.microsoft.com/ws/lh.Lakehouse/Files/inbox/dept_tags.csv" -OutputPath "C:\PAX\out" +``` + +--- + ### Microsoft Agent 365 Parameters These switches add a Microsoft Agent 365 catalog export — a point-in-time inventory of the agents registered in your tenant — produced as a separate `Agent365_.csv` file. The catalog is a snapshot taken at the moment the script runs; the `-StartDate` / `-EndDate` range applies only to audit data, not to the agent catalog. Both switches require an interactive sign-in by an **AI Administrator** or **Global Administrator**, and the tenant must be licensed for **Microsoft Agent 365**. @@ -1289,7 +1323,7 @@ These switches add a Microsoft Agent 365 catalog export — a point-in-time inve **Mutually exclusive with:** `-RollupPlusRaw` **Use When:** -- Producing input for a Copilot Analytics Lab Power BI dashboard (AI-in-One, AI Business Value, or M365 Usage Analytics) +- Producing input for a Copilot Analytics Lab Power BI dashboard (AI-in-One, ValueLens, or M365 Usage Analytics) - You only need the rolled-up output, not the raw audit CSV **Notes:** @@ -1314,38 +1348,89 @@ These switches add a Microsoft Agent 365 catalog export — a point-in-time inve #### `-Dashboard` (string) **Purpose:** Selects which Copilot Analytics Lab dashboard a rollup run produces input for -**Valid values:** `AIO` (AI-in-One), `AIBV` (AI Business Value), `M365` (M365 Usage Analytics) — case-insensitive +**Valid values:** `AIO` (AI-in-One), `ValueLens` (ValueLens), `M365` (M365 Usage Analytics) — case-insensitive **Default:** `AIO` **Use When:** -- Producing **AI Business Value** input (`-Dashboard AIBV`) instead of the default AI-in-One -- Producing **M365 Usage Analytics** input (`-Dashboard M365`) +- Producing **ValueLens** input (`-Dashboard ValueLens`) instead of the default AI-in-One +- Producing **M365 Usage Analytics** input (`-Dashboard M365`) **Behavior:** - **Default `AIO`:** A CopilotInteraction-only rollup produces AI-in-One output — byte-for-byte identical to prior script versions. -- **`AIBV`:** Same CopilotInteraction data + Entra/MAC licensing, emitted in the AI Business Value output profile. -- **`M365`:** Auto-enables `-IncludeM365Usage` and runs the M365 Usage Bundle processor. +- **`ValueLens`:** Same CopilotInteraction data + Entra/MAC licensing, with the same output schema and append behavior as before. +- **`M365`:** Auto-enables `-IncludeM365Usage` and runs the M365 Usage Bundle processor. - **Auto-rollup:** Supplying `-Dashboard` without `-Rollup` / `-RollupPlusRaw` auto-enables `-Rollup`. - **Omitting `-Dashboard`** reproduces prior behavior exactly (AIO for CopilotInteraction, M365 when `-IncludeM365Usage` is present). -- **Persisted to the checkpoint** (`rollupDashboard`) and restored on `-Resume` (last-write-wins). +- **The selected dashboard is remembered across a `-Resume`**, so a resumed run keeps producing the dashboard it started with. **Incompatibilities:** -- `-Dashboard AIO` / `AIBV` cannot be combined with `-IncludeM365Usage` (different source data and processor) — PAX exits with an explicit error. +- `-Dashboard AIO` / `ValueLens` cannot be combined with `-IncludeM365Usage` (different source data and processor) — PAX exits with an explicit error. **Example:** ```powershell -# AI Business Value dashboard input (auto-enables -Rollup) -.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-04-01 -EndDate 2026-04-30 -Dashboard AIBV +# ValueLens dashboard input (auto-enables -Rollup) +.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-04-01 -EndDate 2026-04-30 -Dashboard ValueLens ``` --- + + #### `-FillerLabel` (string) -**Purpose:** Controls how empty parent slots are labelled in the org / manager hierarchy columns of the AI-in-One / AI Business Value rollup Users output +**Purpose:** Controls how empty parent slots are labelled in the org / manager hierarchy columns of the AI-in-One / ValueLens rollup Users output **Valid values:** `Self`, `RepeatManager`, `Fixed` (omit the switch for the default blank behavior) — case-insensitive **Default:** Unset — empty hierarchy level slots above a person are left blank **Applies to:** The `Level0_Name` … `Level14_Name` columns in the rollup Users output (the structural columns are never affected) @@ -1813,7 +1898,7 @@ Graph API mode requests scopes conditionally based on the switches you pass. The | **Graph: Organization.Read.All** | Tenant/organization context, license metadata | `-IncludeUserInfo` or `-OnlyUserInfo` | ✅ Yes | ✅ Yes | — N/A | | **Graph: GroupMember.Read.All** | Group lookup and membership expansion (least privilege) | `-GroupNames` | ✅ Yes | ✅ Yes | — N/A | | **Graph: Sites.ReadWrite.All** | Resolve and upload to SharePoint destination folder | `-OutputPath` is a SharePoint URL | ✅ Yes | ✅ Yes | — N/A | -| **Graph: Files.ReadWrite.All** | Create / replace / resume uploads to the SharePoint folder | `-OutputPath` is a SharePoint URL | ✅ Yes | ✅ Yes | — N/A | +| **Graph: Files.ReadWrite.All** *(optional — not requested)* | Accepted alternative to `Sites.ReadWrite.All` for create / replace / resume uploads; not requested by the tool | Optional — `Sites.ReadWrite.All` alone authorizes SharePoint output | Optional | Optional | — N/A | | **Azure role: Storage Blob Data Contributor** | Write into the OneLake `Files/` area of the destination lakehouse | `-OutputPath` is a Fabric lakehouse URL | ✅ Required on user / managed identity | ✅ Required on service principal | — N/A | | **Fabric portal role: Contributor (or higher)** | Workspace access in the Fabric portal | `-OutputPath` is a Fabric lakehouse URL | ✅ Required | ✅ Required | — N/A | | **Fabric tenant setting: "Service principals can use Fabric APIs"** | Allows non-interactive identities to call Fabric/OneLake | `-OutputPath` is a Fabric lakehouse URL (when using `-Auth AppRegistration` or `-Auth ManagedIdentity`) | — | ✅ Must be enabled by a Fabric admin | — N/A | @@ -2026,7 +2111,7 @@ Sign in using the managed identity attached to the Azure resource that is runnin - A system-assigned or user-assigned managed identity on the Azure resource hosting PAX. - The identity granted the same Microsoft Graph application permissions an `AppRegistration` would need (at minimum `AuditLogsQuery.Read.All`, plus any conditional scopes for the switches you use — see the Permissions tables above). - When `-OutputPath` is a Fabric lakehouse URL: the identity also needs `Storage Blob Data Contributor` (Azure role on the OneLake storage), the **Contributor** role on the Fabric workspace, and the Fabric tenant setting *Service principals can use Fabric APIs* enabled. - - When `-OutputPath` is a SharePoint URL: the identity also needs `Sites.ReadWrite.All` and `Files.ReadWrite.All`, plus Edit/Contribute on the destination folder. + - When `-OutputPath` is a SharePoint URL: the identity also needs `Sites.ReadWrite.All` (`Files.ReadWrite.All` is an accepted alternative but is not requested), plus Edit/Contribute on the destination folder. - If multiple identities are attached to the host (for example, both a system-assigned and one or more user-assigned identities), set the `AZURE_CLIENT_ID` environment variable to the client ID of the one PAX should use. - **Works in:** Graph API mode only; automatically blocked when `-UseEOM` is supplied. - **Automation suitability:** Strongly preferred over `AppRegistration` for any workload that already runs inside Azure — no secret rotation, no certificate management. @@ -2140,11 +2225,11 @@ https://contoso.sharepoint.com/sites/AuditTeam/Shared Documents/PAX-Output | Layer | What you need | Why | |---|---|---| -| Microsoft Graph application or delegated permission | `Sites.ReadWrite.All` | Resolve the site and library | -| Microsoft Graph application or delegated permission | `Files.ReadWrite.All` | Create, replace, and resume uploads of output files | +| Microsoft Graph application or delegated permission | `Sites.ReadWrite.All` | Resolve the site and library, and create, replace, and resume uploads of output files | +| Microsoft Graph application or delegated permission | `Files.ReadWrite.All` *(optional alternative — not requested)* | Accepted alternative to `Sites.ReadWrite.All`; the tool does not request it | | SharePoint folder permission | Edit or Contribute on the destination folder | Standard SharePoint write access for the identity running PAX | -For unattended runs using `-Auth AppRegistration` or `-Auth ManagedIdentity`, the same two Graph permissions must be granted to the service principal / managed identity as **application permissions** with tenant admin consent, and the identity must additionally hold Edit/Contribute on the folder. +For unattended runs using `-Auth AppRegistration` or `-Auth ManagedIdentity`, the same Graph permission (`Sites.ReadWrite.All`) must be granted to the service principal / managed identity as an **application permission** with tenant admin consent, and the identity must additionally hold Edit/Contribute on the folder. ### More examples @@ -2198,7 +2283,7 @@ $clientSecret = ConvertTo-SecureString $env:PAX_CLIENT_SECRET -AsPlainText -Forc |---|---|---| | `Could not resolve SharePoint folder...` at the very start of the run | The URL is a sharing link, view page, or query-string URL, not the folder path | Re-read the *How to get the right URL* section above and copy the address-bar URL from inside the folder. | | `Access denied` to the folder during the pre-flight check | The identity has Graph permissions but no SharePoint folder permission | Add Edit/Contribute on the destination folder for the identity, then re-run. | -| `Sites.ReadWrite.All` or `Files.ReadWrite.All` listed as missing on consent | App registration / managed identity lacks one of the two required Graph permissions | Add the missing permission and grant admin consent. | +| `Sites.ReadWrite.All` listed as missing on consent | The app registration / managed identity lacks the required `Sites.ReadWrite.All` Graph permission (`Files.ReadWrite.All` is an accepted alternative and is not itself required) | Add `Sites.ReadWrite.All` and grant admin consent. | | A long run uploads most files but the last big one fails | Almost always a transient network or auth blip | Re-run with the same parameters. PAX uses checkpoint/resume on the audit side and resumable upload on the destination side; you will not pay the full cost again. | | The folder is empty after PAX prints "Run complete" | Check the script log file — PAX always uploads its own log last; if the log is in the folder, the run succeeded. If you are looking in the wrong folder, double-check the URL you passed. | — | @@ -2544,12 +2629,12 @@ elseif ($LASTEXITCODE -eq 20) { Write-Host 'Circuit breaker tripped – investig -Deidentify ` -OutputPath "C:\Exports\\" -# AI Business Value rollup — Users output includes org/manager-hierarchy columns +# ValueLens rollup — Users output includes org/manager-hierarchy columns ./PAX_Purview_Audit_Log_Processor.ps1 ` -StartDate 2025-10-01 ` -EndDate 2025-10-31 ` -Rollup ` - -Dashboard AIBV ` + -Dashboard ValueLens ` -OutputPath "C:\Exports\\" # Rollup with empty org-level columns filled with a fixed label @@ -2566,12 +2651,12 @@ elseif ($LASTEXITCODE -eq 20) { Write-Host 'Circuit breaker tripped – investig -StartDate 2025-10-01 ` -EndDate 2025-10-31 ` -Rollup ` - -Dashboard AIBV ` + -Dashboard ValueLens ` -Deidentify ` -OutputPath "C:\Exports\\" ``` -See [Deidentification (Anonymized Output)](#deidentification-anonymized-output) and [Org / Manager Hierarchy](#org--manager-hierarchy-ai-in-one--ai-business-value) for full details. +See [Deidentification (Anonymized Output)](#deidentification-anonymized-output) and [Org / Manager Hierarchy](#org--manager-hierarchy-ai-in-one--valuelens) for full details. @@ -3436,20 +3521,22 @@ When `-Rollup` or `-RollupPlusRaw` is specified, PAX runs an **embedded Python p | Run shape | `-Dashboard` | Embedded processor | Inputs consumed | Target Copilot Analytics Lab dashboard | | --- | --- | --- | --- | --- | | **CopilotInteraction-only** (default activity type, or `-ActivityTypes 'CopilotInteraction'`) | `AIO` *(default)* | `Purview_CopilotInteraction_Processor` | Purview CSV **+** Entra users CSV (`EntraUsers_MAClicensing_.csv`) | **AI-in-One (AIO)** | -| **CopilotInteraction-only** | `AIBV` | `Purview_CopilotInteraction_Processor` | Purview CSV **+** Entra users CSV (`EntraUsers_MAClicensing_.csv`) | **AI Business Value (AIBV)** | -| **`-IncludeM365Usage`** (or `-Dashboard M365`) | `M365` | `Purview_M365_Usage_Bundle_Explosion_Processor` | Combined Purview CSV (single file) | **M365 Usage Analytics** | +| **CopilotInteraction-only** | `ValueLens` | `Purview_CopilotInteraction_Processor` | Purview CSV **+** Entra users CSV (`EntraUsers_MAClicensing_.csv`) | **ValueLens** | +| **`-IncludeM365Usage`** (or `-Dashboard M365`) | `M365` | `Purview_M365_Usage_Bundle_Explosion_Processor` | Combined Purview CSV (single file) | **M365 Usage Analytics** | -> The CopilotInteraction processor produces **AIO** output by default. Pass `-Dashboard AIBV` to produce the **AI Business Value** output instead from the same audit + Entra/MAC licensing data — the data PAX collects is identical; only the shape of the rolled-up output differs. See [Dashboard Selection](#dashboard-selection) below. +> The CopilotInteraction processor produces **AIO** output by default. Pass `-Dashboard ValueLens` to produce the **ValueLens** output instead from the same audit + Entra/MAC licensing data — the data PAX collects is identical; only the shape of the rolled-up output differs. See [Dashboard Selection](#dashboard-selection) below. ### Dashboard Selection -The `-Dashboard` parameter chooses which Copilot Analytics Lab dashboard a rollup run produces input for. It accepts one of three values and **defaults to `AIO`**, so existing rollup commands are unchanged. +The `-Dashboard` parameter chooses which Copilot Analytics Lab dashboard a rollup run produces input for. It accepts one of three values and **defaults to `AIO`**, so existing rollup commands are unchanged. | Want this dashboard | Use | What runs | | --- | --- | --- | | **AI-in-One (AIO)** — the default | `-Rollup` *(or `-Rollup -Dashboard AIO`)* | CopilotInteraction processor, AIO output profile | -| **AI Business Value (AIBV)** | `-Rollup -Dashboard AIBV` | CopilotInteraction processor, AIBV output profile | -| **M365 Usage Analytics** | `-Rollup -IncludeM365Usage` *(or `-Rollup -Dashboard M365`)* | M365 Usage Bundle processor | +| **ValueLens** | `-Rollup -Dashboard ValueLens` | CopilotInteraction processor, same output as before (name-only change) | +| **M365 Usage Analytics** | `-Rollup -IncludeM365Usage` *(or `-Rollup -Dashboard M365`)* | M365 Usage Bundle processor | + +> **ValueLens is a name-only change.** It uses the same source data, output schema, append behavior, and de-identification as before. Existing checkpoints resume as ValueLens automatically, without customer intervention. **Defaults and conveniences:** @@ -3457,9 +3544,9 @@ The `-Dashboard` parameter chooses which Copilot Analytics Lab dashboard a rollu - **`-Dashboard M365`** automatically enables `-IncludeM365Usage` (the M365 dashboard consumes the M365 usage bundle). - **`-Dashboard` supplied without `-Rollup` / `-RollupPlusRaw`** automatically enables `-Rollup` (never `-RollupPlusRaw`). - **AIO output is unchanged** from prior script versions — the default produces byte-for-byte the same files it did before `-Dashboard` existed. -- The selected dashboard is persisted to the checkpoint and restored on `-Resume`, so a resumed AIBV run stays AIBV. +- The selected dashboard is persisted to the checkpoint and restored on `-Resume`, so a resumed ValueLens run stays ValueLens. -> **`-Dashboard AIO` / `AIBV` cannot be combined with `-IncludeM365Usage`.** The Copilot dashboards (AIO/AIBV) and the M365 Usage Analytics dashboard read different source data **and** run different processors, so PAX rejects the combination with an explicit error — choose `-Dashboard M365` (or drop `-IncludeM365Usage`) instead. +> **`-Dashboard AIO` / `ValueLens` cannot be combined with `-IncludeM365Usage`.** The Copilot dashboards (AIO/ValueLens) and the M365 Usage Analytics dashboard read different source data **and** run different processors, so PAX rejects the combination with an explicit error — choose `-Dashboard M365` (or drop `-IncludeM365Usage`) instead. ### Switches @@ -3493,7 +3580,7 @@ When the rollup feature is active, PAX prints a cyan banner near the start of th ### Checkpoint Persistence -Rollup configuration is persisted in the checkpoint JSON (`rollupMode` ∈ `None | Rollup | RollupPlusRaw`, `processorMode`, and `rollupDashboard` ∈ `AIO | AIBV | M365`). On resume, the saved values take precedence (last-write-wins) and the script re-derives the runtime processor selection so a resumed run produces the same Power BI input file as the original — a resumed AIBV run stays AIBV rather than reverting to the AIO default. An explicit `-Dashboard` on the resume command line overrides the saved value. +The selected dashboard and rollup mode are saved with the run's checkpoint and restored on resume, so a resumed run keeps producing the same dashboard. Existing checkpoints created before the rename resume visibly as ValueLens; no customer intervention is required. An explicit `-Dashboard` on the resume command line overrides the saved value where permitted. ### Blocked Combinations @@ -3502,15 +3589,58 @@ The rollup feature is intentionally narrow in scope. The script exits with an ex - `-UseEOM` (PowerShell 5.1 path) - `-OnlyUserInfo` - `-ExcludeCopilotInteraction` **without** `-IncludeM365Usage` -- `-Dashboard AIO` or `-Dashboard AIBV` together with `-IncludeM365Usage` (the Copilot dashboards and the M365 dashboard use different source data and processors — pick one) +- `-Dashboard AIO` or `-Dashboard ValueLens` together with `-IncludeM365Usage` (the Copilot dashboards and the M365 dashboard use different source data and processors — pick one) ### Output Files Rolled-up CSVs are written to the same directory as the raw Purview CSV (default: `./output/`). File names follow the embedded processor's own naming conventions and are the exact files expected by the Copilot Analytics Lab Power BI templates — **do not rename them**. See [Output Files & Schema](#output-files--schema) for the surrounding directory layout. -### Org / Manager Hierarchy (AI-in-One / AI Business Value) + + +### Org / Manager Hierarchy (AI-in-One / ValueLens) + +When a CopilotInteraction rollup runs (the **AI-in-One** and **ValueLens** dashboards), PAX automatically enriches the rolled-up **Users** output with each person's place in the organization chart. This requires no extra switch — it is part of every AIO / ValueLens rollup. **How it tracks the hierarchy.** PAX already collects each user's manager when it enriches Entra data (the `-IncludeUserInfo` step that the rollup auto-enables). From those manager links, the processor reconstructs the full reporting tree: it walks from each person up through their managers to the top of their chain, works out how deep in the org each person sits, and counts how many people report to them — both directly and across their whole sub-organization. The result is a set of ready-to-use hierarchy columns on the Users output, so you can build org-based views in Power BI without modelling the tree yourself. @@ -3571,7 +3701,7 @@ When you point a rollup run at an existing rollup file with `-AppendFile`, PAX m **What this means for you:** an append file you started with an **earlier version** needs a **one-time re-baseline** — generate a fresh rollup file with the current version once and use that as your new append target. From then on, every `-AppendFile` run reconciles and grows correctly. **No data is lost** in the process: your original file is preserved as-is, and this run's data is safely written to the new file. -> **Note (v1.11.12):** if you have been appending onto an **AIO** rolled-up interactions file created before v1.11.12, expect this one-time re-baseline on your next run — the fan-out-safe merge needs a stable user-identity column that older AIO files do not carry. Your existing file is left untouched and the run writes a fresh, timestamped file to re-baseline from. AIBV files created in v1.11.11 already carry the needed identity column. +> **Note (v1.11.12):** if you have been appending onto an **AIO** rolled-up interactions file created before v1.11.12, expect this one-time re-baseline on your next run — the fan-out-safe merge needs a stable user-identity column that older AIO files do not carry. Your existing file is left untouched and the run writes a fresh, timestamped file to re-baseline from. ValueLens files created in v1.11.11 already carry the needed identity column. **The rolled-up Users dimension always uploads.** When a rollup run also produces the Users dimension (the org / licensing companion to the interactions file), that Users file is uploaded correctly in **all four** combinations of interactions-append and Users destination — whether the interactions stream is appending or not, and whether the Users destination is `-OutputPathUserInfo` or `-AppendUserInfo`. It uploads exactly once in every case. @@ -3584,8 +3714,8 @@ When you point a rollup run at an existing rollup file with `-AppendFile`, PAX m # Raw CSV(s) deleted on success; only the rollup output remains. .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -Rollup -# AI Business Value (AIBV) dashboard — same data, AIBV output profile. -.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -Rollup -Dashboard AIBV +# ValueLens dashboard — same data and output as before (name-only change). +.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -Rollup -Dashboard ValueLens # Same as the AIO example but keep the raw Purview + Entra users CSVs alongside the rollup output. .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -RollupPlusRaw @@ -3594,8 +3724,8 @@ When you point a rollup run at an existing rollup file with `-AppendFile`, PAX m # -Rollup deletes the raw combined CSV after the rollup output is produced. .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -IncludeM365Usage -Rollup -# AIBV rollup with org-level columns filled with a fixed label where chains are shallow. -.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -Rollup -Dashboard AIBV -FillerLabel Fixed -FillerLabelText "Assistive Directs" +# ValueLens rollup with org-level columns filled with a fixed label where chains are shallow. +.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -Rollup -Dashboard ValueLens -FillerLabel Fixed -FillerLabelText "Assistive Directs" # Anonymized AIO rollup — org-hierarchy structure preserved, names tokenized. .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate '2026-04-01' -EndDate '2026-04-30' -Rollup -Deidentify @@ -3603,7 +3733,7 @@ When you point a rollup run at an existing rollup file with `-AppendFile`, PAX m ### Best Practices -1. **Pick your dashboard with `-Dashboard`.** A CopilotInteraction-only run defaults to **AI-in-One (AIO)**; add `-Dashboard AIBV` for **AI Business Value**; use `-IncludeM365Usage` (or `-Dashboard M365`) for **M365 Usage Analytics**. The Copilot dashboards (AIO/AIBV) and the M365 dashboard cannot be produced in the same run. +1. **Pick your dashboard with `-Dashboard`.** A CopilotInteraction-only run defaults to **AI-in-One (AIO)**; add `-Dashboard ValueLens` for **ValueLens**; use `-IncludeM365Usage` (or `-Dashboard M365`) for **M365 Usage Analytics**. The Copilot dashboards (AIO/ValueLens) and the M365 dashboard cannot be produced in the same run. 2. **Prefer `-RollupPlusRaw` for first-time validation.** Keeping the raw CSV lets you spot-check the rollup output against the source data before deleting raws on subsequent runs. 3. **Don't rename output files.** The Copilot Analytics Lab templates load files by name pattern. Renaming will break the data refresh. 4. **Don't repurpose rollup outputs.** The schemas are tuned for the named Power BI templates. For ad-hoc analytics, BI ingestion outside the Copilot Analytics Lab, or custom data warehouses, use the raw Purview CSV instead. @@ -4091,6 +4221,33 @@ To prevent data loss during authentication failures or interruptions, PAX saves > ⚠️ **Important:** Do not delete the `.pax_incremental` folder during an active run or before resuming an interrupted run, as it contains your retrieved data. +### Fabric Durable Resume Mirror, Exit Codes, and Unattended (Azure-hosted) Execution + +**Local scratch.** On remote-output runs (SharePoint or Microsoft Fabric / OneLake), every artifact is written to a local scratch folder first (`$env:TEMP\PAX_\`; on Linux/container hosts this resolves under `/tmp`) and delivered afterward. If remote delivery or a requested append cannot complete, the local working copy is preserved for recovery rather than deleted, and the run reports a non-zero exit result. + +**Fabric durable resume mirror.** On a Microsoft Fabric / OneLake run, the resume artifacts (checkpoint, incremental data, partial output) are also mirrored to durable OneLake storage at `Files/.pax_resume//`, so an interrupted run can be resumed even if the local host is ephemeral (for example a restarted container). Local and SharePoint runs keep their resume artifacts next to the script only. + +**Process exit codes.** PAX returns a distinct process exit code so schedulers and unattended hosts can act on the outcome: + +| Exit code | Meaning | +|-----------|---------| +| `0` | Success. | +| `1` | Unexpected fatal error (the run did not complete). | +| `10` | Completeness / result-limit reached (re-run with `-AutoCompleteness`). | +| `20` | Reliability circuit breaker tripped (investigate throttling). | +| `30` | Entra user-directory fetch failed or was incomplete. | +| `40` | Completed with gaps — a partition failed terminally or a window reached the subdivision-depth safeguard. Lowest precedence. | + +The complete reference, including precedence and whether `-Resume` applies, is in the script's built-in help (`Get-Help ... -Full`, under `.NOTES`). + +**Azure Container Apps and other unattended hosts should evaluate the PAX process exit result.** A fatal error returns exit `1` rather than a success result, so configure the job to treat any non-zero exit code as a failure and to inspect the run log and the preserved local scratch for diagnosis. + +**Operational guidance.** + +- Run a **one-day** export first to validate configuration and permissions before launching a large historical load. +- Schedule ongoing incremental collection as **separate** runs that write to append targets (`-AppendFile` / `-AppendUserInfo`), rather than repeatedly re-collecting the full history. +- Do **not** run concurrent PAX jobs against the **same** checkpoint or the **same** append targets — a checkpoint and its append targets belong to a single run at a time. + ### Token Refresh Prompts When using delegated authentication (WebLogin/DeviceCode), PAX uses **reactive** token refresh detection with a **silent-first** approach. Instead of prompting at a fixed time interval, the script monitors for 401 Unauthorized errors indicating the token has actually expired. When detected: @@ -4320,7 +4477,7 @@ Because anonymization is deterministic, everything that depends on matching iden - **Joins** between files (for example, activity records to the Entra Users file) still line up, because a given person carries the same token everywhere. - **Per-user counts and rollups** are unchanged — the number of distinct users, interactions per user, and adoption rollups are identical to a non-anonymized run. -- **The org / manager hierarchy is preserved.** In the Power BI rollup, the hierarchy columns are built on a stable internal key rather than on names, so the **structure** of the org chart — levels, management chains, and report counts — is byte-for-byte identical with or without `-Deidentify`; only the name columns are tokenized. See [Org / Manager Hierarchy](#org--manager-hierarchy-ai-in-one--ai-business-value) for details. +- **The org / manager hierarchy is preserved.** In the Power BI rollup, the hierarchy columns are built on a stable internal key rather than on names, so the **structure** of the org chart — levels, management chains, and report counts — is byte-for-byte identical with or without `-Deidentify`; only the name columns are tokenized. See [Org / Manager Hierarchy](#org--manager-hierarchy-ai-in-one--valuelens) for details. ### Where it applies @@ -4341,8 +4498,8 @@ The transformation happens **on the host before anything is written or uploaded* ``` ```powershell -# Anonymized Power BI rollup input (AI Business Value) -.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-04-01 -EndDate 2026-04-30 -Dashboard AIBV -Deidentify -OutputPath C:\Temp\ +# Anonymized Power BI rollup input (ValueLens) +.\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-04-01 -EndDate 2026-04-30 -Dashboard ValueLens -Deidentify -OutputPath C:\Temp\ ``` @@ -4377,7 +4534,7 @@ The transformation happens **on the host before anything is written or uploaded*
📄 Bring your own user/organization directory from a CSV instead of pulling from Entra -**What it does.** `-UserInfoFile` lets you supply the user and organization directory from a **CSV file you provide** instead of pulling it live from Microsoft Entra. Point it at a **local path, a SharePoint document, or a Microsoft Fabric / OneLake file** — the same destination types PAX supports everywhere else. The directory you supply is then used **everywhere the Entra Users export is used** — the standalone `EntraUsers_MAClicensing_.csv`, org / manager hierarchy, the rolled-up **Users dimension** for the AI-in-One and AI Business Value dashboards, de-identification (`-Deidentify`), and upload to your chosen destination — end to end. Simply passing `-UserInfoFile` is enough; you do not also have to request user enrichment separately. +**What it does.** `-UserInfoFile` lets you supply the user and organization directory from a **CSV file you provide** instead of pulling it live from Microsoft Entra. Point it at a **local path, a SharePoint document, or a Microsoft Fabric / OneLake file** — the same destination types PAX supports everywhere else. The directory you supply is then used **everywhere the Entra Users export is used** — the standalone `EntraUsers_MAClicensing_.csv`, org / manager hierarchy, the rolled-up **Users dimension** for the AI-in-One and ValueLens dashboards, de-identification (`-Deidentify`), and upload to your chosen destination — end to end. Simply passing `-UserInfoFile` is enough; you do not also have to request user enrichment separately. **Why use it.** Use your own file when you already maintain an authoritative user/org list (for example, an HR extract), when you want organization data the live Entra pull doesn't carry, or when you want to run enrichment against a curated set of people without querying the directory. @@ -4397,8 +4554,8 @@ The run reports — on screen and in the log — how many users came from your f # Enrich using a directory you supply, instead of pulling from Entra ./PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-06-01 -EndDate 2026-06-02 -UserInfoFile "C:\Data\my_users.csv" -# Directory file on SharePoint, feeding an AI Business Value rollup -./PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-06-01 -EndDate 2026-06-02 -Rollup -Dashboard AIBV -UserInfoFile "https://contoso.sharepoint.com/sites/Analytics/Shared%20Documents/my_users.csv" +# Directory file on SharePoint, feeding a ValueLens rollup +./PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2026-06-01 -EndDate 2026-06-02 -Rollup -Dashboard ValueLens -UserInfoFile "https://contoso.sharepoint.com/sites/Analytics/Shared%20Documents/my_users.csv" ```
@@ -4527,7 +4684,7 @@ Comprehensive user profile data per user, automatically deduplicated by UserPrin | `OrgLevel_3Label` | Org level 3 (HR systems) | (null) | | ... (additional extended attributes) | ... | ... | -> **In a Power BI rollup (AI-in-One / AI Business Value), the Users output gains additional org / manager-hierarchy columns** — each person's `OrgLevel`, `Manager_UserKey`, `TopOfChain_UserKey`, `HierarchyPath`, `IsManager`, `DirectReports`, `TotalReports`, and `Level0`…`Level14` columns — derived from the manager data shown above. These appear only in the rolled-up Users output; see [Org / Manager Hierarchy](#org--manager-hierarchy-ai-in-one--ai-business-value). +> **In a Power BI rollup (AI-in-One / ValueLens), the Users output gains additional org / manager-hierarchy columns** — each person's `OrgLevel`, `Manager_UserKey`, `TopOfChain_UserKey`, `HierarchyPath`, `IsManager`, `DirectReports`, `TotalReports`, and `Level0`…`Level14` columns — derived from the manager data shown above. These appear only in the rolled-up Users output; see [Org / Manager Hierarchy](#org--manager-hierarchy-ai-in-one--valuelens). **License Detection Logic:** @@ -5436,7 +5593,7 @@ These are **suggestions**, not requirements, and are common to any read-only Pow ### Copilot Analytics Lab -The Copilot Analytics Lab is the central landing page for PAX-compatible Power BI templates, dashboards, and companion analytics tooling — including the AI-in-One (AIO) Dashboard, the AI Business Value (AIBV) Dashboard, the Copilot Chat & Agent Intelligence Dashboards, and the broader ROI / adoption / governance visualization library. +The Copilot Analytics Lab is the central landing page for PAX-compatible Power BI templates, dashboards, and companion analytics tooling — including the AI-in-One (AIO) Dashboard, the ValueLens Dashboard, the Copilot Chat & Agent Intelligence Dashboards, and the broader ROI / adoption / governance visualization library. - **[Copilot Analytics Lab](https://microsoft.github.io/CopilotAnalyticsLabs/)** — Single entry point for downstream visualization and analysis assets that consume PAX output diff --git a/release_notes/.gitkeep b/release_notes/.gitkeep index 725eac7..49b1f17 100644 --- a/release_notes/.gitkeep +++ b/release_notes/.gitkeep @@ -1 +1 @@ -# Last updated: 2026-07-06 (Purview v1.11.14, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.36) \ No newline at end of file +# Last updated: 2026-07-31 (Purview v1.11.15, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.37) \ No newline at end of file diff --git a/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md b/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md index f57a1b8..c26138f 100644 --- a/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md +++ b/release_notes/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Release_Note_v1.11.x.md @@ -2,8 +2,8 @@ ## Release Information -- **Latest Version:** 1.11.14 -- **Latest Release Date:** July 6, 2026 +- **Latest Version:** 1.11.15 +- **Latest Release Date:** July 31, 2026 - **Released By:** Microsoft Copilot Growth ROI Advisory Team (copilot-roi-advisory-team-gh@microsoft.com) --- @@ -12,16 +12,52 @@ Download the script below. For questions or issues, refer to the documentation. -- **PAX Purview Audit Log Processor Script v1.11.14:** [PAX_Purview_Audit_Log_Processor_v1.11.14.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.14/PAX_Purview_Audit_Log_Processor_v1.11.14.ps1) +- **PAX Purview Audit Log Processor Script v1.11.15:** [PAX_Purview_Audit_Log_Processor_v1.11.15.ps1](https://github.com/microsoft/PAX/releases/download/purview-v1.11.15/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1) - **Documentation v1.11.x (Markdown):** [PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md](https://github.com/microsoft/PAX/blob/release/release_documentation/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_Documentation_v1.11.x.md) --- ## Overview +### v1.11.15 + +Version 1.11.15 is **both a feature release and a reliability & correctness release.** As a feature release it renames one dashboard selector value to **ValueLens** and adds hybrid directory enrichment (`-UserInfoSupplement`); as a reliability release it delivers a broad set of correctness fixes that apply to normal runs, so existing workflows benefit. The **AI Solutions Intelligence Dashboard (AISID)** implementation is included in this release, but AISID remains **under development**: its customer entry points are gated in v1.11.15, no AISID customer run can authenticate, collect, or publish in this version, and availability is planned for a later PAX script version. See [What's New → v1.11.15](#v11115-1), [Bug Fixes → v1.11.15](#v11115-2), and [Known Considerations → v1.11.15](#v11115-3) for the full detail. + +#### AI Solutions Intelligence Dashboard (AISID) — Under Development, Entry Points Gated + +The AISID implementation is included in v1.11.15, but AISID is still **under development and is not available for customer use in this version**. The `-Dashboard AISID`, `-OutputPathDefenderUsage`, `-AppendDefenderUsage`, and `-DisableAISIDDeltaCache` entry points are gated: supplying any of them reports that AISID is under development and stops the run before any sign-in, collection, or publication, so an AISID customer run cannot contact a service or write output in this version. PAX has no Defender-only execution switch. The completed AISID engineering described in this release note is retained as implementation groundwork, and the entry points are planned to become available in a later PAX script version. See [What's New → v1.11.15](#v11115-1). + +#### ValueLens — Renamed Dashboard Selector + +The customer-facing value for one dashboard selector is renamed to **ValueLens**; select it with `-Dashboard ValueLens`. This is a name-only change — the data, output schema, append behavior, de-identification, and output files are unchanged — and existing checkpoints resume automatically as ValueLens with no customer intervention. + +#### Partition-Failure and Subdivision Reliability (Exit Code 40) + +On ordinary partitioned exports, a partition that keeps failing the same way now stops instead of retrying indefinitely, and a defensive depth limit stops runaway time-window subdivision; both situations are reported in the end-of-run summary with a `-Resume` hint. A run that finishes with such gaps returns a distinct "completed with gaps" process exit code (`40`) at lowest precedence, so automation can tell it apart from a clean success without disturbing the existing exit codes. + +#### SharePoint and Large-File Handling Corrections + +SharePoint output no longer over-requests permissions or emits a false missing-permission warning; a large existing SharePoint append target downloads memory-safely; an advisory appears before merging an unusually large append target; large single-file uploads use a corrected, bounded-memory path; and an upload failure preserves the local copy and is reflected in the run's outcome. + +#### Faster Interactions/Fact Append on Large Targets + +Appending a run into an existing Interactions/Fact CSV now completes substantially faster on large targets, and the improvement grows with the size of the target. The result is unchanged in every respect — the merged output is byte-for-byte identical, the same fixed memory budget applies, temporary disk space is still used during the merge and removed afterward, and there is no row limit or truncation. If the faster path is ever unavailable on a given machine, the original path is used automatically, so results and compatibility are preserved. See [Bug Fixes → v1.11.15](#v11115-2). + +#### Faster Fact Continuity Preparation for Append Rollups + +Before an append rollup begins, PAX prepares continuity keys from the existing Fact file so retained interactions keep their stable `Message_Id` and thread identifiers. That preparation now uses a disk-backed accelerator for supported identifier forms, completing substantially faster while keeping memory bounded. Seed contents, first-occurrence behavior, identifier casing, and continuity results remain unchanged. If the existing file contains an unsupported key form, PAX automatically uses the established compatibility path with no customer action or configuration required. Temporary working files are removed when the step finishes, and a failure leaves the existing append target untouched. See [Bug Fixes → v1.11.15](#v11115-2). + + + +User filtering now applies consistently on every directory-backed pass — first pass and any retry pass — so a run scoped only with `-GroupNames` no longer collects more broadly than intended. Runs that supply neither `-UserIds` nor `-GroupNames` are unchanged. See [Bug Fixes → v1.11.15](#v11115-2). + +#### AISID Completion, Resume, and Data Correctness (Implementation Groundwork) + +The behavior below is completed AISID implementation retained for a later PAX script version; because the AISID entry points are gated in v1.11.15, it is not reachable by a customer run in this version. An AISID run reports its outcome plainly — complete, complete-with-empty, complete-with-unavailable, completed-with-gaps, or interrupted — and cannot report a clean success when a required file is missing, unreadable, or undelivered. Resumed AISID runs restore the original run's dashboard, destination, cache setting, and collection window and give a truthful failure status if resume validation fails, and the dashboard's data-set measures are computed consistently and deterministically. See [What's New → v1.11.15](#v11115-1) and [Bug Fixes → v1.11.15](#v11115-2). + ### v1.11.14 -Version 1.11.14 is a reliability release that fixes two remote-output delivery defects. Both fixes restore intended behavior — there are no new switches, no output-schema changes, and no change to how any run is invoked; runs that do not use the affected remote-output paths behave exactly as in v1.11.13. The first fix ensures the Microsoft Agent 365 catalog CSV is actually delivered to a SharePoint or Microsoft Fabric / OneLake destination (previously it could be generated but not uploaded). The second ensures a custom-named per-stream output file is delivered to its own per-stream destination rather than the primary output location. This release also carries early groundwork for a future AI Solutions Intelligence Dashboard (AISID) capability that is **not available in this version** — see [Known Considerations → v1.11.14](#v11114-3). +Version 1.11.14 is a reliability release that fixes two remote-output delivery defects. Both fixes restore intended behavior — there are no new switches, no output-schema changes, and no change to how any run is invoked; runs that do not use the affected remote-output paths behave exactly as in v1.11.13. The first fix ensures the Microsoft Agent 365 catalog CSV is actually delivered to a SharePoint or Microsoft Fabric / OneLake destination (previously it could be generated but not uploaded). The second ensures a custom-named per-stream output file is delivered to its own per-stream destination rather than the primary output location. This release also carries early groundwork for the AI Solutions Intelligence Dashboard (AISID) capability; AISID remains under development and its customer entry points are gated in v1.11.15, with availability planned for a later PAX script version — see [What's New → v1.11.15](#v11115-1). #### Microsoft Agent 365 Catalog CSV Delivered to Remote Destinations @@ -275,9 +311,35 @@ New sixth value on the `-Auth` ValidateSet for Azure-hosted headless execution ( ## What's New +### v1.11.15 + +- **AI Solutions Intelligence Dashboard (AISID) — implementation included, entry points gated (`-Dashboard AISID`).** The AISID implementation ships in v1.11.15, but AISID is still under development and is **not available for customer use in this version**: supplying `-Dashboard AISID` reports that AISID is under development and stops the run before any sign-in, collection, or publication. The retained implementation runs the full Microsoft Purview + Microsoft Entra + Microsoft Defender pipeline and produces the input set for the Copilot Analytics Lab **AI Solutions Intelligence Dashboard**, reusing the AI-in-One-shaped CopilotInteraction rollup for its Purview/Entra tables and layering Microsoft Defender signals on top. Everything described in this AISID group is implementation groundwork retained for a later PAX script version. Runs that do not select AISID are unchanged. + +- **Dedicated AISID destination — three switches (all gated in v1.11.15).** Each of the three switches below is a gated AISID entry point in this version: supplying it reports that AISID is under development and stops the run. The behavior described for each is retained implementation groundwork for a later PAX script version. + - **`-OutputPathDefenderUsage `** — folder-only destination for the entire AISID output set. Supply a folder path, SharePoint folder URL, or Microsoft Fabric / OneLake folder URL (a file leaf is rejected); the storage tier is inferred exactly like the other `-OutputPath*` switches and must match the run's other destinations. Only meaningful with `-Dashboard AISID`. + - **`-AppendDefenderUsage `** — the append counterpart of `-OutputPathDefenderUsage`. Merges this run's AISID output into an existing set in the target folder rather than overwriting it: the fact tables reconcile on each table's key (overlapping rows update, new rows are added, existing rows are preserved) and the dimension tables (`EntraUsers.csv`, `ai_solutions_catalog.csv`) are regenerated each run. Mutually exclusive with `-OutputPathDefenderUsage` (supply exactly one of the pair per run). Only meaningful with `-Dashboard AISID`. + - **`-DisableAISIDDeltaCache`** — turns off the incremental result cache used by the off-hours-geography signal, forcing a full re-collection of its window on this run. Only meaningful with `-Dashboard AISID`. + +- **12 fixed-name output files.** An AISID run delivers exactly 12 files to the `-OutputPathDefenderUsage` / `-AppendDefenderUsage` destination, each loaded by the dashboard model under a fixed name: `ai_activity_sessions.csv`, `ai_offhours_geo.csv`, `ai_file_proximity.csv`, `ai_oauth_consents.csv`, `ai_sso_signins.csv`, `ai_client_channel.csv`, `ai_copilot_usage_graph.csv`, `ai_appgov_alerts.csv`, `ai_cloud_discovery.csv`, `ai_mda_sessions.csv`, `ai_solutions_catalog.csv`, and `EntraUsers.csv`. The three Microsoft Defender for Cloud Apps files are compatibility tables in this release; their collectors are not included yet. A first run writes their exact headers, while an append run preserves any valid prior files byte-for-byte. All 12 go to the AISID destination — never the primary Purview output location. + +- **Remote-safe delivery.** On a SharePoint or Microsoft Fabric / OneLake run, the AISID writers stage each file to the local scratch folder first, then a single end-of-run upload sweep ships each of the 12 files to the AISID destination exactly once. The internal `.aisid_cache` delta-cache folder is scratch only — it is never uploaded and is never a dashboard artifact. + +- **Explicit completion states.** Every AISID run ends in one of five states, reported in the run log: **complete** (all 12 files produced with data), **complete-with-empty** (a query succeeded with zero rows), **complete-with-unavailable** (a signal is not populated but its required header-only file is delivered), **completed-with-gaps** (one or more required files could not be produced), or **interrupted** (collection stopped before finishing, for example an authentication quit). A run that ends with gaps — including an AISID upload that fails to deliver — exits with a non-zero exit code (`40`), so schedulers and automation can detect it. + +- **`-Dashboard ValueLens` — renamed selector.** The customer-facing dashboard value formerly exposed publicly is now **ValueLens**; select it with `-Dashboard ValueLens`. It is a name-only change — same data, output schema, append behavior, de-identification, and output files — and the former public value is no longer accepted for new commands. Existing checkpoints resume automatically as ValueLens with no customer intervention. + +- **Bring Your Own Directory — hybrid enrichment (`-UserInfoSupplement`).** Alongside the existing `-UserInfoFile` (which *replaces* the Entra directory with a CSV), the new `-UserInfoSupplement` *enriches* the live directory: PAX fetches the live Entra `/users` directory normally, then appends the columns of a customer-provided CSV to each matching Entra row (an Entra-left join on `UserPrincipalName`). Every Entra user is preserved; supplemental rows that match no Entra user are reported and excluded (never added as directory users), and this alone never marks a run "completed with gaps". `UserPrincipalName` is the only required supplemental column and is a join key only — it is never copied into the output, so identity always comes from Entra. Supplemental column names, order, blanks, and values are preserved unchanged and are additive-only (a name collision with any Entra/PAX-owned column, a blank/duplicate UPN, or a duplicate header is a fatal input error). The supplemental input may be a local `.csv`, a SharePoint `.csv` URL, or a Fabric `/Files/.../*.csv` URL. Supplemental columns flow into the normal EntraUsers CSV and the AIO/ValueLens Rollup Users dimension, but not into the AISID `EntraUsers.csv` fixed 14-column schema. With `-Deidentify`, Entra-derived identity fields are de-identified as usual while supplemental values pass through unchanged — PAX does not inspect, mask, or de-identify them, and shows a prominent notice before processing. Auto-enables `-IncludeUserInfo`; compatible with `-UserIds`, `-GroupNames`, `-OnlyUserInfo`, AIO/ValueLens/AISID, `-AppendUserInfo`, and `-Deidentify`; mutually exclusive with `-UserInfoFile`, `-UseEOM`, and `-RAWInputCSV`. + + +- **New "completed with gaps" exit code (`40`).** For unattended and scheduled use, PAX now returns a distinct process exit code `40` when a run finishes but one or more partitions failed terminally, a time window reached the subdivision-depth safeguard, or (on an AISID run) a required file could not be produced or delivered — so automation can tell "finished, but check the summary" apart from a clean success. The existing exit codes are unchanged and keep priority; `40` is the lowest-precedence signal. A full exit-code reference was added to the script's built-in help. + +- **Advisory before merging an unusually large append target (`PAX_APPEND_WARN_BYTES`).** Before merging into an existing append target, PAX checks the target's size without downloading it and, if it is larger than a set threshold (1 GB by default), prints a one-line advisory that a very large target may lengthen the merge and require temporary disk space and additional I/O. The append/merge processes the target within a fixed memory budget by spilling to a temporary disk-backed store rather than holding the whole target in memory. It is advisory only — the run always continues and a legitimately large target is never blocked — and it is silently skipped if the size can't be read or the target doesn't exist yet. The threshold can be changed with the `PAX_APPEND_WARN_BYTES` environment variable (in bytes), which controls only the advisory threshold and does not cap processing. + +- **AISID runs use the standard operational surface (retained implementation).** In the retained implementation an AISID run participates in the same run log, end-of-run output roster, parameter snapshot, checkpoint/resume, and metrics as every other run, and delivers its files to Local, SharePoint, or Microsoft Fabric / OneLake destinations through the same remote-delivery path. This behavior is not reachable in v1.11.15 because the AISID entry points are gated. + ### v1.11.14 -*Not applicable — v1.11.14 is a reliability release and introduces no new customer-facing features. Its two remote-output delivery fixes are described under [Overview → v1.11.14](#v11114) and [Bug Fixes → v1.11.14](#v11114-2). Early groundwork for a future AI Solutions Intelligence Dashboard (AISID) capability is noted under [Known Considerations → v1.11.14](#v11114-3); it is not available in this version.* +*Not applicable — v1.11.14 is a reliability release and introduces no new customer-facing features. Its two remote-output delivery fixes are described under [Overview → v1.11.14](#v11114) and [Bug Fixes → v1.11.14](#v11114-2). Early groundwork for the AI Solutions Intelligence Dashboard (AISID) capability shipped in this release; AISID remains under development and its customer entry points are gated in v1.11.15, with availability planned for a later PAX script version — see [What's New → v1.11.15](#v11115-1).* ### v1.11.13 @@ -721,6 +783,114 @@ Excel filenames, Excel tab names, and the `EntraUsers_*` / `Agent365_*` filename ## Bug Fixes +### v1.11.15 + +
+(v1.11.15) Graph / Purview collection reliability + +- **A partition that keeps failing the same way now stops instead of retrying forever.** Under a persistent server-side failure, a partition that fails the same way enough times in a row is now treated as a terminal failure: it is dropped from the retry queue so the run can finish, and it is listed in the end-of-run summary with its time window, the kind of failure, and a `-Resume` hint. Transient sign-in / token forgiveness was tightened so it fires only on genuine authentication errors — not on unrelated text that merely contains "401" (such as a record count) — and genuine authentication failures are now labelled as such in the failure report. +- **A safeguard prevents runaway time-window subdivision.** When a single window returns more records than the service releases at once, PAX splits it into smaller windows and retries; genuinely high-volume tenants that need many levels of splitting continue to work unchanged. A defensive depth limit now stops the splitting if it ever reaches an unreasonable depth without resolving; a window that reaches the limit is stopped, logged with its bounds and a `-Resume` hint, and reported in its own section of the end-of-run summary. +- **Gaps return a distinct exit code without overriding higher-priority codes.** A run that finishes with one or more terminal partition failures and/or a window that reached the subdivision safeguard now returns exit `40` (completed with gaps). The existing exit codes keep priority — a hard record-limit result, an open reliability circuit breaker, or a directory-fetch shortfall still report their own codes — and a full exit-code reference was added to the script's built-in help. + +
+ +
+(v1.11.15) -UserIds / -GroupNames user filtering + +- **User filtering now applies to every directory-backed pass.** When a run is scoped with `-UserIds`, `-GroupNames`, or both, the resolved set of users — direct user IDs plus everyone expanded from the named groups — is now applied consistently on **both** the first pass and any retry pass. Previously a run scoped only with `-GroupNames` could skip that filtering on the directory-backed path and collect more broadly than intended; it now honors the group membership on every pass. Runs that supply neither option are unaffected and continue to collect without user filtering. _(The live `-UserIds` / `-GroupNames` end-to-end check against a tenant is still pending; the behavior is verified by the automated test suite.)_ + +
+ +
+(v1.11.15) SharePoint permissions, downloads, and append targets + +- **SharePoint output no longer over-requests permissions.** When writing to SharePoint, PAX previously also asked for `Files.ReadWrite.All` on top of `Sites.ReadWrite.All`, even though `Sites.ReadWrite.All` alone authorizes every SharePoint operation PAX performs. The unneeded scope has been removed, which also clears a **false "missing required permission" warning** that appeared for identities granted only `Sites.ReadWrite.All` — those runs were already fully functional. Apps previously granted both scopes keep working and do not need to re-consent. +- **Large SharePoint downloads no longer risk running out of memory.** When a run needs to pull an existing large remote file back down first (for example an append target that has grown large), the download now streams straight to disk instead of being held in memory in full, so its memory use is flat and independent of file size; a not-yet-present remote file is still treated as a first run. +- **Upload failures preserve the local copy and are reported.** If any file fails to upload, the run is reported as completed-with-gaps (exit `40`) and the local working copy is kept so nothing is lost. The run's final remote log records the run's terminal result, the local recovery location, and the exit information before its own last upload, so the delivered log reflects how the run actually ended. +- **Corrected large-file upload request; bounded memory.** Starting a large-file upload session to SharePoint now uses the corrected structured request body, and large files are streamed in bounded-size chunks so memory use stays flat regardless of file size. +- **Faster Interactions/Fact append on large targets.** The record reading that previously dominated the Interactions/Fact append/merge on large histories has been optimized, so the step now finishes in a fraction of the time it used to take, with the improvement growing as the target grows. The result is unchanged in every respect: the merged output is byte-for-byte identical, the same fixed memory budget applies, temporary disk space is still used during the merge and removed afterward, and there is no row limit or truncation. If the optimized reader is ever unavailable on a given machine, the step falls back to the original reader automatically, so results and compatibility are preserved. + +
+ +
+(v1.11.15) Faster, bounded-memory Fact continuity preparation + +- **Large append rollups prepare continuity keys substantially faster.** PAX now uses a disk-backed accelerator when preparing the existing Fact file's `Message_Id` and thread continuity maps. Working memory remains bounded as the target grows, and progress remains visible throughout the step. +- **Continuity results are unchanged.** The accelerated path preserves the exact seed contents, first-occurrence selection, original identifier casing, stable integer values, property order, and `ThreadKey` preference with `ThreadId` fallback. +- **Compatibility is automatic.** If a Fact file contains a key or record form outside the accelerated path's supported domain, PAX automatically uses the established PowerShell preparation path for that run. No switch or customer action is required. +- **Temporary work is cleaned up and failures remain data-safe.** Temporary script, database, journal, and seed files are removed when preparation ends. Neither continuity seed is published unless both complete successfully, and a preparation failure leaves the existing append target and prior outputs untouched. + +
+ +
+(v1.11.15) Stable user keys across append runs + +- **A user seen only in activity keeps their key.** The retained-key list used when appending to existing output is now built from the existing Users file **and** the existing activity file. Previously it was built from the Users file alone, so a user with activity but no directory row had no key retained: a later run could give that key to a different person and mint a second key for the original person. Any user who is new this run is now given a key above every key already in use. +- **Every user in the activity data resolves to a Users row.** A user with activity but no directory row this run receives a minimal Users row carrying only their identity and the key already reserved for them. Every other field is left blank and nothing is invented. When a real directory row for that person arrives in a later run it replaces the minimal row, and the key does not change. +- **A prior mapping that disagrees with itself stops the append.** If the existing files already hold one person under two keys, or one key claimed by two people, PAX refuses to build the retained-key list, leaves both existing files exactly as they were, and reports the run as completed with gaps (exit `40`). +- **The activity file and the Users file are published together, or neither is.** Before either replaces its existing target, the two are checked against each other: every key in the activity data must resolve to exactly one Users row, and that row must be the same person. If the check does not pass, neither file is published, both existing targets remain byte-for-byte as they were, and this run's output is preserved for review. +- **The Users union reports its progress.** Folding this run's users into the existing Users file now reports entry, aggregate progress, and completion or failure, so a long union never looks idle. The records carry aggregate numbers only, and the file produced is identical whether or not progress is reported. + +
+ +
+(v1.11.15) Clearer wording while audit queries are being created + +- **The wait line distinguishes local queue from total outstanding.** The line shown while audit queries are being created reported a single `queued` count, which read as though it covered everything still outstanding. It now reports `localQueued` — the items still waiting in the group being reported on — alongside `pending`, everything not yet finished. This is wording only: the counts, the work performed, the number of requests issued, and the pace of the run are unchanged. + +
+ +
+(v1.11.15) Clear guidance for restricted PowerShell language mode + +- **Restricted hosts stop before initialization.** PAX now checks the PowerShell language mode before creating a log, loading modules, signing in, or writing output. A host that is not running in `FullLanguage` mode stops with exit `1` and directs the user to contact their IT or security team for an approved PowerShell environment or to have PAX allowlisted. The message also clarifies that `ExecutionPolicy Bypass` does not change the language mode. + +
+ +
+(v1.11.15) AISID integration & data correctness (implementation groundwork) + +- **AISID entry points are gated in v1.11.15.** Everything in this group is completed implementation retained for a later PAX script version. AISID is still under development; `-Dashboard AISID`, `-OutputPathDefenderUsage`, `-AppendDefenderUsage`, and `-DisableAISIDDeltaCache` are gated, and no AISID customer run can authenticate, collect, or publish in this version. PAX has no Defender-only execution switch. +- **Remote destinations stage locally.** On a SharePoint or Microsoft Fabric / OneLake run, the AISID writers stage each file to a local working folder rather than treating a remote URL as a local path, and existing AISID history is read from the AISID (Defender-usage) destination rather than the Purview destination. All 12 files route to the AISID destination and upload exactly once per run. +- **Truthful completion.** An empty-only run reports complete-with-empty and stays a clean success; a run whose required files are failed, incomplete, missing, unreadable, or undelivered cannot report a clean success and instead finishes completed-with-gaps (exit `40`). +- **Off-hours / geography.** Country and off-hours measures are derived only from real count values and ignore the service's metadata annotations, and the primary country is chosen deterministically (highest count, with a stable tie-break), so figures are consistent from run to run. +- **File proximity.** The size estimate that decides how finely the file-proximity window is divided now requires a real numeric value; if it can't be read, that data set is treated as failed instead of being mistaken for very high volume, so a single unreadable estimate can no longer trigger runaway window splitting. +- **Copilot usage.** Each interaction contributes its actual number of user prompts — a response-only record counts as zero and a record with several prompts counts them all — and interaction timestamps are interpreted as UTC regardless of the machine's local time zone, so the same records always fall in the same collection window. +- **Resume validation.** A resumed AISID run restores the original run's dashboard, destination, cache setting, and window, re-checks destination safety, and gives a truthful failure status if resume validation fails rather than a false "interrupted" or "cancelled" result. + +
+ +
+(v1.11.15) Fabric checkpoint mirroring and unattended-host exit result + +- **Fabric checkpoint mirroring now sends exact file bytes.** Small checkpoint files and final partial chunks are sent to Microsoft Fabric / OneLake as true byte arrays, a short final read is filled before transmission, and PAX verifies that every source byte was appended before asking OneLake to flush the file. Replacing an existing checkpoint path starts from a newly created file, a file that fails to send remains eligible for retry, and durable mirror state advances only after the complete create, append, and flush sequence succeeds. This corrects checkpoint-mirroring failures that surfaced as uploaded data not being contiguous or the append position not matching the file length. +- **An unexpected fatal error now returns a non-zero exit result.** A fatal error that reaches the top-level handler returns exit `1` instead of falling through to a success result, so an unattended host — for example an Azure Container Apps Job — no longer treats a failed run as successful. Local scratch and Fabric resume data are preserved for diagnosis and recovery, and the intentional-interruption, completed-with-gaps (`40`), and other established exit results remain distinct. + +
+ +
+(v1.11.15) Microsoft Graph audit-query lifecycle and duplicate-safe recovery + +- **Audit queries are created and owned by the main run, not by its worker threads.** Creating each partition's Purview audit query is now performed once by the run itself before any worker starts, and the created query identifier is recorded durably before collection begins. A worker never creates a query; it only polls, pages, and retrieves. This removes the duplicate-query and lost-query-identifier behavior that could leave completed server-side queries unretrieved. +- **A query whose creation outcome is uncertain is never blindly re-submitted.** If PAX cannot prove a submission was rejected — for example a transient service error or a dropped connection — it records the uncertainty durably and re-checks the service by exact query name before doing anything else. A matching existing query is adopted and reused; if more than one could match, the run stops rather than guessing. A second submission is only ever made after the service has been re-checked and no matching query was found. +- **The audit-query interface version is pinned to the one proven end to end.** The query interface version is no longer inferred from a single service listing, which could select a version that then failed to return status or records for a query the Purview portal showed as complete. A retry no longer discards a live query identifier based on message text; the existing query is reused, and genuinely stuck windows remain bounded by the established terminal-failure safeguard. +- **Resume state is written durably before it is relied on.** A checkpoint update that authorizes new work is written, re-read, and verified before the run continues; if the write cannot be proven durable the run stops with a failure result instead of proceeding. A failed publication restores the previous resume artifacts exactly rather than leaving a partially replaced set behind, and a successful restore replaces the resume artifacts as an exact set so stale files cannot be picked up later. +- **Bounded memory for record reading and append processing.** The Interactions/Fact, Users, session-statistics, and Microsoft 365 append merges, the CSV record reader, and the large-file processing stages work within a fixed memory budget by spilling to temporary disk storage, so memory use no longer grows with the size of the accumulated history. Temporary space is released when the step finishes, results are unchanged, and there is no row limit or truncation. + +
+ +
+(v1.11.15) Directory scoping, rollup fields, and run messaging + +- **`-GroupNames` now scopes the directory output as well as the audit records.** A run scoped with `-GroupNames` previously still published the full tenant directory in its Entra/Users output on ordinary runs. The resolved user set is now applied to every directory destination — the Entra CSV, the rollup Users input, the append merge, and remote delivery — so a group-scoped run publishes only the users in scope. Runs that supply neither `-UserIds` nor `-GroupNames` are unchanged. +- **`Organization` now shows the readable department name.** In the Users output the dashboards read, `Organization` is taken from the human-readable department value in the directory export. If that export also contains a separate column literally named `Organization` holding a department *number*, that number no longer displaces the readable name — it is preserved alongside it as `Organization_Id`. Organization slicers in the existing dashboards now receive readable text with no dashboard changes, and exports that contain only a department column are unaffected. +- **The AI-in-One Users output uses the exact field names the model expects.** The AI-in-One Users output emits `DisplayName`, `Country`, and `Email` with the exact casing the dashboard model binds to, keeps `mail` alongside `Email`, backfills a blank canonical field from an equivalent source column, and keeps headers unique. De-identification applies to these fields exactly as before. +- **Application-only and managed-identity runs no longer announce a second sign-in.** When Microsoft Agent 365 information is included, an application-registration or managed-identity run now reports that the Agent 365 step reuses the existing application context instead of announcing an interactive sign-in step that never happens. +- **Token-refresh status is reported truthfully.** The long-run token-refresh heartbeat now reports an explicit state at every point in the run, so it no longer shows an unknown elapsed time before the first refresh has occurred. +- **New guidance for scheduled refresh in the Power BI Service.** Written guidance now distinguishes the two settings most often confused when configuring a scheduled refresh: the authentication method (`OAuth2`) and the privacy level (`Organizational`). + +
+ ### v1.11.14 - **(v1.11.14) Microsoft Agent 365 catalog CSV not delivered to remote destinations.** On a remote-output run (SharePoint or Microsoft Fabric / OneLake) that included the Microsoft Agent 365 catalog, the catalog CSV could be generated — the run log even showed it "written" with its destination — yet the file did not appear at the remote destination, while the other artifacts (audit CSV, Entra Users, run log) uploaded normally. The catalog CSV was the only customer-facing artifact never registered with the end-of-run upload step; it was included only when its filename happened to carry the run timestamp, so a catalog routed to a specifically named remote destination was skipped by the upload step and its temporary local copy was removed during normal cleanup. The catalog CSV is now registered with the upload step in every remote-output mode, exactly like the other artifacts, so it is delivered in the same final upload; a genuine upload failure now also preserves the local copy for retry, the same as for every other artifact. On remote runs the write-log line now states the file is staged locally and queued for upload rather than implying it is already at the destination. Local-output runs are unchanged. @@ -851,9 +1021,21 @@ The following authentication and certificate-handling fixes apply to `-Auth AppR ## Known Considerations +### v1.11.15 + +- **(v1.11.15) AISID is under development and its entry points are gated.** The AISID implementation is included in v1.11.15, but AISID is not available for customer use in this version. Supplying `-Dashboard AISID`, `-OutputPathDefenderUsage`, `-AppendDefenderUsage`, or `-DisableAISIDDeltaCache` reports that AISID is under development and stops the run before any sign-in, collection, or publication, so no AISID customer run can authenticate, collect, or publish. PAX has no Defender-only execution switch. Availability is planned for a later PAX script version. The AISID behavior described elsewhere in this release note is retained implementation groundwork and is not reachable in this version. +- **(v1.11.15) AISID signal availability depends on tenant licensing.** Individual AI Solutions Intelligence Dashboard (AISID) signals draw on Microsoft Defender and Microsoft Entra data your tenant may or may not be licensed for. A signal whose source is not available to the tenant is reported as *unavailable* and the run completes with a **complete-with-unavailable** status rather than failing — see [What's New → v1.11.15](#v11115-1). No action is needed for signals your tenant is not licensed to use. +- **(v1.11.15) Advanced-hunting retention limits the AISID window.** The AISID data sets collect the date range you request intersected with the last 30 days that Microsoft advanced hunting keeps queryable by default. A requested range partly older than that window is trimmed to the available part with a single notice; a range entirely outside it stops before contacting any service. Your Purview activity export still covers the full requested range — only the dashboard's own data sets are limited to the available window. +- **(v1.11.15) Complete-with-unavailable is not a gap.** A signal your tenant is not licensed for is reported as unavailable and the run completes with a **complete-with-unavailable** status; this is distinct from a real gap (a required file that failed, is incomplete, or was not delivered), which reports **completed-with-gaps** and returns exit `40`. +- **(v1.11.15) ValueLens migration.** The former public dashboard selector value is no longer accepted for new commands — use `-Dashboard ValueLens`. Existing checkpoints resume automatically as ValueLens with no customer intervention; this is a name-only change with no effect on data, schema, or output files. +- **(v1.11.15) The large-append-target warning is advisory.** The pre-merge size check is advisory only and never blocks a run. The Interactions/Fact and Users append/merge now processes its target within a fixed memory budget by spilling to a temporary disk-backed store, so a very large target no longer needs to be held in memory in full, though it can require temporary disk space and additional I/O proportional to the target size. The `PAX_APPEND_WARN_BYTES` environment variable (in bytes) controls only the advisory threshold — it does not cap processing. +- **(v1.11.15) `-UserIds` / `-GroupNames` filtering — live validation deferred.** The corrected user-filtering behavior is proven by the automated test suite, but its end-to-end validation against a live tenant is still pending. Treat it as automated-test proven rather than live-confirmed until that check is completed. +- **(v1.11.15) Very large SharePoint uploads — live validation deferred.** SharePoint files above 250 MiB use the resumable upload-session path. The current correction to that path is memory-safe and proven against automated / mock tests, but end-to-end delivery of a single file larger than 250 MiB has not yet been confirmed against a live service; treat very large single-file uploads as not-yet-live-verified. Recent live SharePoint deliveries have likewise not yet been independently audited. +- **(v1.11.15) Fabric checkpoint and unattended-host exit corrections — live validation recommended.** The Fabric checkpoint byte-transmission correction and the unattended-host fatal-exit result are proven against the automated test suite but have not yet been confirmed by a live end-to-end run on a hosted schedule. Confirm the end-to-end result — including the exit code an Azure Container Apps Job observes — against your own environment before relying on it broadly. + ### v1.11.14 -- **(v1.11.14) Looking ahead — AI Solutions Intelligence Dashboard (AISID) under active development:** PAX is working toward support for the AI Solutions Intelligence Dashboard (AISID), a future capability that will enrich the Purview and Entra dataset with Microsoft Defender signals about how AI solutions are used across the organization. It is **not available in this version**: selecting `-Dashboard AISID` exits immediately with a notice and does nothing, and the related `-OutputPathDefenderUsage` / `-AppendDefenderUsage` options have no effect. Full AISID functionality is planned for an upcoming release. No action is needed today. +- **(v1.11.14) Looking ahead — AI Solutions Intelligence Dashboard (AISID):** In v1.11.14 PAX carried early groundwork for the AI Solutions Intelligence Dashboard (AISID), a capability that enriches the Purview and Entra dataset with Microsoft Defender signals about how AI solutions are used across the organization. **AISID remains under development.** Its customer entry points — `-Dashboard AISID`, `-OutputPathDefenderUsage`, `-AppendDefenderUsage`, and `-DisableAISIDDeltaCache` — are gated in v1.11.15, and availability is planned for a later PAX script version — see [What's New → v1.11.15](#v11115-1). ### v1.11.13 diff --git a/script_archive/.gitkeep b/script_archive/.gitkeep index 725eac7..49b1f17 100644 --- a/script_archive/.gitkeep +++ b/script_archive/.gitkeep @@ -1 +1 @@ -# Last updated: 2026-07-06 (Purview v1.11.14, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.36) \ No newline at end of file +# Last updated: 2026-07-31 (Purview v1.11.15, Graph v1.0.1, CopilotInteractions v2.0.0, PAX Umbrella v1.0.37) \ No newline at end of file diff --git a/script_archive/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1 b/script_archive/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1 new file mode 100644 index 0000000..3b6b164 --- /dev/null +++ b/script_archive/Purview_Audit_Log_Processor/PAX_Purview_Audit_Log_Processor_v1.11.15.ps1 @@ -0,0 +1,48381 @@ +# Portable Audit eXporter (PAX) - Purview Audit Log Processor +# Requirements: PowerShell 7+ for default Graph API mode; PowerShell 5.1 supported ONLY with -UseEOM (serial Exchange Online Management mode, no parallel query/explosion). +# Default Activity Type: CopilotInteraction (captures ALL M365 Copilot usage including all M365 apps and Teams meetings) +# DSPM for AI activity types (specified via -ActivityTypes): AIInteraction, ConnectedAIAppInteraction, AIAppInteraction +# MIXED FREE/PAYG Activity Types: AIInteraction (currently Microsoft platforms only), ConnectedAIAppInteraction (Microsoft + third-party) +# PAYG Activity Types: AIAppInteraction (third-party AI via network DLP) +# ---PAYG only applies to third-party AI apps/agents audit records and never applies to any audit records generated by Microsoft AI apps/agents +# NOTE: Graph API mode automatically detects v1.0 (GA) or beta endpoints at runtime (no config needed) +# NOTE: Uses operationFilters with operation names for ALL activity types (e.g., "CopilotInteraction", "AIInteraction") +# See: https://learn.microsoft.com/en-us/office/office-365-management-api/office-365-management-activity-api-schema +<# +.SYNOPSIS + Export Microsoft Purview audit logs for Microsoft 365 Copilot and DSPM for AI activity types. + + *** WARNING: SENSITIVE DATA — CUSTOMER RESPONSIBILITY *** + + The audit data exported by this script is highly sensitive. Output may contain user + identifiers (UPN, email, GUID), file/site/resource paths, conversation and message IDs, + agent identifiers, prompt/response metadata (timestamps, lengths, classifications), and + other personally identifiable information drawn directly from your tenant's Unified + Audit Log. + + - Data is NOT hashed, masked, redacted, anonymized, or de-identified in any way. + Records are exported in their raw, attributable form exactly as Microsoft Purview + returns them. + - Outputs (CSV/Excel/JSON metrics, checkpoint files, logs) may contain confidential + business content, regulated data (PII, PHI, financial, IP), and end-user + communications. + - The customer (you / your organization) is solely responsible for the secure handling, + storage, transmission, retention, disclosure, access control, and deletion of all + data produced by this script, and for ensuring its use complies with all applicable + laws, regulations, contractual obligations, and internal policies — including but + not limited to GDPR, HIPAA, CCPA, employee monitoring laws, works-council + agreements, and data-residency requirements. + - Microsoft has no visibility into, control over, or responsibility for the data + customers extract using this tool or how that data is subsequently used, shared, or + stored. Microsoft disclaims any and all liability arising from or related to + customer use of this script and its output. + - Treat all output files as Highly Confidential. Restrict access to authorized + personnel with a documented business need. Encrypt at rest and in transit. Apply + tenant DLP / sensitivity labels as appropriate. + +.DESCRIPTION + Modes: + Standard - One row per audit record (raw CopilotEventData JSON preserved) + + Graph API Version Selection: + - PAX automatically detects the correct Microsoft Graph security/auditLog endpoint version + for your tenant (currently v1.0 with automatic fallback to beta if the v1.0 endpoint is + unavailable). Detection runs once per session. + - A single-line banner in the run output confirms which endpoint version is in use. + - No command-line switch is required; the selection is fully automatic. + + Parallel Explosion Processing: + - After data retrieval, explosion of records into rows can be parallelized + - Automatic when >500 records (uses job queue with ~1000 records per chunk) + - Control via -ExplosionThreads: 0=auto (2-8 threads, capped at min(ProcessorCount,8)), 1=serial, 2-32=explicit + - Output is identical to serial mode (same columns, data, row count; only order may differ) + + Reliability & Resilience: + - Automatic retry logic: Up to 3 attempts per partition with smart cooldown + - End-of-run summary: Shows Complete/Incomplete/Failed partitions with QueryIds + - Partial success: Continues processing with successful partitions if some fail + - Query naming: PAX_Query__PartX/Total visible in Purview UI + - Unified concurrency: MaxConcurrency parameter controls both EOM and Graph API modes (default: 10) + - Date-range accuracy: Client-side trimming ensures output contains only records within + [StartDate, EndDate), compensating for Purview API date-range bleed + - Checkpoint & Resume: All auth modes automatically save progress to checkpoint files, + enabling resumption after Ctrl+C, network failures, or any interruption via -Resume + + Filtering: + -AgentId : Filter to records matching specific AgentId value(s) + -AgentsOnly : Filter to records with any AgentId present (mutually exclusive with -ExcludeAgents) + -ExcludeAgents : Filter to records WITHOUT AgentId (mutually exclusive with -AgentId/-AgentsOnly) + -PromptFilter + Prompt : Only export messages where Message_isPrompt = True + Response : Only export messages where Message_isPrompt = False + Both : Export messages with either True or False isPrompt values + Null : Only export messages with null/undefined isPrompt values (rare) + Note: PromptFilter uses two-stage filtering for optimal performance: + Stage 1 (Pre-filter): Filters records before explosion based on message content + Stage 2 (Message-level): Filters individual messages during explosion + + -UserIds : Filter to specific user identifier(s) + LIVE MODE: SERVER-SIDE filtering at Purview (efficient, no unnecessary data transfer) + + Accepted formats: + • User Principal Name (UPN): "john.doe@contoso.com" + • SMTP Address: "john.doe@contoso.com" + • User GUID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + Examples: + -UserIds "john.doe@contoso.com" + -UserIds "john.doe@contoso.com","jane.smith@contoso.com","bob.jones@contoso.com" + + -GroupNames : Filter to members of distribution/security group(s) + LIVE MODE ONLY: Groups automatically expanded to individual users after authentication. + - EOM mode: uses Get-DistributionGroupMember (Exchange Online RBAC) + - Graph API mode: reads /groups and /groups/{id}/transitiveMembers via Graph REST (requires GroupMember.Read.All) + + Accepted formats (LIVE MODE only): + • Group Display Name: "Executive Leadership Team" + • Group Email (Alias): "exec-team@contoso.com" + • Group GUID: "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + • Distinguished Name: "CN=ExecTeam,OU=Groups,DC=contoso,DC=com" + Examples: + -GroupNames "Executive Leadership" + -GroupNames "exec-team@contoso.com" + -GroupNames "Engineering Managers","Product Leads","Sales Directors" + Note: Groups are expanded once after authentication + + Combining UserIds + GroupNames (LIVE MODE ONLY): + • When both are specified, the script combines and deduplicates the user lists + • Example: -UserIds "ceo@contoso.com" -GroupNames "Board of Directors" + Pulls records for the CEO plus all expanded board members (duplicates removed) + + COMBINING FILTERS - Powerful Use Cases: + + All filters can be combined for highly targeted data extraction. Filter application order: + + FILTER APPLICATION ORDER: + 1. User/Group filtering (server-side via -UserIds) + 2. Agent filtering (AgentsOnly, AgentId, or ExcludeAgents) + 3. PromptFilter (Prompt, Response, Both, or Null) + + NOTE: Applying User/Group filtering first improves performance by reducing the dataset before subsequent filters. + + TWO-FILTER COMBINATIONS: + + User + Agent: + Use Case: Analyze specific user(s) interactions with Copilot agents + Example: "Show me all agent usage by our power users" + Command: -UserIds "poweruser@contoso.com" -AgentsOnly + + User + PromptFilter: + Use Case: Focus on conversation patterns (prompts/responses) for specific users + Example: "Show me only the questions asked by the executive team" + Command: -GroupNames "Executive Team" -PromptFilter Prompt + Result: Removes resource-only explosion rows, keeps only message data + + Agent + PromptFilter: + Use Case: Analyze agent conversation quality, prompt engineering effectiveness + Example: "Show me all prompts sent to our custom declarative agent" + Command: -AgentId "CopilotStudio.Declarative.abc123" -PromptFilter Prompt + + THREE-FILTER COMBINATION: + + User + Agent + PromptFilter: + Use Case: Deep-dive conversation analysis for specific users with agents + Example: "Show me all questions the sales team asked our Sales Copilot agent" + Command: -GroupNames "Sales Team" -AgentId "SalesCopilot.Agent" -PromptFilter Prompt + Benefits: + • Server-side filtering reduces data transfer (live mode) + • Agent filter removes non-agent interactions + • PromptFilter removes responses and resource-only rows + • Result: Clean dataset of just sales team questions to the agent + + PowerShell 7+ required for default Graph API mode. PowerShell 5.1 is supported ONLY when running in -UseEOM mode (serial Exchange Online Management cmdlets, no parallelism). + +.EXECUTIONPOLICY + No internal execution policy bypass. Use external host invocation if needed: + pwsh.exe -ExecutionPolicy Bypass -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 + # PS 5.1 host (EOM mode only): + powershell.exe -ExecutionPolicy Bypass -File .\PAX_Purview_Audit_Log_Processor.ps1 -UseEOM -StartDate 2025-10-01 -EndDate 2025-10-02 + +.POWERSHELLVERSIONS + PowerShell 7+ required (default Graph API mode; download: https://aka.ms/powershell). + PowerShell 5.1 supported ONLY with -UseEOM (serial EOM mode; no parallel query/explosion). + +.EXAMPLE + # Basic export with auto-generated timestamped filename + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath C:\Temp\ +.EXAMPLE + # PowerShell 5.1 (EOM mode only — serial, no parallelization) + powershell -File .\PAX_Purview_Audit_Log_Processor.ps1 -UseEOM -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only records with agents present + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -AgentsOnly -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only records WITHOUT agents + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -ExcludeAgents -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only prompt messages (Message_isPrompt = True) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -PromptFilter Prompt -OutputPath C:\Temp\ +.EXAMPLE + # Filter to only response messages (Message_isPrompt = False) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -PromptFilter Response -OutputPath C:\Temp\ +.EXAMPLE + # Combine filters: agents + prompts only + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -AgentsOnly -PromptFilter Prompt -OutputPath C:\Temp\ +.EXAMPLE + # Filter to specific users + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "john.doe@contoso.com","jane.smith@contoso.com" -OutputPath C:\Temp\ +.EXAMPLE + # Emit metrics JSON alongside CSV (auto-generated timestamped filename) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-01 -EmitMetricsJson -OutputPath C:\Temp\ +.EXAMPLE + # Emit metrics JSON to custom path + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-01 -EmitMetricsJson -MetricsPath C:\Temp\purview_metrics_20251001.json -OutputPath C:\Temp\ +.EXAMPLE + # AutoCompleteness remediation workflow: first run incomplete (exit code 10), second run resolves saturated windows + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-05 -EndDate 2025-10-05 -EmitMetricsJson -OutputPath C:\Temp\ + # (Exit code 10 indicates saturated windows remain) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-05 -EndDate 2025-10-05 -AutoCompleteness -EmitMetricsJson -OutputPath C:\Temp\ +.EXAMPLE + # Filter to security group members (automatically expanded) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -GroupNames "Executive Leadership" -OutputPath C:\Temp\ +.EXAMPLE + # Filter to multiple groups + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -GroupNames "Executive Team","Engineering Managers" -OutputPath C:\Temp\ +.EXAMPLE + # Graph API: SharePoint/OneDrive document activity with record & service filters + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-12-01 -EndDate 2025-12-02 -ActivityTypes FileAccessed,FilePreviewed -RecordTypes sharePointFileOperation -ServiceTypes SharePoint,OneDrive -OutputPath C:\Temp\ +.EXAMPLE + # Microsoft 365 usage bundle (Exchange, SharePoint, OneDrive, Teams, Forms, Stream, Planner, PowerApps) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeM365Usage -CombineOutput -OutputPath C:\Temp\ +.EXAMPLE + # Rollup: CopilotInteraction-only run, deletes raw CSVs after rolling up + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -Rollup -OutputPath C:\Temp\ +.EXAMPLE + # Rollup + Raw: CopilotInteraction-only run, keeps raw CSVs alongside rollup output + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -RollupPlusRaw -OutputPath C:\Temp\ +.EXAMPLE + # Rollup with a fixed label in the unused deeper org-hierarchy levels (AI-in-One / ValueLens) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -Rollup -FillerLabel Fixed -FillerLabelText "Assistive Directs" -OutputPath C:\Temp\ +.EXAMPLE + # Deidentify: anonymize all identities in the output so it can be shared safely (CSV only) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeM365Usage -RollupPlusRaw -Deidentify -OutputPath C:\Temp\ +.EXAMPLE + # Rollup: M365 usage bundle run, deletes raw CSV after rolling up + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeM365Usage -Rollup -OutputPath C:\Temp\ +.EXAMPLE + # Export with execution telemetry for performance analysis + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeTelemetry -OutputPath C:\Temp\ +.EXAMPLE + # Include Microsoft Agent 365 catalog alongside the audit run (separate Agent365_.csv) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-11-01 -EndDate 2025-11-02 -IncludeAgent365Info -OutputPath C:\Temp\ -OutputPathAgent365Info C:\Temp\ +.EXAMPLE + # Export ONLY the Microsoft Agent 365 catalog (skips audit pull; -Force auto-confirms the preflight) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyAgent365Info -Force -OutputPathAgent365Info C:\Temp\ +.EXAMPLE + # IMPORTANT — AISID IS COMING SOON + # The AI Solutions Intelligence Dashboard (AISID) is currently under development and is not + # available for customer use in this PAX version. The -Dashboard AISID, -OutputPathDefenderUsage, + # -AppendDefenderUsage, and -DisableAISIDDeltaCache entry points are temporarily gated. The + # completed help below is retained as a preview of the capability planned for a later PAX script + # version. + # + # AISID dashboard — a full Purview + Entra + Defender run that produces the AI Solutions Intelligence + # Dashboard data set. Enables the Defender/AISID pipeline and adds ThreatHunting.Read.All at sign-in + # (delegated) / requires it pre-consented (app-only). Requires -IncludeUserInfo (with its own + # destination); -Rollup is auto-enabled for the Purview/Entra output. The AISID CSVs land in the + # dedicated -OutputPathDefenderUsage folder. + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-31 -Dashboard AISID -IncludeUserInfo -OutputPath C:\Reports\ -OutputPathUserInfo C:\Reports\ -OutputPathDefenderUsage C:\Reports\AISID\ +.EXAMPLE + # Combine individual users and groups + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "ceo@contoso.com" -GroupNames "Board of Directors" -OutputPath C:\Temp\ +.EXAMPLE + # COMBINING FILTERS: User + PromptFilter (conversation focus, removes resource-only rows) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "poweruser@contoso.com" -PromptFilter Both -OutputPath C:\Temp\ +.EXAMPLE + # COMBINING FILTERS: Group + Agent (team adoption of specific agent) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -GroupNames "Sales Team" -AgentsOnly -OutputPath C:\Temp\ +.EXAMPLE + # COMBINING FILTERS: User + Agent + PromptFilter (prompts sent to agents by specific users) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -UserIds "analyst@contoso.com" -AgentId "DataAnalysis.Agent" -PromptFilter Prompt -OutputPath C:\Temp\ + +.EXAMPLE + # APPENDING: Append to existing CSV file (relative filename in OutputPath directory) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-15 -EndDate 2025-10-16 -AppendFile "MyReport.csv" -OutputPath C:\Temp\ +.EXAMPLE + # APPENDING: Append to existing CSV file (full path) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-15 -EndDate 2025-10-16 -AppendFile "C:\Data\Audit\CopilotActivity.csv" +.EXAMPLE + # APPENDING: Append with single activity type to existing CSV + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-15 -EndDate 2025-10-16 -AppendFile "CopilotOnly.csv" -ActivityTypes CopilotInteraction -OutputPath C:\Temp\ + +.EXAMPLE + # REMOTE OUTPUT: Upload all run artifacts to a SharePoint document library folder + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath "https://contoso.sharepoint.com/sites/Analytics/Shared Documents/PAX" +.EXAMPLE + # REMOTE OUTPUT: Upload all run artifacts to a Microsoft Fabric Lakehouse as Delta tables (interactive auth) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse" +.EXAMPLE + # REMOTE OUTPUT: Headless run with managed identity writing to OneLake (designed for Azure Container Apps Jobs / VMs / Functions) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -Auth ManagedIdentity -TenantId "" -ClientId "" -OutputPath "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse" +.EXAMPLE + # REMOTE OUTPUT: Headless run with app registration uploading to SharePoint + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -Auth AppRegistration -TenantId "" -ClientId "" -ClientCertificateThumbprint "" -OutputPath "https://contoso.sharepoint.com/sites/Analytics/Shared Documents/PAX" +.EXAMPLE + # PER-DATA-TYPE DESTINATIONS: Send Purview audit to a Fabric Lakehouse, the Users CSV to a different lakehouse table folder, and the run log to a Files subfolder + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -OutputPath "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse" -OutputPathUserInfo "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse/Tables" -OutputPathLog "https://onelake.dfs.fabric.microsoft.com/Analytics/PAX.Lakehouse/Files/logs" + +.EXAMPLE + # CSV export with separate files per activity type (default behavior) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -ActivityTypes CopilotInteraction,ConnectedAIAppInteraction + +.EXAMPLE + # CSV export with combined output (single file with all activity types) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-02 -ActivityTypes CopilotInteraction,ConnectedAIAppInteraction -CombineOutput + +.EXAMPLE + # Resume interrupted operation (auto-discover checkpoint in OutputPath) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -Resume -OutputPath C:\Temp\ + +.EXAMPLE + # Resume from specific checkpoint file + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -Resume "C:\Temp\.pax_checkpoint_20251215_143022.json" + +.EXAMPLE + # Resume with Force (use most recent checkpoint without prompting) + pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -Resume -OutputPath C:\Temp\ -Force + +.NOTES + Reliability Features: + - Automatic retry: Failed partitions retried up to 3 times with smart delays + - Status tracking: Each partition tracked with QueryId and QueryName throughout execution + - Partial success: Script continues with successful data even if some partitions fail + - End summary: Detailed report showing Complete/Incomplete/Failed partitions + - Query names: Visible in Purview as PAX_Query__PartX/Total + Example: PAX_Query_20241101_0000-20241101_0100_Part27/134 + + Concurrency Control: + - MaxConcurrency (default: 10): single parameter that applies to both modes. + • EOM mode: limits concurrent serial queries. + • Graph API mode: limits concurrent partition execution. + - Example: -MaxConcurrency 6 (reduces from the default 10 for rate-limit sensitive environments). + + Graph API Endpoint Selection: + PAX automatically selects the correct Microsoft Graph security/auditLog endpoint for the + run. The v1.0 (GA) endpoint is preferred; if v1.0 is not available for your tenant PAX + falls back to the beta endpoint. Selection runs once per session and is remembered for the + rest of the run. A single-line banner in the run output confirms which endpoint is in use. + No command-line switch is required. + + Example End-of-Run Summary: + ═══════════════════════════════════════════════════════════════ + QUERY SUBMISSION SUMMARY + ═══════════════════════════════════════════════════════════════ + Total Partitions: 134 + Sent and Complete: 131 + [!] Sent but Incomplete: 2 + ✗ Never Sent: 1 + ═══════════════════════════════════════════════════════════════ + + Exit Codes: + A normal run ends by returning ONE of these process codes (read $LASTEXITCODE + in automation, e.g. if ($LASTEXITCODE -ne 0) { Send-Alert }): + + 0 Clean success - run completed with no limit, failure, or gap. + 10 Record limit - a partition hit the Graph 1,000,000 / EOM 10,000 + record limit and -AutoCompleteness was NOT used. + 20 Circuit breaker - repeated block failures tripped the reliability + circuit breaker; collection stopped early. + 30 Directory fetch - Entra user-directory (org / manager) enrichment + could not be fully retrieved; audit data still written. + 40 Completed with gaps - run finished but one or more partitions failed + terminally and/or a window hit the subdivision-depth + guard; see the QUERY SUBMISSION SUMMARY for detail. + + 1 Fatal / aborted - a precondition or validation error (bad dates, + missing prerequisites, unrecoverable startup error) + stopped the run. A request for a feature that is not available + in this version also returns 1. Displaying help returns 0. + + Precedence: when a run qualifies for more than one non-zero outcome, the first + that applies wins, in this order - + 20 (circuit breaker) > 10 (record limit, unless -AutoCompleteness) > + 30 (directory fetch) > 40 (completed with gaps) > 0 (success). + Code 1 is an immediate abort taken BEFORE this end-of-run decision, so it is + not part of the ordering. + + -Resume applicability: + - 0 : nothing to resume (a clean run clears its checkpoint). + - 10 : re-run with -AutoCompleteness to resolve saturated windows; -Resume + also continues an interrupted run. + - 20 : yes - checkpointed partitions are re-processed on -Resume. + - 30 : re-run to re-attempt directory enrichment (audit data already saved). + - 40 : yes - terminally-failed and subdivision-limited windows are retried + on -Resume (the summary prints a -Resume hint for them). + - 1 : usually not resumable (abort occurs before a checkpoint exists). + + Checkpoint & Resume: + All authentication modes automatically create checkpoint files to preserve progress + during long-running operations. This enables resumption after Ctrl+C, network failures, + token expiry, or any interruption without losing completed work. + + Checkpoint Behavior: + • Created automatically for ALL auth modes (WebLogin, DeviceCode, AppRegistration) + • Saved to OutputPath as .pax_checkpoint_.json + • Updated after each partition completes + • Deleted automatically on successful completion + • Stores ALL processing parameters for complete state restoration + + Token Refresh: + • AppRegistration mode: + - Proactive refresh at ~45-50 minutes (before token expiry) + - Reactive refresh on 401 Unauthorized (backup) + - Fully automatic, silent re-authentication using stored credentials + • Interactive modes (WebLogin/DeviceCode): + - Reactive only: Triggered when 401 Unauthorized detected + - Attempts silent refresh first (using SDK cached refresh token) + - Prompts user only if silent refresh fails + • 403 Forbidden errors: Indicates permissions issue, NOT token expiry + - Token refresh will not help; check AuditLogsQuery.Read.All consent and roles + + Resume Mode (Standalone): + The -Resume switch restores ALL settings from the checkpoint file. + No other processing parameters can be specified with -Resume. + + Allowed with -Resume: + • -Force Use most recent checkpoint without prompting + • -Auth Override authentication method + • -TenantId Override tenant ID (for AppRegistration) + • -ClientId Override client ID (for AppRegistration) + • -ClientSecret Provide client secret (for AppRegistration) + • -UserIds Override the checkpoint's user scope + • -GroupNames Override the checkpoint's group scope + + Usage Examples: + • -Resume Auto-discover checkpoint in current directory + • -Resume "path\to\file" Use specific checkpoint file + • -Resume -Force Use most recent checkpoint without prompting + • -Resume -Auth DeviceCode Resume with different auth method + + API Record Limits & Auto-Subdivision: + Graph API 1,000,000 Record Limit: + The Microsoft Graph security/auditLog API has a hard limit of 1,000,000 records per query. + PAX automatically detects when this limit is reached and handles it gracefully. + + Detection: + • Identified when a partition returns exactly 1,000,000 records with no nextLink + • Warning displayed: "[SUBDIVISION] Partition X/Y - Fetched 1,000,000 records (Graph API limit reached)" + + Auto-Subdivision: + • Uses same BlockHours subdivision algorithm as EOM 10K limit handling + • Partition time window is halved and re-queried recursively + • Minimum window: 0.016667 hours (1 minute) - cannot subdivide below this + • If minimum reached, warning displayed and available records returned + + Recommendations for High-Volume Tenants: + • Use smaller -BlockHours (e.g., 0.25 or 0.1) for very active tenants + • Consider shorter date ranges for initial exports + • Monitor "[SUBDIVISION]" messages to tune BlockHours + + EOM 10,000 Record Limit: + The Exchange Online Management (EOM) Search-UnifiedAuditLog cmdlet returns maximum 10,000 + records per query. PAX automatically subdivides time windows when this limit is reached. + + Required Permissions Matrix: + The runtime startup banner emits a tailored, per-run version of this matrix + (highlighting only what THIS invocation needs). The static reference below lists + every permission PAX may require, grouped by feature, with the trigger switch + that turns it on. Each line is tagged so you know WHERE to grant it: + + [App-only] = Microsoft Graph application permission, granted on the + app registration (-Auth AppRegistration) or on the managed + identity's service principal (-Auth ManagedIdentity). + Requires admin consent. + [Delegated] = Microsoft Graph delegated permission, consented at + interactive sign-in by the signed-in user. + [Role] = Entra ID directory role assigned to the signed-in user + (NOT a Graph scope; granted in the Entra admin center). + [Azure RBAC] = Azure role assignment on an Azure resource (NOT a Graph + scope; granted in the Azure portal / az role assignment). + [Exchange] = Exchange Online RBAC role / role group (NOT a Graph scope; + granted in the Exchange admin center). + + Audit query (always required for normal runs; skipped only by -OnlyUserInfo): + [App-only / Delegated] AuditLogsQuery.Read.All + (umbrella - all audit record types) + + M365 usage bundle (only when -IncludeM365Usage is set): + [App-only / Delegated] AuditLogsQuery-Exchange.Read.All + [App-only / Delegated] AuditLogsQuery-SharePoint.Read.All + [App-only / Delegated] AuditLogsQuery-OneDrive.Read.All + + Entra directory + license enrichment (only when -IncludeUserInfo or -OnlyUserInfo is set): + [App-only / Delegated] User.Read.All (read /users) + [App-only / Delegated] Organization.Read.All (read /subscribedSkus) + + Group expansion (only when -GroupNames is set): + [App-only / Delegated] GroupMember.Read.All (read /groups + /groups/{id}/transitiveMembers) + + AISID dashboard - Defender / AI Solutions hunting (only when -Dashboard AISID is set): + [App-only / Delegated] ThreatHunting.Read.All + (POST /security/runHuntingQuery - Defender advanced hunting) + NOTE: delegated modes (-Auth WebLogin / DeviceCode) request this scope at sign-in in the + SAME consent as everything else (no separate login, no second prompt). App-only + modes (-Auth Credential / AppRegistration / Silent / ManagedIdentity) do NOT + request it at runtime: grant ThreatHunting.Read.All as an APPLICATION permission + with admin consent on the app registration / managed-identity service principal + ahead of time (a one-time customer prerequisite). + + Remote output - SharePoint (only when an -OutputPath* value is a SharePoint URL): + [App-only / Delegated] Sites.ReadWrite.All (resolve site/drive via /sites + /drives; + create folder via POST children; PUT / createUploadSession; download) + NOTE: Sites.ReadWrite.All alone is sufficient for every SharePoint endpoint used in + both delegated and application auth. Files.ReadWrite.All is an acceptable alternative + for the drive/file writes but is NOT requested and is not required. + ALSO: the destination user / app / managed identity must have at least + Member access on the target SharePoint site (granted in SharePoint, + not via Graph permissions). + + Remote output - Microsoft Fabric / OneLake (only when an -OutputPath* value is a OneLake URL): + [Azure RBAC] Storage Blob Data Contributor on the Fabric workspace + (assigned at workspace scope to the audit identity) + [Fabric] Contributor on the Fabric workspace + (granted in the Fabric portal: Workspace settings -> Manage access) + [Tenant] "Service principals can use Fabric APIs" tenant setting must be + enabled (Fabric Admin portal -> Tenant settings -> Developer settings) + when running with -Auth AppRegistration or -Auth ManagedIdentity. + NOTE: OneLake uses an INDEPENDENT access token (audience https://storage.azure.com/) + acquired via the Az.Accounts module. Microsoft Graph permissions do NOT + apply to OneLake writes. + + EOM-mode audit query (only when -UseEOM is set): + [Exchange] View-Only Audit Logs role + -- OR -- + [Exchange] Compliance Management role group + -- OR -- + [Exchange] Organization Management role group + -- OR -- + [Exchange] Custom RBAC role granting the Search-UnifiedAuditLog cmdlet + NOTE: EOM mode does NOT use Microsoft Graph scopes for the audit query itself. + + Managed identity bootstrap (only when -Auth ManagedIdentity is set): + All [App-only] scopes triggered by the switches above must be admin-consented + on the managed identity's service principal as APPLICATION permissions (not + delegated). The fabric_resources/Prereqs/Grant-PAXPermissions.ps1 script automates + this one-time grant, including the Storage Blob Data Contributor RBAC role + when an -OutputPath* value is a OneLake URL. + + Remote Output Setup (-OutputPath* with SharePoint or OneLake URLs): + PAX can write all run artifacts (CSV / XLSX / log / metrics / checkpoint) to either a + SharePoint document library folder or a Microsoft Fabric Lakehouse (Delta tables under + Tables/ plus operational artifacts under Files/) instead of a local directory. Storage + tier is inferred from the path form supplied to each -OutputPath* switch. The destination + must already exist (or its parent must, for SharePoint subfolder creation) before the run + starts. PAX will not provision the Fabric workspace, the Lakehouse, or the SharePoint site + for you. + + One-time setup for a SharePoint -OutputPath* destination: + 1. Identify (or create) the target SharePoint site and document library folder. + 2. Grant the identity that runs PAX (the signed-in user for interactive auth, OR + the app registration / managed identity for unattended auth) the following + Microsoft Graph application or delegated permission, matching your -Auth choice: + • Sites.ReadWrite.All + 3. For app-only auth (-Auth AppRegistration / ManagedIdentity), have an admin + consent the permissions in the Entra ID portal. + 4. Pass the folder URL exactly as it appears in the browser address bar: + -OutputPath "https://.sharepoint.com/sites//[/]" + Subfolders that do not yet exist below the document library root are created + automatically on first use. + + One-time setup for a Microsoft Fabric / OneLake -OutputPath* destination: + 1. Install the Az.Accounts PowerShell module on the machine that runs PAX: + Install-Module Az.Accounts -Scope CurrentUser + (Already present on Azure Cloud Shell and the official PowerShell container + image used by the ACA Job runbook.) + 2. In the Microsoft Fabric portal (https://app.fabric.microsoft.com): + a. Create or identify the target Fabric workspace. + b. Inside that workspace, create or identify the target Lakehouse + (or Warehouse). PAX writes to the item's Files area, not Tables. + c. Add the identity that runs PAX (user, app registration, or managed + identity) to the workspace as Contributor (or higher). Member, + Admin, and workspace-owner roles also work; Viewer does NOT. + 3. In the Azure portal (the Fabric workspace surfaces as an Azure resource + under the Microsoft.Fabric provider): + • Assign the same identity the Azure RBAC role + "Storage Blob Data Contributor" on the Fabric workspace (or on the + Fabric capacity / subscription scope, if you want it to apply more + broadly). This role is what authorizes the underlying OneLake DFS + write calls; the Fabric workspace role alone is not sufficient. + 4. For service-principal or managed-identity runs (-Auth AppRegistration / + ManagedIdentity) the Fabric tenant administrator must also enable, in the + Fabric Admin portal under "Tenant settings → Developer settings": + • "Service principals can use Fabric APIs" + (scoped to the security group that contains your service principal / + managed identity, or tenant-wide). + 5. Pass the OneLake URL exactly in the form shown in the Lakehouse "Properties" + pane, ABFS URL → switch the scheme to https and the host to the DFS endpoint: + -OutputPath "https://onelake.dfs.fabric.microsoft.com//.Lakehouse" + Regional aliases of the form "https://-onelake.dfs.fabric.microsoft.com/..." + are also accepted. The item may be given either by name (.Lakehouse) or by its + GUID with no suffix (the form the Fabric portal shows in the item URL, + e.g. ...//); both are accepted. Tables land under + /Tables; operational artifacts under /Files (the run log + location is controllable via -OutputPathLog). + 6. Schemas-mode Lakehouses: Microsoft Fabric's default new-Lakehouse template + organizes tables as /Tables// (the default schema + is 'dbo'). To land Delta tables under a specific schema on a Schemas-mode + Lakehouse, include the schema in the -OutputPath* URL: + -OutputPath "https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Tables/dbo" + Non-Schemas Lakehouses (tables directly under Tables/) accept either + the lakehouse root or the .../Tables form unchanged. + + Common pitfalls: + • Forgetting the "Storage Blob Data Contributor" Azure RBAC role: produces + HTTP 403 (AuthorizationFailure) on the very first OneLake write. The Fabric + Contributor role alone is not enough. + • Forgetting the tenant setting "Service principals can use Fabric APIs": + produces HTTP 401 / 403 for app-only and managed-identity runs even when + every other permission is correct. + • On long runs, the OneLake access token is refreshed automatically; no + customer action is required to keep multi-hour runs alive. + +.PARAMETER StartDate + Start date for audit log query in live mode (format: yyyy-MM-dd or MM/dd/yyyy). + +.PARAMETER EndDate + End date for audit log query in live mode (format: yyyy-MM-dd or MM/dd/yyyy). + +.PARAMETER OutputPath + Destination for the Purview audit output (raw, rollup, or event-level CSV / XLSX). Targets the + Purview audit data stream only — the EntraUsers CSV is controlled by -OutputPathUserInfo. + Storage tier is inferred from the path form: + drive-rooted absolute path → Local; https://*.sharepoint* URL → SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse* URL → Fabric. UNC paths are rejected. + Accepts either a folder (PAX auto-generates the basename) or a fully qualified path including + basename. When omitted, defaults to $PSScriptRoot for Local tier. Paired with -AppendFile + (both switches act on the Purview audit CSV). + +.PARAMETER OutputPathUserInfo + Destination override for the EntraUsers / MAC licensing CSV. Same tier-inference and form + rules as -OutputPath. Independent of -OutputPath so the Users CSV can land in a different + folder, SharePoint library, or Fabric lakehouse from the Purview audit output (subject to + the tier-consistency rule: all -OutputPath* values must resolve to the same storage tier). + When omitted, the Users CSV lands beside -OutputPath. Paired with -AppendUserInfo. + +.PARAMETER OutputPathAgent365Info + Destination override for the Microsoft Agent 365 catalog CSV. Same tier-inference and form + rules as -OutputPath. Independent of -OutputPath so the Agent 365 CSV can land in a different + folder, SharePoint library, or Fabric lakehouse from the Purview audit output (subject to + the tier-consistency rule: all -OutputPath* values must resolve to the same storage tier). + When omitted, the Agent 365 CSV lands beside -OutputPath. Paired with -AppendAgent365Info. + +.PARAMETER OutputPathLog + Destination override for the run log. Accepts Local, SharePoint, or Fabric Files/ targets + (Fabric Tables/* URLs are rejected — logs are not tabular). When omitted, the log lands in + $PSScriptRoot for Local runs and in the active remote destination's run-log folder for + SharePoint / Fabric runs. + +.PARAMETER OutputPathDefenderUsage + AVAILABILITY: this entry point is temporarily gated. AISID is under development and is not available + for customer use in this PAX version; supplying this switch reports that notice and stops the run. + Availability is planned for a later PAX script version. The behavior described below is retained as + a preview. + This switch is tied to -Dashboard AISID and is the destination for the AISID (AI Solutions Intelligence + Dashboard) output set — the Defender/AISID CSVs, including the solutions catalog, are written here. + FOLDER-ONLY: supply a folder path, a + SharePoint folder URL, or a Fabric OneLake folder URL; a file leaf is rejected. Storage tier is + inferred exactly like the other -OutputPath* switches and must match the run's other destinations. + Mutually exclusive with -AppendDefenderUsage. Only meaningful with -Dashboard AISID; supplying it + without -Dashboard AISID is rejected. + +.PARAMETER FlatDepth + Maximum JSON flatten depth for exploding CopilotEventData and AuditData (default 120). + + The script automatically generates descriptive filenames based on: + • Activity types being exported + • Export mode (CSV vs Excel, combined vs separate) + • Current timestamp (yyyyMMdd_HHmmss format) + + Examples of auto-generated filenames: + • Purview_Audit_UsageActivity_CopilotInteraction_20251110_143022.csv (single activity type) + • Purview_Audit_UsageActivity_CombinedActivityTypes_20251110_143022.csv (multiple activity types with -CombineOutput) + • Purview_Audit_MultiTab_20251110_143022.xlsx + + When an -OutputPath* value is a SharePoint or OneLake URL, all output files, directory paths, + and log file paths are reported in the run banners and progress messages using the + remote SharePoint or Fabric URL — not the temporary local scratch path used internally + during the run. + + Note: OutputPath accepts ONLY directory paths, not filenames. + Use -AppendFile parameter to specify a custom filename for appending to existing files. + +.PARAMETER Auth + Authentication method. Options: + • WebLogin – Interactive browser authentication + • DeviceCode – Device code flow for headless scenarios + • Credential – Legacy username/password prompt or GRAPH_* env vars + • Silent – Managed identity or pre-cached token + • AppRegistration – Service principal using client secret or certificate + • ManagedIdentity – Azure-hosted managed identity (system-assigned by default; + pass the user-assigned client ID via the AZURE_CLIENT_ID + environment variable when applicable). Designed for headless + execution inside Azure (Container Apps Job, VM, App Service, + Functions, etc.). Requires the same Microsoft Graph application + permissions as -Auth AppRegistration consented to the managed + identity, and (when an -OutputPath* value is a OneLake URL) + the Azure RBAC role + Storage Blob Data Contributor on the OneLake workspace. + +.PARAMETER TenantId + Azure AD tenant ID (GUID). Required for -Auth AppRegistration unless GRAPH_TENANT_ID + environment variable is set. + +.PARAMETER ClientId + Azure AD app registration client ID (GUID). Required for -Auth AppRegistration unless + GRAPH_CLIENT_ID environment variable is set. + +.PARAMETER ClientSecret + Client secret value for app registration authentication. You can pass it directly, + convert from a secure string, or set it through GRAPH_CLIENT_SECRET. + +.PARAMETER ClientCertificateThumbprint + Thumbprint of a certificate located in the CurrentUser or LocalMachine "My" store. + Used when -Auth AppRegistration should authenticate with a certificate instead of a + client secret. Optional environment variable: GRAPH_CLIENT_CERT_THUMBPRINT. + +.PARAMETER ClientCertificateStoreLocation + Certificate store to search when using ClientCertificateThumbprint. Valid values: + CurrentUser (default) or LocalMachine. + +.PARAMETER ClientCertificatePath + Path to a PFX file containing the certificate for app registration auth. Optional + environment variable: GRAPH_CLIENT_CERT_PATH. + +.PARAMETER ClientCertificatePassword + Password for the PFX file specified by ClientCertificatePath. Accepts secure string or + plain text (converted internally). Optional environment variable: + GRAPH_CLIENT_CERT_PASSWORD. + +.PARAMETER BlockHours + Time window size in hours for each audit log query block. + Range: 0.016667 to 24 hours. Default: 0.5 (30 minutes) + +.PARAMETER PartitionHours + Time partition size in hours for Graph API parallel processing. + Range: 1-72. Default: 0 (auto-calculated based on date range and MaxPartitions) + +.PARAMETER MaxPartitions + Maximum number of time partitions for parallel query execution. + Range: 1-1000. Default: 160 + +.PARAMETER ResultSize + Maximum records returned per Search-UnifiedAuditLog query. + Range: 1 to 10000. Default: 10000 + +.PARAMETER PacingMs + Delay in milliseconds between audit log queries for throttling control. + Range: 0 to 10000. Default: 0 + +.PARAMETER ActivityTypes + Array of Purview audit activity types to query. + Default: @('CopilotInteraction') + Examples: CopilotInteraction, ConnectedAIAppInteraction, AIInteraction, AIAppInteraction + Note: CopilotInteraction captures ALL Microsoft 365 Copilot usage including Teams meetings (AppHost="Teams") + +.PARAMETER RecordTypes + Filter audit records by record type (Graph API mode only). + Accepts one or more record type names that map to Microsoft 365 workload categories. + Examples: SharePointFileOperation, ExchangeItem, MicrosoftTeams, AzureActiveDirectory + + BEHAVIOR: + • Standard mode: Filters server-side to specified record types + • With -IncludeM365Usage: Merged with curated M365 usage bundle record types + + LIMITATIONS: + • Graph API mode only (not supported with -UseEOM) + + Use with -ActivityTypes and -ServiceTypes for precise workload targeting. + +.PARAMETER ServiceTypes + Filter audit records by service/workload (Graph API mode only). + Accepts one or more service names representing Microsoft 365 workloads. + Examples: Exchange, SharePoint, OneDrive, MicrosoftTeams, AzureActiveDirectory + + BEHAVIOR: + • Standard mode: Filters server-side to specified services + • Note: Multiple services may cause separate queries per service + + CRITICAL LIMITATIONS: + • Graph API mode only (not supported with -UseEOM) + • IGNORED when -IncludeM365Usage is active: The M365 usage bundle intentionally + sets ServiceTypes to null for optimal single-pass query performance. + Your -ServiceTypes values will be silently overridden. + + RECOMMENDATION: For M365 usage scenarios, use -IncludeM365Usage without -ServiceTypes. + For targeted workload queries without M365 bundle, use -ServiceTypes with -ActivityTypes. + +.PARAMETER ExplodeArrays + [Deprecated] This switch is retained for backward compatibility and is no longer supported. + +.PARAMETER ExplodeDeep + [Deprecated] This switch is retained for backward compatibility and is no longer supported. + +.PARAMETER RAWInputCSV + [Deprecated] This switch is retained for backward compatibility and is no longer supported. + +.PARAMETER MaxConcurrency + Maximum concurrent queries/partitions (1-10). + - EOM mode: limits concurrent serial queries + - Graph API mode: limits concurrent partition execution + Default: 10 (Microsoft Purview enforces a 10 concurrent search job limit per user account) + +.PARAMETER EnableParallel + Force enable parallel processing (overrides ParallelMode setting). + +.PARAMETER MaxParallelGroups + Maximum activity groups processed concurrently in parallel mode. + Range: 0 to 50. Default: 8 + +.PARAMETER ParallelMode + Parallel processing mode. Options: Off, On, Auto (default) + Auto: Enables parallel for PowerShell 7+ environments automatically. + +.PARAMETER ExplosionThreads + Number of threads for parallel explosion processing (post-retrieval phase). + 0 (default): Auto-detect based on CPU cores (2 to min(ProcessorCount, 8) threads) + 1: Force serial processing (disable parallel explosion) + 2-32: Explicit thread count (ValidateRange 0-32; auto path caps at 8) + Requires PowerShell 7+. Falls back to serial on PS5. + +.PARAMETER DisableAdaptive + Disable adaptive safeguards (memory/latency/concurrency smoothing). + +.PARAMETER ProgressSmoothingAlpha + Weight for smoothing dynamic progress total recalculation. + Range: 0.0 to 1.0. Default: 0.3 (0 = off) + +.PARAMETER HighLatencyMs + Partition average latency threshold (ms) triggering concurrency reduction. + Range: 1000 to 600000. Default: 90000 + +.PARAMETER MemoryPressureMB + Working set (MB) threshold to trigger concurrency reduction. + Range: 256 to 32768. Default: 1500 + +.PARAMETER MaxMemoryMB + Maximum process memory (MB) before flushing the in-memory record buffer to disk. + When exceeded, PAX persists the buffered records to a JSONL working file and releases the + in-memory copy so the run can continue under tight memory budgets. + Range: -1 to 65536. Default: -1 (auto = 75% of system RAM). Use 0 to disable. + +.PARAMETER StatusIntervalSeconds + How often (in seconds) to display a status update during job polling and backpressure waits. + Reduce to see more frequent progress output; increase to reduce console noise on long runs. + Range: 30 to 600. Default: 60 + +.PARAMETER LowLatencyMs + Sustained low latency threshold to consider concurrency step-up. + Range: 100 to 600000. Default: 20000 + +.PARAMETER LowLatencyConsecutive + Required consecutive low-latency groups before step-up. + Range: 1 to 10. Default: 2 + +.PARAMETER ThroughputDropPct + Percentage drop vs baseline required (with high latency) to justify reduction. + Range: 1 to 100. Default: 15 + +.PARAMETER ThroughputSmoothingAlpha + EMA smoothing for throughput baseline. + Range: 0.0 to 1.0. Default: 0.3 + +.PARAMETER AdaptiveConcurrencyCeiling + Upper bound for adaptive concurrency step-ups. + Range: 1 to 50. Default: 6 + +.PARAMETER ExportProgressInterval + Frequency of progress updates during export phase. + Range: 1 to 10000. Default: 10 + +.PARAMETER StreamingSchemaSample + Number of rows sampled before freezing CSV schema in streaming mode (SERIAL MODE ONLY). + Range: 100 to 50000. Default: 5000 + Higher values capture more columns but delay schema freeze. + NOTE: In parallel mode (PS7+), full schema discovery is used instead, scanning ALL rows + for 100% column coverage. This parameter only affects serial mode processing. + +.PARAMETER StreamingChunkSize + Number of rows per write batch in streaming CSV export. + Range: 100 to 50000. Default: 5000 + Lower values reduce memory pressure, higher values improve throughput. + +.PARAMETER AgentId + Filter to records matching specific AgentId value(s). + Example: -AgentId "CopilotStudio.Declarative.abc123" + +.PARAMETER AgentsOnly + Filter to records with any AgentId present (mutually exclusive with -ExcludeAgents). + +.PARAMETER PromptFilter + Filter messages by isPrompt property. + Options: Prompt (True), Response (False), Both (True/False), Null (undefined) + +.PARAMETER CircuitBreakerThreshold + Consecutive block failures before opening circuit breaker. + Range: 1 to 50. Default: 5 + +.PARAMETER CircuitBreakerCooldownSeconds + Cooldown duration (seconds) after circuit breaker trips. + Range: 5 to 3600. Default: 120 + +.PARAMETER BackoffBaseSeconds + Base seconds for exponential backoff between block retries. + Range: 0.1 to 120. Default: 1.0 + +.PARAMETER BackoffMaxSeconds + Maximum cap for exponential backoff delay (seconds). + Range: 1 to 600. Default: 45 + +.PARAMETER ExcludeAgents + Filter to records WITHOUT AgentId (mutually exclusive with -AgentId/-AgentsOnly). + +.PARAMETER UserIds + Filter to specific user identifier(s). + LIVE MODE: Server-side filtering at Purview (efficient). + Accepts UPN, SMTP address, or user GUID. + +.PARAMETER GroupNames + Filter to members of distribution/security group(s). + LIVE MODE ONLY: Groups automatically expanded after authentication. + +.PARAMETER Help + Display script help information. + +.PARAMETER EmitMetricsJson + Emit structured metrics JSON alongside output file. + Default filename: .metrics.json + +.PARAMETER MetricsPath + Override metrics output path. Requires -EmitMetricsJson. + +.PARAMETER AutoCompleteness + Aggressively subdivide windows returning server 10K limit until complete or minimum window reached. + +.PARAMETER IncludeTelemetry + Export execution telemetry CSV alongside audit data (Graph API mode only). + Creates a separate CSV file with one row per partition containing timing and performance metrics. + Useful for analyzing query execution patterns, identifying bottlenecks, and capacity planning. + File naming: _telemetry_.csv + Not available in EOM mode or OnlyUserInfo mode. + +.PARAMETER IncludeCopilotInteraction + Adds CopilotInteraction to the activity list even when you provide a custom -ActivityTypes array. + Useful when combining Copilot telemetry with targeted classic workloads without redefining defaults. + +.PARAMETER IncludeM365Usage + Adds a curated, trimmed Microsoft 365 usage bundle spanning Exchange mail access, + SharePoint/OneDrive file access, Teams chat/messaging, Teams meeting lifecycle, and + Copilot/Connected-AI interaction signals. + + ACTIVITY TYPES INCLUDED: + Exchange: MailItemsAccessed, MailboxLogin, Send + SharePoint/OneDrive (Files): FileAccessed, FileViewed, FilePreviewed, FileModified, + FileDownloaded, FileUploaded + Teams (Chat/Messaging): MessageSent, MessageRead, MessagesListed, ChatRetrieved, + ChatCreated, TeamsSessionStarted + Teams (Meetings): MeetingParticipantJoined, MeetingStarted, MeetingEnded, + MeetingParticipantDetail, MeetingDetail + Copilot / Connected AI: CopilotInteraction, ConnectedAIAppInteraction + + RECORD TYPES INCLUDED: + ExchangeAdmin, ExchangeItem, ExchangeMailbox, SharePointFileOperation, + SharePointSharingOperation, SharePoint, OneDrive, MicrosoftTeams, OfficeNative, + MicrosoftForms, MicrosoftStream, PlannerPlan, PlannerTask, PowerAppsApp + + IMPORTANT: When this switch is active, -ServiceTypes parameter is ignored and set to null. + This ensures optimal single-pass query performance across all Microsoft 365 workloads. + +.PARAMETER Rollup + Run an embedded Python post-processor against the audit run's final CSV + immediately after a successful export, then DELETE the raw CSV(s) so only the rolled-up + output remains. Mutually exclusive with -RollupPlusRaw. + + PURPOSE: These switches exist solely to produce input files for the Microsoft Copilot + Growth ROI Advisory Team's Power BI templates at https://github.com/microsoft/Analytics-Hub. + The rolled-up CSVs are shaped specifically for those templates and are NOT intended for + any other downstream use. For generic analytics exports, omit -Rollup / -RollupPlusRaw + and consume the raw CSV directly. + + The script auto-selects the correct embedded processor and target dashboard: + • CopilotInteraction-only run (default activity type, or -ActivityTypes 'CopilotInteraction'): + Purview_CopilotInteraction_Processor — also auto-enables -IncludeUserInfo and consumes + both the Purview CSV and the Entra users CSV. + Target Analytics-Hub dashboards: AI-in-One and ValueLens. + • -IncludeM365Usage run: Purview_M365_Usage_Bundle_Explosion_Processor — consumes the + combined Purview CSV (-CombineOutput is auto-enabled by -IncludeM365Usage). + Target Analytics-Hub dashboard: M365 Usage Analytics. + + Requires PowerShell 7+ and Python 3.10+. If Python is not on PATH, the script attempts a + per-user silent install (winget Python.Python.3.13 → python.org installer fallback). + + Blocked combinations (script exits with an error): -UseEOM, -ExportWorkbook, + -OnlyUserInfo, -OnlyAgent365Info, -RAWInputCSV, and -ExcludeCopilotInteraction + without -IncludeM365Usage. + +.PARAMETER RollupPlusRaw + Same as -Rollup (produces input files for the Microsoft Copilot Growth ROI + Advisory Team's Power BI templates at https://github.com/microsoft/Analytics-Hub) but + KEEPS the raw CSV(s) on disk alongside the rolled-up output. Mutually exclusive with -Rollup. + +.PARAMETER Deidentify + Anonymize every identifying value in the output using a one-way, deterministic hash, so the + results can be shared without exposing who did what. User principal names / email addresses, + display names, mailbox GUIDs and SIDs, device names, and site / file / resource paths are + replaced with stable tokens — the same identity always maps to the same token, so user counts, + joins, and distinct-resource counts are preserved. Non-identifying fields (job title, company, + department, license, activity counts) are kept as-is. The transformation is irreversible: no + mapping back to the original values is produced or stored. + + Applies to both the raw output and the rolled-up output. CSV output only (cannot be combined + with -ExportWorkbook). When resuming an interrupted run, deidentification is taken from the + original run's checkpoint and cannot be added or removed on the resume command line. + +.PARAMETER FillerLabel + (Power BI rollup only — AI-in-One / ValueLens dashboards.) Chooses how the + manager-hierarchy "level" columns are filled in below a person's own position in the org chart. + The hierarchy columns themselves — each person's chain of managers up to the top of the + organization, their level number, their manager, and their direct-report and total-report + counts — are always included in the rolled-up Users file for these two dashboards. This setting + only changes what appears in the deeper, unused level columns for people who sit near the top: + + null (default) Leave the deeper levels blank. + Self Repeat the person in their deeper levels. + RepeatManager Repeat the person's manager in their deeper levels. + Fixed Show a fixed label of your choice (supply it with -FillerLabelText). + + Requires -Rollup or -RollupPlusRaw. It is not available for the M365 Usage dashboard + (-IncludeM365Usage / -Dashboard M365), which does not include an org hierarchy. + +.PARAMETER FillerLabelText + The label to show in the deeper hierarchy levels when -FillerLabel Fixed is chosen, for + example: -FillerLabel Fixed -FillerLabelText "Assistive Directs". Only used with -FillerLabel Fixed. + +.PARAMETER ExcludeCopilotInteraction + Exclude Microsoft 365 Copilot activity type (CopilotInteraction). + Overrides custom list and default behavior. Use with -ActivityTypes to query only the specified activity types. + +.PARAMETER ExportWorkbook + [Deprecated] This switch is retained for backward compatibility and is no longer supported. + +.PARAMETER AppendFile + Append Purview audit activity rows to an existing Purview audit CSV / XLSX instead of creating + a new timestamped file. Targets the Purview audit data stream only — EntraUsers append is + controlled by -AppendUserInfo. Accepts either a filename (combined with -OutputPath) or a + full path to the existing file. + + **Filename Resolution:** + • Relative filename: -AppendFile "MyReport.csv" → Uses -OutputPath directory + • Full path: -AppendFile "C:\Data\Report.xlsx" → Uses exact path specified + + **Requirements:** + • File must already exist (create it first without -AppendFile) + • File extension must be .csv + • Cannot be used with -OnlyUserInfo (no audit data in scope to append) + • Requires single-file output mode (see Single-File Output Requirements below) + • Data-loss safe: if the existing target cannot be read fully (very large file or key/schema mismatch), + the append aborts and leaves the target untouched rather than overwriting it with only this run's rows. + + **Single-File Output Requirements:** + Must use ONE of these modes to ensure single output file: + 1. Combined CSV: -CombineOutput (all activity types merged into one .csv file) + 2. Single activity type: -ActivityTypes CopilotInteraction (only one activity type selected) + + **CSV Mode Behavior:** + • Union-merges current-run rows with the target file keyed on RecordId (non-rollup) or, + for the rollup Fact CSV, on the full grain + Message_Id_Raw composite key (rollup Fact + rows FAN OUT — many rows can share one Message_Id_Raw, one per distinct grain); rows in + the target but missing from the current run are kept with In_Latest_Append = FALSE. + • Date_Added / Latest_Append_Date / In_Latest_Append provenance columns are maintained + on every row. NOTE: these three columns apply ONLY to the row-identity merges — the + non-rollup raw audit CSV (keyed on RecordId) and the CopilotInteraction rollup Fact + CSV (keyed on the grain + Message_Id_Raw composite). In the rollup Fact CSV a single + Message_Id_Raw can map to MULTIPLE rows (one per distinct grain, e.g. per accessed + resource), so append dedup keys on the whole grain plus Message_Id_Raw — never on + Message_Id_Raw alone (doing so would collapse fan-out rows and lose data). They are + intentionally NOT added to the M365 Usage bundle rollup (-IncludeM365Usage + -Rollup), + whose rows are additive aggregates summed across runs — see the per-row provenance + caveat in the M365 Rollup Anchoring section below. + • Compatible with: Standard (1:1), -Rollup, -RollupPlusRaw. + + **M365 Rollup Anchoring (-IncludeM365Usage + -Rollup / -RollupPlusRaw):** + The embedded M365 Bundle Explosion Processor emits FOUR files into a single output + directory, each sharing the same stem: + • '_Rollup_.csv' — the 14-column aggregated rollup + (UserId, CreationDate, Operation, + Workload, SourceFileExtension, AppHost, + EventCount, ItemsAccessedCount, + CreationTime, MaxCreationTime, + AgentId, AgentName, ContextType, + IsAgentInteraction). + • '_UserStats_.csv' — recomputed per-user metrics (sidecar). + • '_SessionCohort_.csv' — recomputed (UserId, App, Bucket) cohorts (sidecar). + • '_SessionStats_.csv' — per-(UserId, Date, AppHost) + DISTINCTCOUNT(ThreadId) + PromptCount / + AgentPromptCount / ResponseCount / + AgentSessionCount. + Drives the CECopilotPercentile_* + measures from PromptCount semantics. + Only the Rollup file supports the union-merge semantic. Under -AppendFile, the current + run's Rollup is column-tolerant union-merged (keyed on the 9-tuple of UserId / + CreationDate / Operation / Workload / SourceFileExtension / AppHost / AgentId / + AgentName / ContextType) into the customer's target file; the SessionStats file is + separately union-merged on (UserId, CreationDate, AppHost), and the UserStats and + SessionCohort sidecars are regenerated from the merged Rollup (with the merged + SessionStats feeding the CECopilotPercentile_* columns) and anchored off the target + leaf stem (so the same destination URLs are overwritten on every run). + Legacy targets that carry a narrower rollup schema (e.g. a 9- or 10-column file + produced by an earlier processor build) are padded with empty AgentId / AgentName / + ContextType cells and IsAgentInteraction=FALSE during merge so the union retains + a uniform 14-column shape end-to-end. + Provenance columns: the M365 bundle rollup merge is ADDITIVE (on a 9-tuple key match + EventCount / ItemsAccessedCount are summed, CreationTime = min, MaxCreationTime = max, + IsAgentInteraction = OR), so a merged row is a blend of multiple runs and is NOT owned + by any single run. The Date_Added / Latest_Append_Date / In_Latest_Append columns are + therefore intentionally NOT written to the rollup, SessionStats, or the regenerated + UserStats / SessionCohort sidecars (they would be meaningless on a summed row). The + rollup keeps exactly its 14 canonical columns. (Per-row provenance is carried only by + the row-identity merges — see CSV Mode Behavior above.) + Sidecar scope: the UserStats and SessionCohort sidecars are RECOMPUTED from the full + merged Rollup on every append (not appended to), so they always describe the entire + cumulative rollup, not just the latest run's slice. This is required for the window / + percentile / cohort math to stay correct against the full history. + Customer-facing rules: + • -AppendFile MUST point to a '_Rollup.csv' OR '_Rollup_.csv' leaf + (filename / full local path / SharePoint / Fabric URL). Sidecar leaves (_UserStats + / _SessionCohort / _SessionStats) and CopilotInteraction (_Interactions) / M365 + event-level (_Exploded) leaves are rejected at pre-flight. + • The three sidecar files inherit the anchor stem and are written next to the AppendFile + target (same parent folder/URL) using the leaves '_UserStats.csv', + '_SessionCohort.csv', and '_SessionStats.csv'. They + overwrite the same destination URLs on every run, so the destination always has + exactly one current set of all 4 files. + • Renamed AppendFile is fine as long as the leaf still ends in '_Rollup.csv' or + '_Rollup_.csv'. Example: -AppendFile 'MyM365Rollup_Rollup.csv' + produces sidecars 'MyM365Rollup_UserStats.csv' and 'MyM365Rollup_SessionCohort.csv' + at the same parent. + • Prior sidecars with DIFFERENT leaf names (e.g. left over from a previous run before + you renamed the AppendFile target) are NOT auto-deleted. Clean them up manually if you + want zero orphans in the destination namespace. + • For first-time append: run once WITHOUT -AppendFile to produce the initial timestamped + Rollup (plus its sidecars), then point -AppendFile at that Rollup on subsequent runs + (either keep the timestamped leaf or rename to drop the timestamp — both shapes are + accepted). + Fabric Tables/ destinations: the table name follows the AppendFile leaf stem and stays + stable across runs (the customer sets that stem; only the current-run scratch output + carries a fresh timestamp, and the merge anchors back to the fixed AppendFile target). + Convert-CsvToDelta -Mode overwrite replaces table contents with the merged-and-recomputed CSVs. + + **Independent companion switches:** + • -AppendUserInfo → merges into the EntraUsers CSV (auto-enables -IncludeUserInfo) + -AppendFile and -AppendUserInfo may be supplied together; each targets its own data stream + and each is paired with its own -OutputPath* override. + + **Always Timestamped (Never Overwritten):** + Even when using -AppendFile, these files are always timestamped: + • Log files: *_.log + • Telemetry files: *_telemetry_.csv + • Metrics files: *_metrics_.json (unless -MetricsPath specified) + + **Error Scenarios:** + • File not found: Script exits (create initial file first without -AppendFile) + • CSV header mismatch: Script exits with detailed column differences + • Multiple output files would be created: Script exits with single-file output requirement + • -OnlyUserInfo mode: Script exits (no audit data in scope to append) + +.PARAMETER AppendUserInfo + Path to an existing EntraUsers CSV that the current run should merge departed + users into. Specifying -AppendUserInfo automatically turns on -IncludeUserInfo, + so you do not need to set both. Storage tier is inferred from the path form + (drive-rooted absolute path = Local; https://*.sharepoint*/* URL = SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse/* URL = Fabric). The + tier must match every other -OutputPath* / -Append* value supplied to the run. + + **Behavior:** + • The current run's freshly fetched EntraUsers snapshot is union-merged into + the file you supply (rows present in the target but missing from the current + snapshot are preserved as departed users). + • Three provenance columns are maintained on every row: + - Date_Added First run that introduced the user. + - Latest_Append_Date Most recent run that observed the user. + - In_Latest_Append TRUE if the user was present in the latest run. + • A pristine raw EntraUsers snapshot (EntraUsers_MAClicensing_.csv) + is written transiently during the merge step and removed once the union is + committed to the merged target, so the destination folder holds a single + EntraUsers CSV (the merged target) at run-end (parity with -AppendFile's + clean single-file outcome). If the merge step fails the cleanup does not + run; the raw snapshot stays where Save-CsvAtomic wrote it and is + enumerated in the end-of-run output files listing. + • The run-summary reports the merged target via the "Appended to:" line. + + **Pairing rules:** + • Use either -AppendUserInfo (merge) or -OutputPathUserInfo (overwrite/redirect), + not both, for the same destination. + +.PARAMETER UserInfoFile + Path to a customer-provided CSV that supplies the Entra user directory in lieu of the live + Entra /users pull. Storage tier is inferred from the value form (drive-rooted absolute path = + Local; https://*.sharepoint*/* URL = SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse/* URL = Fabric); remote inputs are staged + to local scratch before parsing. The file must be CSV with a header row, at least one data row, + and a UserPrincipalName column (required). DisplayName, Department, JobTitle, ManagerUpn, and + HasLicense are recognized when present; any other columns are preserved as passthrough. Mutually + exclusive with -GroupNames. + +.PARAMETER UserInfoSupplement + Path to a customer-provided CSV that ENRICHES the live Entra user directory (hybrid mode), as + opposed to -UserInfoFile which REPLACES it. The live Entra /users directory is fetched normally, + then the supplemental columns are appended to each matching Entra row. Storage tier is inferred + from the value form (drive-rooted absolute path = Local; https://*.sharepoint*/* URL = SharePoint; + https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse/Files/.../*.csv URL = Fabric); remote inputs + are staged to local scratch before parsing. The file must be a .csv with a header row, at least one + data row, and exactly one UserPrincipalName column (the only required column; matched + case-insensitively). Rows are joined to Entra by UserPrincipalName as an Entra-left join: every + Entra user is preserved, supplemental columns are appended after the existing Entra columns in + their original order, non-matching Entra users receive blank supplemental values, and supplemental + rows that do not match any Entra user are reported and excluded (never added as directory users). + The supplemental UserPrincipalName is a join key only and is never copied into the output — the + output identity always comes from Entra. Supplemental column names/order/values are preserved + without transformation; PAX does not inspect, mask, hash, or de-identify supplemental values (with + -Deidentify, Entra-derived identity fields are still protected but supplemental values pass through + unchanged, and a prominent notice is shown). Supplemental columns flow to the normal EntraUsers CSV + and the AIO/ValueLens Rollup Users dimension, but NOT to the AISID EntraUsers.csv fixed 14-column + dashboard schema. Auto-enables -IncludeUserInfo. Mutually exclusive with -UserInfoFile, -UseEOM, + and -RAWInputCSV; compatible with -UserIds, -GroupNames, -OnlyUserInfo, AIO/ValueLens/AISID, + -AppendUserInfo, and -Deidentify. + +.PARAMETER AppendAgent365Info + Append the Microsoft Agent 365 catalog export into an existing file. Filename rules match + -AppendFile. Auto-enables -IncludeAgent365Info. Incompatible with -OnlyAgent365Info. May be + used standalone or together with -AppendFile. Use either -AppendAgent365Info (merge) or + -OutputPathAgent365Info (overwrite/redirect), not both, for the same destination. + +.PARAMETER AppendDefenderUsage + AVAILABILITY: this entry point is temporarily gated. AISID is under development and is not available + for customer use in this PAX version; supplying this switch reports that notice and stops the run. + Availability is planned for a later PAX script version. The behavior described below is retained as + a preview. + This switch is tied to -Dashboard AISID and appends the AISID (Defender/AI Solutions) output set into + an existing AISID destination folder. + FOLDER-ONLY (points at the folder holding the AISID CSV set); a file leaf is rejected. Mutually + exclusive with -OutputPathDefenderUsage (exactly one of the pair per run). Only meaningful with + -Dashboard AISID. + +.PARAMETER CombineOutput + Combines all activity types into a single output file or tab. + + **Default Behavior (without -CombineOutput):** + • CSV: One separate CSV file per activity type (plus EntraUsers_MAClicensing_.csv if -IncludeUserInfo) + • Excel: Multi-tab workbook (one tab per activity type; EntraUsers tab appended last if -IncludeUserInfo) + + **With -CombineOutput switch:** + • CSV: Single combined activity file named Purview_Audit_UsageActivity_CombinedActivityTypes_.csv (or Purview_Audit_UsageActivity__.csv when only one activity type was queried; plus separate EntraUsers_MAClicensing_.csv if -IncludeUserInfo) + • Excel: First tab named CombinedUsageActivity (no timestamp) with all activity rows; separate EntraUsers tab if -IncludeUserInfo + + Entra user/org data is never merged into the combined activity dataset—always exported separately. + + Recommended: Use combined mode for ingestion pipelines; separated mode for granular analysis. + +.PARAMETER Force + Force execution without interactive prompts. Automatically accepts defaults for: + 1. DSPM for AI Billing Information: Automatically continues when AIAppInteraction / AIInteraction / ConnectedAIAppInteraction are included via -ActivityTypes. + The DSPM informational prompt is ALSO auto-suppressed (without -Force) on -IncludeM365Usage runs that do not include AIAppInteraction, because ConnectedAIAppInteraction is part of the curated M365 usage bundle. The PAYG prompt for AIAppInteraction still fires unless -Force is set. + 2. Conflict Resolution (ExcludeCopilotInteraction): Automatically honors -ExcludeCopilotInteraction when conflict with -ActivityTypes + Use this switch for unattended/automated executions (CI/CD pipelines, scheduled tasks). + +.PARAMETER ClearUncertainCreate + Operator-only SELECTIVE recovery for durable uncertain-create state on resume. An + uncertain create is a first create POST that MAY have been accepted but could not be + proven, so on resume such a partition is reconciled READ-ONLY (never re-POSTed) and, if + still not visible, is left uncertain and its worker is NOT launched. Durable uncertain + state is NEVER auto-cleared - not by elapsed time, a collection miss, a process restart, + or failure-count exhaustion. This parameter takes an EXPLICIT list of partition indexes + (e.g. -ClearUncertainCreate 3,7 clears ONLY partitions 3 and 7); it never performs an + indiscriminate global clear. A selection that is empty, contains a duplicate, or names an + index that is not currently in the durable uncertain-create set is REJECTED and nothing + is cleared. Each selected partition is first reconciled READ-ONLY: if it is still visible + by exact name it is ADOPTED (fetched, never cleared into a duplicate POST); only a + selected partition confirmed NOT visible has its durable marker cleared - selectively and + atomically, before any fresh create - and is then created fresh. Use it only after you + have confirmed no such query exists server-side for the selected indexes, since a fresh + create could otherwise duplicate an accepted-but-invisible query. + +.PARAMETER ClearUncertainContract + Operator-only EXPLICIT-CONTRACT companion to -ClearUncertainCreate for the case where a + single partition index maps to MORE THAN ONE durable uncertain-create contract (Graph + splits a run into one workload pass per service and every pass restarts partition indexes + at 1, so two different contracts can share index 1). When -ClearUncertainCreate is + ambiguous it is REJECTED before any auth or mutation with an actionable message that lists + the candidate ":" tokens (a fingerprint is a non-secret contract hash). + Pass those exact tokens to -ClearUncertainContract to clear EXACTLY the named contract(s): + each token ":" must resolve to exactly one durable uncertain-create + contract; a token that is malformed, names an unknown contract, matches more than one, or is + duplicated is REJECTED and nothing is cleared (fail-closed, no partial mutation). A valid + token clears ONLY that contract - every unselected uncertain contract is preserved exactly. + Like -ClearUncertainCreate, each selected contract is first reconciled READ-ONLY and is only + cleared for a fresh create if it is still not visible by exact name; a still-visible query is + adopted, never cleared into a duplicate POST. The simple -ClearUncertainCreate path + is unchanged for the common unambiguous case. + +.PARAMETER SkipDiagnostics + Skip pre-query capability diagnostics (advanced). + +.PARAMETER SkipVersionCheck + Skip the brief startup check that compares this script's version with the latest published on the + PAX GitHub repo. The check is informational only (never prompts, never auto-updates) and times out + in ~5 seconds if GitHub is unreachable; use this switch to suppress it on offline or locked-down hosts. + +.PARAMETER UseEOM + Use Exchange Online Management mode with Search-UnifiedAuditLog cmdlet. + When specified, queries Purview audit logs via EOM PowerShell module (serial processing only). + Default mode (when omitted) uses Microsoft Graph Security API with parallel processing support. + + PERMISSIONS REQUIRED (EOM mode): + • Exchange Online RBAC Roles: + - View-Only Audit Logs role + - Compliance Management role group + - Organization Management role group + - Or custom role with Search-UnifiedAuditLog cmdlet permission + + PARALLEL PROCESSING: + EOM mode is SERIAL-ONLY. Parallel processing is automatically disabled. + If -EnableParallel or -ParallelMode is specified with -UseEOM, script will exit with error. + + POWERSHELL VERSION: + EOM mode is the only path supported on PowerShell 5.1. The default Graph API mode + requires PowerShell 7+ (for ThreadJob-based parallel query execution). + + +.PARAMETER IncludeUserInfo + Include Entra (Microsoft Entra ID) user directory & Copilot license enrichment (Graph API mode only). + Exports an independent EntraUsers file/tab with: + • Core identity & profile details (name, UPN, job, department, location, organization info) + • Account & sync state (accountEnabled, onPremSync attributes, creation / change stamps) + • Manager expansion (identity + basic role/job fields via $expand=manager) + • Contact & routing (mail, proxyAddresses flattened, preferredLanguage) + • License enrichment (assignedLicenses list + hasLicense boolean for Copilot detection) + + Separation Principle: EntraUsers data is never merged into activity rows; always a distinct artifact. + + Requirements: + • Graph API mode (not supported with -UseEOM) + • Graph permissions (consented at runtime only when this switch is set): + - User.Read.All (read /users for Entra directory + license map) + - Organization.Read.All (read /subscribedSkus for SKU lookup) + • One-time directory + license fetch at startup (typ. +10–20s) + + License Detection Logic: + Dynamic discovery (no hardcoded SKU GUIDs): Copilot service plan IDs are + collected from /subscribedSkus where servicePlanName matches '*COPILOT*'. + hasLicense = true when the user has any assignedPlan with + capabilityStatus == 'Enabled' AND servicePlanId in the discovered set. + Honors admin-disabled service plans inside otherwise-assigned SKUs. + + Performance: Single batched fetch + hashtable lookups; no per-record calls. + + Use Cases: + • License compliance & adoption + • Mapping usage to directory attributes + • Identifying unlicensed usage patterns + + Not available in EOM mode. + +.PARAMETER OnlyUserInfo + Export ONLY Entra user directory and license information (skips all audit log retrieval). + This is a specialized mode for quickly exporting user licensing data without querying audit logs. + + BEHAVIOR: + • Authenticates to Microsoft Graph + • Fetches Entra user directory and license data + • Exports standalone EntraUsers_MAClicensing_.csv file + • Skips all audit log queries (completes in 5-15 seconds vs. minutes/hours) + • Automatically enables -IncludeUserInfo + + OUTPUT: + Single CSV file: EntraUsers_MAClicensing_YYYYMMDD_HHMMSS.csv + Contains 37 columns including: + - Identity fields (UPN, displayName, id, mail) + - Profile data (jobTitle, department, officeLocation) + - Manager hierarchy (manager info expanded) + - License assignments (assignedLicenses + hasCopilotLicense boolean) + + COMPATIBLE PARAMETERS (can be used WITH -OnlyUserInfo): + • -OutputPath : Specify output directory + • -OutputPathUserInfo: Per-data-type destination (overrides -OutputPath for the EntraUsers CSV) + • -AppendUserInfo : Union-merge this run's EntraUsers snapshot into an existing target + • -UserIds : Scope the exported directory to these user principal names + • -GroupNames : Scope the exported directory to the members of these group(s) + • -Auth : Choose authentication method (WebLogin, DeviceCode, etc.) + • -CombineOutput : (Has no effect, but allowed for script compatibility) + • -DisableAdaptive : (Has no effect, but allowed) + • -Debug / -Verbose : Enable diagnostic output + + INCOMPATIBLE PARAMETERS (cannot be used with -OnlyUserInfo): + Audit Retrieval: + • StartDate, EndDate : No audit queries to filter + • ActivityTypes : Cleared by -OnlyUserInfo + • ExcludeCopilotInteraction: Activity type modifier + • BlockHours, PartitionHours, MaxPartitions, ResultSize, PacingMs + • AutoCompleteness : Audit log completeness checks + • StreamingSchemaSample, StreamingChunkSize, ExportProgressInterval + + Filtering: + • AgentId, AgentsOnly, ExcludeAgents: Agent filtering requires audit logs + • PromptFilter : Message filtering requires audit logs + + Parallelization: + • ParallelMode, MaxParallelGroups, MaxConcurrency, EnableParallel + • MaxActivePartitions : Query execution settings + + EOM Mode: + • UseEOM : Exchange Online Management mode + + USE CASES: + 1. License compliance auditing (quick snapshot of all user licenses) + 2. Periodic license data exports for tracking/trending + 3. Standalone user directory exports for cross-referencing + 4. Rapid licensing status checks without audit log overhead + + EXAMPLES: + # Basic user-only export + .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyUserInfo + + # Export to specific directory + .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyUserInfo -OutputPath "D:\UserData\" + + # Use device code auth (for automation/headless scenarios) + .\PAX_Purview_Audit_Log_Processor.ps1 -OnlyUserInfo -Auth DeviceCode + + # App registration auth (client secret) + .\PAX_Purview_Audit_Log_Processor.ps1 -Auth AppRegistration -TenantId "" -ClientId "" -ClientSecret (ConvertTo-SecureString "" -AsPlainText -Force) + + # App registration auth (certificate thumbprint) + .\PAX_Purview_Audit_Log_Processor.ps1 -Auth AppRegistration -TenantId "" -ClientId "" -ClientCertificateThumbprint "" + + PERFORMANCE: + Typical execution time: 5-15 seconds (vs. minutes/hours for audit log queries) + Network traffic: Minimal (only user directory + license API calls) + + NOT AVAILABLE IN EOM MODE: Requires Microsoft Graph API (user directory/licenses not in EOM). + +.PARAMETER IncludeAgent365Info + Adds a Microsoft Agent 365 enrichment phase to the run, producing Agent365_.csv + (columns matching the Microsoft Admin Center "Agent 365" agent catalog export). Runs after the + main audit + EntraUsers phases and sources data from the Microsoft Graph Agent Package + Management API, with one narrow audit lookup per agent for creation date / created-by. + + REQUIREMENTS: + • Tenant enrolled in the Microsoft Agent 365 program (and holding a Microsoft Agent 365 license) + • App-only auth (-Auth AppRegistration certificate/secret, or -Auth ManagedIdentity): the app + registration / managed-identity service principal must be granted + admin-consented the + APPLICATION permissions CopilotPackages.Read.All (and Application.Read.All for developer-name + resolution). No interactive sign-in is performed; the Agent 365 phase reuses the app-only context. + • Delegated auth (-Auth WebLogin / DeviceCode / Credential / Silent): the signed-in caller must + hold AI Administrator or Global Administrator; the Agent 365 scopes are consented at sign-in. + • Requires a destination — supply -OutputPathAgent365Info or -AppendAgent365Info + • The catalog API is currently published at /beta only. + INCOMPATIBLE: -UseEOM, -RAWInputCSV. Compatible with -Resume (the agent phase runs at end of run) and + with all auth modes, including -Auth ManagedIdentity and headless / noninteractive hosts (no prompt). + If the tenant is not enrolled/licensed, the agent phase is skipped with an explanatory banner and the rest of the run completes normally. + +.PARAMETER OnlyAgent365Info + Export ONLY the Microsoft Agent 365 catalog (skips all audit log retrieval and EntraUsers + enrichment), producing only Agent365_.csv. Same requirements as -IncludeAgent365Info + (including app-only support via -Auth AppRegistration or -Auth ManagedIdentity, and delegated + modes). Incompatible with -Resume (there is no audit phase to resume). A Y/N preflight confirms + before any Graph call; pass -Force to auto-confirm. + +.PARAMETER MaxNetworkOutageMinutes + Maximum continuous network outage the script will tolerate during audit log operations (query creation, polling, record retrieval). + Applies to transient network errors: 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout, connection failures. + Script automatically retries failed operations with randomized delays (30-60s) until connectivity is restored or timeout is exceeded. + Clean terminal output shows error summaries with countdown timers; full error details logged to file for troubleshooting. + Progress is preserved - no data loss during network interruptions. + Exceeding this window aborts with clear error message indicating tolerance exceeded. + Default: 30 minutes (adjustable 1-120) + +.PARAMETER Resume + Resume an interrupted operation from a checkpoint file. + Checkpoint files are automatically created during all auth modes to allow resumption + after Ctrl+C, network failures, token expiry, or any interruption. + + IMPORTANT: Resume mode is STANDALONE. + All processing parameters are restored from the checkpoint file. + You cannot specify other parameters with -Resume (except auth overrides). + + USAGE: + -Resume Auto-discover checkpoint in current directory/OutputPath + -Resume "path\to\file" Use specific checkpoint file + + ALLOWED WITH -Resume: + -Force Use most recent checkpoint without prompting + -Auth Override authentication method + -TenantId, -ClientId Override auth credentials (for AppRegistration) + -ClientSecret Provide client secret (for AppRegistration) + -UserIds, -GroupNames Override the checkpoint's user/group scope (fail-closed) + + NOT ALLOWED WITH -Resume: + Any other parameter (dates, activities, explosion settings, -Deidentify, etc.) + These are all restored from the checkpoint to ensure data consistency. In particular, + -Deidentify is taken from the checkpoint — a run started with -Deidentify stays + deidentified when resumed, and deidentification cannot be added or removed on the resume + command line. + + CHECKPOINT LOCATION: + Files are created in OutputPath with pattern: .pax_checkpoint_.json + Checkpoint and partial-output files are always written to the LOCAL filesystem, + even when an -OutputPath* value is a SharePoint or OneLake URL. Resume must be run + from the same machine that produced the checkpoint. + +#> + +param( + [Parameter(Mandatory = $false)] + [string]$StartDate, # Omit BOTH StartDate+EndDate => default [yesterday..today) UTC; omit exactly one => that side stays unbounded ('*'); Replay: optional filter. -Dashboard AISID additionally requires BOTH dates concrete. + + [Parameter(Mandatory = $false)] + [string]$EndDate, # Omit BOTH StartDate+EndDate => default [yesterday..today) UTC; omit exactly one => that side stays unbounded ('*'); Replay: optional filter. -Dashboard AISID additionally requires BOTH dates concrete. + + [Parameter(Mandatory = $false)] + [string]$OutputPath, + + # Per-data-type destination overrides. Storage tier (Local / SharePoint / Fabric) is + # inferred from each path's form: drive-rooted absolute path = Local, https://*.sharepoint.* + # URL = SharePoint, https://*.onelake.dfs.fabric.microsoft.com/*Lakehouse* URL = Fabric. + # UNC paths are rejected. All destinations supplied to a single run must resolve to the + # same storage tier (no mixed Local/SP/Fabric in one invocation). + [Parameter(Mandatory = $false)] + [string]$OutputPathUserInfo, + + [Parameter(Mandatory = $false)] + [string]$OutputPathAgent365Info, + + [Parameter(Mandatory = $false)] + [string]$OutputPathLog, + + # Destination for the entire AISID (AI Solutions Intelligence Dashboard) output set — all 6 Defender/AISID + # CSVs (including the solutions catalog) land here. FOLDER-ONLY: supply a folder path / SharePoint folder URL / + # Fabric OneLake folder URL; a file leaf is rejected. Storage tier is inferred like the other -OutputPath* + # switches and must match the run's other destinations. Only meaningful with -Dashboard AISID. + [Parameter(Mandatory = $false)] + [string]$OutputPathDefenderUsage, + + # TEMPORARILY GATED: AISID is under development and is not available for customer use in this PAX + # version; supplying this switch reports that notice and stops the run. Availability is planned for + # a later PAX script version. The description below is retained as a preview. + # Turns off the incremental result cache used by the off-hours geography signal, forcing a + # full re-collection of its window on this run. Only meaningful with -Dashboard AISID. + [Parameter(Mandatory = $false)] + [switch]$DisableAISIDDeltaCache, + [Parameter(DontShow = $true)] + [switch]$VerifyPartitionStability, + + [Parameter(Mandatory = $false)] + [ValidateSet('WebLogin', 'DeviceCode', 'Credential', 'Silent', 'AppRegistration', 'ManagedIdentity')] + [string]$Auth = 'WebLogin', + + [Parameter(Mandatory = $false)] + [string]$TenantId, + + [Parameter(Mandatory = $false)] + [string]$ClientId, + + [Parameter(Mandatory = $false)] + [string]$ClientSecret, + + [Parameter(Mandatory = $false)] + [string]$ClientCertificateThumbprint, + + [Parameter(Mandatory = $false)] + [ValidateSet('CurrentUser','LocalMachine')] + [string]$ClientCertificateStoreLocation = 'CurrentUser', + + [Parameter(Mandatory = $false)] + [string]$ClientCertificatePath, + + [Parameter(Mandatory = $false)] + [System.Security.SecureString]$ClientCertificatePassword, + + [Parameter(Mandatory = $false)] + [ValidateRange(0.016667, 24)] + [double]$BlockHours = 0.5, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 72)] + [int]$PartitionHours = 0, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 1000)] + [int]$MaxPartitions = 160, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10000)] + [int]$ResultSize = 10000, + + [Parameter(Mandatory = $false)] + [ValidateRange(0, 10000)] + [int]$PacingMs = 0, + + [Parameter(Mandatory = $false)] + [string[]]$ActivityTypes = @('CopilotInteraction'), + + [Parameter(Mandatory = $false)] + [string[]]$RecordTypes, + + [Parameter(Mandatory = $false)] + [string[]]$ServiceTypes, + + [Parameter(Mandatory = $false, DontShow = $true)] + [switch]$ExplodeArrays, + + [Parameter(Mandatory = $false, DontShow = $true)] + [switch]$ExplodeDeep, + [Parameter(Mandatory = $false)] + [int]$FlatDepth = 120, + [Parameter(Mandatory = $false, DontShow = $true)] + [string]$RAWInputCSV, + [Parameter(Mandatory = $false)] + # Controls concurrent execution: EOM mode limits serial queries, Graph API mode limits partition parallelism + [int]$MaxConcurrency = 10, + [Parameter(Mandatory = $false)] + [switch]$EnableParallel, + [Parameter(Mandatory = $false)] + [ValidateRange(0, 50)] + # Allows multiple activity groups to be processed concurrently (aligns with Microsoft's ~10 query safe limit) + [int]$MaxParallelGroups = 8, + [Parameter(Mandatory = $false)] + [ValidateSet('Off', 'On', 'Auto')] + # Default now 'Auto' so that PS 7+ environments engage parallel processing automatically unless explicitly turned Off. + [string]$ParallelMode = 'Auto', + [Parameter(Mandatory = $false)] + [ValidateRange(0, 32)] + # 0=auto-detect (2 to min(ProcessorCount, 8) threads), 1=serial, 2-32=explicit thread count. Requires PS7+. + [int]$ExplosionThreads = 0, + [Parameter(Mandatory = $false)] + [switch]$DisableAdaptive, # Disable adaptive safeguards (memory/latency/concurrency smoothing) + [Parameter(Mandatory = $false)] + [ValidateRange(0.0,1.0)] + [double]$ProgressSmoothingAlpha = 0.3, # Weight for smoothing dynamic progress total recalculation (0 => off) + [Parameter(Mandatory = $false)] + [ValidateRange(1000,600000)] + [int]$HighLatencyMs = 90000, # Partition average latency threshold (ms) triggering mild concurrency reduction + [Parameter(Mandatory = $false)] + [ValidateRange(256,32768)] + [int]$MemoryPressureMB = 1500, # Working set (MB) threshold to trigger mild concurrency reduction + [Parameter(Mandatory = $false)] + [ValidateRange(-1,65536)] + [int]$MaxMemoryMB = -1, # Max process memory (MB) before flushing $allLogs to disk (-1 = auto 75%, 0 = disabled) + [Parameter(Mandatory = $false)] + [ValidateRange(30,600)] + [int]$StatusIntervalSeconds = 60, # How often (seconds) to display status during polling and backpressure waits + [Parameter(Mandatory = $false)] + [ValidateRange(100,600000)] + [int]$LowLatencyMs = 20000, # Sustained low latency threshold to consider concurrency step-up + [Parameter(Mandatory = $false)] + [ValidateRange(1,10)] + [int]$LowLatencyConsecutive = 2, # Required consecutive low-latency groups before step-up + [Parameter(Mandatory = $false)] + [ValidateRange(1,100)] + [int]$ThroughputDropPct = 15, # % drop vs baseline required (with high latency) to justify reduction + [Parameter(Mandatory = $false)] + [ValidateRange(0.0,1.0)] + [double]$ThroughputSmoothingAlpha = 0.3,# EMA smoothing for throughput baseline + [Parameter(Mandatory = $false)] + [ValidateRange(1,50)] + [int]$AdaptiveConcurrencyCeiling = 6, # Upper bound for adaptive step-ups + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10000)] + [int]$ExportProgressInterval = 10, + + # Streaming export is always-on + [Parameter(Mandatory = $false)] + [ValidateRange(100, 50000)] + [int]$StreamingSchemaSample = 5000, + + [Parameter(Mandatory = $false)] + [ValidateRange(100, 50000)] + [int]$StreamingChunkSize = 5000, + + [Parameter(Mandatory = $false)] + [string[]]$AgentId, + + [Parameter(Mandatory = $false)] + [switch]$AgentsOnly, + + [Parameter(Mandatory = $false)] + [ValidateSet('Prompt', 'Response', 'Both', 'Null')] + [string]$PromptFilter, + + # --- Reliability Enhancements (Backoff & Circuit Breaker) --- + [Parameter(Mandatory = $false)] + [ValidateRange(1,50)] + [int]$CircuitBreakerThreshold = 5, # Consecutive block failures before opening circuit breaker + [Parameter(Mandatory = $false)] + [ValidateRange(5,3600)] + [int]$CircuitBreakerCooldownSeconds = 120, # Cooldown duration after breaker trips + [Parameter(Mandatory = $false)] + [ValidateRange(0.1,120)] + [double]$BackoffBaseSeconds = 1.0, # Base seconds for exponential backoff between block retries + [Parameter(Mandatory = $false)] + [ValidateRange(1,600)] + [int]$BackoffMaxSeconds = 45, # Max cap for exponential backoff delay + + [Parameter(Mandatory = $false)] + [switch]$ExcludeAgents, + + [Parameter(Mandatory = $false)] + [string[]]$UserIds, + + [Parameter(Mandatory = $false)] + [string[]]$GroupNames, + + [Parameter(Mandatory = $false)] + [switch]$Help, + + # Emit structured metrics JSON alongside CSV (OutputFile name with .metrics.json) + [Parameter(Mandatory = $false)] + [switch]$EmitMetricsJson, + + # Override metrics output path (optional). If provided and -EmitMetricsJson specified, writes here instead of OutputFile substitution. + [Parameter(Mandatory = $false)] + [string]$MetricsPath, + + # Ensure completeness: aggressively subdivide any window still returning server 10K limit until below threshold or min window reached. + [Parameter(Mandatory = $false)] + [switch]$AutoCompleteness, + + # Include CopilotInteraction explicitly (default when no custom -ActivityTypes provided) + [Parameter(Mandatory = $false)] + [switch]$IncludeCopilotInteraction, + + [Parameter(Mandatory = $false)] + [switch]$IncludeM365Usage, + + # Exclude CopilotInteraction activity type (overrides custom list and default fallback) + [Parameter(Mandatory = $false)] + [switch]$ExcludeCopilotInteraction, + + # [Deprecated] Excel workbook export. + [Parameter(Mandatory = $false, DontShow = $true)] + [switch]$ExportWorkbook, + + # Append data to existing CSV file + # Provide filename (e.g., "MyReport.csv") or full path (e.g., "C:\Data\Report.csv") + [Parameter(Mandatory = $false)] + [string]$AppendFile, + + # Append EntraUsers / MAC licensing data into an existing file. Filename rules match -AppendFile. + # Auto-enables -IncludeUserInfo. Incompatible with -OnlyUserInfo. May be used standalone or together with -AppendFile. + [Parameter(Mandatory = $false)] + [string]$AppendUserInfo, + + # Ingest the Entra user directory from a customer-provided CSV in lieu of the live Entra /users pull. + # Storage tier is inferred from the value form (drive-rooted absolute path = Local; SharePoint URL = + # SharePoint; OneLake Lakehouse URL = Fabric); remote inputs are staged to scratch before parsing. + # CSV only; requires a UserPrincipalName column. Mutually exclusive with -GroupNames. + [Parameter(Mandatory = $false)] + [string]$UserInfoFile, + + # Hybrid directory enrichment: fetch the live Entra /users directory normally, then append the columns + # of a customer-provided supplemental CSV to each matching Entra row (Entra-left join by + # UserPrincipalName). Storage tier is inferred from the value form; remote inputs are staged to scratch + # before parsing. .csv only; requires exactly one UserPrincipalName column (join key only, never copied + # into output). Every Entra user is preserved; unmatched supplemental rows are reported and excluded. + # Auto-enables -IncludeUserInfo. Mutually exclusive with -UserInfoFile / -UseEOM / -RAWInputCSV; + # compatible with -GroupNames. + [Parameter(Mandatory = $false)] + [string]$UserInfoSupplement, + + # Append Agent 365 catalog data into an existing file. Filename rules match -AppendFile. + # Auto-enables -IncludeAgent365Info. Incompatible with -OnlyAgent365Info. May be used standalone or together with -AppendFile. + [Parameter(Mandatory = $false)] + [string]$AppendAgent365Info, + + # Append AISID (Defender/AI Solutions) output into an existing AISID destination. FOLDER-ONLY (points at the + # folder holding the AISID CSV set); a file leaf is rejected. Mutually exclusive with -OutputPathDefenderUsage + # (exactly one of the pair per run). Only meaningful with -Dashboard AISID. + [Parameter(Mandatory = $false)] + [string]$AppendDefenderUsage, + + # Combine all activity types into single output file/tab (CSV or Excel) + # CSV default when omitted: separate files per activity type + # Excel default when omitted: separate tabs per activity type + # Use -CombineOutput switch to merge all activity types into one file/tab + [Parameter(Mandatory = $false)] + [switch]$CombineOutput, + + # Force execution without interactive prompts (PAYG warning, conflict resolution) + [Parameter(Mandatory = $false)] + [switch]$Force, + + # Operator-only SELECTIVE recovery: an explicit list of partition indexes whose durable + # uncertain-create checkpoint state is discarded on resume so those specific partitions are + # created fresh. Durable uncertain state is never auto-cleared (see .PARAMETER + # ClearUncertainCreate); an explicit index selection is the only way to clear it, and a + # still-visible selected partition is adopted rather than cleared into a duplicate POST. + [Parameter(Mandatory = $false)] + [int[]]$ClearUncertainCreate, + + # Operator-only EXPLICIT-CONTRACT companion to -ClearUncertainCreate: an explicit list of + # ":" tokens naming durable uncertain-create contracts to clear on + # resume. It disambiguates the case where one partition index maps to more than one durable + # uncertain contract (distinct fingerprints, same index) - an ambiguous -ClearUncertainCreate + # is rejected and lists the candidate tokens to pass here. A valid token clears ONLY + # that contract; every unselected uncertain contract is preserved exactly (see .PARAMETER + # ClearUncertainContract). A still-visible selected contract is adopted, never re-POSTed. + [Parameter(Mandatory = $false)] + [string[]]$ClearUncertainContract, + + # Skip pre-query capability diagnostics (advanced) + [Parameter(Mandatory = $false)] + [switch]$SkipDiagnostics, + + # Skip the startup GitHub version check (offline / locked-down environments) + [Parameter(Mandatory = $false)] + [switch]$SkipVersionCheck, + + # Use Exchange Online Management mode (Search-UnifiedAuditLog cmdlet, serial-only) + [Parameter(Mandatory = $false)] + [switch]$UseEOM, + + # Include Entra user directory and license information in export (adds separate EntraUsers file/tab; column count may evolve) + [Parameter(Mandatory = $false)] + [switch]$IncludeUserInfo, + + # Export only Entra user directory and license information (skips all audit log retrieval) + [Parameter(Mandatory = $false)] + [switch]$OnlyUserInfo, + + # Include Microsoft Agent 365 enrichment in export (adds separate Agents365 CSV file/Excel tab). + # Sources data from the Microsoft Graph Agent Package Management API + # (https://graph.microsoft.com/beta/copilot/admin/catalog/packages). + # Runs under any auth mode (no auth mode is rejected up-front): app-only + # (-Auth AppRegistration certificate/secret or -Auth ManagedIdentity) uses the application + # permissions CopilotPackages.Read.All + Application.Read.All with no interactive sign-in; + # delegated auth requires the signed-in user to hold AI Administrator or Global Administrator. + # See the .PARAMETER IncludeAgent365Info help for full details. + [Parameter(Mandatory = $false)] + [switch]$IncludeAgent365Info, + + # Export only Microsoft Agent 365 data (skips all audit log retrieval). + # Date created and Created by columns will be left blank (those rely on audit data). + # Supported under app-only auth (-Auth AppRegistration or -Auth ManagedIdentity) as well as + # delegated; no auth mode is rejected up-front (access failures surface as a runtime 403). + [Parameter(Mandatory = $false)] + [switch]$OnlyAgent365Info, + + # Maximum minutes to tolerate continuous network outage during Graph async polling & record retrieval (adaptive backoff). Default 30. + [Parameter(Mandatory = $false)] + [int]$MaxNetworkOutageMinutes = 30, + + # Export Graph API telemetry CSV for partition timing analysis and troubleshooting + [Parameter(Mandatory = $false)] + [switch]$IncludeTelemetry, + + # Rollup post-processor: produce ONLY rolled-up CSV(s) and delete the raw CSV(s) after success. + # Mutually exclusive with -RollupPlusRaw. Requires PowerShell 7+. Auto-installs Python 3.10+ and + # the optional 'orjson' package on first use. Only valid for default (CopilotInteraction-only) runs, + # explicit -ActivityTypes 'CopilotInteraction', or -IncludeM365Usage runs. Not compatible with + # -UseEOM, -ExportWorkbook, -OnlyUserInfo, -OnlyAgent365Info, or -RAWInputCSV. See documentation + # for full gating matrix. + [Parameter(Mandatory = $false)] + [switch]$Rollup, + + # Rollup post-processor: produce BOTH the raw CSV(s) AND the rolled-up CSV(s). Same gating rules + # as -Rollup. Mutually exclusive with -Rollup. + [Parameter(Mandatory = $false)] + [switch]$RollupPlusRaw, + + # Rollup target dashboard selector. Chooses which embedded Python + # post-processor + output profile a rollup run produces: + # AIO (default) - AI-in-One dashboard -> CopilotInteraction processor, --profile aio + # ValueLens - ValueLens dashboard -> CopilotInteraction processor, --profile aibv + # M365 - M365 Usage Analytics -> M365 bundle processor (auto-enables -IncludeM365Usage) + # AISID - AI Solutions Intelligence Dashboard. TEMPORARILY GATED: AISID is under + # development and is not available for customer use in this PAX version; + # selecting it reports that notice and stops the run. Availability is planned + # for a later PAX script version. The description below is retained as a + # preview. Drives a full Purview + Entra + Defender run + # and the Defender/AISID data pipeline (the only trigger for it), requires + # -IncludeUserInfo and a run not narrowed by any -Only* switch, reuses the AIO-shaped + # CopilotInteraction rollup for its Purview/Entra tables (its own Defender-sourced CSVs + # are exported without rollup processing), and adds the ThreatHunting.Read.All Graph + # permission (delegated) / application permission (app-only modes) at sign-in. + # Only meaningful with -Rollup / -RollupPlusRaw; if supplied without either, -Rollup is auto-enabled. + # If omitted: M365 when -IncludeM365Usage is present, otherwise AIO. Case-insensitive. + [Parameter(Mandatory = $false)] + [ValidateSet('AIO', 'ValueLens', 'M365', 'AISID')] + [string]$Dashboard = 'AIO', + + # Deidentify (anonymize) all identifying values in RAW output files using a + # one-way, salted, deterministic, format-preserving hash. Applies to the raw + # audit CSV(s) and the EntraUsers CSV. Rollup outputs are deidentified by the + # embedded Python processor (same salt + algorithm), so -Rollup / -RollupPlusRaw + # rollups are covered there; this switch additionally scrubs any RETAINED raw + # files. No-op for plain -Rollup (raw deleted). Irreversible: no decode map is + # produced or stored. + [Parameter(Mandatory = $false)] + [switch]$Deidentify, + + # Org / manager-hierarchy level-filler for the rolled-up AI-in-One / AI Business + # Value Users output. The hierarchy columns are ALWAYS produced for those two + # dashboards; this only controls what appears in level slots deeper than a user's + # own level: + # null (default) -> leave blank + # Self -> repeat the user + # RepeatManager -> repeat the user's manager + # Fixed -> a literal label supplied via -FillerLabelText "" + # Requires -Rollup or -RollupPlusRaw; not valid with -IncludeM365Usage / -Dashboard M365. + [Parameter(Mandatory = $false)] + [string]$FillerLabel, + + # The literal label text used with -FillerLabel Fixed (e.g. "Assistive Directs"). + # Only valid together with -FillerLabel Fixed. + [Parameter(Mandatory = $false)] + [string]$FillerLabelText, + + # Resume from checkpoint file - HANDLED VIA $args (not param block) to support: + # -Resume (auto-discover checkpoint in OutputPath) + # -Resume "path/to/file" (explicit checkpoint path) + # This parameter captures any remaining arguments for manual -Resume parsing + [Parameter(Mandatory = $false, ValueFromRemainingArguments = $true)] + [string[]]$RemainingArgs +) + +# ============================================================ +# POWERSHELL LANGUAGE MODE PREFLIGHT +# PAX requires FullLanguage mode for its .NET and runtime features. Stop before +# creating a bootstrap log, loading modules, authenticating, or writing output +# when an endpoint application-control policy places this process in another mode. +# ============================================================ +$currentLanguageMode = "$($ExecutionContext.SessionState.LanguageMode)" +if ($currentLanguageMode -ne 'FullLanguage') { + Microsoft.PowerShell.Utility\Write-Host '' + Microsoft.PowerShell.Utility\Write-Host 'PAX cannot run in the current PowerShell language mode.' -ForegroundColor Red + Microsoft.PowerShell.Utility\Write-Host ("Current mode: {0}" -f $currentLanguageMode) -ForegroundColor Yellow + Microsoft.PowerShell.Utility\Write-Host '' + Microsoft.PowerShell.Utility\Write-Host 'PAX requires PowerShell running in FullLanguage mode.' + Microsoft.PowerShell.Utility\Write-Host 'This setting is commonly controlled by your organization through WDAC or AppLocker policy.' + Microsoft.PowerShell.Utility\Write-Host 'Contact your IT or security team for an approved PowerShell environment or to have PAX allowlisted.' + Microsoft.PowerShell.Utility\Write-Host 'ExecutionPolicy Bypass does not change PowerShell language mode.' + Microsoft.PowerShell.Utility\Write-Host '' + exit 1 +} + +# ============================================================ +# CULTURE GUARD — DO NOT REMOVE +# Pin the main thread to InvariantCulture for the entire run. +# Rationale: .NET's custom date/time format string treats ':' not as a +# literal colon but as the culture's DateTimeFormatInfo.TimeSeparator. +# On da-DK and fi-FI hosts TimeSeparator is '.', so an unscoped +# .ToString('yyyy-MM-ddTHH:mm:ss.fffZ') produces 'T20.35.42.000Z', +# which the Purview audit query endpoint rejects with HTTP 500. +# Setting CurrentCulture here makes every downstream .ToString(format), +# Get-Date -Format, and ParseExact($x,$fmt,$null) call in the main +# thread emit / consume invariant ISO 8601. ThreadJob runspaces get +# their own per-thread guard inside $queryJobScriptBlock because +# PowerShell 5.1 does not inherit parent culture into ThreadJobs. +# ============================================================ +[System.Threading.Thread]::CurrentThread.CurrentCulture = [System.Globalization.CultureInfo]::InvariantCulture +[System.Threading.Thread]::CurrentThread.CurrentUICulture = [System.Globalization.CultureInfo]::InvariantCulture + +# ============================================================ +# BOOTSTRAP LOG +# Open a deterministic log file IMMEDIATELY after param() so every Write-Host / +# Write-Log call from this point forward lands on disk. The final log location +# (which depends on -OutputPath / -AppendFile / -OutputPathLog / checkpoint state) +# is resolved much later in the script — at that point the bootstrap log is +# renamed/moved to the final path. The in-memory $script:LogBuffer remains as a +# belt-and-braces fallback if the bootstrap file itself cannot be created +# (read-only volume, permissions, etc.). +# +# Path resolution order (durable bootstrap-log location for Azure Container Instances): +# 1. $env:PAX_BOOTSTRAP_LOG_DIR — operator-supplied durable location. +# In the PAX.Dockerfile this defaults to "/pax-logs" and is declared as a +# VOLUME so operators can mount Azure Files at that path. When mounted, +# pre-flight / auth / param-validation failures leave the bootstrap log +# on the Azure Files share, retrievable without spinning a new container. +# 2. [System.IO.Path]::GetTempPath() — process temp dir, correct on every +# supported host: Windows local ($env:TEMP), Linux container (/tmp), +# macOS dev (/var/folders/.../T/). No host-OS branching required. +# +# The override is validated for write access; if it isn't writable the script +# silently falls back to GetTempPath() so a misconfigured env var never blocks +# a run. Successful migration of the bootstrap log to its final location uses +# Move-Item, which removes the bootstrap source file — the durable directory +# remains for future runs. +# ============================================================ +try { + $candidateDirs = @() + if (-not [string]::IsNullOrWhiteSpace($env:PAX_BOOTSTRAP_LOG_DIR)) { + $candidateDirs += $env:PAX_BOOTSTRAP_LOG_DIR + } + $candidateDirs += [System.IO.Path]::GetTempPath() + + $script:BootstrapLogDir = $null + foreach ($cand in $candidateDirs) { + try { + if (-not (Test-Path -LiteralPath $cand -PathType Container)) { + New-Item -Path $cand -ItemType Directory -Force -ErrorAction Stop | Out-Null + } + # Probe write access with a zero-byte sentinel. + $probe = Join-Path $cand (".pax_writeprobe_{0}_{1}" -f $PID, (Get-Date -Format 'yyyyMMddHHmmssfff')) + Set-Content -LiteralPath $probe -Value '' -ErrorAction Stop + Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue + $script:BootstrapLogDir = $cand + break + } catch { + continue + } + } + if (-not $script:BootstrapLogDir) { throw "No writable bootstrap-log directory available." } + + $script:BootstrapLogName = "PAX_bootstrap_{0}_{1}.log" -f $PID, (Get-Date -Format 'yyyyMMddHHmmss') + $script:LogFile = Join-Path $script:BootstrapLogDir $script:BootstrapLogName + $script:LogFileIsBootstrap = $true + New-Item -Path $script:LogFile -ItemType File -Force -ErrorAction Stop | Out-Null + Add-Content -Path $script:LogFile -Value ("[{0}] [INFO] Bootstrap log opened at {1} (PID {2}, source: {3})." -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $script:LogFile, $PID, $(if ($env:PAX_BOOTSTRAP_LOG_DIR -and $script:BootstrapLogDir -eq $env:PAX_BOOTSTRAP_LOG_DIR) { 'PAX_BOOTSTRAP_LOG_DIR' } else { 'system temp' })) -Encoding UTF8 -ErrorAction SilentlyContinue +} catch { + # Bootstrap log creation failed — leave $script:LogFile null so the existing + # in-memory $script:LogBuffer path (defined further down) absorbs early entries. + $script:LogFile = $null + $script:LogFileIsBootstrap = $false + Microsoft.PowerShell.Utility\Write-Host ("WARNING: Could not create bootstrap log: {0}" -f $_.Exception.Message) -ForegroundColor Yellow +} + +# ============================================================ +# DEPRECATED SWITCH GATE +# Any explicit use of these switches on the command line causes an +# immediate graceful exit. These switches are no longer supported. +# ============================================================ +$script:DeprecatedSwitchesHit = @() +if ($PSBoundParameters.ContainsKey('ExportWorkbook')) { $script:DeprecatedSwitchesHit += '-ExportWorkbook' } +if ($PSBoundParameters.ContainsKey('RAWInputCSV')) { $script:DeprecatedSwitchesHit += '-RAWInputCSV' } +if ($PSBoundParameters.ContainsKey('ExplodeArrays')) { $script:DeprecatedSwitchesHit += '-ExplodeArrays' } +if ($PSBoundParameters.ContainsKey('ExplodeDeep')) { $script:DeprecatedSwitchesHit += '-ExplodeDeep' } +if ($script:DeprecatedSwitchesHit.Count -gt 0) { + foreach ($d in $script:DeprecatedSwitchesHit) { + Microsoft.PowerShell.Utility\Write-Host ("{0} is deprecated and will be removed in a future release." -f $d) -ForegroundColor Yellow + } + exit 0 +} + +# ============================================================ +# AISID AVAILABILITY GATE +# The AI Solutions Intelligence Dashboard (AISID) customer entry points are not +# available in this version. The AISID implementation stays in place; only the +# customer entry points are gated. This guard runs before any module load, +# sign-in, destination probe, output directory creation, checkpoint mutation, +# audit query creation, AISID collection, history reconciliation, upload, or +# local publication, so a gated invocation never contacts a service and never +# writes customer output. It is invoked a second time after checkpoint +# restoration so a stored AISID contract cannot reach a gated path either. +# ============================================================ +$script:AISIDGateBlocked = $false +function script:Test-AISIDEntryPointGated { + param( + [bool]$DashboardIsAISID = $false, + [bool]$DefenderUsageOutputRequested = $false, + [bool]$DefenderUsageAppendRequested = $false, + [bool]$DeltaCacheSwitchRequested = $false + ) + if (-not ($DashboardIsAISID -or $DefenderUsageOutputRequested -or $DefenderUsageAppendRequested -or $DeltaCacheSwitchRequested)) { return $false } + if ($script:AISIDGateBlocked) { return $true } + $script:AISIDGateBlocked = $true + Microsoft.PowerShell.Utility\Write-Host "The AI Solutions Intelligence Dashboard (AISID) is currently under development and is not available for customer use in this PAX version. The AISID customer entry points will become available in a later PAX script version." -ForegroundColor Yellow + Microsoft.PowerShell.Utility\Write-Host "Gated entry points: -Dashboard AISID, -OutputPathDefenderUsage, -AppendDefenderUsage, and -DisableAISIDDeltaCache." -ForegroundColor Yellow + return $true +} +$aisidCliDashboard = ($PSBoundParameters.ContainsKey('Dashboard') -and ([string]$Dashboard).Trim().ToUpperInvariant() -eq 'AISID') +$aisidCliOutput = $PSBoundParameters.ContainsKey('OutputPathDefenderUsage') +$aisidCliAppend = $PSBoundParameters.ContainsKey('AppendDefenderUsage') +$aisidCliCache = ($PSBoundParameters.ContainsKey('DisableAISIDDeltaCache') -and [bool]$DisableAISIDDeltaCache) +if (script:Test-AISIDEntryPointGated -DashboardIsAISID $aisidCliDashboard -DefenderUsageOutputRequested $aisidCliOutput -DefenderUsageAppendRequested $aisidCliAppend -DeltaCacheSwitchRequested $aisidCliCache) { + exit 1 +} + +# ============================================================ +# MANUAL -Resume PARAMETER PARSING +# Enables: -Resume (auto-discover) and -Resume "path" (explicit) +# ============================================================ +$Resume = $null +$ResumeSpecified = $false +if ($RemainingArgs -and $RemainingArgs.Count -gt 0) { + for ($i = 0; $i -lt $RemainingArgs.Count; $i++) { + if ($RemainingArgs[$i] -eq '-Resume') { + $ResumeSpecified = $true + # Check if next argument exists and is not another parameter + if (($i + 1) -lt $RemainingArgs.Count -and $RemainingArgs[$i + 1] -notmatch '^-') { + $Resume = $RemainingArgs[$i + 1] + $i++ # Skip the path argument + } else { + $Resume = '' # Auto-discover mode + } + } + } +} + +# ============================================================ +# AISID AVAILABILITY GATE - stored resume contract (read-only pre-scan) +# When -Resume names an unambiguous checkpoint, its stored contract is inspected +# read-only here - before any module load, sign-in, destination probe, checkpoint +# mutation, audit query, collection, or publication - so a stored AISID contract is +# stopped at the same point as an explicit AISID switch. An ambiguous selection is +# left to the post-restoration guard, which still runs before sign-in, collection, +# checkpoint mutation, and publication. Nothing is created, written, or modified here. +# ============================================================ +if ($ResumeSpecified) { + $aisidResumeProbePath = $null + try { + if (-not [string]::IsNullOrWhiteSpace($Resume)) { + if (Test-Path -LiteralPath $Resume -PathType Leaf) { $aisidResumeProbePath = $Resume } + elseif (Test-Path -LiteralPath $Resume -PathType Container) { + $aisidProbeFound = @(Get-ChildItem -LiteralPath $Resume -Filter '.pax_checkpoint_*.json' -File -ErrorAction SilentlyContinue) + if ($aisidProbeFound.Count -eq 1) { $aisidResumeProbePath = $aisidProbeFound[0].FullName } + } + } else { + $aisidProbeDirs = @() + if (-not [string]::IsNullOrWhiteSpace($OutputPath)) { $aisidProbeDirs += $OutputPath } + $aisidProbeDirs += (Get-Location).Path + foreach ($aisidProbeDir in $aisidProbeDirs) { + if (-not (Test-Path -LiteralPath $aisidProbeDir -PathType Container)) { continue } + $aisidProbeFound = @(Get-ChildItem -LiteralPath $aisidProbeDir -Filter '.pax_checkpoint_*.json' -File -ErrorAction SilentlyContinue) + if ($aisidProbeFound.Count -eq 1) { $aisidResumeProbePath = $aisidProbeFound[0].FullName; break } + if ($aisidProbeFound.Count -gt 1) { break } + } + } + } catch { $aisidResumeProbePath = $null } + if ($aisidResumeProbePath) { + $aisidProbeDash = $false; $aisidProbeOut = $false; $aisidProbeApp = $false; $aisidProbeCache = $false + try { + $aisidProbeData = Get-Content -LiteralPath $aisidResumeProbePath -Raw -ErrorAction Stop | ConvertFrom-Json -AsHashtable -ErrorAction Stop + if ($aisidProbeData) { + if ($aisidProbeData.Contains('aisidWindow') -and $aisidProbeData.aisidWindow) { $aisidProbeDash = $true } + if ($aisidProbeData.Contains('aisidResumeStateVersion')) { $aisidProbeDash = $true } + if ($aisidProbeData.Contains('parameters') -and $aisidProbeData.parameters) { + $aisidProbeParams = $aisidProbeData.parameters + if ($aisidProbeParams.Contains('dashboard') -and ([string]$aisidProbeParams.dashboard).Trim().ToUpperInvariant() -eq 'AISID') { $aisidProbeDash = $true } + if ($aisidProbeParams.Contains('rollupDashboard') -and ([string]$aisidProbeParams.rollupDashboard).Trim().ToUpperInvariant() -eq 'AISID') { $aisidProbeDash = $true } + $aisidProbeOut = ($aisidProbeParams.Contains('outputPathDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$aisidProbeParams.outputPathDefenderUsage)) + $aisidProbeApp = ($aisidProbeParams.Contains('appendDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$aisidProbeParams.appendDefenderUsage)) + $aisidProbeCache = ($aisidProbeParams.Contains('disableAISIDDeltaCache') -and [bool]$aisidProbeParams.disableAISIDDeltaCache) + } + } + } catch { } + if (script:Test-AISIDEntryPointGated -DashboardIsAISID $aisidProbeDash -DefenderUsageOutputRequested $aisidProbeOut -DefenderUsageAppendRequested $aisidProbeApp -DeltaCacheSwitchRequested $aisidProbeCache) { + exit 1 + } + } +} + +# ============================================================ +# PROMOTE AUTH PARAMETERS TO SCRIPT SCOPE +# Enables access from within functions (e.g., Connect-PurviewAudit) +# ============================================================ +$script:TenantId = $TenantId +$script:ClientId = $ClientId +$script:ClientSecret = $ClientSecret +$script:ClientCertificateThumbprint = $ClientCertificateThumbprint +$script:ClientCertificateStoreLocation = $ClientCertificateStoreLocation +$script:ClientCertificatePath = $ClientCertificatePath +$script:ClientCertificatePassword = $ClientCertificatePassword + + function Resolve-CommaSeparatedValues { + param([string[]]$Values) + + if (-not $Values -or $Values.Count -eq 0) { + return $Values + } + + $finalActivityTypes = @() + + # Step 1: Add explicit -ActivityTypes parameter values (if provided and not default) + if ($PSBoundParameters.ContainsKey('ActivityTypes') -and $ActivityTypes) { + foreach ($actType in $ActivityTypes) { + if ($actType -and $actType -ne '') { + $finalActivityTypes += $actType + } + } + if ($finalActivityTypes.Count -gt 0) { + Write-LogHost "Custom ActivityTypes provided: $($finalActivityTypes -join ', ')" -ForegroundColor Gray + } + } + + # Step 3: Add CopilotInteraction when explicitly requested + if ($IncludeCopilotInteraction -and -not ($finalActivityTypes -contains $copilotBaseActivityType)) { + $finalActivityTypes += $copilotBaseActivityType + Write-LogHost "IncludeCopilotInteraction: Adding $copilotBaseActivityType (explicit request)" -ForegroundColor Cyan + } + + # Step 4: Add Microsoft 365 usage bundle when requested + Write-LogHost ("IncludeM365Usage switch present: {0}" -f $IncludeM365Usage.IsPresent) -ForegroundColor DarkGray + if ($IncludeM365Usage) { + $finalActivityTypes += $m365UsageActivityBundle + Write-LogHost ("M365 Usage bundle: Adding {0} activity types across Exchange/SharePoint/OneDrive/Teams" -f $m365UsageActivityBundle.Count) -ForegroundColor Cyan + + # -IncludeM365Usage always implies -CombineOutput. The M365 usage bundle spans 30+ + # activity types across 4 workloads; per-activity-type splits produce unmanageable file + # counts and are incompatible with the rollup post-processor (which consumes a single + # CSV). Auto-enable -CombineOutput once, with a one-line info message (no prompt). + if (-not $CombineOutput) { + $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) + Write-LogHost "M365 Usage mode: -CombineOutput auto-enabled (M365 bundle always produces a single combined output)." -ForegroundColor Cyan + } + + $RecordTypes = @( + if ($RecordTypes) { $RecordTypes } + $m365UsageRecordBundle + ) | Where-Object { $_ } | Select-Object -Unique + if ($RecordTypes.Count -eq 0) { $RecordTypes = $null } + + # CRITICAL: Do NOT set ServiceTypes for M365 usage mode - Graph API should get ALL workloads in single pass + # Multiple serviceFilter values cause unnecessary workload splits (Exchange, SharePoint, OneDrive, Teams) + # Instead, send NO serviceFilter and let Graph API return all workloads in one query per partition + $ServiceTypes = $null + + if ($RecordTypes) { + Write-LogHost "M365 Usage bundle: RecordTypes => $($RecordTypes -join ', ')" -ForegroundColor Gray + } + Write-LogHost "M365 Usage mode: ServiceTypes => NULL (single workload pass, all services combined)" -ForegroundColor Cyan + } + + # Step 5: BASE ACTIVITY TYPE - Add CopilotInteraction as default base type + # This is the core Microsoft 365 Copilot activity type (FREE, included in M365 Copilot licensing) + # Captures ALL M365 Copilot usage including Teams meetings, Word, Excel, PowerPoint, Outlook, etc. + # Auto-add when the user did not explicitly provide -ActivityTypes. + # Exception: Always respect -ExcludeCopilotInteraction (handled in Step 6) + $userProvidedCustomTypes = $PSBoundParameters.ContainsKey('ActivityTypes') + if (-not $ExcludeCopilotInteraction) { + # Auto-add only when no custom types were provided + if (-not $userProvidedCustomTypes) { + # Add CopilotInteraction if not already present + if (-not ($finalActivityTypes -contains $copilotBaseActivityType)) { + $finalActivityTypes = @($copilotBaseActivityType) + $finalActivityTypes + } + } + } + + # Step 6: EXCLUSION OVERRIDE - Remove CopilotInteraction if -ExcludeCopilotInteraction is true + if ($ExcludeCopilotInteraction) { + $finalActivityTypes = $finalActivityTypes | Where-Object { $_ -ne $copilotBaseActivityType } + } + + # Step 7: Final deduplication and validation + $finalActivityTypes = @($finalActivityTypes | Select-Object -Unique) + + return $finalActivityTypes +} + +# ============================================================ +# AI Solutions Catalog (customer-editable dimension table) +# ============================================================ +# Single source of truth for AI-app detection and the dashboard's AI_Solutions +# dimension. To extend coverage for your tenant, add a hashtable entry using the +# same keys as the existing entries; the detection-surface lists +# (CloudAppNames / SignInAppNames) may be partial — any non-empty match classifies +# an event. The catalog is also written to the AI solutions catalog CSV so the +# dashboard's AI_Solutions dimension can consume it directly. +# KEY SEMANTICS: +# AISolution - Primary key; must be unique; appears in every AISID output. +# Category - Enterprise AI | Developer AI | Unmanaged AI +# Vendor - Human-readable vendor/publisher name. +# RiskTier - Enterprise AI | Low | Medium | High (governance tier) +# DefaultDataHandling - Short narrative of the product's default data-handling posture. +# SolutionGroup - Dashboard grouping key ('Microsoft 365 Copilot', 'ChatGPT', +# 'Claude', 'Gemini', 'Perplexity', 'Grok', or 'All Other AI'). +# CloudAppNames - Values matched against Defender CloudAppEvents application names. +# SignInAppNames - Values matched against Entra and Graph sign-in application names. +# NetworkDomains - AI service hostnames matched against Defender device network event URLs +# (empty when no authoritative hostname is known for the entry). +# ============================================================ +$script:AISolutionCatalog = @( + @{ AISolution = 'Microsoft 365 Copilot'; Category = 'Enterprise AI'; Vendor = 'Microsoft'; RiskTier = 'Sanctioned'; DefaultDataHandling = 'Customer controlled, no training on tenant data'; SolutionGroup = 'Microsoft Copilot'; CloudAppNames = @('Microsoft 365 Copilot','Microsoft Copilot','Microsoft 365 Chat'); SignInAppNames = @('Microsoft 365 Copilot','Microsoft Copilot'); NetworkDomains = @('copilot.microsoft.com','copilot.cloud.microsoft') } + @{ AISolution = 'GitHub Copilot'; Category = 'Developer AI'; Vendor = 'GitHub'; RiskTier = 'Sanctioned'; DefaultDataHandling = 'Per enterprise contract'; SolutionGroup = 'Microsoft Copilot'; CloudAppNames = @('GitHub Copilot','GitHub.com'); SignInAppNames = @('GitHub Copilot','GitHub'); NetworkDomains = @() } + @{ AISolution = 'Cursor'; Category = 'Developer AI'; Vendor = 'Anysphere'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Code stored per tier'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Cursor'); SignInAppNames = @('Cursor'); NetworkDomains = @() } + @{ AISolution = 'ChatGPT'; Category = 'Unmanaged AI'; Vendor = 'OpenAI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Training by default on free tier'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('ChatGPT','ChatGPT Enterprise','OpenAI ChatGPT'); SignInAppNames = @('ChatGPT Enterprise','OpenAI'); NetworkDomains = @('chat.openai.com','chatgpt.com','api.openai.com') } + @{ AISolution = 'Claude'; Category = 'Unmanaged AI'; Vendor = 'Anthropic'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Not used for training (paid tiers)'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('Claude','Claude for Work','Anthropic Claude'); SignInAppNames = @('Claude for Work','Anthropic'); NetworkDomains = @('claude.ai','api.anthropic.com') } + @{ AISolution = 'Gemini'; Category = 'Unmanaged AI'; Vendor = 'Google'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Not used for training (Workspace tier)'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('Google Gemini','Gemini'); SignInAppNames = @('Google Gemini'); NetworkDomains = @('gemini.google.com','bard.google.com') } + @{ AISolution = 'Perplexity'; Category = 'Unmanaged AI'; Vendor = 'Perplexity'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Not disclosed in detail'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('Perplexity','Perplexity Enterprise'); SignInAppNames = @('Perplexity Enterprise','Perplexity'); NetworkDomains = @('perplexity.ai') } + @{ AISolution = 'Mistral'; Category = 'Unmanaged AI'; Vendor = 'Mistral AI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'EU-hosted'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Mistral','Mistral AI'); SignInAppNames = @('Mistral'); NetworkDomains = @() } + @{ AISolution = 'HuggingFace'; Category = 'Unmanaged AI'; Vendor = 'Hugging Face'; RiskTier = 'Unsanctioned'; DefaultDataHandling = 'Model-dependent'; SolutionGroup = 'Shadow AI'; CloudAppNames = @('HuggingFace','Hugging Face'); SignInAppNames = @('HuggingFace','Hugging Face'); NetworkDomains = @('huggingface.co') } + @{ AISolution = 'Poe'; Category = 'Unmanaged AI'; Vendor = 'Quora'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Per aggregator model'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Poe'); SignInAppNames = @('Poe'); NetworkDomains = @() } + @{ AISolution = 'Grok'; Category = 'Unmanaged AI'; Vendor = 'xAI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Training by default'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Grok','xAI'); SignInAppNames = @('Grok','xAI'); NetworkDomains = @() } + @{ AISolution = 'DeepSeek'; Category = 'Unmanaged AI'; Vendor = 'DeepSeek'; RiskTier = 'Unresolved'; DefaultDataHandling = 'PRC data residency'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('DeepSeek'); SignInAppNames = @('DeepSeek'); NetworkDomains = @() } + @{ AISolution = 'Character.AI'; Category = 'Unmanaged AI'; Vendor = 'Character.AI'; RiskTier = 'Unresolved'; DefaultDataHandling = 'Public by default'; SolutionGroup = 'Needs Classification'; CloudAppNames = @('Character.AI','Character AI'); SignInAppNames = @('Character.AI','Character AI'); NetworkDomains = @() } +) + +# ---- AI Solutions Catalog helper functions (pure lookups, no I/O) ---- + +function Get-AISIDFilteredCatalog { + <# + .SYNOPSIS + Returns the AI solutions catalog. + #> + [OutputType([System.Collections.IList])] + param() + return $script:AISolutionCatalog +} + +function Get-AISIDCloudAppNames { + <# + .SYNOPSIS + Returns a de-duplicated flat array of cloud-app detection names. + #> + [OutputType([string[]])] + param() + $cat = Get-AISIDFilteredCatalog + return @($cat | ForEach-Object { $_.CloudAppNames } | Where-Object { $_ } | Select-Object -Unique) +} + +function Get-AISIDSignInAppNames { + <# + .SYNOPSIS + Returns a de-duplicated flat array of sign-in application detection names. + #> + [OutputType([string[]])] + param() + $cat = Get-AISIDFilteredCatalog + return @($cat | ForEach-Object { $_.SignInAppNames } | Where-Object { $_ } | Select-Object -Unique) +} + +function Resolve-AISIDFromAppName { + <# + .SYNOPSIS + Resolves an application display name to its AI solutions catalog entry. + Returns $null when no match is found. + .NOTES + Matches against both CloudAppNames and SignInAppNames (case-insensitive). + #> + [OutputType([System.Collections.IDictionary])] + param([Parameter(Mandatory)][string]$AppName) + + if ([string]::IsNullOrWhiteSpace($AppName)) { return $null } + $needle = $AppName.Trim().ToLowerInvariant() + $cat = Get-AISIDFilteredCatalog + foreach ($entry in $cat) { + $names = @() + if ($entry.CloudAppNames) { $names += $entry.CloudAppNames } + if ($entry.SignInAppNames) { $names += $entry.SignInAppNames } + foreach ($n in $names) { + if ($n -and $n.ToLowerInvariant() -eq $needle) { return $entry } + } + } + return $null +} + +function Get-AISIDNetworkDomains { + <# + .SYNOPSIS + Returns a de-duplicated flat array of AI service network domains (hostnames) for host/URL + matching. Built from the catalog's NetworkDomains lists; entries with no known hostname + contribute nothing. + #> + [OutputType([string[]])] + param() + $cat = Get-AISIDFilteredCatalog + return @($cat | ForEach-Object { $_.NetworkDomains } | Where-Object { $_ } | Select-Object -Unique) +} + +function Resolve-AISIDFromDomain { + <# + .SYNOPSIS + Resolves a host / FQDN to its AI solutions catalog entry, or $null when no match is found. + .NOTES + Matches against the catalog's NetworkDomains using endswith semantics, so a subdomain + (for example 'api.openai.com') resolves to a catalog entry that lists a parent domain. + #> + [OutputType([System.Collections.IDictionary])] + param([Parameter(Mandatory)][string]$HostName) + + if ([string]::IsNullOrWhiteSpace($HostName)) { return $null } + $needle = $HostName.Trim().ToLowerInvariant() + $cat = Get-AISIDFilteredCatalog + foreach ($entry in $cat) { + if (-not $entry.NetworkDomains) { continue } + foreach ($d in $entry.NetworkDomains) { + if (-not $d) { continue } + $lc = ([string]$d).ToLowerInvariant() + if ($needle -eq $lc -or $needle.EndsWith('.' + $lc)) { return $entry } + } + } + return $null +} + +# Microsoft advanced hunting keeps data queryable for the last 30 days by default; reaching further +# back requires tenant-side streaming of specific tables into Microsoft Sentinel with extended +# retention, which this script cannot detect or assume. The window is fixed here as the single +# source of truth. Ref: https://learn.microsoft.com/en-us/defender-xdr/advanced-hunting-overview +$script:AISIDLookbackDaysFixed = 30 + +function Get-AISIDEffectiveLookbackDays { + <# + .SYNOPSIS + Returns the AI Solutions Intelligence Dashboard lookback window, in days. + #> + [OutputType([int])] + param() + return $script:AISIDLookbackDaysFixed +} + +function Resolve-AISIDDateWindow { + <##SYN Pure resolver for the AISID requested-vs-available date-window intersection. + Side-effect-free: no Get-Date, no logging, no prompts, no exits, no script-scope reads/writes. + All inputs are concrete UTC instants; requested and available intervals are half-open [start,end). + AvailableStartUtc = ReferenceNowUtc - RetentionDays; AvailableEndUtc = ReferenceNowUtc. + Effective = requested INTERSECT available. Status FULL = effective==requested (no clip); + PARTIAL = intervals overlap but effective differs; NONE = EffectiveStartUtc >= EffectiveEndUtc + (no overlap; an empty requested interval is NONE, never FULL). #> + [CmdletBinding()] + [OutputType([pscustomobject])] + param( + [Parameter(Mandatory)][datetime]$RequestedStartUtc, + [Parameter(Mandatory)][datetime]$RequestedEndUtc, + [Parameter(Mandatory)][datetime]$ReferenceNowUtc, + [int]$RetentionDays = 30 + ) + $reqStart = [datetime]::SpecifyKind($RequestedStartUtc, [System.DateTimeKind]::Utc) + $reqEnd = [datetime]::SpecifyKind($RequestedEndUtc, [System.DateTimeKind]::Utc) + $refNow = [datetime]::SpecifyKind($ReferenceNowUtc, [System.DateTimeKind]::Utc) + $availStart = $refNow.AddDays(-$RetentionDays) + $availEnd = $refNow + $effStart = if ($reqStart -gt $availStart) { $reqStart } else { $availStart } + $effEnd = if ($reqEnd -lt $availEnd) { $reqEnd } else { $availEnd } + $startClipped = ($effStart -ne $reqStart) + $endClipped = ($effEnd -ne $reqEnd) + if ($effStart -ge $effEnd) { + $status = 'NONE'; $wasClipped = $true + $clipReason = 'Requested interval does not overlap the available retention window.' + } + elseif (-not $startClipped -and -not $endClipped) { + $status = 'FULL'; $wasClipped = $false; $clipReason = '' + } + else { + $status = 'PARTIAL'; $wasClipped = $true + $parts = @() + if ($startClipped) { $parts += ("requested start {0:yyyy-MM-ddTHH:mm:ssZ} precedes the {1}-day retention floor {2:yyyy-MM-ddTHH:mm:ssZ}" -f $reqStart, $RetentionDays, $availStart) } + if ($endClipped) { $parts += ("requested end {0:yyyy-MM-ddTHH:mm:ssZ} exceeds the current retention ceiling {1:yyyy-MM-ddTHH:mm:ssZ}" -f $reqEnd, $availEnd) } + $clipReason = ($parts -join '; ') + '.' + } + return [pscustomobject][ordered]@{ + RequestedStartUtc = $reqStart + RequestedEndUtc = $reqEnd + ReferenceNowUtc = $refNow + AvailableStartUtc = $availStart + AvailableEndUtc = $availEnd + EffectiveStartUtc = $effStart + EffectiveEndUtc = $effEnd + Status = $status + WasClipped = $wasClipped + ClipReason = $clipReason + RetentionDays = [int]$RetentionDays + } +} + +function Get-AISIDCollectionBounds { + <##SYN Returns the frozen AISID collection interval [StartUtc, EndUtc) - the resolved + requested-intersect-available effective window - as a two-key hashtable. Every date-bearing + AISID collector reads its bounds from here so the whole run shares one window instant. #> + [CmdletBinding()] + param() + if ($script:AISIDWindow -and $script:AISIDWindow.EffectiveStartUtc -and $script:AISIDWindow.EffectiveEndUtc) { + return @{ + StartUtc = [datetime]::SpecifyKind([datetime]$script:AISIDWindow.EffectiveStartUtc, [System.DateTimeKind]::Utc) + EndUtc = [datetime]::SpecifyKind([datetime]$script:AISIDWindow.EffectiveEndUtc, [System.DateTimeKind]::Utc) + } + } + throw "AISID collection bounds requested before the AISID date window was established (script:AISIDWindow is missing EffectiveStartUtc/EffectiveEndUtc). This is a control-flow invariant violation: every valid fresh or resumed AISID run must resolve and freeze the window before any collector runs." +} + +function Write-AISIDWindowSummary { + <##SYN Prints the one-line AISID collection-window summary (terminal + log) for a completed run. + FULL/PARTIAL only; a NONE run exits before any collection and never reaches here. #> + [CmdletBinding()] + param() + if (-not $script:AISIDWindow) { return } + $w = $script:AISIDWindow + $fmt = 'yyyy-MM-ddTHH:mm:ssZ' + $reqStr = ('[{0} .. {1})' -f ([datetime]$w.RequestedStartUtc).ToString($fmt), ([datetime]$w.RequestedEndUtc).ToString($fmt)) + $effStr = ('[{0} .. {1})' -f ([datetime]$w.EffectiveStartUtc).ToString($fmt), ([datetime]$w.EffectiveEndUtc).ToString($fmt)) + if ([string]$w.Status -eq 'PARTIAL') { + Write-LogHost ("AISID collection window: PARTIAL | requested {0} | effective {1} | reason: {2}" -f $reqStr, $effStr, [string]$w.ClipReason) -ForegroundColor Yellow + } + else { + Write-LogHost ("AISID collection window: FULL | requested {0} | effective {1}" -f $reqStr, $effStr) -ForegroundColor Green + } +} + +# ============================================================ +# AI Solutions Intelligence Dashboard — Graph hunting rate limiter +# ============================================================ +# Microsoft's advanced hunting API guarantees at least 45 calls/min per tenant (higher for +# larger tenants) and enforces a per-tenant CPU budget that returns HTTP 429 when exhausted. +# This limiter paces proactively well under that floor and drains reactively on a rate-limit +# response. Ref: https://learn.microsoft.com/en-us/graph/api/resources/security-api-overview +$script:AISIDTokenBucket = [hashtable]::Synchronized(@{ + IntervalMs = 2000 # Proactive floor: 30 calls/min (headroom under the >=45/min guarantee) + BackoffIntervalMs = 4000 # Slowed cadence after a rate-limit response: 15 calls/min + LastEmit = [DateTime]::MinValue + BackoffUntil = [DateTime]::MinValue + PermitCount = 0 + Lock = [object]::new() +}) + +function Set-AISIDTokenBucketBackoff { + <# + .SYNOPSIS + Slows the hunting-query rate limiter to its backoff cadence for a set duration after a + rate-limit response. + #> + [CmdletBinding()] + param([int]$BackoffDurationSeconds = 60) + $script:AISIDTokenBucket.BackoffUntil = (Get-Date).AddSeconds($BackoffDurationSeconds) + Write-LogHost "[AISID] Rate-limit backoff engaged; hunting-query cadence slowed to $($script:AISIDTokenBucket.BackoffIntervalMs)ms for ${BackoffDurationSeconds}s." -ForegroundColor Yellow +} + +function Invoke-WithAISIDTokenBucket { + <# + .SYNOPSIS + Runs a scriptblock through the hunting-query rate limiter, waiting for the next permit + (base or backoff cadence) before invoking it and returning its result unchanged. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][scriptblock]$ScriptBlock, + [string]$Description = '' + ) + + $bucket = $script:AISIDTokenBucket + $gotLock = $false + try { + [System.Threading.Monitor]::Enter($bucket.Lock, [ref]$gotLock) + $now = Get-Date + $intervalMs = if ($now -lt $bucket.BackoffUntil) { $bucket.BackoffIntervalMs } else { $bucket.IntervalMs } + if ($bucket.LastEmit -ne [DateTime]::MinValue) { + $elapsedMs = ($now - $bucket.LastEmit).TotalMilliseconds + if ($elapsedMs -lt $intervalMs) { + $waitMs = [int]($intervalMs - $elapsedMs) + [System.Threading.Monitor]::Exit($bucket.Lock); $gotLock = $false + if ($waitMs -ge 1000 -and $Description) { + Write-LogHost "[AISID] Waiting ${waitMs}ms for the next query permit ($Description)..." -ForegroundColor DarkGray + } + Start-Sleep -Milliseconds $waitMs + [System.Threading.Monitor]::Enter($bucket.Lock, [ref]$gotLock) + } + } + $bucket.LastEmit = Get-Date + $bucket.PermitCount++ + if ($gotLock) { [System.Threading.Monitor]::Exit($bucket.Lock); $gotLock = $false } + } + catch { + if ($gotLock) { [System.Threading.Monitor]::Exit($bucket.Lock); $gotLock = $false } + throw + } + return (& $ScriptBlock) +} + +function Invoke-GraphHuntingQuery { + <# + .SYNOPSIS + Runs a Kusto advanced-hunting query through Microsoft Graph and returns a normalized + result. Single entry point for the dashboard's hunting queries. + .DESCRIPTION + Posts to the Microsoft Graph advanced-hunting endpoint over the run's existing Graph + session (Invoke-MgGraphRequest), paced by the shared rate limiter and backing off + reactively on a rate-limit response. The collection window is the fixed lookback + (Get-AISIDEffectiveLookbackDays) expressed as an ISO 8601 timespan. + Ref: https://learn.microsoft.com/en-us/graph/api/security-security-runhuntingquery + .OUTPUTS + Hashtable (never throws): Success, Results, Schema, RowCount, TruncationDetected, + ErrorMessage, HttpStatusCode, AuthQuit, Elapsed. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Query, + [string]$Description = '', + [int]$MaxTransientRetries = 3 + ) + + # Advanced hunting returns at most 100,000 rows (and 50 MB) per query; a result at or near + # the row ceiling signals possible silent truncation and is surfaced as a warning. + # Ref: https://learn.microsoft.com/en-us/graph/api/resources/security-api-overview + $rowCeiling = 100000 + $truncationWarnAt = 99000 + + $result = @{ + Success = $false; Results = $null; Schema = $null; RowCount = 0; TruncationDetected = $false + ErrorMessage = ''; HttpStatusCode = 0; AuthQuit = $false; Elapsed = [TimeSpan]::Zero + } + + $lookbackDays = Get-AISIDEffectiveLookbackDays + $endpoint = 'https://graph.microsoft.com/v1.0/security/runHuntingQuery' + $body = @{ Query = $Query; Timespan = "P${lookbackDays}D" } | ConvertTo-Json -Compress -Depth 4 + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $attempt = 0 + $tokenRetryConsumed = $false + + while ($true) { + $attempt++ + + # Keep the shared Graph token fresh through the existing refresh contract. + $refresh = Refresh-GraphTokenIfNeeded + if ($refresh -eq 'Quit') { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $httpStatus = 0; $retryAfter = 0; $errMsg = ''; $payload = $null + try { + $payload = Invoke-WithAISIDTokenBucket -Description $Description -ScriptBlock { + Invoke-MgGraphRequest -Method POST -Uri $endpoint -Body $body -ContentType 'application/json' -OutputType PSObject -ErrorAction Stop + } + $httpStatus = 200 + } + catch { + $errMsg = $_.Exception.Message + $httpStatus = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($httpStatus -eq 0 -and $errMsg -match '429|Too Many Requests|TooManyRequests') { $httpStatus = 429 } + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + } + $result.HttpStatusCode = $httpStatus + + if ($payload -and $httpStatus -ge 200 -and $httpStatus -lt 300) { + $rows = @($payload.results) + $result.Success = $true; $result.Results = $rows; $result.Schema = $payload.schema; $result.RowCount = $rows.Count + $result.TruncationDetected = ($rows.Count -ge $truncationWarnAt) + if ($result.TruncationDetected) { + Write-LogHost "[AISID] Query '$Description' returned $($rows.Count) rows, at or near the $rowCeiling-row limit; results may be truncated." -ForegroundColor Yellow + } + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + if ($httpStatus -eq 401 -and -not $tokenRetryConsumed) { + $tokenRetryConsumed = $true + $r = Refresh-GraphTokenIfNeeded + if ($r -eq 'Quit') { + $result.AuthQuit = $true; $result.ErrorMessage = 'Unauthorized and token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + continue + } + + if ($httpStatus -eq 429) { + $waitSec = if ($retryAfter -gt 0) { $retryAfter } else { 60 } + Write-LogHost "[AISID] Rate limited (HTTP 429); waiting ${waitSec}s before retrying ($Description)." -ForegroundColor Yellow + Set-AISIDTokenBucketBackoff -BackoffDurationSeconds 60 + Start-Sleep -Seconds $waitSec + continue + } + + $transient = ($httpStatus -ge 500 -or $httpStatus -eq 408 -or $httpStatus -eq 0) + if ($transient -and $attempt -le $MaxTransientRetries) { + $backoffSec = [int][Math]::Min(60, [Math]::Pow(2, $attempt)) + Write-LogHost "[AISID] Transient failure (HTTP $httpStatus, attempt $attempt/$MaxTransientRetries); retrying in ${backoffSec}s. $errMsg" -ForegroundColor Yellow + Start-Sleep -Seconds $backoffSec + continue + } + + $result.ErrorMessage = "Graph hunting query failed (HTTP $httpStatus, attempt $attempt): $errMsg" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } +} + +function Get-AISIDHuntingRowEstimate { + <# + .SYNOPSIS + Runs a count-only hunting query and returns the estimated row count for the given KQL, or + -1 when the probe could not be evaluated. Used to decide whether a result set would exceed + the advanced-hunting response cap and therefore needs time-window subdivision. + .OUTPUTS + [long] estimated row count, or -1 on probe failure / auth quit. + #> + [OutputType([long])] + param( + [Parameter(Mandatory)][string]$ProbeQuery, + [string]$Description = 'row-estimate probe' + ) + $r = Invoke-GraphHuntingQuery -Query $ProbeQuery -Description $Description + if ($r.AuthQuit -or -not $r.Success) { return -1 } + $rows = @($r.Results) + if ($rows.Count -eq 0) { return 0 } + # The probe ends in `summarize N = count()`, so the single result row must carry a numeric field + # named exactly 'N'. OData annotation companions ('N@odata.type' = '#Int64', and any top-level + # '@odata.*') are ignored structurally. The count is read ONLY from a real 'N' property; an unrelated + # lone numeric property is NEVER accepted. Missing, non-numeric, or duplicate 'N' => unreadable (-1). + $nProps = @($rows[0].PSObject.Properties | Where-Object { $_.Name -eq 'N' }) + if ($nProps.Count -ne 1) { return -1 } + $long = [long]0 + if ($null -eq $nProps[0].Value -or -not [long]::TryParse([string]$nProps[0].Value, [ref]$long)) { return -1 } + return [long]$long +} + +function Get-AISIDTimePartitions { + <# + .SYNOPSIS + Plans a set of time sub-windows whose individual estimated row counts stay under the + advanced-hunting response cap. Probes the given window via -EstimateProvider; if the + estimate is under the cap the window is returned whole, otherwise it is split into + ceil(estimate / target) equal sub-windows, each planned recursively (re-probed, so a + non-uniform distribution is handled by splitting only the ranges that are actually hot). + Subdivision stops at -FloorMinutes; a floor window still over the cap is returned with + Truncated set and a warning, consistent with the query-level truncation warning. + .OUTPUTS + Array of ordered hashtables @{ StartUtc; EndUtc; Estimate; Truncated }. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][datetime]$StartUtc, + [Parameter(Mandatory)][datetime]$EndUtc, + [Parameter(Mandatory)][scriptblock]$EstimateProvider, + [long]$Cap = 90000, + [long]$Target = 80000, + [int]$FloorMinutes = 60 + ) + + $estimate = [long](& $EstimateProvider $StartUtc $EndUtc) + $spanMinutes = ($EndUtc - $StartUtc).TotalMinutes + + # An unreadable probe (-1) must NEVER be treated as "cannot rule out over-cap" and subdivided: that + # produced an unbounded recursive fan-out down to the minute floor. Return a single window flagged + # EstimateFailed so the collector fails this table explicitly instead of silently truncating/emptying. + if ($estimate -lt 0) { + return , ([ordered]@{ StartUtc = $StartUtc; EndUtc = $EndUtc; Estimate = $estimate; Truncated = $false; EstimateFailed = $true }) + } + + # Under the cap (and a successful probe): take the window whole. + if ($estimate -lt $Cap) { + return , ([ordered]@{ StartUtc = $StartUtc; EndUtc = $EndUtc; Estimate = $estimate; Truncated = $false; EstimateFailed = $false }) + } + # At/over the cap (or an unreadable probe, treated as "cannot rule out over-cap") but already at + # the minimum granularity: accept this window and flag possible truncation. + if ($spanMinutes -le $FloorMinutes) { + $trunc = ($estimate -ge $Cap) + if ($trunc) { + Write-LogHost ("[AISID] FileProximity: window {0:o}..{1:o} still estimates {2} rows at the {3}-minute floor; this window may be truncated at the {4}-row hunting limit." -f $StartUtc, $EndUtc, $estimate, $FloorMinutes, 100000) -ForegroundColor Yellow + } + return , ([ordered]@{ StartUtc = $StartUtc; EndUtc = $EndUtc; Estimate = $estimate; Truncated = $trunc; EstimateFailed = $false }) + } + + # Split into enough parts to bring each comfortably under target (at least 2), but never so many + # that a sub-window would fall below the floor. Each part is re-probed recursively. + $parts = 2 + if ($estimate -gt 0) { $parts = [Math]::Max(2, [int][Math]::Ceiling($estimate / [double]$Target)) } + $maxParts = [Math]::Max(2, [int][Math]::Floor($spanMinutes / [double]$FloorMinutes)) + if ($parts -gt $maxParts) { $parts = $maxParts } + + $out = @() + $totalTicks = ($EndUtc - $StartUtc).Ticks + for ($i = 0; $i -lt $parts; $i++) { + $subStart = $StartUtc.AddTicks([long]($totalTicks * $i / $parts)) + $subEnd = if ($i -eq ($parts - 1)) { $EndUtc } else { $StartUtc.AddTicks([long]($totalTicks * ($i + 1) / $parts)) } + $out += Get-AISIDTimePartitions -StartUtc $subStart -EndUtc $subEnd -EstimateProvider $EstimateProvider -Cap $Cap -Target $Target -FloorMinutes $FloorMinutes + } + return $out +} + +function Test-AISIDLicensing { + <# + .SYNOPSIS + Evaluates, independently per table, which AI Solutions Intelligence Dashboard data sets + this tenant can populate. There is no combined pass/fail — each table stands on its own, + so a tenant with only Microsoft Entra access still collects the two Entra-sourced tables. + .DESCRIPTION + The four advanced-hunting tables (ActivitySessions from CloudAppEvents, OffHoursGeo from + EntraIdSignInEvents, FileProximity, and ClientChannel) are probed directly with a minimal query; a failed probe marks only + that table unavailable. The remaining three tables draw on data the full run already produces — + OAuthConsents and SSOSignIns from the run's Microsoft Entra access, and CopilotUsage from the + CopilotInteraction rollup — so they are eligible whenever the run is authenticated. + .OUTPUTS + Ordered hashtable keyed by table name, each value @{ Eligible; Source; Detail }, plus an + AuthQuit flag. No early return produces an all-or-nothing verdict. + #> + [CmdletBinding()] + param() + + $verdict = [ordered]@{ + ActivitySessions = @{ Eligible = $false; Source = 'Defender advanced hunting'; Detail = '' } + OffHoursGeo = @{ Eligible = $false; Source = 'Defender advanced hunting'; Detail = '' } + FileProximity = @{ Eligible = $false; Source = 'Defender advanced hunting (MDE)'; Detail = '' } + ClientChannel = @{ Eligible = $false; Source = 'Defender advanced hunting (MDE)'; Detail = '' } + OAuthConsents = @{ Eligible = $true; Source = 'Microsoft Entra audit logs'; Detail = "Available from the run's Entra access." } + SSOSignIns = @{ Eligible = $true; Source = 'Microsoft Entra sign-in logs'; Detail = "Available from the run's Entra access." } + CopilotUsage = @{ Eligible = $true; Source = 'Microsoft Purview (CopilotInteraction rollup)'; Detail = "Available from the run's CopilotInteraction rollup." } + AuthQuit = $false + } + + $huntProbes = [ordered]@{ + ActivitySessions = 'CloudAppEvents | take 1 | project Timestamp' + OffHoursGeo = 'EntraIdSignInEvents | take 1 | project Timestamp' + FileProximity = 'DeviceNetworkEvents | take 1 | project Timestamp' + ClientChannel = 'DeviceNetworkEvents | take 1 | project Timestamp' + } + + Write-LogHost '[AISID] Checking which data sets this tenant can populate...' -ForegroundColor Cyan + foreach ($tbl in $huntProbes.Keys) { + $r = Invoke-GraphHuntingQuery -Query $huntProbes[$tbl] -Description "Eligibility probe: $tbl" + if ($r.AuthQuit) { $verdict.AuthQuit = $true } + if ($r.Success) { + $verdict[$tbl].Eligible = $true + $verdict[$tbl].Detail = 'Source data is queryable.' + } + else { + $verdict[$tbl].Detail = "Source data is not available in this tenant (status $($r.HttpStatusCode))." + } + } + + return $verdict +} + +function Write-AISIDLicensingReport { + <# + .SYNOPSIS + Prints a per-table summary of which AI Solutions Intelligence Dashboard data sets will be + collected for this tenant and which are unavailable. Reports table by table; it never + declares the dashboard as a whole unavailable. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][System.Collections.IDictionary]$Verdict) + + $tables = @('ActivitySessions', 'OffHoursGeo', 'FileProximity', 'ClientChannel', 'OAuthConsents', 'SSOSignIns', 'CopilotUsage') + $eligibleCount = @($tables | Where-Object { $Verdict[$_].Eligible }).Count + + Write-LogHost '' + Write-LogHost "[AISID] Data set availability for this tenant ($eligibleCount of $($tables.Count) available):" -ForegroundColor Cyan + foreach ($tbl in $tables) { + $v = $Verdict[$tbl] + $status = if ($v.Eligible) { 'available' } else { 'not available' } + $color = if ($v.Eligible) { 'Green' } else { 'Yellow' } + Write-LogHost ("[AISID] {0,-18} {1,-14} ({2})" -f $tbl, $status, $v.Source) -ForegroundColor $color + } + if ($eligibleCount -lt $tables.Count) { + Write-LogHost '[AISID] Data sets shown as not available depend on Microsoft Defender or Entra features this tenant does not currently expose. The available data sets are collected normally.' -ForegroundColor Gray + } + Write-LogHost '' +} + +# ============================================================ +# AI Solutions Intelligence Dashboard — table export contract + export/merge infrastructure +# ============================================================ +# The dashboard model loads each CSV by a FIXED name from one folder and applies a fixed, +# ordered TransformColumnTypes; a column-order or name mismatch surfaces as a hard Power Query +# refresh error on the customer side, so the header order below is contractual. Column names, +# order, and the merge key grain are sourced from the model schema captured in +# PBIT_GROUND_TRUTH.md (table AI_Activity) and the FileProximity stub contract — never typed +# from memory. The FileProximity entry is the permanent header-only stub (always empty). +$script:AISIDTableSchemas = @{ + ActivitySessions = @{ + FileName = 'ai_activity_sessions.csv' + Columns = @('UPN', 'AISolution', 'YearMonth', 'Sessions', 'ActiveDays', 'EstimatedPrompts', 'DistinctDevices', 'Category', 'RiskTier') + Key = @('UPN', 'AISolution', 'YearMonth') + } + FileProximity = @{ + FileName = 'ai_file_proximity.csv' + Columns = @('Timestamp', 'UPN', 'AISolution', 'YearMonth', 'FileName', 'FolderCategory', 'FolderPath', 'SecondsToAI', 'NameMatchesSensitivePattern', 'FolderMatchesSensitive') + Key = @('Timestamp', 'UPN', 'FileName') + } + OffHoursGeo = @{ + FileName = 'ai_offhours_geo.csv' + Columns = @('UPN', 'YearMonth', 'TotalSessions', 'OffHoursSessions', 'OffHoursPct', 'DistinctCountries', 'AnomalousCountryCount', 'AnomalousCountries') + Key = @('UPN', 'YearMonth') + } + OAuthConsents = @{ + FileName = 'ai_oauth_consents.csv' + Columns = @('UPN', 'AppName', 'YearMonth', 'ConsentCount', 'LastConsent', 'PermissionWeight', 'Permissions') + Key = @('UPN', 'AppName', 'YearMonth') + } + SSOSignIns = @{ + FileName = 'ai_sso_signins.csv' + Columns = @('UPN', 'Application', 'YearMonth', 'SignInCount', 'DistinctDays', 'IsGuest', 'Countries', 'HasConditionalAccess', 'LastSignIn') + Key = @('UPN', 'Application', 'YearMonth') + } + ClientChannel = @{ + FileName = 'ai_client_channel.csv' + Columns = @('AISite', 'Channel', 'YearMonth', 'EventCount') + Key = @('AISite', 'Channel', 'YearMonth') + } + CopilotUsage = @{ + FileName = 'ai_copilot_usage_graph.csv' + Columns = @('UserPrincipalName', 'YearMonth', 'TeamsPrompts', 'WordPrompts', 'ExcelPrompts', 'OutlookPrompts', 'PowerPointPrompts', 'ChatPrompts', 'TotalPrompts', 'ActiveDays', 'LastActivityDate') + Key = @('UserPrincipalName', 'YearMonth') + } + AppGovAlerts = @{ + FileName = 'ai_appgov_alerts.csv' + Columns = @('Timestamp', 'YearMonth', 'UPN', 'AppName', 'AlertType', 'Severity', 'Description') + Key = @() + } + CloudDiscovery = @{ + FileName = 'ai_cloud_discovery.csv' + Columns = @('AIDomain', 'AppCategory', 'YearMonth', 'RiskScore', 'UploadVolumeMB', 'DownloadVolumeMB', 'TransactionCount', 'DistinctUsers', 'SanctionStatus') + Key = @() + } + MDASessions = @{ + FileName = 'ai_mda_sessions.csv' + Columns = @('Timestamp', 'YearMonth', 'UPN', 'AppName', 'ActionType', 'PolicyHit', 'PolicyAction', 'IPAddress', 'CountryCode', 'EventCount') + Key = @() + } + EntraUsers = @{ + FileName = 'EntraUsers.csv' + Columns = @('userPrincipalName', 'displayName', 'department', 'jobTitle', 'city', 'country', 'companyName', 'accountEnabled', 'userType', 'manager_displayName', 'manager_userPrincipalName', 'hasLicense', 'assignedLicenses', 'createdDateTime') + Key = @() # Dimension table: whole file regenerated each run (overwrite); no replace-on-key merge. + } + SolutionsCatalog = @{ + FileName = 'ai_solutions_catalog.csv' + Columns = @('AISolution', 'Category', 'Vendor', 'RiskTier', 'DefaultDataHandling', 'SolutionGroup') + Key = @() # Dimension table: whole file regenerated each run (overwrite); no replace-on-key merge. + } +} + +function script:Get-AISIDScopeUpnSet { + # Resolved AISID collector scope: a case-insensitive set of the target UPNs, or $null when no + # user/group scope was requested (an unfiltered whole-directory run). Fail-closed: a REQUESTED + # scope that resolves to zero users already aborts the run upstream (exit 1), so a non-null empty + # set never occurs here; $null distinctly means "no scope requested" (do not filter). An empty + # scope therefore never widens, and a requested scope never leaks a non-target UPN into output. + $u = @($script:targetUsers) + if ($u.Count -eq 0) { return $null } + $set = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($x in $u) { $s = ([string]$x).Trim(); if ($s) { [void]$set.Add($s) } } + if ($set.Count -eq 0) { return $null } + # Unary comma: return the HashSet itself (with its OrdinalIgnoreCase comparer) instead of letting + # PowerShell enumerate it into a case-sensitive object[] on the way out. + return , $set +} + +# Per-table UPN column for AISID client-side scope enforcement. ClientChannel has no per-user grain +# (scoped at its source query) and SolutionsCatalog is intentionally global, so neither appears here; +# the EntraUsers dimension is scoped separately (target set + manager chain). +$script:AISIDScopeUpnColumn = @{ + ActivitySessions = 'UPN' + FileProximity = 'UPN' + OffHoursGeo = 'UPN' + OAuthConsents = 'UPN' + SSOSignIns = 'UPN' + CopilotUsage = 'UserPrincipalName' +} + +function script:Get-AISIDScopeKqlFilter { + # Server-side scope clause for a collector's hunting query: a KQL '| where in~ (dynamic([...]))' + # fragment. With -Users it covers exactly that batch (used by the batched runner); without -Users it + # covers the full resolved target set. Returns '' when no scope was requested (unfiltered run, + # unchanged behavior). The UPN literals are lower-cased and JSON-escaped. + param([Parameter(Mandatory)][string]$UpnColumn, [string[]]$Users) + $scope = + if ($PSBoundParameters.ContainsKey('Users')) { @($Users) } + else { $set = script:Get-AISIDScopeUpnSet; if ($null -eq $set) { @() } else { @($set) } } + if ($scope.Count -eq 0) { return '' } + $json = ($scope | ForEach-Object { '"' + (([string]$_).ToLowerInvariant() -replace '"', '\"') + '"' }) -join ',' + return ("| where {0} in~ (dynamic([{1}]))" -f $UpnColumn, $json) +} + +function script:Get-AISIDScopeBatches { + # Partitions the resolved target user set into deterministic batches sized by safe serialized request + # length, so a large scope never produces a single oversized KQL 'in~ (dynamic([...]))' array. EVERY + # target user appears in EXACTLY ONE batch (no cap, no first-N, no sampling, no truncation, no + # duplicates); batches are ordered and their union equals the full scope. Returns @() when no scope + # was requested (collectors then run one unfiltered query). + param([int]$MaxSerializedChars = 8000) + $set = script:Get-AISIDScopeUpnSet + if ($null -eq $set) { return @() } + # Deterministic order; force a clean [string[]] so no PSObject-wrapped element leaks into the batches. + [string[]]$users = @(@($set) | ForEach-Object { ([string]$_).ToLowerInvariant() } | Sort-Object -Unique) + $batches = New-Object System.Collections.Generic.List[object] + $start = 0 + while ($start -lt $users.Length) { + $len = 0; $count = 0 + for ($i = $start; $i -lt $users.Length; $i++) { + $add = $users[$i].Length + 3 + if ($count -gt 0 -and ($len + $add) -gt $MaxSerializedChars) { break } + $len += $add; $count++ + } + if ($count -lt 1) { $count = 1 } + [string[]]$batch = $users[$start..($start + $count - 1)] + [void]$batches.Add($batch) + $start += $count + } + return $batches.ToArray() +} + +function script:Invoke-AISIDBatchedHuntingQuery { + # Runs a hunting query once per target-user batch and returns a combined result hashtable + # (Success/AuthQuit/TruncationDetected/Results/ErrorMessage). The QueryBuilder receives the per-batch + # server-side scope clause and returns the full KQL for that batch, so every target user is queried + # exactly once across the batches. When no scope was requested a single unfiltered query runs + # (QueryBuilder called with ''), preserving the original one-query behavior. Any batch AuthQuit or + # failure short-circuits and is surfaced to the caller (fail-closed). Rows are concatenated in batch + # order; per-table dedup/aggregation is the caller's existing responsibility. + param( + [Parameter(Mandatory)][string]$UpnColumn, + [Parameter(Mandatory)][scriptblock]$QueryBuilder, + [Parameter(Mandatory)][string]$Description + ) + $acc = New-Object System.Collections.Generic.List[object] + $out = @{ Success = $true; AuthQuit = $false; TruncationDetected = $false; Results = @(); ErrorMessage = '' } + $batches = @(script:Get-AISIDScopeBatches) + $clauses = if ($batches.Count -eq 0) { @('') } else { @($batches | ForEach-Object { script:Get-AISIDScopeKqlFilter -UpnColumn $UpnColumn -Users $_ }) } + foreach ($clause in $clauses) { + $kql = & $QueryBuilder $clause + $api = Invoke-GraphHuntingQuery -Query $kql -Description $Description + if ($api.AuthQuit) { $out.AuthQuit = $true; $out.Success = $false; return $out } + if (-not $api.Success) { $out.Success = $false; $out.ErrorMessage = [string]$api.ErrorMessage; return $out } + if ($api.TruncationDetected) { $out.TruncationDetected = $true } + foreach ($r in @($api.Results)) { [void]$acc.Add($r) } + } + $out.Results = @($acc.ToArray()) + return $out +} + +function script:Select-AISIDScopedRows { + # Client-side scope enforcement (defense-in-depth behind the server-side KQL filter): drops any row + # whose UPN-column value is not in the resolved target set. Tables with no per-user grain + # (ClientChannel is scoped at its source query; SolutionsCatalog is intentionally global) and the + # EntraUsers dimension (scoped separately, incl. manager chain) are returned unchanged. When no + # scope was requested the rows are returned unchanged. + param([Parameter(Mandatory)][string]$Table, [object[]]$Rows = @()) + $set = script:Get-AISIDScopeUpnSet + if ($null -eq $set) { return @($Rows) } + if (-not $script:AISIDScopeUpnColumn.ContainsKey($Table)) { return @($Rows) } + $col = $script:AISIDScopeUpnColumn[$Table] + $kept = New-Object System.Collections.Generic.List[object] + foreach ($r in @($Rows)) { + if (-not $r) { continue } + $v = ([string]$r.$col).Trim() + if ($v -and $set.Contains($v)) { [void]$kept.Add($r) } + } + return $kept.ToArray() +} + +function script:Get-AISIDContainmentSet { + # The set of allowed UPN-column VALUES for a table's FINAL post-merge scope containment. When + # -Deidentify is on the raw target UPNs are mapped through the SAME deterministic Get-PaxDeidUpn + # helper the rows are hashed with, so already-deidentified history compares against deidentified + # scope keys (never raw-vs-hashed). Returns $null when no scope was requested or the table has no + # per-user grain (ClientChannel / SolutionsCatalog / EntraUsers). + param([Parameter(Mandatory)][string]$Table) + $set = script:Get-AISIDScopeUpnSet + if ($null -eq $set) { return $null } + if (-not $script:AISIDScopeUpnColumn.ContainsKey($Table)) { return $null } + $out = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($u in @($set)) { + $val = if ($script:PaxDeidEnabled) { Get-PaxDeidUpn -Value ([string]$u) } else { [string]$u } + $v = ([string]$val).Trim() + if ($v) { [void]$out.Add($v) } + } + return , $out +} + +function script:Select-AISIDContainedRows { + # Splits merged rows into in-scope and out-of-scope by the table's UPN column, using the deid-aware + # containment set. Returns a hashtable @{ InScope; OutOfScope; Applies }. Applies=$false for + # unscoped runs or non-user-grain tables (rows all treated as in-scope). + param([Parameter(Mandatory)][string]$Table, [object[]]$Rows = @()) + $cs = script:Get-AISIDContainmentSet -Table $Table + if ($null -eq $cs) { return @{ Applies = $false; InScope = @($Rows); OutOfScope = @() } } + $col = $script:AISIDScopeUpnColumn[$Table] + $in = New-Object System.Collections.Generic.List[object] + $offList = New-Object System.Collections.Generic.List[object] + foreach ($r in @($Rows)) { + if (-not $r) { continue } + $v = ([string]$r.$col).Trim() + if ($v -and $cs.Contains($v)) { [void]$in.Add($r) } else { [void]$offList.Add($r) } + } + return @{ Applies = $true; InScope = $in.ToArray(); OutOfScope = $offList.ToArray() } +} + +function script:Test-PaxZeroRecordHeaderOnlyPath { + # Decides whether the zero in-memory-record path enters the header-only-Purview-CSV emit block. TRUE + # when there are no records AND it is neither -OnlyUserInfo (Entra-only) nor a streaming-merge export + # (records stream from JSONL). -Dashboard AISID IS included here (it still needs the header-only + # Purview CSV), but the AISID case CONTINUES past the block (see + # Test-PaxZeroRecordContinuesAfterHeaderOnly) instead of returning, so a zero-record AISID run flows + # through the SAME downstream finalization (Rollup empty Fact + Users -> Entra -> AISID export -> + # completion/roster -> truthful exit) as a populated run. Zero Purview records are a valid empty + # result, never a failure. + param([int]$AllLogsCount, [bool]$OnlyUserInfo, [bool]$UseStreamingMergeForExport) + if ($AllLogsCount -ne 0) { return $false } + if ($OnlyUserInfo) { return $false } + if ($UseStreamingMergeForExport) { return $false } + return $true +} + +function script:Test-PaxZeroRecordContinuesAfterHeaderOnly { + # After the zero-record header-only Purview CSV is emitted, decides whether the run CONTINUES to the + # common downstream finalization (Rollup -> Entra -> AISID -> completion/roster) instead of taking the + # early return. TRUE only for -Dashboard AISID: a zero-record AISID run must still produce the empty + # Rollup Fact, the Rollup Users dimension, the normal Entra output, all twelve AISID CSVs (incl. + # EntraUsers.csv), the completion summary and the output roster. FALSE for every other mode, whose + # zero-record behavior (emit header-only and return) is unchanged. + param([string]$Dashboard) + return ($Dashboard -eq 'AISID') +} + +function Export-AISIDTable { + <# + .SYNOPSIS + Writes one dashboard CSV in the exact contractual column order, with the header row always + present. A table with zero rows still produces a valid header-only file (never omitted, + never an error) so the dashboard model loads it cleanly as empty. + .DESCRIPTION + Folder/destination resolution is the caller's responsibility: the caller passes the already + resolved effective directory (e.g. from Resolve-DataTypePaths for the dashboard destination), + so this writer performs no tier inference or collision handling of its own. Only the columns + named in OrderedColumns are emitted, so incidental provenance fields on the input rows never + leak into the model. + .OUTPUTS + The full path of the file written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][string]$FileName, + [Parameter(Mandatory)][string[]]$OrderedColumns, + [object[]]$Rows = @() + ) + + if (-not (Test-Path -LiteralPath $DestinationDir)) { + New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null + } + $target = Join-Path $DestinationDir $FileName + + $sb = [System.Text.StringBuilder]::new() + # Header row is written unconditionally — this is the safe-empty-table fallback. + [void]$sb.AppendLine(($OrderedColumns -join ',')) + foreach ($row in @($Rows)) { + if (-not $row) { continue } + $fields = foreach ($col in $OrderedColumns) { + $raw = if ($row -is [System.Collections.IDictionary]) { $row[$col] } else { $row.$col } + $sVal = if ($null -eq $raw) { '' } else { [string]$raw } + if ($sVal -match '[",\r\n]') { '"' + ($sVal -replace '"', '""') + '"' } else { $sVal } + } + [void]$sb.AppendLine(($fields -join ',')) + } + [System.IO.File]::WriteAllText($target, $sb.ToString(), (New-Object System.Text.UTF8Encoding($false))) + return $target +} + +function Merge-AISIDTableOnKey { + <# + .SYNOPSIS + Merges new rows into existing rows using replace-on-key semantics: a new row with the same + key overwrites the existing one; counts are never summed. Table-agnostic — the key grain is + supplied by the caller via KeyColumns, not hardcoded to any table. + .OUTPUTS + The merged row set (existing rows whose keys were not re-supplied, plus every new row). + #> + [CmdletBinding()] + param( + [object[]]$ExistingRows = @(), + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + + $index = [ordered]@{} + foreach ($r in @($ExistingRows)) { if ($r) { $index[(& $keyOf $r)] = $r } } + foreach ($r in @($NewRows)) { if ($r) { $index[(& $keyOf $r)] = $r } } + return @($index.Values) +} + +function Merge-AISIDTableAppendDedup { + <# + .SYNOPSIS + Merges new event rows into existing event rows using append-and-deduplicate semantics: every + distinct key is kept exactly once, existing history is preserved, and a re-observed event (a new + row whose key already exists) is dropped rather than duplicated or overwritten. Used for + event-grain tables where each row is an immutable observation and history accumulates across runs. + Table-agnostic — the key grain is supplied by the caller via KeyColumns, not hardcoded. + .OUTPUTS + The merged row set: all existing rows, plus each new row whose key was not already present. + #> + [CmdletBinding()] + param( + [object[]]$ExistingRows = @(), + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + + # Existing history is added first; a re-observed event (a NewRows entry whose key already exists) is + # neither re-added nor overwritten, so event history only ever grows and never double-counts. + $index = [ordered]@{} + foreach ($r in @($ExistingRows)) { if ($r) { $k = & $keyOf $r; if (-not $index.Contains($k)) { $index[$k] = $r } } } + foreach ($r in @($NewRows)) { if ($r) { $k = & $keyOf $r; if (-not $index.Contains($k)) { $index[$k] = $r } } } + return @($index.Values) +} + +# ───────────────────────────────────────────────────────────────────────────── +# AISID bounded staged history reconciliation (Blocker 2) +# The array-based merges above receive the ENTIRE prior history as an in-memory +# array (Get-AISIDExistingRows loads the whole CSV) held simultaneously with the +# keyed index — multiple copies of accumulated history. The streamed variants +# below reconcile against the prior history WITHOUT materializing it as an array: +# the prior CSV is read one row at a time through the pipeline and only one row +# per key (the output grain) is retained. Semantics are byte-for-byte identical to +# the array merges (replace-on-key = last wins; append-dedup = first wins; +# first-appearance output order), so callers see the same result set. A read that +# fails (locked / permission / malformed — as opposed to a confirmed-absent file) +# propagates so the caller preserves the prior file rather than overwriting it. +# ───────────────────────────────────────────────────────────────────────────── +function Test-AISIDCsvHasDataRow { + <# + .SYNOPSIS + Returns $true when a local CSV exists and has at least one data row (header + >=1), reading at + most the first two lines. Bounded — never loads the file. A confirmed-absent file returns $false; + any other read failure propagates so the caller can preserve prior state. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$Path) + if (-not (Test-Path -LiteralPath $Path)) { return $false } + $reader = $null + try { + $reader = [System.IO.StreamReader]::new($Path, [System.Text.UTF8Encoding]::new($false)) + $null = $reader.ReadLine() # header + $second = $reader.ReadLine() # first data row (if any) + return ($null -ne $second) + } + finally { + if ($reader) { $reader.Dispose() } + } +} + +function Merge-AISIDTableOnKeyStreamed { + <# + .SYNOPSIS + Bounded replace-on-key merge: streams the prior history CSV row-by-row into a keyed index (one + row per key), then applies the new rows (a new row overwrites the existing one on the same key). + Identical result to Merge-AISIDTableOnKey, but the full prior-history array is never held. When + the prior file is absent or header-only (no data rows), the new rows are returned unchanged — + matching the caller's "merge only when prior history exists" guard exactly. + #> + [CmdletBinding()] + param( + [string]$ExistingCsvPath, + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + $index = [ordered]@{} + $existingCount = 0 + if ($ExistingCsvPath -and (Test-Path -LiteralPath $ExistingCsvPath)) { + Import-Csv -LiteralPath $ExistingCsvPath | ForEach-Object { $index[(& $keyOf $_)] = $_; $existingCount++ } + } + if ($existingCount -eq 0) { return @($NewRows) } + foreach ($r in @($NewRows)) { if ($r) { $index[(& $keyOf $r)] = $r } } + try { Write-PaxMemoryObservation -Stage 'AISIDHistoryReconcile' -RowsProcessed ([long]$index.Count) -Force } catch {} + return @($index.Values) +} + +function Merge-AISIDTableAppendDedupStreamed { + <# + .SYNOPSIS + Bounded append-and-dedup merge: streams the prior history CSV row-by-row into a keyed index + (first occurrence wins), then applies each new row only if its key is not already present. + Identical result to Merge-AISIDTableAppendDedup, but the full prior-history array is never held. + Whether to merge-vs-stub remains the caller's decision (prior history present OR a collected run). + #> + [CmdletBinding()] + param( + [string]$ExistingCsvPath, + [object[]]$NewRows = @(), + [Parameter(Mandatory)][string[]]$KeyColumns + ) + $keyOf = { + param($r) + ($KeyColumns | ForEach-Object { + $val = if ($r -is [System.Collections.IDictionary]) { $r[$_] } else { $r.$_ } + ([string]$val).ToLowerInvariant() + }) -join '|' + } + $index = [ordered]@{} + if ($ExistingCsvPath -and (Test-Path -LiteralPath $ExistingCsvPath)) { + Import-Csv -LiteralPath $ExistingCsvPath | ForEach-Object { $k = & $keyOf $_; if (-not $index.Contains($k)) { $index[$k] = $_ } } + } + foreach ($r in @($NewRows)) { if ($r) { $k = & $keyOf $r; if (-not $index.Contains($k)) { $index[$k] = $r } } } + try { Write-PaxMemoryObservation -Stage 'AISIDHistoryReconcile' -RowsProcessed ([long]$index.Count) -Force } catch {} + return @($index.Values) +} + +function Get-AISIDExistingRows { + <# + .SYNOPSIS + Reads the rows already present in a dashboard CSV so a replace-on-key export can merge new rows + into the accumulated history instead of overwriting it. Resolves the destination tier (local, + SharePoint, or Fabric/OneLake) and returns the parsed rows. + .DESCRIPTION + Critically distinguishes a genuinely absent file (a normal first-run condition -> empty set) from a + read that FAILED for another reason on a file that may well exist (locked/permission-denied/partial + or malformed CSV locally; auth/throttle/network or corrupt download remotely). Only a confirmed + absence returns empty; any other failure is thrown so the caller can preserve the existing file + rather than overwrite it with an empty-merge result (which would reproduce the original data-loss + bug from a different cause). Remote not-found is detected as an HTTP 404 via the same + $_.Exception.Response.StatusCode.value__ inspection used elsewhere for remote 404 handling. + + The reader is merge-mode-agnostic: it only returns the existing rows. The caller decides how to + combine them, so a future grain (for example event-level append-and-dedup) can reuse it unchanged. + .OUTPUTS + An array of existing rows (one object per CSV row), or an empty array when the file is confirmed + absent. Throws on any non-not-found read failure. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][string]$FileName, + [string]$ParentOverride + ) + + $mode = $script:RemoteOutputMode + + if (-not $mode -or $mode -eq 'None') { + # Local tier. A genuinely absent file is a normal first-run condition -> empty. Any OTHER failure + # (locked file, permission denied, malformed/partial CSV) must NOT masquerade as "no history" -- + # Import-Csv is left uncaught so it propagates to the caller, which preserves the existing file. + $path = Join-Path $DestinationDir $FileName + if (-not (Test-Path -LiteralPath $path)) { return @() } + return @(Import-Csv -LiteralPath $path) + } + + # Remote tier (SharePoint / Fabric-OneLake). Download to a temp path via the established remote-read + # functions, then parse. Only a confirmed HTTP 404 (file not yet present) is treated as an empty + # first-run set; every other failure (401/403/429/5xx/network, or an unparseable status) propagates so + # the caller preserves prior state rather than merging as if empty. + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("aisid_read_" + [System.Guid]::NewGuid().ToString('N') + ".csv") + try { + try { + switch ($mode) { + 'SharePoint' { Get-RemoteFile-SharePoint -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + 'Fabric' { Get-RemoteFile-OneLake -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { + Write-Verbose ("Get-AISIDExistingRows: remote '{0}' not found (404); treating as first run." -f $FileName) + return @() + } + # Not a confirmed absence -- surface the real failure; the caller preserves prior state. + throw + } + if (-not (Test-Path -LiteralPath $tmp)) { return @() } + return @(Import-Csv -LiteralPath $tmp) + } + finally { + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + } +} + +function Get-AISIDExistingCsvToScratch { + <# + .SYNOPSIS + Downloads the remote (SharePoint / Fabric-OneLake) prior-history CSV to a local scratch file and + returns the scratch path so it can be streamed through the bounded merge — the downloaded file is + never parsed into an in-memory array. Returns $null when the remote file is confirmed absent + (HTTP 404 -> first run). Any other failure (401/403/429/5xx/network/unparseable status) throws so + the caller preserves prior remote state. The CALLER owns cleanup of the returned scratch path + (removed on success, preserved for recovery on failure) — this mirrors the memory-safe + download-to-local behavior of Get-AISIDExistingRows without the whole-file Import-Csv. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$FileName, + [string]$ParentOverride + ) + $mode = $script:RemoteOutputMode + if (-not $mode -or $mode -eq 'None') { return $null } + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("aisid_hist_" + [System.Guid]::NewGuid().ToString('N') + ".csv") + try { + switch ($mode) { + 'SharePoint' { Get-RemoteFile-SharePoint -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + 'Fabric' { Get-RemoteFile-OneLake -RelativeName $FileName -DestinationPath $tmp -ParentOverride $ParentOverride } + } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + if ($status -eq 404) { + Write-Verbose ("Get-AISIDExistingCsvToScratch: remote '{0}' not found (404); treating as first run." -f $FileName) + return $null + } + throw + } + if (-not (Test-Path -LiteralPath $tmp)) { return $null } + try { Write-PaxMemoryObservation -Stage 'AISIDRemoteHistoryDownloaded' -Force } catch {} + return $tmp +} + +function Export-AISIDCompatibilityTable { + <# + .SYNOPSIS + Publishes a required dashboard compatibility table without collecting new rows. A valid prior + file is copied byte-for-byte into the new run bundle; only a confirmed-absent prior file becomes + a header-only table. Read or header failures throw so prior data is never replaced by a false empty. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Table, + [Parameter(Mandatory)][string]$DestinationDir, + [hashtable]$ExistingRows = @{}, + [string]$HistoryDir = '', + [string]$ParentOverride = '' + ) + $schema = $script:AISIDTableSchemas[$Table] + if (-not $schema) { throw "Unknown AISID compatibility table: $Table" } + if ($ExistingRows.ContainsKey($Table) -and @($ExistingRows[$Table]).Count -gt 0) { + return Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows @($ExistingRows[$Table]) + } + + $source = $null + $remoteScratch = $false + if ($script:RemoteOutputMode -and $script:RemoteOutputMode -ne 'None' -and $ParentOverride) { + $source = Get-AISIDExistingCsvToScratch -FileName $schema.FileName -ParentOverride $ParentOverride + $remoteScratch = [bool]$source + } + elseif ($HistoryDir) { + $candidate = Join-Path $HistoryDir $schema.FileName + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $source = $candidate } + } + + if (-not $source) { + return Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows @() + } + + $expectedHeader = $schema.Columns -join ',' + $actualHeader = Get-Content -LiteralPath $source -TotalCount 1 -ErrorAction Stop + if ($actualHeader -cne $expectedHeader) { + throw ("{0}: prior compatibility file has an invalid header; prior file left unchanged" -f $Table) + } + if (-not (Test-Path -LiteralPath $DestinationDir)) { New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null } + $target = Join-Path $DestinationDir $schema.FileName + [System.IO.File]::Copy($source, $target, $true) + if ($remoteScratch -and (Test-Path -LiteralPath $source)) { Remove-Item -LiteralPath $source -Force -ErrorAction SilentlyContinue } + return $target +} + +function ConvertTo-AISIDInvariantUtcTimestamp { + <# + .SYNOPSIS + Serializes a timestamp value to an invariant, culture-independent ISO-8601 UTC string ending in + 'Z' (round-trip 7-digit fractional form, e.g. 2026-07-01T08:47:59.0000000Z). Format-only: the + instant is preserved, only its textual representation is normalized. Unparseable input is + returned as its original string so no value is silently dropped. + .OUTPUTS + [string] + #> + [CmdletBinding()] + param([Parameter()] $Value) + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $fmt = 'yyyy-MM-ddTHH:mm:ss.fffffffZ' + if ($null -eq $Value) { return '' } + if ($Value -is [datetime]) { return ([datetime]$Value).ToUniversalTime().ToString($fmt, $inv) } + if ($Value -is [System.DateTimeOffset]) { return ([System.DateTimeOffset]$Value).UtcDateTime.ToString($fmt, $inv) } + $s = [string]$Value + if ([string]::IsNullOrWhiteSpace($s)) { return '' } + $parsed = [datetime]::MinValue + $styles = [System.Globalization.DateTimeStyles]::AdjustToUniversal -bor [System.Globalization.DateTimeStyles]::AssumeUniversal + if ([datetime]::TryParse($s, $inv, $styles, [ref]$parsed)) { return $parsed.ToString($fmt, $inv) } + return $s +} + +function Get-AISIDFileProximity { + <# + .SYNOPSIS + Collects the File Proximity data set: files created or changed within five minutes of an AI + website visit on the same device — a temporal-proximity signal (temporal correlation only; not + proof of exfiltration). Sourced from Defender + DeviceNetworkEvents (AI site visits) joined to DeviceFileEvents (file activity). Event-grain; the + collection window is subdivided automatically so no single hunting query approaches the + advanced-hunting response cap. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $domains = @(Get-AISIDNetworkDomains) + if ($domains.Count -eq 0) { + Write-LogHost '[AISID] FileProximity: no AI network domains in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $domainJson = ($domains | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + # Collection window: the frozen requested-intersect-available effective interval (half-open). + $fpBounds = Get-AISIDCollectionBounds + $startUtc = $fpBounds.StartUtc + $endUtc = $fpBounds.EndUtc + + # Builds the DeviceNetworkEvents x DeviceFileEvents 5-minute-proximity join for one time window. + # The API Timespan already caps the source at the fixed lookback; the window bounds here subdivide + # that lookback so no single response approaches the row cap. -CountOnly returns the post-join row + # count for the estimate probe; the full form emits the contractual columns (with the visited URL, + # which is resolved to an AI solution client-side below). + # Target-user scope pushed server-side onto the joined events (before the tail projects/aggregates). + $fpScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'InitiatingProcessAccountUpn' + $buildQuery = { + param([datetime]$WinStart, [datetime]$WinEnd, [switch]$CountOnly, [string]$ScopeClause = $fpScope) + $ws = $WinStart.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $we = $WinEnd.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $tail = if ($CountOnly) { + '| summarize N = count()' + } + else { +@' +| extend SecondsToAI = datetime_diff('second', FileTimestamp, AITimestamp) +| extend UPN = tolower(InitiatingProcessAccountUpn) +| extend YearMonth = format_datetime(AITimestamp, 'yyyy-MM') +| extend FolderCategory = case( + FolderPath has 'Desktop', 'Desktop', + FolderPath has 'Downloads', 'Downloads', + FolderPath has 'Documents', 'Documents', + FolderPath has 'OneDrive', 'OneDrive', + FolderPath has 'SharePoint', 'SharePoint', + 'Other') +| extend NameMatchesSensitivePattern = iff(FileName has_any (SensitiveNamePatterns), 1, 0) +| extend FolderMatchesSensitive = iff(FolderPath has_any (SensitiveFolderPatterns), 1, 0) +| project Timestamp = AITimestamp, RemoteUrl, UPN, YearMonth, FileName, FolderCategory, FolderPath, + SecondsToAI, NameMatchesSensitivePattern, FolderMatchesSensitive +'@ + } +@" +let AIDomains = dynamic([$domainJson]); +let SensitiveNamePatterns = dynamic(["confidential","secret","password","credential","private","restricted","internal","draft","salary","ssn","pii","financial","budget","forecast","strategy","merger","acquisition","termination","layoff","patent"]); +let SensitiveFolderPatterns = dynamic(["confidential","restricted","hr","legal","finance","executive","board","compliance","audit","security"]); +let WinStart = datetime($ws); +let WinEnd = datetime($we); +let AIVisits = + DeviceNetworkEvents + | where Timestamp between (WinStart .. WinEnd) + | where ActionType == 'ConnectionSuccess' + | where RemoteUrl has_any (AIDomains) + | project AITimestamp = Timestamp, DeviceId, RemoteUrl; +let FileAccess = + DeviceFileEvents + | where Timestamp between (WinStart .. (WinEnd + 5m)) + | where ActionType in ('FileCreated', 'FileModified', 'FileRenamed', 'FileCopied') + | where FileName !endswith '.tmp' and FileName !endswith '.log' + | project FileTimestamp = Timestamp, DeviceId, FileName, FolderPath, InitiatingProcessAccountUpn; +AIVisits +| join kind=inner FileAccess on DeviceId +| where FileTimestamp between (AITimestamp .. (AITimestamp + 5m)) +$ScopeClause +$tail +"@ + } + + # Plan sub-windows so no single query approaches the response cap; each range is re-probed, so a + # non-uniform distribution only subdivides the ranges that are actually hot. + $estimateProvider = { + param($a, $b) + $probe = & $buildQuery -WinStart $a -WinEnd $b -CountOnly + Get-AISIDHuntingRowEstimate -ProbeQuery $probe -Description 'FileProximity row-estimate' + } + $partitions = @(Get-AISIDTimePartitions -StartUtc $startUtc -EndUtc $endUtc -EstimateProvider $estimateProvider) + + # A partition flagged EstimateFailed means the row-estimate probe was unreadable; fail the table + # explicitly rather than silently emitting an empty/truncated file (and never retry unbounded). + if (@($partitions | Where-Object { $_.EstimateFailed }).Count -gt 0) { + $result.ErrorMessage = 'FileProximity row-estimate probe returned an unreadable result; collection aborted to avoid an unbounded estimate retry and a false-empty output.' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + Write-LogHost ("[AISID] FileProximity: planned {0} time partition(s) for collection." -f $partitions.Count) -ForegroundColor DarkGray + + $rawRows = New-Object System.Collections.Generic.List[object] + foreach ($p in $partitions) { + if ($p.Truncated) { $result.TruncationDetected = $true } + # Batched per partition: for a large scope the source events are filtered per user-batch so every + # target user's events are queried exactly once; event-grain rows are concatenated (each event's + # InitiatingProcessAccountUpn belongs to exactly one batch, so there is no cross-batch duplication). + $__pStart = $p.StartUtc; $__pEnd = $p.EndUtc + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'InitiatingProcessAccountUpn' -Description 'FileProximity' -QueryBuilder ({ param($__scope) & $buildQuery -WinStart $__pStart -WinEnd $__pEnd -ScopeClause $__scope }.GetNewClosure()) + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "FileProximity query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if ($api.TruncationDetected) { $result.TruncationDetected = $true } + foreach ($row in @($api.Results)) { if ($row) { $rawRows.Add($row) } } + } + $result.RawRowCount = $rawRows.Count + + # Client-side: resolve each event's AI solution from the visited host (catalog-driven, mirroring how + # ActivitySessions resolves its application name), then emit the contractual event-grain columns. + $rows = New-Object System.Collections.Generic.List[object] + foreach ($r in $rawRows) { + $hostName = $null + $ru = [string]$r.RemoteUrl + if (-not [string]::IsNullOrWhiteSpace($ru)) { + $parsed = $null + if ([Uri]::TryCreate($ru, [UriKind]::Absolute, [ref]$parsed)) { $hostName = $parsed.Host } + else { $hostName = ((($ru -replace '^[A-Za-z][A-Za-z0-9+.\-]*://', '') -replace '/.*$', '') -replace ':\d+$', '') } + } + $entry = if ($hostName) { Resolve-AISIDFromDomain -HostName $hostName } else { $null } + $solution = if ($entry) { [string]$entry.AISolution } else { 'Other AI' } + $rows.Add([pscustomobject][ordered]@{ + Timestamp = (ConvertTo-AISIDInvariantUtcTimestamp -Value $r.Timestamp) + UPN = [string]$r.UPN + AISolution = $solution + YearMonth = [string]$r.YearMonth + FileName = [string]$r.FileName + FolderCategory = [string]$r.FolderCategory + FolderPath = [string]$r.FolderPath + SecondsToAI = [int]$r.SecondsToAI + NameMatchesSensitivePattern = [int]$r.NameMatchesSensitivePattern + FolderMatchesSensitive = [int]$r.FolderMatchesSensitive + }) + } + + $result.Rows = $rows.ToArray() + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function Get-AISIDClientChannel { + <# + .SYNOPSIS + Collects the Client Channel data set: AI website activity split by access channel — browser vs. + scripted/API client vs. desktop app — per AI site per month. Sourced from Defender + DeviceNetworkEvents; the AI sites are filtered by the catalog's network domains, and each event's + site and channel are labelled server-side (the channel is inferred from initiating-process + metadata, not a definitive client attribution). Aggregate-grain (one row per site/channel/month), so a + single hunting query stays well under the advanced-hunting response cap. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $domains = @(Get-AISIDNetworkDomains) + if ($domains.Count -eq 0) { + Write-LogHost '[AISID] ClientChannel: no AI network domains in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $domainJson = ($domains | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + + # The catalog's network domains drive the AI-site filter (mirroring FileProximity). The site and + # channel labels use the authoritative source mapping — visited host -> canonical site, initiating + # process -> Browser/API/Desktop — applied server-side. The shared hunting-query timespan bounds the + # collection window; the aggregate collapses to one row per site/channel/month, so no subdivision is + # needed. (The site case() carries the full source mapping; only catalog-filtered hosts reach it.) + $ccBounds = Get-AISIDCollectionBounds + $ccStart = $ccBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ccEnd = $ccBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + # Target-user scope pushed server-side onto the source events BEFORE the site/channel aggregate + # collapses UPN away (ClientChannel has no per-user output grain, so this is its only scope point). + $ccScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'InitiatingProcessAccountUpn' + # Batched: a large scope is split into size-bounded batches; the source events are filtered per batch + # (every target user's events queried exactly once) and the per-batch aggregates are RE-AGGREGATED + # below by (AISite, Channel, YearMonth) so counts sum deterministically without duplicate keys. + $null = $ccScope + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'InitiatingProcessAccountUpn' -Description 'ClientChannel' -QueryBuilder ({ + param($__scope) +@" +let AIDomains = dynamic([$domainJson]); +let WinStart = datetime($ccStart); +let WinEnd = datetime($ccEnd); +DeviceNetworkEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where ActionType == 'ConnectionSuccess' +| where RemoteUrl has_any (AIDomains) +$__scope +| extend AISite = case( + RemoteUrl has 'copilot.microsoft.com' or RemoteUrl has 'copilot.cloud.microsoft', 'copilot.microsoft.com', + RemoteUrl has 'chat.openai.com' or RemoteUrl has 'chatgpt.com', 'chatgpt.com', + RemoteUrl has 'api.openai.com', 'api.openai.com', + RemoteUrl has 'claude.ai', 'claude.ai', + RemoteUrl has 'api.anthropic.com', 'api.anthropic.com', + RemoteUrl has 'gemini.google.com' or RemoteUrl has 'bard.google.com', 'gemini.google.com', + RemoteUrl has 'perplexity.ai', 'perplexity.ai', + RemoteUrl has 'midjourney.com', 'midjourney.com', + RemoteUrl has 'huggingface.co', 'huggingface.co', + RemoteUrl has 'stability.ai', 'stability.ai', + RemoteUrl has 'jasper.ai', 'jasper.ai', + RemoteUrl has 'grammarly.com', 'grammarly.com', + RemoteUrl has 'notion.so', 'notion.so', + RemoteUrl has 'firefly.adobe.com', 'firefly.adobe.com', + RemoteUrl has 'runwayml.com', 'runwayml.com', + RemoteUrl has 'canva.com', 'canva.com', + RemoteUrl has 'app.synthesia.io', 'app.synthesia.io', + RemoteUrl) +| extend Channel = case( + InitiatingProcessFileName has_any ('chrome.exe', 'msedge.exe', 'firefox.exe', 'brave.exe', 'safari', 'opera.exe', 'iexplore.exe', 'Chrome', 'Safari', 'Firefox'), 'Browser', + InitiatingProcessFileName has_any ('python', 'node', 'java', 'curl', 'powershell', 'pwsh', 'cmd.exe', 'bash', 'dotnet', 'go'), 'API', + 'Desktop') +| extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') +| summarize EventCount = count() by AISite, Channel, YearMonth +"@ + }.GetNewClosure()) + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "ClientChannel query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $rawRows = @($api.Results) + $result.RawRowCount = $rawRows.Count + $result.TruncationDetected = [bool]$api.TruncationDetected + + # Re-aggregate across batches: the same (AISite, Channel, YearMonth) can appear in multiple batches + # (disjoint user subsets), so EventCount is SUMMED per key — deterministic, no duplicate aggregation. + $ccAgg = [ordered]@{} + foreach ($r in $rawRows) { + if (-not $r) { continue } + $k = '{0}|{1}|{2}' -f [string]$r.AISite, [string]$r.Channel, [string]$r.YearMonth + if (-not $ccAgg.Contains($k)) { + $ccAgg[$k] = [pscustomobject][ordered]@{ AISite = [string]$r.AISite; Channel = [string]$r.Channel; YearMonth = [string]$r.YearMonth; EventCount = 0 } + } + $ccAgg[$k].EventCount = [int]$ccAgg[$k].EventCount + [int]$r.EventCount + } + + $result.Rows = @($ccAgg.Values) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function ConvertTo-AISIDUtcInstant { + <# + .SYNOPSIS + Converts an audit event timestamp to a Kind=Utc [datetime] WITHOUT any machine-timezone + dependency, for AISID window filtering and per-instant (YearMonth / ActiveDays / LastActivityDate) + grouping. + .DESCRIPTION + An explicit Z or numeric offset is authoritative. A [datetime]'s Kind is honored: Utc is returned + as-is; Local is converted with .ToUniversalTime(); Unspecified (offset-free) is treated as UTC under + one explicit, documented policy -- NEVER silently as host-local time. A [datetimeoffset] uses its own + offset (.UtcDateTime). Strings are parsed with the INVARIANT culture and AssumeUniversal|AdjustToUniversal + styles so a trailing Z or offset governs and an offset-free string is taken as UTC. A [datetime] is + NEVER stringified and reparsed through the current culture (the prior defect: that produced a + host-timezone-dependent shift). Malformed/empty input fails clearly. + .OUTPUTS + [bool] -- $true and sets [ref]$Utc to a Kind=Utc [datetime] on success; $false on null/malformed input. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][AllowNull()]$Value, [Parameter(Mandatory)][ref]$Utc) + + if ($null -eq $Value) { return $false } + if ($Value -is [datetimeoffset]) { $Utc.Value = $Value.UtcDateTime; return $true } + if ($Value -is [datetime]) { + switch ([string]$Value.Kind) { + 'Utc' { $Utc.Value = $Value; return $true } + 'Local' { $Utc.Value = $Value.ToUniversalTime(); return $true } + default { $Utc.Value = [datetime]::SpecifyKind($Value, [System.DateTimeKind]::Utc); return $true } + } + } + $s = [string]$Value + if ([string]::IsNullOrWhiteSpace($s)) { return $false } + $dto = [datetimeoffset]::MinValue + $styles = [System.Globalization.DateTimeStyles]::AssumeUniversal -bor [System.Globalization.DateTimeStyles]::AdjustToUniversal + if ([datetimeoffset]::TryParse($s, [System.Globalization.CultureInfo]::InvariantCulture, $styles, [ref]$dto)) { + $Utc.Value = $dto.UtcDateTime + return $true + } + return $false +} + +function Get-AISIDCopilotUsage { + <# + .SYNOPSIS + Builds the Copilot Usage data set (per-user, per-month prompt counts across the six MODELED + surfaces: Teams / Word / Excel / Outlook / PowerPoint / Chat) from the CURRENT run's RAW Purview + CopilotInteraction CSV. No second audit search and no Graph call: it streams the raw CSV, parses + each AuditData JSON, and pivots AuditData.CopilotEventData.AppHost against a fixed, case-insensitive + allowlist. This is deliberately the six modeled surfaces ONLY - it is NOT comprehensive Microsoft 365 + Copilot usage; any AppHost that cannot map to one of the six columns (Security Copilot, Copilot Studio, + M365 Admin Center, and other unmodeled hosts) is excluded from every surface AND from TotalPrompts, so + TotalPrompts always reconciles to the six-surface sum. + .NOTES + Source path is passed explicitly (-RawPurviewCsv = the current run's raw Purview CSV, still present and + identified at the post-rollup AISID call). AuditData.CreationTime is the CopilotUsage event timestamp. + Fails gracefully (Success=$false) only for CopilotUsage when the source is absent/unreadable or every + record is unusable, so the export falls that one table back to a header-only file. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$RawPurviewCsv) + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + if ([string]::IsNullOrWhiteSpace($RawPurviewCsv) -or -not (Test-Path -LiteralPath $RawPurviewCsv -PathType Leaf)) { + $result.ErrorMessage = 'CopilotUsage: raw Purview CopilotInteraction CSV not available; cannot build the Copilot Usage data set.' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + # AISID effective collection window (frozen). Records outside [start, end) are excluded before grouping. + $cuBounds = Get-AISIDCollectionBounds + $cuStartUtc = $cuBounds.StartUtc + $cuEndUtc = $cuBounds.EndUtc + + # AppHost -> surface (six modeled surfaces ONLY). Case-insensitive exact match after trim. Any AppHost + # absent here is EXCLUDED from every surface AND from TotalPrompts (so Total reconciles to the six-surface + # sum). No wildcards; no unknown-to-Chat. Keys are lowercase for case-insensitive lookup. + $surfaceOf = @{ + 'teams' = 'TeamsPrompts' + 'word' = 'WordPrompts'; 'wordoncanvas' = 'WordPrompts' + 'excel' = 'ExcelPrompts' + 'outlook' = 'OutlookPrompts'; 'outlookoncanvas' = 'OutlookPrompts'; 'outlooksidepane' = 'OutlookPrompts' + 'powerpoint' = 'PowerPointPrompts'; 'powerpointoncanvas' = 'PowerPointPrompts' + 'bizchat' = 'ChatPrompts'; 'bing' = 'ChatPrompts'; 'office' = 'ChatPrompts'; 'edge' = 'ChatPrompts' + 'm365app' = 'ChatPrompts'; 'm365chat' = 'ChatPrompts'; 'officecopilotnotebook' = 'ChatPrompts' + 'officecopilotsearchanswer' = 'ChatPrompts'; 'onenotecopilotnotebook' = 'ChatPrompts' + } + + $agg = [ordered]@{} + $rawCount = 0; $skipped = 0; $included = 0; $outOfWindow = 0; $responseOnly = 0 + $exSec = 0; $exStudio = 0; $exAdmin = 0; $exOther = 0 + try { + Import-Csv -LiteralPath $RawPurviewCsv | ForEach-Object { + $rawCount++ + $d = $null + try { $d = $_.AuditData | ConvertFrom-Json -ErrorAction Stop } catch { $d = $null } + if (-not $d) { $skipped++; return } + $upnRaw = [string]$d.UserId + if ([string]::IsNullOrWhiteSpace($upnRaw)) { $skipped++; return } + $dtU = [datetime]::MinValue + # Convert the audit event timestamp to a UTC instant with NO machine-timezone dependency (an explicit + # Z/offset is authoritative; a DateTime Kind is honored; offset-free => UTC by documented policy). + # Never stringify-and-reparse through the current culture (the prior host-timezone-dependent defect). + if (-not (ConvertTo-AISIDUtcInstant -Value $d.CreationTime -Utc ([ref]$dtU))) { $skipped++; return } + # AISID window filter: keep only records whose UTC instant is in the frozen half-open interval. + if ($dtU -lt $cuStartUtc -or $dtU -ge $cuEndUtc) { $outOfWindow++; return } + $appHostRaw = ([string]$d.CopilotEventData.AppHost).Trim() + $key = $appHostRaw.ToLowerInvariant() + if (-not $surfaceOf.ContainsKey($key)) { + if ($appHostRaw -like 'SecurityCopilot-*') { $exSec++ } + elseif ($key -eq 'copilot studio') { $exStudio++ } + elseif ($key -eq 'm365admincenter') { $exAdmin++ } + else { $exOther++ } + return + } + $surface = $surfaceOf[$key] + $included++ + # Count actual user-prompt messages in this record (CopilotEventData.Messages[].isPrompt === $true). A + # response-only record (zero prompt messages) contributes NOTHING to any surface, TotalPrompts, ActiveDays, + # or LastActivityDate; a record with N prompt messages contributes N (no one-prompt-per-record assumption). + $promptCount = 0 + $msgs = $d.CopilotEventData.Messages + if ($null -ne $msgs) { foreach ($m in @($msgs)) { if ($m -and $m.isPrompt -eq $true) { $promptCount++ } } } + if ($promptCount -le 0) { $responseOnly++; return } + $upn = $upnRaw.ToLowerInvariant() + $ym = $dtU.ToString('yyyy-MM') + $day = $dtU.ToString('yyyy-MM-dd') + $aggKey = '{0}|{1}' -f $upn, $ym + if (-not $agg.Contains($aggKey)) { + $agg[$aggKey] = @{ + UserPrincipalName = $upn; YearMonth = $ym + TeamsPrompts = 0; WordPrompts = 0; ExcelPrompts = 0; OutlookPrompts = 0; PowerPointPrompts = 0; ChatPrompts = 0 + TotalPrompts = 0; Days = (New-Object 'System.Collections.Generic.HashSet[string]'); LastDay = '' + } + } + $b = $agg[$aggKey] + $b[$surface] = [int]$b[$surface] + $promptCount + $b.TotalPrompts = [int]$b.TotalPrompts + $promptCount + [void]$b.Days.Add($day) + if ($day -gt [string]$b.LastDay) { $b.LastDay = $day } + } + } + catch { + $result.ErrorMessage = "CopilotUsage: failed to stream raw Purview CSV: $($_.Exception.Message)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $result.RawRowCount = $rawCount + + if ($rawCount -gt 0 -and ($rawCount - $skipped) -eq 0) { + $result.ErrorMessage = "CopilotUsage: all $rawCount raw record(s) were malformed/unusable; falling back to header-only for this table." + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $rows = New-Object System.Collections.Generic.List[object] + foreach ($b in $agg.Values) { + $rows.Add([pscustomobject][ordered]@{ + UserPrincipalName = [string]$b.UserPrincipalName + YearMonth = [string]$b.YearMonth + TeamsPrompts = [int]$b.TeamsPrompts + WordPrompts = [int]$b.WordPrompts + ExcelPrompts = [int]$b.ExcelPrompts + OutlookPrompts = [int]$b.OutlookPrompts + PowerPointPrompts = [int]$b.PowerPointPrompts + ChatPrompts = [int]$b.ChatPrompts + TotalPrompts = [int]$b.TotalPrompts + ActiveDays = [int]$b.Days.Count + LastActivityDate = [string]$b.LastDay + }) + } + + if ($skipped -gt 0) { + Write-LogHost ("[AISID] CopilotUsage: skipped {0} malformed/incomplete record(s)." -f $skipped) -ForegroundColor DarkYellow + } + $exTotal = $exSec + $exStudio + $exAdmin + $exOther + Write-LogHost ("[AISID] CopilotUsage: raw={0} in-scope={1} response-only={9} out-of-window={8} excluded={2} (SecurityCopilot-={3}, Copilot Studio={4}, M365AdminCenter={5}, Other unmodeled={6}); user-month rows={7}." -f $rawCount, $included, $exTotal, $exSec, $exStudio, $exAdmin, $exOther, $rows.Count, $outOfWindow, $responseOnly) -ForegroundColor DarkGray + + $result.Rows = $rows.ToArray() + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function New-AISIDFileProximityStub { + <# + .SYNOPSIS + Emits the permanent header-only file-proximity CSV. This table is always empty by design: + the dashboard model loads the file on every refresh, but no query, licensing check, or Graph + call is involved — the header alone satisfies the model's fixed schema. + .OUTPUTS + The full path of the stub file written. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$DestinationDir) + + $schema = $script:AISIDTableSchemas['FileProximity'] + return Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows @() +} + +function Get-AISIDActivitySessions { + <# + .SYNOPSIS + Collects the Activity Sessions data set from cloud-app activity and returns enriched rows in + the contractual column shape. Server-side aggregation keeps the result small; catalog-driven + enrichment (application name -> AI solution / category / risk tier) is applied client-side and + collapsed to one row per (UPN, AISolution, YearMonth). + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; TruncationDetected = $false + AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $cloudAppNames = @(Get-AISIDCloudAppNames) + if ($cloudAppNames.Count -eq 0) { + Write-LogHost '[AISID] ActivitySessions: no cloud-app names in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + # The collection window is bounded by the shared hunting-query timespan; the query filters to the + # catalog's cloud-app names, resolves each event's canonical user principal name from the directory + # identity data (falling back to the raw account object id when the directory has no match for that + # identity), and aggregates per user/app/month server-side. + $appFilterJson = ($cloudAppNames | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + $asBounds = Get-AISIDCollectionBounds + $asStart = $asBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $asEnd = $asBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $asScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'UPN' + # Batched: a large target scope is split into deterministic size-bounded batches; the query runs once + # per batch (every target user queried exactly once) and the rows are concatenated. $asScope is + # retained for structural parity but the per-batch clause ($__scope) is what the runner injects. + $null = $asScope + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'UPN' -Description 'ActivitySessions' -QueryBuilder ({ + param($__scope) +@" +let AppFilter = dynamic([$appFilterJson]); +let UserUpns = IdentityInfo + | summarize take_any(AccountUpn) by AccountObjectId; +let WinStart = datetime($asStart); +let WinEnd = datetime($asEnd); +CloudAppEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where Application in~ (AppFilter) +| lookup kind=leftouter UserUpns on AccountObjectId +| extend UPN = tolower(coalesce(AccountUpn, AccountObjectId)) +| where isnotempty(UPN) +$__scope +| summarize + Sessions = count(), + ActiveDays = dcount(format_datetime(Timestamp, 'yyyy-MM-dd')), + EstimatedPrompts = countif(ActionType in ("MessageSent", "SearchPerformed", "AppAccessedViaAPI")), + DistinctDevices = dcount(DeviceType) + by UPN, AppNameMatched = Application, YearMonth = format_datetime(Timestamp, 'yyyy-MM') +"@ + }.GetNewClosure()) + + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "ActivitySessions query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $rawRows = @($api.Results) + $result.RawRowCount = $rawRows.Count + $result.TruncationDetected = [bool]$api.TruncationDetected + + # Enrich each raw row and collapse catalog aliases that resolve to the same solution. Distinct-count + # metrics (ActiveDays, DistinctDevices) cannot be re-derived exactly once aliases collapse, so take + # the max as a safe upper bound; additive count metrics (Sessions, EstimatedPrompts) sum, since each + # raw app alias contributes a disjoint set of events. + $agg = [ordered]@{} + foreach ($r in $rawRows) { + if (-not $r) { continue } + $appName = [string]$r.AppNameMatched + $resolved = Resolve-AISIDFromAppName -AppName $appName + $solution = if ($resolved) { $resolved.AISolution } else { $appName } + $category = if ($resolved) { $resolved.Category } else { 'Unknown' } + $riskTier = if ($resolved) { $resolved.RiskTier } else { 'Unknown' } + $key = '{0}|{1}|{2}' -f ([string]$r.UPN).ToLowerInvariant(), $solution, [string]$r.YearMonth + + if (-not $agg.Contains($key)) { + $agg[$key] = [ordered]@{ + UPN = [string]$r.UPN; AISolution = $solution; YearMonth = [string]$r.YearMonth + Sessions = 0; ActiveDays = 0; EstimatedPrompts = 0; DistinctDevices = 0 + Category = $category; RiskTier = $riskTier + } + } + $b = $agg[$key] + $b.Sessions = [int]$b.Sessions + [int]$r.Sessions + $b.ActiveDays = [Math]::Max([int]$b.ActiveDays, [int]$r.ActiveDays) + $b.DistinctDevices = [Math]::Max([int]$b.DistinctDevices, [int]$r.DistinctDevices) + $b.EstimatedPrompts = [int]$b.EstimatedPrompts + [int]$r.EstimatedPrompts + } + + $result.Rows = @($agg.Values | ForEach-Object { [pscustomobject]$_ }) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function script:Get-AISIDScratchRoot { + # ONE run-specific internal AISID scratch root under OS temp. Unique per run — its leaf carries the + # global ScriptRunTimestamp, the process id, a per-run random nonce, AND a hash of the resolved user + # scope — so no two runs, and no two user scopes, ever share a scratch root or an off-hours/geography + # cache. Created on first use; deleted wholesale after a successful publication + # (Remove-AISIDScratchRoot). It is NEVER placed under the customer output folder, so a successful run + # leaves no cache/scratch behind. + if (-not $script:AISIDScratchNonce) { $script:AISIDScratchNonce = [System.Guid]::NewGuid().ToString('N').Substring(0, 12) } + $scopeSrc = (@($script:targetUsers) | ForEach-Object { ([string]$_).Trim().ToLowerInvariant() } | Where-Object { $_ } | Sort-Object -Unique) -join '|' + $scopeHash = ([BitConverter]::ToString([System.Security.Cryptography.SHA256]::Create().ComputeHash([System.Text.Encoding]::UTF8.GetBytes($scopeSrc)))).Replace('-', '').Substring(0, 8) + $leaf = 'PAX_AISID_{0}_{1}_{2}_{3}' -f $global:ScriptRunTimestamp, $PID, $script:AISIDScratchNonce, $scopeHash + return (Join-Path ([System.IO.Path]::GetTempPath()) $leaf) +} + +function script:Get-AISIDCacheDir { + # The off-hours/geography delta cache lives beneath the run-specific scratch root + # (Get-AISIDScratchRoot) in OS temp, NEVER in the customer output folder. Because the scratch root is + # unique per run, the cache is intra-run only — it is never reused between independent runs, and one + # scope can never read another scope's cache. The '.aisid_cache' leaf is created under the returned + # directory by Save-AISIDOffHoursGeoCache. The DestinationDir parameter is retained for call-site + # compatibility but intentionally ignored (the cache location is no longer destination-keyed). + param([Parameter()][AllowEmptyString()][string]$DestinationDir = '') + return (script:Get-AISIDScratchRoot) +} + +function script:Remove-AISIDScratchRoot { + # Recursively delete the run-specific AISID scratch root after a successful publication and twelve-CSV + # verification. Returns $true only when the root no longer exists on disk. A residual scratch root is a + # completed-with-gaps condition (the caller sets AISIDHadGaps -> exit 40 and logs the residual path) — + # a successful AISID run leaves no cache or scratch directory anywhere. + param([Parameter()][AllowEmptyString()][string]$Root = '') + if ([string]::IsNullOrWhiteSpace($Root)) { return $true } + if (-not (Test-Path -LiteralPath $Root)) { return $true } + try { [System.GC]::Collect(); [System.GC]::WaitForPendingFinalizers() } catch {} + try { Remove-Item -LiteralPath $Root -Recurse -Force -ErrorAction Stop } catch {} + return (-not (Test-Path -LiteralPath $Root)) +} + +function script:Get-AISIDFabricRunFolderUrl { + # Defect-3: build the CANONICAL Fabric AISID run-folder URL from Resolve-FabricTarget's structured + # result so the twelve fixed-name CSV bundle ALWAYS lives under Files/ (never Tables/) and the produced + # URL round-trips through Resolve-FabricTarget. Root, GUID-root, /Files[/], and /Tables[/] + # parents all resolve to '//Files/[/]'. Tables inputs route + # the operational CSV bundle under Files/ (TablesPath is intentionally ignored), consistent with the + # established operational-artifact routing policy (Send-FileToOneLake). Never appends the leaf to the + # raw parent URL. Returns '' when the parent cannot be parsed (caller treats '' as no remote target). + param([Parameter(Mandatory)][string]$ParentUrl, [Parameter(Mandatory)][string]$Leaf) + if ([string]::IsNullOrWhiteSpace($ParentUrl)) { return '' } + try { + $r = Resolve-FabricTarget -Url $ParentUrl + $filesBase = "$($r.FilesystemBase)/$($r.ItemFull)/Files" + $rel = if ($r.FilesPath) { "$($r.FilesPath)/$Leaf" } else { [string]$Leaf } + return "$filesBase/$rel" + } + catch { return '' } +} + +function script:Resolve-AISIDRunFolder { + # Canonical AISID run-folder resolver: every AISID run publishes the twelve fixed-name CSVs into a + # per-run 'AISID_' child folder beneath the DefenderUsage destination, on ALL + # tiers (Local / SharePoint / Fabric). One resolver drives writes, existing-history reads, completion + # checks, upload destination construction, upload registration, roster, metrics/snapshots, checkpoint/ + # resume, recovery, and AppendDefenderUsage, so every consumer agrees on one location. The twelve + # filenames inside are NEVER renamed. Optional test-injection of the resolved DefenderUsage dir and + # remote parent keeps the resolver pure for focused tests. + param( + [string]$ParentOverride = '', + [string]$EffectiveDirOverride = $null, + [string]$RemoteParentOverride = $null + ) + $leaf = 'AISID_' + $global:ScriptRunTimestamp + $mode = if ($script:RemoteOutputMode) { [string]$script:RemoteOutputMode } else { 'None' } + $effDir = + if (-not [string]::IsNullOrEmpty($EffectiveDirOverride)) { [string]$EffectiveDirOverride } + else { $du = script:Resolve-DataTypePaths -DataType 'DefenderUsage'; if ($du -and $du.EffectiveDir) { [string]$du.EffectiveDir } else { '' } } + $localFolder = if ($effDir) { Join-Path $effDir $leaf } else { $leaf } + $remoteParent = + if ($ParentOverride) { [string]$ParentOverride } + elseif (-not [string]::IsNullOrEmpty($RemoteParentOverride)) { [string]$RemoteParentOverride } + elseif ($script:DestParentUrl -and $script:DestParentUrl['DefenderUsage']) { [string]$script:DestParentUrl['DefenderUsage'] } + else { '' } + # Defect-3: on Fabric the AISID bundle URL is built from Resolve-FabricTarget's structured result so + # it ALWAYS lands under Files/ and round-trips through Resolve-FabricTarget (a root/Tables parent + # appended raw would be an invalid third segment). SharePoint / other tiers keep the folder-URL join. + $remoteUrl = + if ($mode -eq 'None' -or -not $remoteParent) { '' } + elseif ($mode -eq 'Fabric') { script:Get-AISIDFabricRunFolderUrl -ParentUrl $remoteParent -Leaf $leaf } + else { (($remoteParent -replace '/+$', '') + '/' + $leaf) } + $staging = Join-Path (script:Get-AISIDScratchRoot) $leaf + $writeDir = if ($mode -ne 'None') { $staging } else { $localFolder } + $display = if ($mode -ne 'None') { $remoteUrl } else { $localFolder } + return [pscustomobject][ordered]@{ + RunLeaf = $leaf + Mode = $mode + LocalRunFolder = $localFolder + RemoteRunFolderUrl = $remoteUrl + RemoteScratchStaging = $staging + WriteDir = $writeDir + DisplayPath = $display + } +} + +function script:Get-AISIDUploadPlan { + # Blocker-1: builds the AUTHORITATIVE remote AISID upload set from the run-specific staging directory + # ($script:AISIDRunFolder.RemoteScratchStaging) — never from the general run scratch dir. For each + # expected fixed-name leaf it REQUIRES the staged file to exist under $StagingDir and takes its full + # path from there, so an unrelated same-named file elsewhere can never impersonate an AISID artifact. + # Returns the resolved staged file objects (exactly one per present leaf, deduped case-insensitively), + # the set of missing leaves, and Complete=$true only when every expected leaf was found in staging. + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$StagingDir, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$ExpectedLeafs + ) + $files = New-Object System.Collections.Generic.List[object] + $missing = New-Object System.Collections.Generic.List[string] + $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($leaf in @($ExpectedLeafs)) { + if (-not $leaf) { continue } + if (-not $seen.Add($leaf)) { continue } + $full = if ($StagingDir) { Join-Path $StagingDir $leaf } else { '' } + if ($full -and (Test-Path -LiteralPath $full -PathType Leaf)) { $files.Add((Get-Item -LiteralPath $full)) } + else { [void]$missing.Add($leaf) } + } + return [pscustomobject][ordered]@{ + Files = @($files.ToArray()) + MissingLeafs = @($missing.ToArray()) + Complete = ($missing.Count -eq 0 -and $files.Count -gt 0) + } +} + +function script:Resolve-AISIDUploadParentUrl { + # Blocker-1: the per-run AISID child URL ($script:AISIDRemoteRunFolderUrl) is the upload parent for + # every AISID fixed-name leaf on every remote tier (SharePoint / Fabric); all other artifacts keep + # their default per-data-type parent. Isolated so the upload sweep's AISID routing is behaviorally + # testable without reconstructing the sweep. + param([Parameter(Mandatory)][string]$Leaf, [Parameter()][AllowEmptyString()][string]$DefaultParentUrl = '') + if ($script:AISIDUploadLeafs -and $script:AISIDUploadLeafs.Contains($Leaf) -and $script:AISIDRemoteRunFolderUrl) { + return [string]$script:AISIDRemoteRunFolderUrl + } + return [string]$DefaultParentUrl +} + +function script:New-AISIDRunFolderOwnership { + # Blocker-5: acquire EXCLUSIVE ownership of a local AISID run folder atomically. A one-second run + # timestamp cannot distinguish two concurrent processes, so Test-Path+CreateDirectory is not a valid + # ownership decision (both can win the race). This uses an atomic FileMode.CreateNew marker in the + # PARENT folder — the OS guarantees exactly one creator — as the ownership gate, THEN creates the run + # folder. Returns { Acquired=[bool]; Reason=[string]; MarkerPath=[string] }. An already-existing run + # folder (a prior non-resume run) or an existing marker (a concurrent/failed run) is a collision and + # fails closed — this never deletes or overwrites another run's folder or marker. + param([Parameter(Mandatory)][string]$RunFolder) + $parent = Split-Path -Parent $RunFolder + $leaf = Split-Path -Leaf $RunFolder + if ([string]::IsNullOrWhiteSpace($parent)) { $parent = (Get-Location).Path } + try { [void][System.IO.Directory]::CreateDirectory($parent) } + catch { return [pscustomobject]@{ Acquired = $false; Reason = ('parent-uncreatable: ' + $_.Exception.Message); MarkerPath = '' } } + # An existing run folder is a collision (no resume-identity verification is available here). + if (Test-Path -LiteralPath $RunFolder) { return [pscustomobject]@{ Acquired = $false; Reason = 'run-folder-exists'; MarkerPath = '' } } + $marker = Join-Path $parent ('.aisid_own_' + $leaf) + $fs = $null + try { + $fs = [System.IO.File]::Open($marker, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + $bytes = [System.Text.Encoding]::UTF8.GetBytes(('{0}|{1}|{2}' -f $PID, $global:ScriptRunTimestamp, ((Get-Date).ToUniversalTime().ToString('o')))) + $fs.Write($bytes, 0, $bytes.Length) + } + catch [System.IO.IOException] { + if ($fs) { $fs.Dispose() } + return [pscustomobject]@{ Acquired = $false; Reason = 'ownership-marker-exists'; MarkerPath = $marker } + } + catch { + if ($fs) { $fs.Dispose() } + return [pscustomobject]@{ Acquired = $false; Reason = ('marker-uncreatable: ' + $_.Exception.Message); MarkerPath = $marker } + } + finally { if ($fs) { $fs.Dispose() } } + # Marker acquired exclusively — create the run folder now. + try { [void][System.IO.Directory]::CreateDirectory($RunFolder) } + catch { + # Defect-3 fail-closed: the marker was acquired but the run folder could not be created. Remove the + # marker immediately and verify its absence so NO unregistered/orphan marker is left behind. If the + # marker itself cannot be removed, surface a structured cleanup failure that still names it. + $__rfErr = $_.Exception.Message + $__removed = $false + try { Remove-Item -LiteralPath $marker -Force -ErrorAction Stop; $__removed = -not (Test-Path -LiteralPath $marker) } catch { $__removed = $false } + if ($__removed) { return [pscustomobject]@{ Acquired = $false; Reason = ('run-folder-uncreatable: ' + $__rfErr); MarkerPath = ''; CleanupFailed = $false } } + return [pscustomobject]@{ Acquired = $false; Reason = ('run-folder-uncreatable; marker cleanup FAILED: ' + $__rfErr); MarkerPath = $marker; CleanupFailed = $true } + } + return [pscustomobject]@{ Acquired = $true; Reason = 'acquired'; MarkerPath = $marker; CleanupFailed = $false } +} + +function script:Get-AISIDRemoteFolderProbe { + # Blocker-4: read-only existence probe for a remote AISID run-folder URL. Returns + # { Accessible=[bool]; Exists=[bool] }. A confirmed HTTP 404 is Accessible=$true/Exists=$false (normal + # first run); a 2xx is Exists=$true; any auth/throttle/network/5xx/unparseable failure is + # Accessible=$false so the caller fails closed. NEVER creates or modifies the folder — a metadata GET + # (SharePoint driveItem) / directory HEAD (Fabric DFS) only. Uses the established target resolution + # and transport. Isolated so the collision decision is behaviorally testable via injection. + param( + [Parameter(Mandatory)][string]$Url, + [Parameter(Mandatory)][ValidateSet('SharePoint', 'Fabric')][string]$Mode + ) + try { + if ($Mode -eq 'SharePoint') { + $resolved = Resolve-SharePointTarget -Url $Url + $folderPath = if ($resolved.FolderPath) { $resolved.FolderPath } else { '' } + $metaUri = if ($folderPath) { + $esc = ($folderPath -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${esc}" + } + else { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root" } + $graphToken = Get-GraphAccessToken + if (-not $graphToken) { return [pscustomobject]@{ Accessible = $false; Exists = $false } } + try { + $null = Invoke-WebRequest -Uri $metaUri -Method GET -Headers @{ Authorization = "Bearer $graphToken" } -UseBasicParsing -ErrorAction Stop + return [pscustomobject]@{ Accessible = $true; Exists = $true } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return [pscustomobject]@{ Accessible = $true; Exists = $false } } + return [pscustomobject]@{ Accessible = $false; Exists = $false } + } + } + else { + $resolved = Resolve-FabricTarget -Url $Url + $dirPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$($resolved.FilesPath)" + try { + $null = Invoke-FabricWebRequest -Uri $dirPath -Method HEAD + return [pscustomobject]@{ Accessible = $true; Exists = $true } + } + catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return [pscustomobject]@{ Accessible = $true; Exists = $false } } + return [pscustomobject]@{ Accessible = $false; Exists = $false } + } + } + } + catch { return [pscustomobject]@{ Accessible = $false; Exists = $false } } +} + +function script:Test-AISIDRemoteRunFolderState { + # Blocker-4: classify a remote AISID run-folder URL as 'Absent' | 'Exists' | 'Inaccessible' from the + # read-only probe, distinguishing a confirmed absence (safe to create/upload) from a confirmed + # existing folder (collision -> fail closed, never touched) and from a lookup/transport/authorization + # failure (fail closed instead of assuming absence). The caller blocks on 'Exists' and 'Inaccessible'. + param( + [Parameter(Mandatory)][string]$Url, + [Parameter(Mandatory)][ValidateSet('SharePoint', 'Fabric')][string]$Mode + ) + try { + $probe = script:Get-AISIDRemoteFolderProbe -Url $Url -Mode $Mode + if ($null -eq $probe -or $probe.Accessible -ne $true) { return 'Inaccessible' } + if ($probe.Exists -eq $true) { return 'Exists' } + return 'Absent' + } + catch { return 'Inaccessible' } +} + +function script:New-AISIDRemoteRunFolderSharePoint { + # Defect-1/2: EXCLUSIVELY create the exact timestamped SharePoint AISID child folder via a single POST + # to the parent's /children with '@microsoft.graph.conflictBehavior'='fail'. The POST is the + # AUTHORITATIVE ownership decision (2xx = exclusive create; 409 / nameAlreadyExists = collision), so + # two same-second runs cannot both proceed. NEVER uses 'replace'/'rename'; NEVER treats an existing + # folder as success; NEVER modifies or deletes an existing folder. Not routed through + # Test-RemoteDestination (which targets the parent destination, not this timestamped child). Returns + # { State='Acquired'|'Collision'|'Inaccessible'; Reason }. + param([Parameter(Mandatory)][string]$Url) + $resolved = Resolve-SharePointTarget -Url $Url + $folderPath = if ($resolved.FolderPath) { [string]$resolved.FolderPath } else { '' } + if (-not $folderPath) { return [pscustomobject]@{ State = 'Inaccessible'; Reason = 'resolved child URL carries no folder path' } } + # Defect-2: Resolve-SharePointTarget derives FolderPath from Uri.AbsolutePath, whose segments are + # ALREADY percent-encoded (e.g. 'PAX%20Exports/Sub%20Folder'). Re-escaping them would double-encode + # ('%2520'). Canonicalize each segment EXACTLY ONCE: decode to its display value, then encode that + # once. The Graph colon-path is built from the single-encoded parent segments; the POST body child + # 'name' is the DECODED display value (Graph stores the literal name). This round-trip preserves + # spaces, '#', '%', apostrophes, Unicode, '+', and nested paths without ever creating a literal '%20'. + $segs = @($folderPath -split '/' | Where-Object { $_ -ne '' }) + if ($segs.Count -eq 0) { return [pscustomobject]@{ State = 'Inaccessible'; Reason = 'resolved child URL carries no folder path' } } + $leafDisplay = [System.Uri]::UnescapeDataString($segs[-1]) + $parentSegs = if ($segs.Count -gt 1) { $segs[0..($segs.Count - 2)] } else { @() } + $childrenUri = if ($parentSegs.Count -gt 0) { + $parentEnc = ($parentSegs | ForEach-Object { [System.Uri]::EscapeDataString([System.Uri]::UnescapeDataString($_)) }) -join '/' + "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${parentEnc}:/children" + } + else { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root/children" } + $body = @{ name = $leafDisplay; folder = @{}; '@microsoft.graph.conflictBehavior' = 'fail' } + try { + $null = Invoke-MgGraphRequest -Method POST -Uri $childrenUri -Body $body -ErrorAction Stop + return [pscustomobject]@{ State = 'Acquired'; Reason = 'exclusive create (conflictBehavior=fail)' } + } + catch { + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($status -eq 0) { try { if ($_.Exception.Response.StatusCode -is [System.Net.HttpStatusCode]) { $status = [int]$_.Exception.Response.StatusCode } } catch {} } + $detail = '' + try { if ($_.ErrorDetails -and $_.ErrorDetails.Message) { $detail = [string]$_.ErrorDetails.Message } } catch {} + if ([string]::IsNullOrEmpty($detail)) { $detail = [string]$_.Exception.Message } + if ($status -eq 409 -or $detail -match 'nameAlreadyExists') { return [pscustomobject]@{ State = 'Collision'; Reason = ('folder already exists (HTTP {0})' -f $status) } } + return [pscustomobject]@{ State = 'Inaccessible'; Reason = ('create failed (HTTP {0}): {1}' -f $status, $detail) } + } +} + +function script:New-AISIDRemoteRunFolderFabric { + # Defect-2: EXCLUSIVELY create the exact timestamped Fabric AISID child directory via ADLS Gen2 + # "Create Path" (PUT ?resource=directory) with a conditional 'If-None-Match: *' header, so an + # already-present directory is a CONFLICT (fail closed) rather than an idempotent success. The PUT is + # the authoritative ownership decision — a prior read-only HEAD is never relied upon. NEVER overwrites + # or reuses an existing directory. Returns { State='Acquired'|'Collision'|'Inaccessible'; Reason }. + param([Parameter(Mandatory)][string]$Url) + $resolved = Resolve-FabricTarget -Url $Url + $dirUri = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$($resolved.FilesPath)?resource=directory" + try { + $null = Invoke-FabricWebRequest -Uri $dirUri -Method PUT -Headers @{ 'If-None-Match' = '*' } + return [pscustomobject]@{ State = 'Acquired'; Reason = 'exclusive create (If-None-Match:*)' } + } + catch { + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + $detail = [string]$_.Exception.Message + if ($status -eq 409 -or $status -eq 412 -or $detail -match 'PathAlreadyExists') { return [pscustomobject]@{ State = 'Collision'; Reason = ('directory already exists (HTTP {0})' -f $status) } } + return [pscustomobject]@{ State = 'Inaccessible'; Reason = ('create failed (HTTP {0}): {1}' -f $status, $detail) } + } +} + +function script:Acquire-AISIDRemoteRunFolderOwnership { + # Defect-3: single dispatcher that acquires EXCLUSIVE ownership of the remote AISID run folder BEFORE + # any upload. The read-only probe (Test-AISIDRemoteRunFolderState) runs first as an early diagnostic + # only; the tier-specific exclusive CREATE is authoritative — if the probe reports Absent but the + # create reports Collision, the result is Collision (fail closed). Returns + # { Acquired=[bool]; State='Acquired'|'Collision'|'Inaccessible'; Mode; Url; Reason; ProbeState }. + # The caller proceeds (stages/uploads) only when Acquired is $true. + param( + [Parameter(Mandatory)][string]$Url, + [Parameter(Mandatory)][ValidateSet('SharePoint', 'Fabric')][string]$Mode + ) + $probeState = try { script:Test-AISIDRemoteRunFolderState -Url $Url -Mode $Mode } catch { 'Inaccessible' } + $create = if ($Mode -eq 'SharePoint') { script:New-AISIDRemoteRunFolderSharePoint -Url $Url } else { script:New-AISIDRemoteRunFolderFabric -Url $Url } + return [pscustomobject]@{ + Acquired = ($create.State -eq 'Acquired') + State = $create.State + Mode = $Mode + Url = $Url + Reason = $create.Reason + ProbeState = $probeState + } +} + +function script:Resolve-AISIDHistorySource { + # Blocker-3: the AISID history-read source, SEPARATE from the new output run folder. Prior history is + # read ONLY from an explicitly supplied -AppendDefenderUsage prior bundle; a plain + # -OutputPathDefenderUsage run (or no destination) reads NO prior history (never the fresh output + # folder). Returns { LocalDir=[string]; RemoteParentUrl=[string] } with at most one non-empty per tier + # (Local -> LocalDir; SharePoint/Fabric -> RemoteParentUrl). The supplied prior bundle is only READ; + # the reconciled result is written into the new AISID_ child. Overrides keep it testable. + param( + [Nullable[bool]]$AppendBoundOverride = $null, + [string]$EffectiveDirOverride = $null, + [string]$RemoteParentOverride = $null + ) + $mode = if ($script:RemoteOutputMode) { [string]$script:RemoteOutputMode } else { 'None' } + $bound = + if ($null -ne $AppendBoundOverride) { [bool]$AppendBoundOverride } + elseif ($script:AppendIsBound -and $script:AppendIsBound.ContainsKey('DefenderUsage')) { [bool]$script:AppendIsBound['DefenderUsage'] } + else { $false } + if (-not $bound) { return [pscustomobject]@{ LocalDir = ''; RemoteParentUrl = '' } } + if ($mode -eq 'None') { + $eff = + if (-not [string]::IsNullOrEmpty($EffectiveDirOverride)) { [string]$EffectiveDirOverride } + else { $du = try { script:Resolve-DataTypePaths -DataType 'DefenderUsage' } catch { $null }; if ($du -and $du.EffectiveDir) { [string]$du.EffectiveDir } else { '' } } + return [pscustomobject]@{ LocalDir = $eff; RemoteParentUrl = '' } + } + $rp = + if (-not [string]::IsNullOrEmpty($RemoteParentOverride)) { [string]$RemoteParentOverride } + elseif ($script:DestParentUrl -and $script:DestParentUrl['DefenderUsage']) { [string]$script:DestParentUrl['DefenderUsage'] } + else { '' } + return [pscustomobject]@{ LocalDir = ''; RemoteParentUrl = $rp } +} + +function Get-AISIDOffHoursGeoCache { + <# + .SYNOPSIS + Returns cached off-hours/geography raw rows when a same-day cache for the same collection + window exists; otherwise returns $null. This delta cache is scoped to the off-hours/geography + signal alone — no other data set reads or writes it. + .OUTPUTS + The cached raw row set, or $null on any miss / unreadable cache. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][int]$LookbackDays, + [string]$WindowStartUtc = '', + [string]$WindowEndUtc = '' + ) + + $cacheFile = Join-Path (Join-Path $DestinationDir '.aisid_cache') 'offhoursgeo.json' + if (-not (Test-Path -LiteralPath $cacheFile)) { return $null } + try { $payload = Get-Content -LiteralPath $cacheFile -Raw | ConvertFrom-Json } catch { return $null } + if (-not $payload) { return $null } + if ([int]$payload.LookbackDays -ne $LookbackDays) { return $null } + # Cache identity includes the exact effective window bounds: same day + different bounds = miss, + # and a legacy LookbackDays-only cache (no WindowStartUtc/WindowEndUtc) is always a miss. Both sides + # are normalized to the invariant UTC ISO-8601 form first, because ConvertFrom-Json auto-parses an + # ISO-8601 'Z' string back into a [datetime] whose culture-formatted [string] would never string-equal + # the ISO literal (a same-run cache would otherwise always false-miss). + if ((ConvertTo-AISIDInvariantUtcTimestamp -Value $payload.WindowStartUtc) -ne (ConvertTo-AISIDInvariantUtcTimestamp -Value $WindowStartUtc)) { return $null } + if ((ConvertTo-AISIDInvariantUtcTimestamp -Value $payload.WindowEndUtc) -ne (ConvertTo-AISIDInvariantUtcTimestamp -Value $WindowEndUtc)) { return $null } + $cachedUtc = $null + try { $cachedUtc = [datetime]$payload.CachedUtc } catch { return $null } + # Same-window reuse is valid only within the same UTC calendar day. + if ($cachedUtc.ToUniversalTime().Date -ne (Get-Date).ToUniversalTime().Date) { return $null } + return @($payload.Rows) +} + +function Save-AISIDOffHoursGeoCache { + <# + .SYNOPSIS + Persists off-hours/geography raw rows to the off-hours/geography-scoped delta cache. Failures + are non-fatal (a run must never break because the cache could not be written). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][int]$LookbackDays, + [string]$WindowStartUtc = '', + [string]$WindowEndUtc = '', + [object[]]$Rows = @() + ) + + $cacheDir = Join-Path $DestinationDir '.aisid_cache' + try { + if (-not (Test-Path -LiteralPath $cacheDir)) { New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null } + $payload = [ordered]@{ + LookbackDays = $LookbackDays + WindowStartUtc = $WindowStartUtc + WindowEndUtc = $WindowEndUtc + CachedUtc = (Get-Date).ToUniversalTime().ToString('o') + Rows = @($Rows) + } + $json = $payload | ConvertTo-Json -Depth 6 + [System.IO.File]::WriteAllText((Join-Path $cacheDir 'offhoursgeo.json'), $json, (New-Object System.Text.UTF8Encoding($false))) + } + catch { + Write-LogHost "[AISID] OffHoursGeo cache write failed (non-fatal): $($_.Exception.Message)" -ForegroundColor Yellow + } +} + +function Get-AISIDCountryCountMap { + <# + .SYNOPSIS + Converts an advanced-hunting CountryCounts bag into a hashtable of country-key -> integer + session count. OData annotation properties emitted by the Graph security hunting API + ('@odata.type' = '#microsoft.graph.security.dynamicColumnValue', and any per-field + '@odata.type' companion) are structurally ignored, never cast; any entry whose + value is not a whole number is skipped rather than guessed. Metadata therefore can never + enter the map and can never raise an Int32 conversion error. + .OUTPUTS + [hashtable] of [string] country -> [int] count (possibly empty). + #> + [CmdletBinding()] + param([Parameter()][AllowNull()]$CountryCounts) + + $map = @{} + if ($null -eq $CountryCounts) { return $map } + + if ($CountryCounts -is [System.Collections.IDictionary]) { + $pairs = @($CountryCounts.Keys | ForEach-Object { [pscustomobject]@{ Name = [string]$_; Value = $CountryCounts[$_] } }) + } + else { + $pairs = @($CountryCounts.PSObject.Properties | ForEach-Object { [pscustomobject]@{ Name = $_.Name; Value = $_.Value } }) + } + + foreach ($p in $pairs) { + $key = [string]$p.Name + if ([string]::IsNullOrWhiteSpace($key)) { continue } + if ($key -like '*@odata*') { continue } + $parsed = [long]0 + if ([long]::TryParse([string]$p.Value, [ref]$parsed)) { $map[$key] = [int]$parsed } + } + return $map +} + +function Get-AISIDOffHoursGeo { + <# + .SYNOPSIS + Collects the off-hours / geography data set from Entra sign-in hunting events and returns rows + in the contractual column order. Uses the off-hours/geography-scoped delta cache unless + DisableDeltaCache is set, in which case a full fresh query always runs and the cache is neither + read nor written. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, CacheHit, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param( + [switch]$DisableDeltaCache, + [string]$CacheDir + ) + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0; CacheHit = $false + TruncationDetected = $false; AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $signInAppNames = @(Get-AISIDSignInAppNames) + if ($signInAppNames.Count -eq 0) { + Write-LogHost '[AISID] OffHoursGeo: no sign-in app names in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + + $lookbackDays = Get-AISIDEffectiveLookbackDays + $appFilterJson = ($signInAppNames | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + # The collection window is bounded by the shared hunting-query timespan; the query filters to the + # catalog's sign-in app names and aggregates per user/month, with a per-country breakdown so the + # "primary country vs. anomalous" split can be computed client-side without a second round-trip. + $ohgBounds = Get-AISIDCollectionBounds + $ohgStart = $ohgBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ohgEnd = $ohgBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ohgScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'UPN' + # Batched: a large scope is split into size-bounded batches; both the outer and the joined subquery + # receive the SAME per-batch scope clause so every target user is queried exactly once and the + # per-user rows are concatenated across batches. $ohgScope retained for structural parity. + $null = $ohgScope + $ohgQueryBuilder = ({ + param($__scope) +@" +let AppFilter = dynamic([$appFilterJson]); +let WinStart = datetime($ohgStart); +let WinEnd = datetime($ohgEnd); +EntraIdSignInEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where Application in~ (AppFilter) +| extend UPN = tolower(AccountUpn) +| where isnotempty(UPN) +$__scope +| extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') +| extend CountryKey = iff(isempty(Country), 'Unknown', Country) +| summarize CountryCount = count() by UPN, YearMonth, CountryKey +| summarize + TotalSessions = sum(CountryCount), + DistinctCountries = dcount(CountryKey), + CountryCounts = make_bag(pack(CountryKey, CountryCount), 200) + by UPN, YearMonth +| join kind=inner ( + EntraIdSignInEvents + | where Timestamp >= WinStart and Timestamp < WinEnd + | where Application in~ (AppFilter) + | extend UPN = tolower(AccountUpn) + | where isnotempty(UPN) +$__scope + | extend Hour = datetime_part('hour', Timestamp) + | extend IsOffHours = iff(Hour < 7 or Hour > 19, 1, 0) + | extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') + | summarize OffHoursSessions = sum(IsOffHours) by UPN, YearMonth + ) on UPN, YearMonth +| project UPN, YearMonth, TotalSessions, OffHoursSessions, DistinctCountries, CountryCounts +"@ + }.GetNewClosure()) + + + $rawRows = $null + # A resumed AISID run whose stored cache-window identity cannot be trusted forces a guaranteed cache miss on READ (the original + # run's destination/cache identity cannot be trusted to match), so the window is re-queried fresh. + # The switch semantics of -DisableAISIDDeltaCache are unchanged; the cache WRITE below still runs. + if (-not $DisableDeltaCache -and -not $script:AISIDResumeForceCacheMiss -and $CacheDir) { + $cached = Get-AISIDOffHoursGeoCache -DestinationDir $CacheDir -LookbackDays $lookbackDays -WindowStartUtc $ohgStart -WindowEndUtc $ohgEnd + if ($null -ne $cached) { + $rawRows = @($cached); $result.CacheHit = $true + Write-LogHost "[AISID] OffHoursGeo: delta cache hit ($($rawRows.Count) rows)." -ForegroundColor DarkGray + } + } + + if ($null -eq $rawRows) { + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'UPN' -Description 'OffHoursGeo' -QueryBuilder $ohgQueryBuilder + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "OffHoursGeo query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $rawRows = @($api.Results) + $result.TruncationDetected = [bool]$api.TruncationDetected + if (-not $DisableDeltaCache -and $CacheDir) { + Save-AISIDOffHoursGeoCache -DestinationDir $CacheDir -LookbackDays $lookbackDays -WindowStartUtc $ohgStart -WindowEndUtc $ohgEnd -Rows $rawRows + } + } + + $result.RawRowCount = @($rawRows).Count + + # Client-side enrichment: off-hours percentage, and anomalous countries = every non-primary country. + $rows = New-Object System.Collections.Generic.List[object] + foreach ($r in $rawRows) { + if (-not $r) { continue } + $total = [int]$r.TotalSessions + $off = [int]$r.OffHoursSessions + $pct = if ($total -gt 0) { [Math]::Round(($off / [double]$total), 4) } else { 0 } + + $countryMap = Get-AISIDCountryCountMap -CountryCounts $r.CountryCounts + $anomalous = @() + if ($countryMap.Count -gt 1) { + # Deterministic primary: highest session count; ties broken by alphabetically-first country key. + $primary = ($countryMap.GetEnumerator() | Sort-Object @{ Expression = { $_.Value }; Descending = $true }, @{ Expression = { $_.Key }; Descending = $false } | Select-Object -First 1).Key + $anomalous = @($countryMap.Keys | Where-Object { $_ -ne $primary } | Sort-Object) + } + + $rows.Add([pscustomobject][ordered]@{ + UPN = [string]$r.UPN + YearMonth = [string]$r.YearMonth + TotalSessions = $total + OffHoursSessions = $off + OffHoursPct = $pct + DistinctCountries = [int]$r.DistinctCountries + AnomalousCountryCount = $anomalous.Count + AnomalousCountries = ($anomalous -join ';') + }) + } + $result.Rows = $rows.ToArray() + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +# Severity-tier reference for OAuth permission scopes. The tiering signal is anchored on Microsoft's own +# admin-consent-requirement model: application permissions always require admin consent, many high-privilege +# delegated permissions require admin consent, and delegated permissions that do not require admin consent are +# lower-privilege by design. See: +# https://learn.microsoft.com/en-us/entra/identity-platform/permissions-consent-overview +# https://learn.microsoft.com/en-us/graph/permissions-overview +# https://learn.microsoft.com/en-us/graph/permissions-reference +# The admin-consent facts are Microsoft-documented; the specific High/Medium/Low assignment per scope is +# informed practitioner judgment layered on top of that signal, not a Microsoft-official severity ranking. +# Any scope not listed here resolves to 'Unclassified' (never a numeric guess). +$script:AISIDPermissionRiskCatalog = @{ + # High: broad directory/role/app control, tenant-wide write, or mail send/read-write (all admin-consent). + 'Directory.ReadWrite.All' = 'High' + 'RoleManagement.ReadWrite.Directory' = 'High' + 'Application.ReadWrite.All' = 'High' + 'AppRoleAssignment.ReadWrite.All' = 'High' + 'User.ReadWrite.All' = 'High' + 'Group.ReadWrite.All' = 'High' + 'Sites.FullControl.All' = 'High' + 'Files.ReadWrite.All' = 'High' + 'Mail.ReadWrite' = 'High' + 'Mail.Send' = 'High' + 'Directory.AccessAsUser.All' = 'High' + 'full_access_as_app' = 'High' + # Medium: tenant-wide read of sensitive data, or scoped write of user content (typically admin-consent). + 'Directory.Read.All' = 'Medium' + 'User.Read.All' = 'Medium' + 'Group.Read.All' = 'Medium' + 'Mail.Read' = 'Medium' + 'Files.Read.All' = 'Medium' + 'Sites.Read.All' = 'Medium' + 'Calendars.ReadWrite' = 'Medium' + 'Contacts.ReadWrite' = 'Medium' + 'People.Read.All' = 'Medium' + 'Chat.Read.All' = 'Medium' + # Low: self-scoped or low-privilege scopes that do not require admin consent by design. + 'User.Read' = 'Low' + 'User.ReadBasic.All' = 'Low' + 'openid' = 'Low' + 'profile' = 'Low' + 'email' = 'Low' + 'offline_access' = 'Low' + 'Calendars.Read' = 'Low' + 'Contacts.Read' = 'Low' + 'Files.Read' = 'Low' + 'Mail.ReadBasic' = 'Low' +} + +function Resolve-AISIDPermissionTier { + <# + .SYNOPSIS + Maps a single Graph permission scope string to a severity tier via the reference catalog. Any scope + not present in the catalog resolves to 'Unclassified' — this is the only fallback, and it is never a + numeric guess. + .OUTPUTS + One of: 'High', 'Medium', 'Low', 'Unclassified'. + #> + [CmdletBinding()] + param([Parameter(Mandatory)][AllowEmptyString()][string]$Scope) + + $key = $Scope.Trim() + if ($key -and $script:AISIDPermissionRiskCatalog.ContainsKey($key)) { + return $script:AISIDPermissionRiskCatalog[$key] + } + return 'Unclassified' +} + +function Get-AISIDPermissionWeight { + <# + .SYNOPSIS + Computes the integer PermissionWeight for one consent grant's set of permission scopes. Each scope is + mapped to a severity tier and scored (High=3, Medium=2, Low=1, Unclassified=0); the row's weight is the + HIGHEST score across all granted scopes, because one severe scope defines the risk regardless of how + many trivial scopes accompany it. A grant with no scopes returns 0 without ever entering the tier + lookup; the companion Permissions column distinguishes 'no scopes' (empty) from 'only unrecognized + scopes' (populated) at the same weight of 0. + .OUTPUTS + [int] weight (0-3). + #> + [CmdletBinding()] + param([string[]]$Permissions = @()) + + $tierScore = @{ High = 3; Medium = 2; Low = 1; Unclassified = 0 } + + # A single source value may carry one or several scope tokens; split on common delimiters and strip any + # surrounding quote/bracket characters so an exact catalog-key match is not defeated by formatting. + $tokens = New-Object System.Collections.Generic.List[string] + foreach ($p in @($Permissions)) { + if (-not $p) { continue } + foreach ($t in ($p -split '[,;\s]+')) { + $t = $t.Trim('"', "'", '[', ']') + if ($t) { $tokens.Add($t) } + } + } + if ($tokens.Count -eq 0) { return 0 } + + $max = 0 + foreach ($t in $tokens) { + $score = $tierScore[(Resolve-AISIDPermissionTier -Scope $t)] + if ($score -gt $max) { $max = $score } + } + return $max +} + +function Get-AISIDOAuthConsents { + <# + .SYNOPSIS + Collects OAuth consent activity from the Entra audit log (directoryAudits) on the existing + Graph session, correlates it to catalog AI apps, and aggregates to one row per + (UPN, AppName, YearMonth). This table is cache-free by design (the delta cache is + off-hours/geography-only) and issues no hunting query. + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0 + TruncationDetected = $false; AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $aiAppNames = @(Get-AISIDSignInAppNames) + $aiAppLower = @{} + foreach ($n in $aiAppNames) { $aiAppLower[$n.ToLowerInvariant()] = $true } + + $ocBounds = Get-AISIDCollectionBounds + $startIso = $ocBounds.StartUtc.ToString('o') + $endIso = $ocBounds.EndUtc.ToString('o') + # Consent-related operations recorded in the Entra audit log. + $ops = @('Consent to application', 'Add delegated permission grant', 'Add app role assignment to service principal') + $opClause = ($ops | ForEach-Object { "activityDisplayName eq '$_'" }) -join ' or ' + $filter = "($opClause) and activityDateTime ge $startIso and activityDateTime lt $endIso" + $encoded = [System.Uri]::EscapeDataString($filter) + $uri = "https://graph.microsoft.com/v1.0/auditLogs/directoryAudits?`$filter=$encoded&`$top=200" + + $collected = New-Object System.Collections.Generic.List[object] + try { + while ($uri) { + # Refresh on the existing session before each page to avoid mid-pagination 401s. + $refresh = Refresh-GraphTokenIfNeeded + if ($refresh -eq 'Quit') { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $resp = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + if ($resp.value) { foreach ($rec in $resp.value) { $collected.Add($rec) } } + $uri = $resp.'@odata.nextLink' + } + } + catch { + $result.ErrorMessage = "OAuthConsents Graph query failed: $($_.Exception.Message)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $rawRecords = @($collected.ToArray()) + $result.RawRowCount = $rawRecords.Count + + # Client-side catalog match + aggregate by (UPN, AppName, YearMonth). + $agg = [ordered]@{} + foreach ($rec in $rawRecords) { + if (-not $rec) { continue } + $target = if ($rec.targetResources) { @($rec.targetResources)[0] } else { $null } + $appName = if ($target -and $target.displayName) { [string]$target.displayName } else { $null } + if (-not $appName) { continue } + + $resolved = Resolve-AISIDFromAppName -AppName $appName + if (-not ($resolved -or $aiAppLower.ContainsKey($appName.ToLowerInvariant()))) { continue } + $solution = if ($resolved) { $resolved.AISolution } else { $appName } + + $upn = $null + if ($rec.initiatedBy -and $rec.initiatedBy.user -and $rec.initiatedBy.user.userPrincipalName) { + $upn = [string]$rec.initiatedBy.user.userPrincipalName + } + if (-not $upn) { continue } + $upn = $upn.ToLowerInvariant() + + $ts = $null; try { $ts = [datetime]$rec.activityDateTime } catch {} + if (-not $ts) { continue } + $tsUtc = $ts.ToUniversalTime() + $yearMonth = $tsUtc.ToString('yyyy-MM') + + $perms = @() + # Granted scopes come from the consent event's DelegatedPermissionGrant.Scope modified property + # (TargetResources[].ModifiedProperties[].NewValue) — the field-tested source for the actual granted + # scope strings. The value can arrive wrapped in quotes/brackets, so surrounding quote/bracket + # characters are trimmed before it is stored (the per-token tokenizer strips them again defensively). + if ($rec.targetResources) { + foreach ($tr in @($rec.targetResources)) { + if (-not $tr.modifiedProperties) { continue } + foreach ($mp in @($tr.modifiedProperties)) { + $dn = if ($mp.displayName) { [string]$mp.displayName } elseif ($mp.DisplayName) { [string]$mp.DisplayName } else { '' } + if ($dn -ne 'DelegatedPermissionGrant.Scope') { continue } + $nv = if ($mp.newValue) { [string]$mp.newValue } elseif ($mp.NewValue) { [string]$mp.NewValue } else { '' } + $nv = $nv.Trim().Trim('"', "'", '[', ']').Trim() + if ($nv) { $perms += $nv } + } + } + } + + $key = '{0}|{1}|{2}' -f $upn, $solution, $yearMonth + if (-not $agg.Contains($key)) { + $agg[$key] = [ordered]@{ + UPN = $upn; AppName = $solution; YearMonth = $yearMonth + ConsentCount = 0; LastConsent = $tsUtc + PermSet = (New-Object System.Collections.Generic.HashSet[string]) + } + } + $b = $agg[$key] + $b.ConsentCount = [int]$b.ConsentCount + 1 + if ($tsUtc -gt [datetime]$b.LastConsent) { $b.LastConsent = $tsUtc } + foreach ($p in $perms) { [void]$b.PermSet.Add($p) } + } + + $rows = foreach ($b in $agg.Values) { + $permArr = @($b.PermSet) + [pscustomobject][ordered]@{ + UPN = $b.UPN + AppName = $b.AppName + YearMonth = $b.YearMonth + ConsentCount = [int]$b.ConsentCount + LastConsent = ([datetime]$b.LastConsent).ToString('o') + PermissionWeight = Get-AISIDPermissionWeight -Permissions $permArr + Permissions = ($permArr -join ';') + } + } + $result.Rows = @($rows) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function Get-AISIDSSOSignIns { + <# + .SYNOPSIS + Collects SSO sign-in activity from Entra sign-in hunting events, aggregated per + (UPN, Application, YearMonth). Cache-free by design (the delta cache is off-hours/geography-only). + .OUTPUTS + Hashtable: Success, Rows, RawRowCount, TruncationDetected, AuthQuit, ErrorMessage, Elapsed. + #> + [CmdletBinding()] + param() + + $result = @{ + Success = $false; Rows = @(); RawRowCount = 0 + TruncationDetected = $false; AuthQuit = $false; ErrorMessage = ''; Elapsed = [TimeSpan]::Zero + } + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + $signInAppNames = @(Get-AISIDSignInAppNames) + if ($signInAppNames.Count -eq 0) { + Write-LogHost '[AISID] SSOSignIns: no sign-in app names in catalog; skipping query.' -ForegroundColor Yellow + $result.Success = $true; $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $appFilterJson = ($signInAppNames | ForEach-Object { '"' + ($_ -replace '"', '\"') + '"' }) -join ',' + # Guest is inferred from the external-identity marker in the UPN; conditional-access applicability is + # inferred from a non-empty conditional-access evaluation status. Both are advisory client-side reads. + $ssoBounds = Get-AISIDCollectionBounds + $ssoStart = $ssoBounds.StartUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ssoEnd = $ssoBounds.EndUtc.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ') + $ssoScope = script:Get-AISIDScopeKqlFilter -UpnColumn 'UPN' + # Batched: a large scope is split into size-bounded batches; the query runs once per batch (every + # target user queried exactly once) and the per-user rows are concatenated. $ssoScope retained for + # structural parity; the per-batch clause ($__scope) is what the runner injects. + $null = $ssoScope + $api = script:Invoke-AISIDBatchedHuntingQuery -UpnColumn 'UPN' -Description 'SSOSignIns' -QueryBuilder ({ + param($__scope) +@" +let AppFilter = dynamic([$appFilterJson]); +let WinStart = datetime($ssoStart); +let WinEnd = datetime($ssoEnd); +EntraIdSignInEvents +| where Timestamp >= WinStart and Timestamp < WinEnd +| where Application in~ (AppFilter) +| extend UPN = tolower(AccountUpn) +| where isnotempty(UPN) +$__scope +| extend YearMonth = format_datetime(Timestamp, 'yyyy-MM') +| extend DayKey = format_datetime(Timestamp, 'yyyy-MM-dd') +| extend IsGuestFlag = iff(UPN has '#ext#', 'True', 'False') +| extend CAApplied = iff(ConditionalAccessStatus == 'success' or ConditionalAccessStatus == 'failure', 'Yes', 'No') +| extend CountryKey = iff(isempty(Country), 'Unknown', Country) +| summarize + SignInCount = count(), + DistinctDays = dcount(DayKey), + IsGuest = max(IsGuestFlag), + HasConditionalAccess = max(CAApplied), + Countries = make_set(CountryKey, 100), + LastSignIn = max(Timestamp) + by UPN, Application, YearMonth +"@ + }.GetNewClosure()) + if ($api.AuthQuit) { + $result.AuthQuit = $true; $result.ErrorMessage = 'Graph token refresh returned Quit' + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + if (-not $api.Success) { + $result.ErrorMessage = "SSOSignIns query failed: $($api.ErrorMessage)" + $sw.Stop(); $result.Elapsed = $sw.Elapsed; return $result + } + $rawRows = @($api.Results) + $result.RawRowCount = $rawRows.Count + $result.TruncationDetected = [bool]$api.TruncationDetected + + $rows = foreach ($r in $rawRows) { + if (-not $r) { continue } + $countries = @() + if ($r.Countries) { + if ($r.Countries -is [System.Array]) { $countries = @($r.Countries | ForEach-Object { [string]$_ }) } + else { $countries = @([string]$r.Countries) } + } + $last = '' + if ($r.LastSignIn) { try { $last = ([datetime]$r.LastSignIn).ToUniversalTime().ToString('o') } catch { $last = [string]$r.LastSignIn } } + [pscustomobject][ordered]@{ + UPN = [string]$r.UPN + Application = [string]$r.Application + YearMonth = [string]$r.YearMonth + SignInCount = [int]$r.SignInCount + DistinctDays = [int]$r.DistinctDays + IsGuest = [string]$r.IsGuest + Countries = ($countries -join ';') + HasConditionalAccess = [string]$r.HasConditionalAccess + LastSignIn = $last + } + } + $result.Rows = @($rows) + $result.Success = $true + $sw.Stop(); $result.Elapsed = $sw.Elapsed + return $result +} + +function Get-AISIDSolutionsCatalogExport { + <# + .SYNOPSIS + Projects the in-memory AI solutions reference catalog to the dashboard's dimension CSV columns. + This is dimension data — the whole file is regenerated each run (overwrite); there is no + time-series grain and no per-row key merge. + .OUTPUTS + An array of rows in the contractual column order. + #> + [CmdletBinding()] + param() + + $rows = foreach ($entry in $script:AISolutionCatalog) { + if (-not $entry) { continue } + [pscustomobject][ordered]@{ + AISolution = [string]$entry.AISolution + Category = [string]$entry.Category + Vendor = [string]$entry.Vendor + RiskTier = [string]$entry.RiskTier + DefaultDataHandling = [string]$entry.DefaultDataHandling + SolutionGroup = [string]$entry.SolutionGroup + } + } + return @($rows) +} + +function Get-AISIDEntraUsersProjection { + <# + .SYNOPSIS + Projects the already-collected general EntraUsers export data down to the dashboard's fixed + column set, in contractual order. Reads in-memory data only — it performs no directory query of + its own and leaves the general export untouched. Dimension data: the whole file is regenerated + each run (overwrite); no per-row key merge. + .OUTPUTS + An array of rows in the contractual column order. + #> + [CmdletBinding()] + param([object[]]$SourceRows = @($script:EntraUsersData)) + + $SourceRows = @($SourceRows) + + # Target-user scope for the EntraUsers dimension: when a user/group scope was requested, restrict + # the projection to the target users PLUS their full manager chain (managers of managers, up to the + # top). No non-target UPN reaches the dimension except those manager-chain rows, which the dashboard + # hierarchy needs to resolve. Unscoped runs project the source rows unchanged. + $scope = script:Get-AISIDScopeUpnSet + if ($null -ne $scope) { + $byUpn = @{} + foreach ($u in $SourceRows) { + if (-not $u) { continue } + $g = if ($u -is [System.Collections.IDictionary]) { [string]$u['userPrincipalName'] } else { [string]$u.userPrincipalName } + $g = ([string]$g).Trim() + if ($g) { $byUpn[$g.ToLowerInvariant()] = $u } + } + $keep = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($t in @($scope)) { + $cur = ([string]$t).Trim() + $guard = 0 + while ($cur -and $keep.Add($cur) -and $guard -lt 10000) { + $guard++ + $row = $byUpn[$cur.ToLowerInvariant()] + if (-not $row) { break } + $mgr = if ($row -is [System.Collections.IDictionary]) { [string]$row['manager_userPrincipalName'] } else { [string]$row.manager_userPrincipalName } + $cur = ([string]$mgr).Trim() + } + } + $SourceRows = @($SourceRows | Where-Object { + $g = if ($_ -is [System.Collections.IDictionary]) { [string]$_['userPrincipalName'] } else { [string]$_.userPrincipalName } + $g = ([string]$g).Trim() + $g -and $keep.Contains($g) + }) + } + + $rows = foreach ($u in @($SourceRows)) { + if (-not $u) { continue } + $get = { + param($n) + if ($u -is [System.Collections.IDictionary]) { $u[$n] } else { $u.$n } + } + [pscustomobject][ordered]@{ + userPrincipalName = [string](& $get 'userPrincipalName') + displayName = [string](& $get 'displayName') + department = [string](& $get 'department') + jobTitle = [string](& $get 'jobTitle') + city = [string](& $get 'city') + country = [string](& $get 'country') + companyName = [string](& $get 'companyName') + accountEnabled = [string](& $get 'accountEnabled') + userType = [string](& $get 'userType') + manager_displayName = [string](& $get 'manager_displayName') + manager_userPrincipalName = [string](& $get 'manager_userPrincipalName') + hasLicense = [string](& $get 'hasLicense') + assignedLicenses = [string](& $get 'assignedLicenses') + createdDateTime = [string](& $get 'createdDateTime') + } + } + return @($rows) +} + +function Export-AISIDPilotTables { + <# + .SYNOPSIS + Processes the five query-backed data sets (Activity Sessions, Off-Hours/Geography, File + Proximity, OAuth Consents, SSO Sign-Ins) through the shared export/merge infrastructure, + producing the full dashboard output set. Each query-backed table is collected only when its + per-table licensing verdict is eligible; an ineligible or empty table still yields a valid + header-only CSV. File Proximity is event-grain (append-and-dedup merge); when it is ineligible + or its query fails and no prior history exists, the header-only stub is emitted as its fallback. + The off-hours/geography delta cache is used unless DisableDeltaCache is set. + .OUTPUTS + Ordered hashtable of written paths per table, plus AuthQuit. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [Parameter(Mandatory)][System.Collections.IDictionary]$Verdict, + [switch]$DisableDeltaCache, + [hashtable]$ExistingRows = @{}, + [string]$ParentOverride, + [string]$HistoryDir = '', + [string]$RawPurviewCsv + ) + + $out = [ordered]@{ + ActivitySessions = $null; OffHoursGeo = $null; OAuthConsents = $null; SSOSignIns = $null + FileProximity = $null; ClientChannel = $null; CopilotUsage = $null; AppGovAlerts = $null; CloudDiscovery = $null; MDASessions = $null + EntraUsers = $null; SolutionsCatalog = $null; AuthQuit = $false + ReadFailures = (New-Object System.Collections.Generic.List[string]) + Failed = (New-Object System.Collections.Generic.List[string]) + TableStatus = [ordered]@{} + } + # A1 state contract: every dashboard table gets an EXPLICIT terminal state before completeness is + # evaluated. Seed each as Incomplete ("collector not reached") so a table skipped by an early exit is + # never silently treated as clean; per-table processing below overwrites this with the decided state. + foreach ($t in @('ActivitySessions', 'OffHoursGeo', 'FileProximity', 'OAuthConsents', 'SSOSignIns', 'ClientChannel', 'CopilotUsage', 'AppGovAlerts', 'CloudDiscovery', 'MDASessions', 'SolutionsCatalog', 'EntraUsers')) { + $out.TableStatus[$t] = [ordered]@{ State = 'Incomplete'; Reason = 'collector not reached' } + } + + # Table -> collector. Each collector returns the standard result hashtable (Success/Rows/AuthQuit). + # Off-Hours/Geography is the only cache-aware collector; every other table is cache-free. + $collect = { + param($table) + switch ($table) { + 'ActivitySessions' { Get-AISIDActivitySessions } + 'OffHoursGeo' { Get-AISIDOffHoursGeo -DisableDeltaCache:$DisableDeltaCache -CacheDir (script:Get-AISIDCacheDir -DestinationDir $DestinationDir) } + 'OAuthConsents' { Get-AISIDOAuthConsents } + 'SSOSignIns' { Get-AISIDSSOSignIns } + 'ClientChannel' { Get-AISIDClientChannel } + 'CopilotUsage' { Get-AISIDCopilotUsage -RawPurviewCsv $RawPurviewCsv } + } + } + + foreach ($table in @('ActivitySessions', 'OffHoursGeo', 'OAuthConsents', 'SSOSignIns', 'ClientChannel', 'CopilotUsage')) { + $schema = $script:AISIDTableSchemas[$table] + $rows = @() + $state = 'Unavailable'; $reason = 'licensing ineligible' + if ($Verdict[$table] -and $Verdict[$table].Eligible) { + try { + $q = & $collect $table + if ($q.AuthQuit) { $out.AuthQuit = $true } + if ($q.Success) { $rows = @($q.Rows); if (@($rows).Count -gt 0) { $state = 'Succeeded'; $reason = 'collected rows' } else { $state = 'Empty'; $reason = 'query succeeded with zero rows' } } + else { $state = 'Failed'; $reason = 'collector returned Success=false'; [void]$out.Failed.Add($table) } + } + catch { + # Per-table isolation: an UNEXPECTED throw (as opposed to a Success=$false return) is + # caught here so the remaining tables in the loop still run. $rows stays empty, so the + # read-before-merge + export below fall this table back to its safe-empty / history- + # preserving output exactly as an ineligible or Success=$false table does. + Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); falling back to safe-empty output for this table." -f $table, $_.Exception.Message) -ForegroundColor Yellow + $state = 'Failed'; $reason = ("collector threw: {0}" -f $_.Exception.Message) + [void]$out.Failed.Add($table) + } + } + # Ineligible or empty: $rows stays empty; Export-AISIDTable still writes a header-only CSV. + + # Client-side target-user scope enforcement (defense-in-depth behind the server-side KQL + # filter): when a user/group scope was requested, drop any newly collected row whose UPN is + # not in the resolved target set so no non-target user reaches this run's output. No-op for + # unscoped runs and for non-user-grain / global tables (ClientChannel, SolutionsCatalog) and + # the separately-scoped EntraUsers dimension. + $rows = script:Select-AISIDScopedRows -Table $table -Rows $rows + + # Deidentify this table's NEW rows BEFORE the read-before-merge below, so on-disk (already + # deidentified) rows and these newly-collected rows compare on matching keys during merge. + # No-op when -Deidentify is off. CopilotUsage is now deidentified here too: its source is the + # IDENTIFIED raw Purview CSV (read before any raw scrub), so its UserPrincipalName must pass once + # through the shared Get-PaxDeidUpn path to match the deidentified EntraUsers key. + $rows = ConvertTo-AISIDDeidentifiedRows -Table $table -Rows $rows + + # ClientChannel scope provenance: ClientChannel aggregates away UPN, so a prior ClientChannel + # bundle cannot be proven to share this run's user scope. For a SCOPED run, refuse to merge + # ClientChannel history whose scope provenance cannot be established — fail closed (don't merge, + # don't publish, preserve prior, AISID gap) rather than silently mix tenant-wide and scoped + # aggregates. + if ($table -eq 'ClientChannel' -and $null -ne (script:Get-AISIDScopeUpnSet)) { + $__ccHasPrior = + ($ExistingRows.ContainsKey($table) -and @($ExistingRows[$table]).Count -gt 0) -or + ((-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') -and $HistoryDir -and (Test-AISIDCsvHasDataRow -Path (Join-Path $HistoryDir $schema.FileName))) -or + ($script:RemoteOutputMode -and $script:RemoteOutputMode -ne 'None' -and $ParentOverride) + if ($__ccHasPrior) { + Write-LogHost "[AISID] ClientChannel: a scoped run cannot prove the prior ClientChannel bundle shares this user scope (aggregates carry no UPN); refusing to merge and leaving the prior file unchanged (AISID gap)." -ForegroundColor Yellow + [void]$out.Failed.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Failed'; Reason = 'scoped ClientChannel append: prior scope provenance unprovable; prior file left unchanged' } + $script:AISIDHadGaps = $true + continue + } + } + + # Read-before-merge for replace-on-key tables: reconcile against the accumulated history already + # on disk so prior periods are preserved instead of being overwritten with only this run's rows. + # Overwrite/dimension tables (empty Key) never enter this path. An explicitly injected ExistingRows + # entry takes precedence over the on-disk read. On the LOCAL tier the prior history is streamed + # row-by-row through the bounded merge (never materialized as an array); the remote tier still + # downloads-and-reads (inherent) then array-merges. + if ($schema.Key -and $schema.Key.Count -gt 0) { + if ($ExistingRows.ContainsKey($table)) { + $existing = @($ExistingRows[$table]) + if (@($existing).Count -gt 0) { + $rows = Merge-AISIDTableOnKey -ExistingRows $existing -NewRows $rows -KeyColumns $schema.Key + } + } + elseif (-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') { + # Blocker-3: prior history comes from the SEPARATE supplied prior-bundle folder ($HistoryDir), + # never from the fresh output run folder ($DestinationDir). No $HistoryDir + # (-OutputPathDefenderUsage without -AppendDefenderUsage) => no prior history read; this run's + # rows stand alone. A read failure (locked/malformed, not a confirmed absence) propagates so the + # prior file is left untouched this run. + $__histPath = if ($HistoryDir) { Join-Path $HistoryDir $schema.FileName } else { '' } + if ($__histPath -and (Test-Path -LiteralPath $__histPath)) { + try { + $rows = Merge-AISIDTableOnKeyStreamed -ExistingCsvPath $__histPath -NewRows $rows -KeyColumns $schema.Key + } + catch { + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f $table, $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Incomplete'; Reason = 'existing file unreadable; prior file left untouched' } + continue + } + } + } + else { + # Remote tier: prior history from the supplied prior-bundle URL ($ParentOverride); none when + # empty (no -AppendDefenderUsage). Download prior history to a local scratch file (memory-safe), + # then stream the scratch through the SAME bounded merge. Scratch is removed on success and + # preserved for recovery on failure. + if ($ParentOverride) { + $scratchPath = $null + try { + $scratchPath = Get-AISIDExistingCsvToScratch -FileName $schema.FileName -ParentOverride $ParentOverride + $rows = Merge-AISIDTableOnKeyStreamed -ExistingCsvPath $scratchPath -NewRows $rows -KeyColumns $schema.Key + } + catch { + if ($scratchPath -and (Test-Path -LiteralPath $scratchPath)) { Write-LogHost ("[AISID] {0}: remote history scratch preserved for recovery: {1}" -f $table, $scratchPath) -ForegroundColor DarkYellow } + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f $table, $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Incomplete'; Reason = 'existing file unreadable; prior file left untouched' } + continue + } + if ($scratchPath -and (Test-Path -LiteralPath $scratchPath)) { Remove-Item -LiteralPath $scratchPath -Force -ErrorAction SilentlyContinue } + } + } + } + + # Final post-merge scope containment (defense-in-depth): after the read-before-merge, verify NO + # row in the merged result falls outside the effective target scope. History (injected, streamed + # local, or downloaded remote) that reintroduced a non-target user is caught here; when + # -Deidentify is on the comparison uses deidentified scope keys. If any out-of-scope row survives, + # the merged file is NOT published, the prior file is left unchanged, the table is marked + # Failed/Incomplete, and an AISID gap (exit 40) is flagged — a scoped run never emits a non-target + # user and never overwrites prior history with a scope it cannot prove. + $__cont = script:Select-AISIDContainedRows -Table $table -Rows $rows + if ($__cont.Applies -and @($__cont.OutOfScope).Count -gt 0) { + Write-LogHost ("[AISID] {0}: {1} row(s) outside the requested user scope survived the history merge; refusing to publish and leaving the prior file unchanged (AISID gap)." -f $table, @($__cont.OutOfScope).Count) -ForegroundColor Yellow + [void]$out.Failed.Add($table) + $out.TableStatus[$table] = [ordered]@{ State = 'Failed'; Reason = 'post-merge scope containment failed; prior file left unchanged' } + $script:AISIDHadGaps = $true + continue + } + + $out[$table] = Export-AISIDTable -DestinationDir $DestinationDir -FileName $schema.FileName -OrderedColumns $schema.Columns -Rows $rows + $out.TableStatus[$table] = [ordered]@{ State = $state; Reason = $reason } + } + + # File Proximity: event-grain, append-and-dedup. Same licensing-gated pattern as the query-backed + # tables above (eligibility -> collect -> read-before-merge -> merge -> export), but with the + # append+dedup merge so accumulated event history only ever grows. The header-only stub is the + # fallback only when the table is ineligible or its query fails AND no prior history exists. + $fpSchema = $script:AISIDTableSchemas['FileProximity'] + $fpRows = @() + $fpCollected = $false + $fpState = 'Unavailable'; $fpReason = 'licensing ineligible' + if ($Verdict['FileProximity'] -and $Verdict['FileProximity'].Eligible) { + try { + $fpQ = Get-AISIDFileProximity + if ($fpQ.AuthQuit) { $out.AuthQuit = $true } + if ($fpQ.Success) { $fpRows = @($fpQ.Rows); $fpCollected = $true; if (@($fpRows).Count -gt 0) { $fpState = 'Succeeded'; $fpReason = 'collected rows' } else { $fpState = 'Empty'; $fpReason = 'query succeeded with zero rows' } } else { $fpState = 'Failed'; $fpReason = 'collector returned Success=false'; [void]$out.Failed.Add('FileProximity') } + } + catch { + # Per-table isolation: an unexpected throw is caught so the dimension tables still run. + # $fpCollected stays $false and $fpRows empty -> the append-dedup / stub path below falls + # back to this table's safe output (stub when no prior history, else history preserved). + Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); falling back to safe-empty output for this table." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + $fpState = 'Failed'; $fpReason = ("collector threw: {0}" -f $_.Exception.Message) + [void]$out.Failed.Add('FileProximity') + } + } + # Blocker-2: client-side target-user scope enforcement on FileProximity's NEW rows (same order as the + # generic loop: scope BEFORE deidentify) so no non-target user reaches this run's output. No-op when + # unscoped. Then deidentify the NEW rows before the append-dedup read/merge (no-op when off). + $fpRows = script:Select-AISIDScopedRows -Table 'FileProximity' -Rows $fpRows + $fpRows = ConvertTo-AISIDDeidentifiedRows -Table 'FileProximity' -Rows $fpRows + $fpExisting = @() + $fpReadFailed = $false + $fpHasHistory = $false + $fpUseStreamed = $false + $fpStreamedPath = $null + $fpScratchPath = $null + if ($ExistingRows.ContainsKey('FileProximity')) { + $fpExisting = @($ExistingRows['FileProximity']) + $fpHasHistory = (@($fpExisting).Count -gt 0) + } + elseif (-not $script:RemoteOutputMode -or $script:RemoteOutputMode -eq 'None') { + # Blocker-3: FileProximity prior event history comes from the SEPARATE supplied prior-bundle folder + # ($HistoryDir), never from the fresh output run folder. No $HistoryDir => no prior history. + try { + $fpStreamedPath = if ($HistoryDir) { Join-Path $HistoryDir $fpSchema.FileName } else { '' } + if ($fpStreamedPath) { + $fpHasHistory = Test-AISIDCsvHasDataRow -Path $fpStreamedPath + $fpUseStreamed = $true + } + } + catch { + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add('FileProximity') + $fpState = 'Incomplete'; $fpReason = 'existing file unreadable; prior file left untouched' + $fpReadFailed = $true + } + } + else { + # Remote tier: prior event history from the supplied prior-bundle URL ($ParentOverride); none when + # empty (no -AppendDefenderUsage). Download prior history to a local scratch file (memory-safe), + # then stream the scratch through the bounded append-dedup merge — never parsed into an array. + if ($ParentOverride) { + try { + $fpScratchPath = Get-AISIDExistingCsvToScratch -FileName $fpSchema.FileName -ParentOverride $ParentOverride + $fpStreamedPath = $fpScratchPath + $fpHasHistory = ($fpScratchPath -and (Test-AISIDCsvHasDataRow -Path $fpScratchPath)) + $fpUseStreamed = $true + } + catch { + # Not a confirmed absence: leave the prior file untouched this run rather than risk an + # empty-merge overwrite of accumulated event history. + if ($fpScratchPath -and (Test-Path -LiteralPath $fpScratchPath)) { Write-LogHost ("[AISID] {0}: remote history scratch preserved for recovery: {1}" -f 'FileProximity', $fpScratchPath) -ForegroundColor DarkYellow } + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add('FileProximity') + $fpState = 'Incomplete'; $fpReason = 'existing file unreadable; prior file left untouched' + $fpReadFailed = $true + } + } + } + if (-not $fpReadFailed) { + if ($fpHasHistory -or $fpCollected) { + if ($fpUseStreamed) { + try { + $fpMerged = Merge-AISIDTableAppendDedupStreamed -ExistingCsvPath $fpStreamedPath -NewRows $fpRows -KeyColumns $fpSchema.Key + } + catch { + if ($fpScratchPath -and (Test-Path -LiteralPath $fpScratchPath)) { Write-LogHost ("[AISID] {0}: remote history scratch preserved for recovery: {1}" -f 'FileProximity', $fpScratchPath) -ForegroundColor DarkYellow } + Write-LogHost ("[AISID] {0}: existing data could not be read ({1}); leaving the prior file untouched this run." -f 'FileProximity', $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add('FileProximity') + $fpState = 'Incomplete'; $fpReason = 'existing file unreadable; prior file left untouched' + $fpReadFailed = $true + } + } + else { + $fpMerged = Merge-AISIDTableAppendDedup -ExistingRows $fpExisting -NewRows $fpRows -KeyColumns $fpSchema.Key + } + if (-not $fpReadFailed) { + # Blocker-2: final post-merge scope containment (deidentify-aware) — history (injected, streamed + # local, or downloaded remote) that reintroduced a non-target user is caught here. If any + # out-of-scope row survives, DO NOT publish; mark Failed; preserve the prior file; flag an AISID + # gap. A scoped run never emits a non-target FileProximity row. + $__fpCont = script:Select-AISIDContainedRows -Table 'FileProximity' -Rows $fpMerged + if ($__fpCont.Applies -and @($__fpCont.OutOfScope).Count -gt 0) { + Write-LogHost ("[AISID] FileProximity: {0} row(s) outside the requested user scope survived the history merge; refusing to publish and leaving the prior file unchanged (AISID gap)." -f @($__fpCont.OutOfScope).Count) -ForegroundColor Yellow + [void]$out.Failed.Add('FileProximity') + $fpState = 'Failed'; $fpReason = 'post-merge scope containment failed; prior file left unchanged' + $script:AISIDHadGaps = $true + } + else { + $out.FileProximity = Export-AISIDTable -DestinationDir $DestinationDir -FileName $fpSchema.FileName -OrderedColumns $fpSchema.Columns -Rows $fpMerged + } + } + } + else { + # Ineligible or query-failed with no prior history: emit the header-only stub fallback. + $out.FileProximity = New-AISIDFileProximityStub -DestinationDir $DestinationDir + } + } + # Remove the downloaded remote history scratch on success; on a read/merge failure it is preserved + # above for recovery. + if (-not $fpReadFailed -and $fpScratchPath -and (Test-Path -LiteralPath $fpScratchPath)) { + Remove-Item -LiteralPath $fpScratchPath -Force -ErrorAction SilentlyContinue + } + $out.TableStatus['FileProximity'] = [ordered]@{ State = $fpState; Reason = $fpReason } + + # Dimension tables: always regenerated in full (overwrite), independent of any licensing verdict and + # with no per-row key merge. Both project already-collected in-memory data. Each collector call is + # isolated so an unexpected throw falls back to a header-only file and never skips the other table. + $catSchema = $script:AISIDTableSchemas['SolutionsCatalog'] + $catRows = @() + $catState = 'Succeeded'; $catReason = 'dimension regenerated' + try { $catRows = @(Get-AISIDSolutionsCatalogExport) } + catch { Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); the dimension FAILED (recorded as a gap)." -f 'SolutionsCatalog', $_.Exception.Message) -ForegroundColor Red; $catState = 'Failed'; $catReason = ("collector threw: {0}" -f $_.Exception.Message); [void]$out.Failed.Add('SolutionsCatalog') } + if ($catState -eq 'Succeeded' -and @($catRows).Count -eq 0) { $catState = 'Empty'; $catReason = 'dimension produced zero rows' } + $catRows = ConvertTo-AISIDDeidentifiedRows -Table 'SolutionsCatalog' -Rows $catRows + $out.SolutionsCatalog = Export-AISIDTable -DestinationDir $DestinationDir -FileName $catSchema.FileName -OrderedColumns $catSchema.Columns -Rows $catRows + $out.TableStatus['SolutionsCatalog'] = [ordered]@{ State = $catState; Reason = $catReason } + + $euSchema = $script:AISIDTableSchemas['EntraUsers'] + $euRows = @() + $euState = 'Succeeded'; $euReason = 'dimension regenerated' + try { $euRows = @(Get-AISIDEntraUsersProjection) } + catch { Write-LogHost ("[AISID] {0}: collector threw an unexpected error ({1}); the dimension FAILED (recorded as a gap)." -f 'EntraUsers', $_.Exception.Message) -ForegroundColor Red; $euState = 'Failed'; $euReason = ("collector threw: {0}" -f $_.Exception.Message); [void]$out.Failed.Add('EntraUsers') } + if ($euState -eq 'Succeeded' -and @($euRows).Count -eq 0) { $euState = 'Empty'; $euReason = 'dimension produced zero rows' } + # F3 backstop (defense-in-depth): for a SCOPED AISID run the requested target(s) are REQUIRED in the + # EntraUsers dimension. The early user-scope gate already fails closed when an explicit target is absent + # from the directory; this second line of defense catches a resolved target that is nevertheless missing + # from the projected rows and records it as a real gap (Failed) — NOT a factual Empty. Only the explicit + # target set is required (manager-chain rows merely accompany it); an unscoped run keeps ordinary Empty. + if ($euState -ne 'Failed') { + $euScope = script:Get-AISIDScopeUpnSet + if ($null -ne $euScope) { + $euProjected = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($er in @($euRows)) { + if (-not $er) { continue } + $eg = if ($er -is [System.Collections.IDictionary]) { [string]$er['userPrincipalName'] } else { [string]$er.userPrincipalName } + $eg = ([string]$eg).Trim() + if ($eg) { [void]$euProjected.Add($eg) } + } + $euMissingTargets = @(@($euScope) | Where-Object { $t = ([string]$_).Trim(); $t -and -not $euProjected.Contains($t) }) + if ($euMissingTargets.Count -gt 0) { + Write-LogHost ("[AISID] EntraUsers: required scoped target(s) absent from the directory projection ({0}); the dimension FAILED (recorded as a gap), not a factual empty." -f (@($euMissingTargets) -join ', ')) -ForegroundColor Red + $euState = 'Failed'; $euReason = ("required scoped target(s) not present in projection: {0}" -f (@($euMissingTargets) -join ', ')) + [void]$out.Failed.Add('EntraUsers') + } + } + } + $euRows = ConvertTo-AISIDDeidentifiedRows -Table 'EntraUsers' -Rows $euRows + $out.EntraUsers = Export-AISIDTable -DestinationDir $DestinationDir -FileName $euSchema.FileName -OrderedColumns $euSchema.Columns -Rows $euRows + $out.TableStatus['EntraUsers'] = [ordered]@{ State = $euState; Reason = $euReason } + + # The dashboard loads these MDA-tier tables unconditionally. Until their collectors are available, + # preserve valid prior history byte-for-byte; when none exists, publish exact header-only compatibility + # files so tenants without those data sources refresh cleanly. + foreach ($stubTable in @('AppGovAlerts', 'CloudDiscovery', 'MDASessions')) { + try { + $out[$stubTable] = Export-AISIDCompatibilityTable -Table $stubTable -DestinationDir $DestinationDir -ExistingRows $ExistingRows -HistoryDir $HistoryDir -ParentOverride $ParentOverride + $out.TableStatus[$stubTable] = [ordered]@{ State = 'Unavailable'; Reason = 'data source not implemented; prior history preserved or header-only compatibility file emitted' } + } + catch { + Write-LogHost ("[AISID] {0}: existing compatibility data could not be preserved ({1}); leaving the prior file unchanged." -f $stubTable, $_.Exception.Message) -ForegroundColor Yellow + [void]$out.ReadFailures.Add($stubTable) + $out.TableStatus[$stubTable] = [ordered]@{ State = 'Incomplete'; Reason = 'existing compatibility file unreadable or invalid; prior file left unchanged' } + } + } + return $out +} + +function ConvertTo-AISIDDeidentifiedRows { + <# + .SYNOPSIS + Deidentifies the PII columns of an AISID table's newly-collected rows, reusing the shared + deterministic Get-PaxDeid* helpers (same salt/algorithm as the main pipeline and the Python + rollup, so a given real identity maps to the identical token everywhere and cross-table joins + survive). No new hashing logic is introduced. + .DESCRIPTION + No-op when -Deidentify is off ($script:PaxDeidEnabled = $false): the input rows are returned + unchanged (same reference). Only the columns named in the per-table map are transformed; every + other column is copied verbatim and column order is preserved (the source row is never mutated). + Tables with NO map entry are returned unchanged: ClientChannel and SolutionsCatalog carry no PII. + CopilotUsage IS deidentified in this path now: its source is the run's IDENTIFIED raw Purview CSV, + so its UserPrincipalName is hashed once through the shared Get-PaxDeid* path to the identical token + used everywhere. Applied to a table's NEW rows BEFORE the read-before-merge/merge so on-disk (already + deidentified) rows and these rows compare on matching keys. Org/geo attributes (department, + jobTitle, city, country, companyName, ...) are intentionally retained, matching the main + pipeline's established Invoke-PaxRawDeidentify policy. + .OUTPUTS + The rows with mapped PII columns deidentified, or the input unchanged (off / no map entry). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Table, + [object[]]$Rows = @() + ) + if (-not $script:PaxDeidEnabled) { return $Rows } + + # Per-table PII column -> Get-PaxDeid* helper kind. Direct identifiers only. ClientChannel and + # SolutionsCatalog carry no PII and are deliberately absent; CopilotUsage IS mapped here now. + $deidMap = @{ + ActivitySessions = [ordered]@{ UPN = 'Upn' } + OffHoursGeo = [ordered]@{ UPN = 'Upn' } + OAuthConsents = [ordered]@{ UPN = 'Upn' } + SSOSignIns = [ordered]@{ UPN = 'Upn' } + FileProximity = [ordered]@{ UPN = 'Upn'; FileName = 'File'; FolderPath = 'Resource' } + CopilotUsage = [ordered]@{ UserPrincipalName = 'Upn' } + EntraUsers = [ordered]@{ userPrincipalName = 'Upn'; displayName = 'Name'; manager_userPrincipalName = 'Upn'; manager_displayName = 'Name' } + } + $colMap = $deidMap[$Table] + if (-not $colMap) { return $Rows } + + $out = foreach ($row in $Rows) { + if (-not $row) { continue } + # Ordered clone: preserves column order and never mutates the source row object. + $clone = [ordered]@{} + foreach ($p in $row.PSObject.Properties) { $clone[$p.Name] = $p.Value } + foreach ($col in $colMap.Keys) { + if ($clone.Contains($col) -and -not [string]::IsNullOrEmpty([string]$clone[$col])) { + $sv = [string]$clone[$col] + $clone[$col] = switch ($colMap[$col]) { + 'Upn' { Get-PaxDeidUpn -Value $sv } + 'Name' { Get-PaxDeidName -Value $sv } + 'File' { Get-PaxDeidFile -Value $sv } + 'Resource' { Get-PaxDeidResource -Value $sv } + } + } + } + [pscustomobject]$clone + } + return @($out) +} + +function Invoke-AISIDPilotExport { + <# + .SYNOPSIS + End-of-run orchestrator for the AI Solutions Intelligence Dashboard (AISID) export: evaluates + per-table licensing, reports it, and writes all twelve dashboard CSVs through the shared + export/merge infrastructure. Intended to be called exactly once, only under -Dashboard AISID, + from the point in the rollup post-processing where the CopilotInteraction Fact CSV has just + been written and BEFORE any -AppendFile merge/delete can consume it (so Get-AISIDCopilotUsage's + source is guaranteed present). + .DESCRIPTION + Whole-call failure isolation: any unexpected error is caught, logged loudly, and swallowed so + the AISID export can never abort the run or disturb the Purview/rollup output already on disk. + Per-table handling (each collector's licensing-ineligible/empty result and each read failure -> + safe header-only fallback / ReadFailures) is owned by Export-AISIDPilotTables; this outer guard + covers Test-AISIDLicensing and any orchestration-level error. + .OUTPUTS + The Export-AISIDPilotTables result hashtable (written paths per table + AuthQuit), or $null on + an isolated failure. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$DestinationDir, + [switch]$DisableDeltaCache, + [string]$RawPurviewCsv, + [string]$ParentOverride, + [string]$HistoryDir = '' + ) + try { + $verdict = Test-AISIDLicensing + Write-AISIDLicensingReport -Verdict $verdict + return Export-AISIDPilotTables -DestinationDir $DestinationDir -Verdict $verdict -DisableDeltaCache:$DisableDeltaCache -RawPurviewCsv $RawPurviewCsv -ParentOverride $ParentOverride -HistoryDir $HistoryDir + } + catch { + Write-LogHost ("[AISID] Dashboard export failed (non-fatal; the run's Purview/rollup output is unaffected): {0}" -f $_.Exception.Message) -ForegroundColor Red + return $null + } +} + +function Get-AISIDCompletionStatus { + <# + .SYNOPSIS + Evaluates whether the AISID dashboard export completed truthfully, using the per-table state + contract (TableStatus: Succeeded / Empty / Unavailable / Failed / Incomplete) produced by + Export-AISIDPilotTables cross-checked against the twelve contractual dashboard CSVs (each must + exist with its exact header). A collector Failed/Incomplete state, a missing/invalid required + file, an auth-quit interruption, or an orchestration-level null result is a GAP and must never + be reported as clean. Header-only output alone never decides WHY a table is empty — the + collector's recorded state does. + .OUTPUTS + [pscustomobject] with Status, HadGaps, AllTablesTerminal, CleanCompletion, Missing, Failed, + Incomplete, Unavailable, Empty, Succeeded, AuthQuit, ExportErrored, PerTable. + #> + [CmdletBinding()] + param([object]$Result, [Parameter(Mandatory)][string]$DestinationDir) + + $schemas = $script:AISIDTableSchemas + $order = @('ActivitySessions', 'OffHoursGeo', 'FileProximity', 'OAuthConsents', 'SSOSignIns', 'ClientChannel', 'CopilotUsage', 'AppGovAlerts', 'CloudDiscovery', 'MDASessions', 'SolutionsCatalog', 'EntraUsers') + $exportErrored = ($null -eq $Result) + $ts = if ($Result -and $Result.TableStatus) { $Result.TableStatus } else { $null } + + $missing = @(); $failed = @(); $incomplete = @(); $unavailable = @(); $empty = @(); $succeeded = @() + $perTable = [ordered]@{} + $allTerminal = (-not $exportErrored) + + foreach ($t in $order) { + $sch = $schemas[$t] + # Collector-recorded state (authoritative for WHY). Absent => the export never decided this table. + $state = 'Incomplete'; $reason = 'export did not return a state for this table' + if ($ts -and $ts.Contains($t)) { $state = [string]$ts[$t].State; $reason = [string]$ts[$t].Reason } + else { $allTerminal = $false } + + # File presence + header validity (the dashboard model binds each CSV by a fixed header). + $fileOk = $false; $fileNote = 'missing' + if ($sch) { + $path = Join-Path $DestinationDir $sch.FileName + if (Test-Path -LiteralPath $path) { + $header = $null + try { $header = (Get-Content -LiteralPath $path -TotalCount 1 -ErrorAction Stop) } catch { $header = $null } + if (-not [string]::IsNullOrEmpty($header) -and ($header.Trim() -eq ($sch.Columns -join ','))) { $fileOk = $true; $fileNote = 'present' } + else { $fileNote = 'invalid header' } + } + } + if (-not $fileOk) { $missing += $sch.FileName } + + # A Succeeded/Empty/Unavailable state that lost its file is a contradiction -> treat as a gap. + $effective = $state + if (-not $fileOk -and $state -in @('Succeeded', 'Empty', 'Unavailable')) { $effective = 'Failed'; $reason = ("recorded {0} but output file is {1}" -f $state, $fileNote) } + + switch ($effective) { + 'Succeeded' { $succeeded += $t } + 'Empty' { $empty += $t } + 'Unavailable' { $unavailable += $t } + 'Failed' { $failed += $t } + default { $incomplete += $t } + } + $perTable[$t] = [ordered]@{ State = $effective; Reason = $reason; File = $fileNote } + } + + $authQuit = [bool]($Result -and $Result.AuthQuit) + $hadGaps = $exportErrored -or ($failed.Count -gt 0) -or ($incomplete.Count -gt 0) -or ($missing.Count -gt 0) -or $authQuit + $cleanCompletion = (-not $hadGaps) -and $allTerminal -and (($succeeded.Count + $empty.Count + $unavailable.Count) -eq $order.Count) + $status = if ($authQuit) { 'interrupted' } elseif ($hadGaps) { 'completed-with-gaps' } elseif ($unavailable.Count -gt 0) { 'complete-with-unavailable' } elseif ($empty.Count -gt 0) { 'complete-with-empty' } else { 'complete' } + + Write-LogHost '[AISID] ===== Dashboard completion summary =====' -ForegroundColor Cyan + Write-LogHost ("[AISID] Required dashboard files present: {0}/{1}" -f ($order.Count - @($missing | Sort-Object -Unique).Count), $order.Count) -ForegroundColor Gray + if ($succeeded.Count -gt 0) { Write-LogHost ("[AISID] Succeeded: {0}" -f ($succeeded -join ', ')) -ForegroundColor Green } + if ($empty.Count -gt 0) { Write-LogHost ("[AISID] Empty (query succeeded, zero rows): {0}" -f ($empty -join ', ')) -ForegroundColor Gray } + if ($unavailable.Count -gt 0) { Write-LogHost ("[AISID] Unavailable (licensing ineligible): {0}" -f ($unavailable -join ', ')) -ForegroundColor Gray } + if ($failed.Count -gt 0) { Write-LogHost ("[AISID] FAILED (real data gap, NOT a factual empty): {0}" -f (@($failed | Sort-Object -Unique) -join ', ')) -ForegroundColor Red } + if ($incomplete.Count -gt 0) { Write-LogHost ("[AISID] Incomplete (collector not reached / existing file unreadable): {0}" -f (@($incomplete | Sort-Object -Unique) -join ', ')) -ForegroundColor Yellow } + if (@($missing | Sort-Object -Unique).Count -gt 0) { Write-LogHost ("[AISID] Missing/invalid required files: {0}" -f (@($missing | Sort-Object -Unique) -join ', ')) -ForegroundColor Red } + if ($authQuit) { Write-LogHost '[AISID] Collection was interrupted (authentication quit) before all data sets finished.' -ForegroundColor Red } + if ($exportErrored) { Write-LogHost '[AISID] Dashboard export failed at the orchestration level (isolated error); output is incomplete.' -ForegroundColor Red } + Write-LogHost ("[AISID] Overall AISID status: {0}" -f $status) -ForegroundColor Cyan + + return [pscustomobject]@{ + Status = $status; HadGaps = $hadGaps; AllTablesTerminal = $allTerminal; CleanCompletion = $cleanCompletion + Missing = @($missing | Sort-Object -Unique); Failed = @($failed | Sort-Object -Unique); Incomplete = @($incomplete | Sort-Object -Unique) + Unavailable = @($unavailable); Empty = @($empty); Succeeded = @($succeeded); AuthQuit = $authQuit; ExportErrored = $exportErrored; PerTable = $perTable + } +} + +function Send-PromptNotification { + <# + .SYNOPSIS + Plays a system beep to alert user that a prompt requires attention. + .DESCRIPTION + Useful when user is working in other windows and needs to be notified + when a prompt appears that requires input. + #> + + try { + # Play 3 short beeps to get attention + [Console]::Beep(800, 200) # 800Hz for 200ms + Start-Sleep -Milliseconds 100 + [Console]::Beep(1000, 200) # 1000Hz for 200ms + Start-Sleep -Milliseconds 100 + [Console]::Beep(1200, 300) # 1200Hz for 300ms (slightly longer final beep) + } + catch { + # Silently fail if beep not supported (e.g., some server environments) + } +} + +# Validate -OnlyUserInfo parameter compatibility +if ($OnlyUserInfo) { + $incompatibleParams = @() + + # Date filtering parameters + if ($PSBoundParameters.ContainsKey('StartDate')) { $incompatibleParams += " - StartDate (not applicable for user-only export)" } + if ($PSBoundParameters.ContainsKey('EndDate')) { $incompatibleParams += " - EndDate (not applicable for user-only export)" } + + # Activity configuration parameters + if ($PSBoundParameters.ContainsKey('ActivityTypes')) { $incompatibleParams += " - ActivityTypes (cleared by -OnlyUserInfo)" } + if ($IncludeM365Usage) { $incompatibleParams += " - IncludeM365Usage (activity type modifier)" } + if ($ExcludeCopilotInteraction) { $incompatibleParams += " - ExcludeCopilotInteraction (activity type modifier)" } + + # Audit retrieval settings + if ($PSBoundParameters.ContainsKey('BlockHours') -and $BlockHours -ne 0.5) { $incompatibleParams += " - BlockHours (audit query partitioning)" } + if ($PSBoundParameters.ContainsKey('PartitionHours') -and $PartitionHours -ne 0) { $incompatibleParams += " - PartitionHours (audit query partitioning)" } + if ($PSBoundParameters.ContainsKey('MaxPartitions') -and $MaxPartitions -ne 160) { $incompatibleParams += " - MaxPartitions (audit query limits)" } + if ($PSBoundParameters.ContainsKey('ResultSize') -and $ResultSize -ne 10000) { $incompatibleParams += " - ResultSize (audit query page size)" } + if ($PSBoundParameters.ContainsKey('PacingMs') -and $PacingMs -ne 0) { $incompatibleParams += " - PacingMs (audit query throttling)" } + if ($AutoCompleteness) { $incompatibleParams += " - AutoCompleteness (audit log completeness checks)" } + if ($PSBoundParameters.ContainsKey('StreamingSchemaSample') -and $StreamingSchemaSample -ne 5000) { $incompatibleParams += " - StreamingSchemaSample (audit record schema sampling)" } + if ($PSBoundParameters.ContainsKey('StreamingChunkSize') -and $StreamingChunkSize -ne 5000) { $incompatibleParams += " - StreamingChunkSize (audit streaming batch size)" } + if ($PSBoundParameters.ContainsKey('ExportProgressInterval') -and $ExportProgressInterval -ne 10) { $incompatibleParams += " - ExportProgressInterval (audit export progress)" } + + # Filtering parameters + if ($PSBoundParameters.ContainsKey('AgentId')) { $incompatibleParams += " - AgentId (audit record filtering)" } + if ($AgentsOnly) { $incompatibleParams += " - AgentsOnly (audit record filtering)" } + if ($ExcludeAgents) { $incompatibleParams += " - ExcludeAgents (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('PromptFilter')) { $incompatibleParams += " - PromptFilter (audit record content filtering)" } + if ($PSBoundParameters.ContainsKey('RecordTypes')) { $incompatibleParams += " - RecordTypes (audit record filtering)" } + if ($PSBoundParameters.ContainsKey('ServiceTypes')) { $incompatibleParams += " - ServiceTypes (audit record filtering)" } + + # Processing mode parameters + if ($ExplodeArrays) { $incompatibleParams += " - ExplodeArrays (audit record array expansion)" } + if ($ExplodeDeep) { $incompatibleParams += " - ExplodeDeep (audit record deep expansion)" } + if ($PSBoundParameters.ContainsKey('RAWInputCSV')) { $incompatibleParams += " - RAWInputCSV (offline audit replay mode)" } + + # Parallel processing parameters + if ($EnableParallel) { $incompatibleParams += " - EnableParallel (parallel audit query execution)" } + if ($PSBoundParameters.ContainsKey('MaxConcurrency') -and $MaxConcurrency -ne 10) { $incompatibleParams += " - MaxConcurrency (concurrent query/partition limit)" } + if ($PSBoundParameters.ContainsKey('MaxParallelGroups') -and $MaxParallelGroups -ne 8) { $incompatibleParams += " - MaxParallelGroups (parallel activity group limit)" } + if ($PSBoundParameters.ContainsKey('ParallelMode') -and $ParallelMode -ne 'Auto') { $incompatibleParams += " - ParallelMode (parallel processing mode)" } + if ($DisableAdaptive) { $incompatibleParams += " - DisableAdaptive (adaptive concurrency controls)" } + if ($PSBoundParameters.ContainsKey('ProgressSmoothingAlpha') -and $ProgressSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ProgressSmoothingAlpha (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('HighLatencyMs') -and $HighLatencyMs -ne 90000) { $incompatibleParams += " - HighLatencyMs (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('MemoryPressureMB') -and $MemoryPressureMB -ne 1500) { $incompatibleParams += " - MemoryPressureMB (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('LowLatencyMs') -and $LowLatencyMs -ne 20000) { $incompatibleParams += " - LowLatencyMs (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('LowLatencyConsecutive') -and $LowLatencyConsecutive -ne 2) { $incompatibleParams += " - LowLatencyConsecutive (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('ThroughputDropPct') -and $ThroughputDropPct -ne 15) { $incompatibleParams += " - ThroughputDropPct (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('ThroughputSmoothingAlpha') -and $ThroughputSmoothingAlpha -ne 0.3) { $incompatibleParams += " - ThroughputSmoothingAlpha (adaptive tuning)" } + if ($PSBoundParameters.ContainsKey('AdaptiveConcurrencyCeiling') -and $AdaptiveConcurrencyCeiling -ne 6) { $incompatibleParams += " - AdaptiveConcurrencyCeiling (adaptive tuning)" } + + # Reliability parameters (audit-specific) + if ($PSBoundParameters.ContainsKey('CircuitBreakerThreshold') -and $CircuitBreakerThreshold -ne 5) { $incompatibleParams += " - CircuitBreakerThreshold (block failure circuit breaker)" } + if ($PSBoundParameters.ContainsKey('CircuitBreakerCooldownSeconds') -and $CircuitBreakerCooldownSeconds -ne 120) { $incompatibleParams += " - CircuitBreakerCooldownSeconds (circuit breaker cooldown)" } + if ($PSBoundParameters.ContainsKey('BackoffBaseSeconds') -and $BackoffBaseSeconds -ne 1.0) { $incompatibleParams += " - BackoffBaseSeconds (block retry backoff)" } + if ($PSBoundParameters.ContainsKey('BackoffMaxSeconds') -and $BackoffMaxSeconds -ne 45) { $incompatibleParams += " - BackoffMaxSeconds (block retry max backoff)" } + + # Alternative modes + if ($UseEOM) { $incompatibleParams += " - UseEOM (Exchange Online Management mode incompatible with Graph Entra enrichment)" } + + # Output combination parameters + if ($CombineOutput) { $incompatibleParams += " - CombineOutput (only relevant with multiple activity types)" } + if ($AppendFile) { $incompatibleParams += " - AppendFile (appending user-only data to existing audit output not supported)" } + + if ($incompatibleParams.Count -gt 0) { + Write-Host "" + Write-Host "ERROR: The -OnlyUserInfo switch cannot be used with the following parameters:" -ForegroundColor Red + Write-Host "" + $incompatibleParams | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } + Write-Host "" + Write-Host "The -OnlyUserInfo switch exports only Entra user directory and license information (no audit logs)." -ForegroundColor Cyan + Write-Host "" + Write-Host "Compatible parameters:" -ForegroundColor Green + Write-Host " - OutputPath (where to save the file)" -ForegroundColor White + Write-Host " - UserIds / GroupNames (scope the exported directory to these users / group members)" -ForegroundColor White + Write-Host " - Auth (authentication method: WebLogin, DeviceCode, Credential, Silent)" -ForegroundColor White + Write-Host " - Force (bypass interactive prompts)" -ForegroundColor White + Write-Host " - MaxNetworkOutageMinutes (network resilience for Graph API calls)" -ForegroundColor White + Write-Host " - EmitMetricsJson (track Entra retrieval metrics)" -ForegroundColor White + Write-Host " - MetricsPath (custom metrics output location)" -ForegroundColor White + Write-Host " - SkipDiagnostics (skip pre-query capability checks)" -ForegroundColor White + Write-Host "" + Write-Host "Please remove the incompatible parameters and try again." -ForegroundColor Cyan + Write-Host "" + exit 1 + } + + # If validation passes, configure for user-only export + Write-Host "" + Write-Host "INFO: -OnlyUserInfo mode enabled. Skipping all audit log retrieval, exporting only Entra user data." -ForegroundColor Green + Write-Host "" + $IncludeUserInfo = $true + $ActivityTypes = @() +} + +# PAX4A-GUARD-BEGIN +# -UserInfoFile / -GroupNames MUTUAL EXCLUSIVITY +# -UserInfoFile supplies the Entra user directory from a customer-provided CSV (bypassing the +# live /users pull). -GroupNames expands audit records against the LIVE directory. Supplying both +# is contradictory. Hard-stop here in the early parameter-validation stage so it fires on EVERY +# entry path (fresh and -Resume) BEFORE any -GroupNames consumer downstream (the RequiredScopes +# builder and the permissions banner). No partial work is done. +if ($PSBoundParameters.ContainsKey('UserInfoFile') -and $PSBoundParameters.ContainsKey('GroupNames')) { + Write-Host "" + Write-Host "ERROR: -UserInfoFile and -GroupNames cannot both be supplied." -ForegroundColor Red + Write-Host " -UserInfoFile ingests the Entra user directory from a customer-provided CSV." -ForegroundColor Yellow + Write-Host " -GroupNames expands audit records against the live Entra directory." -ForegroundColor Yellow + Write-Host " Supply exactly ONE: -UserInfoFile for a supplied directory, or -GroupNames for group filtering." -ForegroundColor Yellow + Write-Host "" + exit 1 +} +# PAX4A-GUARD-END + +# PAX4D-AUTOENABLE-BEGIN +# -UserInfoFile auto-enables -IncludeUserInfo (FU6): every -UserInfoFile export path (file naming, +# the multi-tab workbook, and the directory substitution seam) is gated on $IncludeUserInfo, so a +# customer need only pass -UserInfoFile. Mirrors the -OnlyUserInfo auto-enable above; fires on every +# entry path before any $IncludeUserInfo consumer. No effect when -UserInfoFile is unset. +if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile) -and -not $IncludeUserInfo) { + $IncludeUserInfo = $true +} +# PAX4D-AUTOENABLE-END + +# PAX-UIS-GUARD-BEGIN +# -UserInfoSupplement is the HYBRID directory-enrichment mode: the live Entra directory is fetched +# normally and the supplemental CSV columns are appended by an Entra-left join. It is distinct from +# -UserInfoFile (which REPLACES the directory), so the two are mutually exclusive. It also requires the +# live /users pull, so it is incompatible with -UseEOM (no Graph directory) and -RAWInputCSV (offline +# replay). Unlike -UserInfoFile, it is COMPATIBLE with -GroupNames. These hard-stops fire on every entry +# path (fresh and -Resume) before any consumer. It auto-enables -IncludeUserInfo. +$script:UserInfoSupplementActive = ($PSBoundParameters.ContainsKey('UserInfoSupplement') -and -not [string]::IsNullOrWhiteSpace($UserInfoSupplement)) +if ($script:UserInfoSupplementActive) { + if ($PSBoundParameters.ContainsKey('UserInfoFile') -and -not [string]::IsNullOrWhiteSpace($UserInfoFile)) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement and -UserInfoFile cannot both be supplied." -ForegroundColor Red + Write-Host " -UserInfoSupplement ENRICHES the live Entra directory (hybrid); -UserInfoFile REPLACES it." -ForegroundColor Yellow + Write-Host " Supply exactly ONE." -ForegroundColor Yellow + Write-Host "" + exit 1 + } + if ($UseEOM) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement requires the Graph /users directory and is not supported with -UseEOM." -ForegroundColor Red + Write-Host "" + exit 1 + } + if ($PSBoundParameters.ContainsKey('RAWInputCSV') -and -not [string]::IsNullOrWhiteSpace($RAWInputCSV)) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement enriches a live directory fetch and cannot be combined with -RAWInputCSV (offline replay)." -ForegroundColor Red + Write-Host "" + exit 1 + } + if (-not $IncludeUserInfo) { $IncludeUserInfo = $true } +} +# PAX-UIS-GUARD-END + +# Canonical maps for Graph filter normalization +$recordTypeCanonicalMap = @{ + 'azureactivedirectory' = 'AzureActiveDirectory' + 'azureactivedirectoryaccountlogon' = 'AzureActiveDirectoryAccountLogon' + 'azureactivedirectorystslogon' = 'AzureActiveDirectoryStsLogon' + 'exchangeadmin' = 'ExchangeAdmin' + 'exchangeitem' = 'ExchangeItem' + 'exchangemailbox' = 'ExchangeMailbox' + 'sharepointfileoperation' = 'SharePointFileOperation' + 'sharepointsharingoperation' = 'SharePointSharingOperation' + 'sharepoint' = 'SharePoint' + 'onedrive' = 'OneDrive' + 'microsoftteams' = 'MicrosoftTeams' +} + +$serviceCanonicalMap = @{ + 'azureactivedirectory' = 'AzureActiveDirectory' + 'exchange' = 'Exchange' + 'sharepoint' = 'SharePoint' + 'onedrive' = 'OneDrive' + 'teams' = 'Teams' +} + +# Normalize optional Graph filter passthrough parameters (dedupe & trim) +# Split ActivityTypes if provided as comma-separated string +if ($ActivityTypes) { + $processedActivityTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $ActivityTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedActivityTypes.Add($token) + } + } + $ActivityTypes = @( + $processedActivityTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($ActivityTypes.Count -eq 0) { + $ActivityTypes = $null + } +} + +if ($RecordTypes) { + $processedRecordTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $RecordTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedRecordTypes.Add($token) + } + } + $RecordTypes = @( + $processedRecordTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($RecordTypes.Count -eq 0) { + $RecordTypes = $null + } else { + $RecordTypes = @( + foreach ($rt in $RecordTypes) { + $key = $rt.ToLowerInvariant() + if ($recordTypeCanonicalMap.ContainsKey($key)) { $recordTypeCanonicalMap[$key] } else { $rt } + } + ) | Select-Object -Unique + } +} + +if ($ServiceTypes) { + $processedServiceTypes = New-Object System.Collections.Generic.List[string] + foreach ($value in $ServiceTypes) { + if ($null -eq $value) { continue } + $raw = $value.ToString() + foreach ($piece in ($raw -split ',')) { + $token = $piece.Trim(" '""`t") + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $processedServiceTypes.Add($token) + } + } + $ServiceTypes = @( + $processedServiceTypes | + ForEach-Object { $_.Trim() } | + Where-Object { $_ } | + Select-Object -Unique + ) + if ($ServiceTypes.Count -eq 0) { + $ServiceTypes = $null + } else { + $ServiceTypes = @( + foreach ($svc in $ServiceTypes) { + $key = $svc.ToLowerInvariant() + if ($serviceCanonicalMap.ContainsKey($key)) { $serviceCanonicalMap[$key] } else { $svc } + } + ) | Select-Object -Unique + } +} + +# Mapping of audit record types to supported workloads for Graph security audit queries +$recordTypeWorkloadMap = @{ + 'azureActiveDirectory' = @('AzureActiveDirectory') + 'azureActiveDirectoryAccountLogon' = @('AzureActiveDirectory') + 'azureActiveDirectoryStsLogon' = @('AzureActiveDirectory') + 'exchangeAdmin' = @('Exchange') + 'exchangeItem' = @('Exchange') + 'exchangeMailbox' = @('Exchange') + 'sharePointFileOperation' = @('SharePoint','OneDrive') + 'sharePointSharingOperation' = @('SharePoint','OneDrive') + 'sharePoint' = @('SharePoint','OneDrive') + 'onedrive' = @('OneDrive') + 'microsoftTeams' = @('Teams') + # M365 usage record types: Process in first workload pass to avoid creating additional passes + # These record types capture cross-workload activities (Office apps, Forms, Stream, Planner, PowerApps) + # Mapping to Exchange ensures they run in the first service-filtered pass + 'officeNative' = @('Exchange') + 'microsoftForms' = @('Exchange') + 'microsoftStream' = @('Exchange') + 'plannerPlan' = @('Exchange') + 'plannerTask' = @('Exchange') + 'powerAppsApp' = @('Exchange') +} + +$serviceOperationMap = @{ + 'AzureActiveDirectory' = @('UserLoggedIn','UserLoginFailed','AdminLoggedIn','ResetUserPassword','AddRegisteredUser','UpdateUser','ChangedUserSetting') + 'Exchange' = @('MailItemsAccessed','Send','SendOnBehalf','SoftDelete','HardDelete','MoveToDeletedItems','CopyToFolder','AddMailboxPermission','RemoveMailboxPermission') + 'SharePoint' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'OneDrive' = @('FileAccessed','FileDownloaded','FileUploaded','FileModified','FileDeleted','FileMoved','SharingInvitationCreated','SharingInvitationAccepted','SharedLinkCreated','SharingRevoked','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'Teams' = @('TeamMemberAdded','TeamMemberRemoved','ChannelAdded','ChannelDeleted','ChannelMessageSent','ChannelMessageDeleted','TeamDeleted','TeamArchived','AddMemberToUnifiedGroup','RemoveMemberFromUnifiedGroup') + 'MicrosoftForms' = @('CreateForm','EditForm','DeleteForm','ViewForm','CreateResponse','SubmitResponse','ViewResponse','DeleteResponse') + 'MicrosoftStream' = @('StreamModified','StreamViewed','StreamDeleted','StreamDownloaded') + 'MicrosoftPlanner' = @('PlanCreated','PlanDeleted','PlanModified','TaskCreated','TaskDeleted','TaskModified','TaskAssigned','TaskCompleted') + 'PowerApps' = @('LaunchedApp','CreatedApp','EditedApp','DeletedApp','PublishedApp') +} + +$copilotBaseActivityType = 'CopilotInteraction' +$m365UsageServiceBundle = @('Exchange','SharePoint','OneDrive','Teams') +$m365UsageRecordBundle = @('ExchangeAdmin','ExchangeItem','ExchangeMailbox','SharePointFileOperation','SharePointSharingOperation','SharePoint','OneDrive','MicrosoftTeams','OfficeNative','MicrosoftForms','MicrosoftStream','PlannerPlan','PlannerTask','PowerAppsApp') +# Curated, trimmed M365 usage operations targeted at the Analytics-Hub M365 Usage Analytics +# dashboard. Scope: Exchange mail access, SharePoint/OneDrive file access, Teams chat/messaging, +# Teams meeting lifecycle, and Copilot/Connected-AI interaction signals. +$m365UsageActivityBundle = @( + # === Exchange / Email === + 'MailItemsAccessed','MailboxLogin','Send', + + # === SharePoint / OneDrive - File access === + 'FileAccessed','FileViewed','FilePreviewed','FileModified','FileDownloaded','FileUploaded', + + # === Teams - Chat / Messaging === + 'MessageSent','MessageRead','MessagesListed','ChatRetrieved','ChatCreated','TeamsSessionStarted', + + # === Teams - Meeting lifecycle === + 'MeetingParticipantJoined','MeetingStarted','MeetingEnded','MeetingParticipantDetail','MeetingDetail', + + # === Copilot / Connected AI === + 'CopilotInteraction','ConnectedAIAppInteraction' +) | Select-Object -Unique + +# Script version constant (must appear after param/help to keep param() valid as first executable block) +$ScriptVersion = '1.11.15' + +function Invoke-PaxVersionCheck { + # Informational, non-blocking, failure-isolated version check against the public PAX repo. + # Reads versions.json from the release branch, compares the purview script version, and prints + # a single info line. Never prompts, never throws, capped at ~5s if the server is unreachable. + param([string]$CurrentVersion) + $repoUrl = 'https://github.com/microsoft/PAX' + try { + $verUrl = 'https://raw.githubusercontent.com/microsoft/PAX/release/versions.json' + $resp = Invoke-RestMethod -Uri $verUrl -TimeoutSec 5 -ErrorAction Stop + $latest = [string]$resp.products.purview.version + $relDate = [string]$resp.lastUpdated + if ($latest -and ([version]$latest) -gt ([version]$CurrentVersion)) { + $line = " Update available: PAX v$latest" + if ($relDate) { $line += " (released $relDate)" } + $line += " - you are on v$CurrentVersion. Latest: $repoUrl" + Write-LogHost $line -ForegroundColor Cyan + } else { + Write-LogHost " Version check: you are on the latest PAX version (v$CurrentVersion). $repoUrl" -ForegroundColor DarkGray + } + } catch { + Write-LogHost " Version check skipped: the PAX GitHub repo was not reachable (offline or blocked). Latest: $repoUrl" -ForegroundColor DarkGray + } +} + +# --- Initialize/Clear persistent script variables to prevent cross-run contamination --- +# Note: Script-scoped variables persist across multiple script invocations in the same PowerShell session +$script:partitionStatus = $null +$script:processedJobIds = $null +$script:shownJobMessages = $null + +# --- Microsoft 365 Copilot license detection --- +# No hardcoded SKU GUIDs. Copilot service plans are discovered dynamically at runtime +# from /subscribedSkus by matching servicePlanName against the pattern '*COPILOT*'. +# Per-user eligibility is then computed from the user's assignedPlans where +# capabilityStatus == 'Enabled' and servicePlanId is in the discovered set. +# See Get-UserLicenseData(). + +# --- Synchronized Timestamp & OutputPath Validation --- + +# Generate synchronized timestamp for all output files in this run +$global:ScriptRunTimestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + +# --- Logging Helper Functions (defined early for use throughout script) --- +# $script:LogFile was already initialized to a bootstrap temp-path file at the top +# of the script body (right after param()). DO NOT re-null it here; that would +# discard the early-startup log entries already captured. The bootstrap file is +# renamed to the final location once $OutputFile is resolved (see the relocation +# block further down). The in-memory $script:LogBuffer remains a fallback for the +# rare case where bootstrap creation failed (e.g. read-only temp). +if (-not (Get-Variable -Name LogFile -Scope Script -ErrorAction SilentlyContinue)) { + $script:LogFile = $null +} +$script:LogBuffer = New-Object System.Collections.Generic.List[string] + +function Write-Log { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [$Level] $Message" + Microsoft.PowerShell.Utility\Write-Host $Message + try { + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# Log-file-only writer: never echoes to host. Used for diagnostic detail (temp paths, +# argument vectors, stack traces) that should be captured in the run log but not +# clutter the customer-facing console. +function Write-LogFile { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$Level = "INFO") + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [$Level] $Message" + try { + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# --- Shared memory / stage observability --------------------------------------- +# Lightweight, additive heartbeat: samples the parent process working set and +# managed heap, an optional Python child process working set, rows/pages +# processed, and temporary storage bytes, then appends a single diagnostic line +# to the existing run log (Write-LogFile). No new required output file is +# created, and it NEVER surfaces identifying row values. Emission is throttled to +# the progress interval (default 30s) unless -Force is given. Every metric read +# is guarded so a missing child process or an inaccessible counter can never +# crash the run. This is observation only; it changes no success/failure result. +function Write-PaxMemoryObservation { + [CmdletBinding()] + param( + [string] $Stage = '', + [AllowNull()] [System.Nullable[int]] $ChildPid = $null, + [AllowNull()] [System.Nullable[long]] $RowsProcessed = $null, + [AllowNull()] [System.Nullable[long]] $PagesProcessed = $null, + [string] $TempDir = '', + [switch] $Force + ) + try { + $intervalSec = 30 + try { if ($script:ProgressIntervalSeconds -and [int]$script:ProgressIntervalSeconds -gt 0) { $intervalSec = [int]$script:ProgressIntervalSeconds } } catch {} + $nowUtc = [DateTime]::UtcNow + if (-not $Force) { + $last = $script:PaxLastMemObsUtc + if ($last -and (($nowUtc - $last).TotalSeconds -lt $intervalSec)) { return } + } + $script:PaxLastMemObsUtc = $nowUtc + + $proc = [System.Diagnostics.Process]::GetCurrentProcess() + $parentWs = [long]$proc.WorkingSet64 + $heap = [long][System.GC]::GetTotalMemory($false) + $childWs = $null + if ($null -ne $ChildPid) { + try { $cp = [System.Diagnostics.Process]::GetProcessById([int]$ChildPid); $childWs = [long]$cp.WorkingSet64 } catch { $childWs = $null } + } + $tempBytes = $null + if ($TempDir -and (Test-Path -LiteralPath $TempDir)) { + try { $tempBytes = [long](@(Get-ChildItem -LiteralPath $TempDir -Recurse -File -ErrorAction SilentlyContinue | Measure-Object -Property Length -Sum).Sum) } catch { $tempBytes = $null } + } + $lastProg = $null + try { $lastProg = $script:PaxLastProgressUtc } catch {} + + $parts = New-Object System.Collections.Generic.List[string] + $parts.Add(("ts={0}" -f $nowUtc.ToString('o'))) + if ($Stage) { $parts.Add("stage=$Stage") } + $parts.Add("pid=$($proc.Id)") + $parts.Add(("parentWS_MB={0:N1}" -f ($parentWs / 1MB))) + $parts.Add(("heap_MB={0:N1}" -f ($heap / 1MB))) + if ($null -ne $ChildPid) { $parts.Add("childPID=$ChildPid") } + if ($null -ne $childWs) { $parts.Add(("childWS_MB={0:N1}" -f ($childWs / 1MB))) } + if ($null -ne $RowsProcessed) { $parts.Add("rows=$RowsProcessed") } + if ($null -ne $PagesProcessed) { $parts.Add("pages=$PagesProcessed") } + if ($null -ne $tempBytes) { $parts.Add("tempBytes=$tempBytes") } + if ($lastProg) { try { $parts.Add("lastProgress=$($lastProg.ToString('o'))") } catch {} } + try { Write-LogFile ("MEMOBS " + ($parts -join ' ')) } catch {} + + # Optional in-memory metrics sink (append-only) when the run maintains one. + try { + if ($null -ne $script:PaxMemObsList) { + [void]$script:PaxMemObsList.Add([ordered]@{ ts = $nowUtc.ToString('o'); stage = $Stage; pid = $proc.Id; parentWSBytes = $parentWs; heapBytes = $heap; childPID = $ChildPid; childWSBytes = $childWs; rows = $RowsProcessed; pages = $PagesProcessed; tempBytes = $tempBytes }) + } + } catch {} + } + catch {} +} + +function Write-LogHost { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Message, [string]$ForegroundColor = "White") + Microsoft.PowerShell.Utility\Write-Host $Message -ForegroundColor $ForegroundColor + try { + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $logEntry = "[$timestamp] [INFO] $Message" + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $logEntry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($logEntry) | Out-Null } + } catch {} +} + +# Mirror Write-Host to log file with matching signature +function global:Write-Host { + [CmdletBinding()] + param( + [Parameter(Position=0, ValueFromPipeline=$true, ValueFromRemainingArguments=$true)] + $Object, + [object] $Separator, + [ConsoleColor] $ForegroundColor, + [ConsoleColor] $BackgroundColor, + [switch] $NoNewLine + ) + process { + Microsoft.PowerShell.Utility\Write-Host @PSBoundParameters + try { + # Compose message + $msgItems = @($Object) + $msg = ($msgItems | Out-String).TrimEnd() + if ($msg) { + $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $entry = "[$timestamp] [INFO] $msg" + if ($script:LogFile) { Add-Content -Path $script:LogFile -Value $entry -Encoding UTF8 -ErrorAction SilentlyContinue } + else { $script:LogBuffer.Add($entry) | Out-Null } + } + } catch {} + } +} + +function Get-MaskedUsername { + <# + .SYNOPSIS + Masks a username or email address for secure display in logs and screenshots. + + .DESCRIPTION + Converts "admin@contoso.com" to "a******n@contoso.com" to prevent accidental + credential exposure in terminal output, screenshots, or log files. + + Preserves first and last character of local part, masks middle with 6 asterisks. + Returns original string if input is null, empty, or doesn't contain "@". + + .PARAMETER Username + The username or email address to mask + + .OUTPUTS + Masked string (e.g., "a******n@contoso.com") + + .EXAMPLE + Get-MaskedUsername -Username "admin@contoso.com" + Returns: "a******n@contoso.com" + #> + + param( + [Parameter(Mandatory = $false)] + [string]$Username + ) + + if ([string]::IsNullOrWhiteSpace($Username)) { + return $Username + } + + # Only mask if it looks like an email address + if ($Username -notmatch '@') { + return $Username + } + + $parts = $Username -split '@' + if ($parts.Count -ne 2) { + return $Username + } + + $localPart = $parts[0] + $domain = $parts[1] + + # Handle very short usernames + if ($localPart.Length -le 2) { + return "$($localPart[0])******@$domain" + } + + $first = $localPart[0] + $last = $localPart[$localPart.Length - 1] + $masked = "$first******$last@$domain" + + return $masked +} + +# ============================================================ +# DEIDENTIFICATION (-Deidentify): one-way, salted, deterministic, format-preserving. +# PowerShell deidentifies RAW output files only (the embedded Python processors +# deidentify rollup outputs). The salt, HMAC-SHA256 algorithm, normalization +# (Trim + lower-invariant), lowercase hex, and output formats below are BYTE-FOR-BYTE +# identical to the deid_* helpers in the CopilotInteraction and M365 Python +# processors (PAX deidentify spec) so the same identity hashes to the same token +# across all engines. OFF (default) = no-op: every helper returns its input unchanged. +# Irreversible: no decode map is produced or stored. DO NOT CHANGE THE SALT. +# ============================================================ +$script:PaxDeidEnabled = [bool]$Deidentify +$script:PaxDeidSaltBytes = [System.Text.Encoding]::UTF8.GetBytes('PAX-Deidentify-Salt-v1-DO-NOT-CHANGE-7f3c1e9b2d846050a1c4e8b3') +$script:PaxDeidDomain = 'deidentified.domain' +$script:PaxDeidCache = @{} +$script:PaxDeidMarkerSuffix = '@deidentified.domain' # tail of every deidentified UPN/email token + +# -Deidentify is CSV-only: the deprecated Excel workbook path is never scrubbed, so a +# deidentified workbook would silently leak identifiable data. Block the combination. +if ($script:PaxDeidEnabled -and $ExportWorkbook) { + Write-Host "" -ForegroundColor Red + Write-Host "ERROR: -Deidentify cannot be combined with -ExportWorkbook (Excel)." -ForegroundColor Red + Write-Host "Deidentification scrubs CSV output only; the workbook path is not anonymized and would" -ForegroundColor Yellow + Write-Host "leak identifiable data. Re-run with CSV output (omit -ExportWorkbook) to deidentify." -ForegroundColor Yellow + exit 1 +} + +function Get-PaxDeidHex { + param([string]$Value, [int]$Length) + $norm = $Value.Trim().ToLowerInvariant() + $hmac = [System.Security.Cryptography.HMACSHA256]::new($script:PaxDeidSaltBytes) + try { + $bytes = $hmac.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($norm)) + } finally { + $hmac.Dispose() + } + $sb = [System.Text.StringBuilder]::new(64) + foreach ($b in $bytes) { [void]$sb.Append($b.ToString('x2')) } + return $sb.ToString().Substring(0, $Length) +} + +function Get-PaxDeidUpn { + # UPN / email -> <12hex>@deidentified.domain + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "upn`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = (Get-PaxDeidHex -Value $Value -Length 12) + '@' + $script:PaxDeidDomain + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidName { + # Person / device display name -> <12hex> + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "name`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = Get-PaxDeidHex -Value $Value -Length 12 + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidGuid { + # GUID -> deterministic GUID shape xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "guid`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $h = Get-PaxDeidHex -Value $Value -Length 32 + $v = '{0}-{1}-{2}-{3}-{4}' -f $h.Substring(0,8), $h.Substring(8,4), $h.Substring(12,4), $h.Substring(16,4), $h.Substring(20,12) + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidSid { + # SID -> deterministic S-1-5-21---- shape + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "sid`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $h = Get-PaxDeidHex -Value $Value -Length 32 + $d1 = [Convert]::ToUInt32($h.Substring(0,8), 16) + $d2 = [Convert]::ToUInt32($h.Substring(8,8), 16) + $d3 = [Convert]::ToUInt32($h.Substring(16,8), 16) + $d4 = [Convert]::ToUInt32($h.Substring(24,8), 16) + $v = "S-1-5-21-$d1-$d2-$d3-$d4" + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidToken { + # Opaque id (employeeId, immutableId) -> <12hex> + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "tok`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = Get-PaxDeidHex -Value $Value -Length 12 + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidResource { + # Resource URL -> site_<12hex> (whole-string hash; preserves distinct-count) + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "res`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = 'site_' + (Get-PaxDeidHex -Value $Value -Length 12) + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidFile { + # File / document name -> file_<12hex> + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "file`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $v = 'file_' + (Get-PaxDeidHex -Value $Value -Length 12) + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidProxy { + # proxyAddresses entry(ies) -> keep smtp:/SMTP: prefix + deidentified email. + # Handles ';'-delimited multi-value fields. + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $parts = $Value -split ';' + $out = foreach ($entry in $parts) { + if ([string]::IsNullOrEmpty($entry)) { + $entry + } elseif ($entry.Contains(':')) { + $idx = $entry.IndexOf(':') + $entry.Substring(0, $idx) + ':' + (Get-PaxDeidUpn -Value $entry.Substring($idx + 1)) + } else { + Get-PaxDeidUpn -Value $entry + } + } + return ($out -join ';') +} + +function Get-PaxDeidIp { + # IP address -> deterministic, format-preserving token. IPv4 -> a.b.c.d (4 octets from the + # first 4 HMAC bytes); IPv6 -> 8 hextets from the 16-byte HMAC. Stays a valid-looking IP so + # downstream schema/parsers are unaffected. No-op when off or empty. + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $k = "ip`0$Value" + if ($script:PaxDeidCache.ContainsKey($k)) { return $script:PaxDeidCache[$k] } + $h = Get-PaxDeidHex -Value $Value -Length 32 + if ($Value.Contains(':')) { + $groups = for ($i = 0; $i -lt 8; $i++) { $h.Substring($i * 4, 4) } + $v = $groups -join ':' + } else { + $octets = for ($i = 0; $i -lt 4; $i++) { [Convert]::ToInt32($h.Substring($i * 2, 2), 16) } + $v = $octets -join '.' + } + $script:PaxDeidCache[$k] = $v + return $v +} + +function Get-PaxDeidByType { + # Dispatch a single value to the type-appropriate Get-PaxDeid* helper. Used by the JSON + # scrubber so the same helper set (and tokens) applies to nested AuditData leaves. + param([string]$Value, [string]$Type) + switch ($Type) { + 'Upn' { Get-PaxDeidUpn -Value $Value } + 'Name' { Get-PaxDeidName -Value $Value } + 'Guid' { Get-PaxDeidGuid -Value $Value } + 'Sid' { Get-PaxDeidSid -Value $Value } + 'Token' { Get-PaxDeidToken -Value $Value } + 'Resource' { Get-PaxDeidResource -Value $Value } + 'File' { Get-PaxDeidFile -Value $Value } + 'Proxy' { Get-PaxDeidProxy -Value $Value } + 'Ip' { Get-PaxDeidIp -Value $Value } + default { $Value } + } +} + +function Get-PaxDeidJsonMap { + # PATH-AWARE map of AuditData JSON leaf -> deid type. Path uses '.' for nested objects and + # '[]' for array elements (e.g. Folders[].FolderItems[].Subject). Path-awareness is required + # because the same leaf name means different things by location: root 'Id' is the audit + # RecordId (KEEP) while Item.Id / FolderItems[].Id are item identifiers (scrub). Only the keys + # below are altered; every other field is preserved verbatim (app/tenant/org identities, + # operations, types, timestamps, etc. are intentionally absent and therefore kept). + @{ + # Identity (UPN / SID / GUID) + 'UserId' = 'Upn' + 'MailboxOwnerUPN' = 'Upn' + 'LogonUserSid' = 'Sid' + 'MailboxOwnerSid' = 'Sid' + 'MailboxGuid' = 'Guid' + 'UserKey' = 'Guid' + 'TokenObjectId' = 'Guid' + 'SessionId' = 'Guid' + 'AppAccessContext.AADSessionId' = 'Guid' + 'AppAccessContext.UniqueTokenId' = 'Token' + # IP addresses + 'ClientIP' = 'Ip' + 'ClientIPAddress' = 'Ip' + 'ActorIpAddress' = 'Ip' + # Resources / files (rollup-aligned) + 'SiteUrl' = 'Resource' + 'SourceRelativeUrl' = 'Resource' + 'SourceFileName' = 'File' + 'CopilotEventData.ThreadId' = 'Token' + 'CopilotEventData.AccessedResources[].SiteUrl' = 'Resource' + 'CopilotEventData.AccessedResources[].Name' = 'File' + # Mail item identifiers + content + 'Folders[].FolderItems[].InternetMessageId' = 'Token' + 'Folders[].FolderItems[].Id' = 'Token' + 'Folders[].FolderItems[].ImmutableId' = 'Token' + 'Folders[].FolderItems[].Subject' = 'Token' + 'Folders[].Id' = 'Token' + 'Item.InternetMessageId' = 'Token' + 'Item.Id' = 'Token' + 'Item.ImmutableId' = 'Token' + 'Item.Subject' = 'Token' + 'Item.Attachments' = 'Token' + 'Item.ParentFolder.Id' = 'Token' + } +} + +function Invoke-PaxDeidJsonNode { + # Recursively scrub a System.Text.Json JsonNode in place: any STRING leaf whose path is in + # $Map is replaced with its deid token; every other leaf/structure is left untouched. Keeps + # strings as strings (no ConvertFrom-Json date auto-conversion), preserving full fidelity. + param( + [System.Text.Json.Nodes.JsonNode]$Node, + [string]$Path, + [hashtable]$Map + ) + if ($null -eq $Node) { return } + if ($Node -is [System.Text.Json.Nodes.JsonObject]) { + $keys = @($Node.GetEnumerator() | ForEach-Object { $_.Key }) + foreach ($key in $keys) { + $child = $Node[$key] + $childPath = if ($Path) { "$Path.$key" } else { $key } + if ($child -is [System.Text.Json.Nodes.JsonValue]) { + if ($child.GetValueKind() -eq [System.Text.Json.JsonValueKind]::String -and $Map.ContainsKey($childPath)) { + $orig = $child.GetValue[string]() + $new = Get-PaxDeidByType -Value $orig -Type $Map[$childPath] + if ($new -ne $orig) { $Node[$key] = [System.Text.Json.Nodes.JsonValue]::Create($new) } + } + } else { + Invoke-PaxDeidJsonNode -Node $child -Path $childPath -Map $Map + } + } + } + elseif ($Node -is [System.Text.Json.Nodes.JsonArray]) { + $arrPath = "$Path[]" + for ($i = 0; $i -lt $Node.Count; $i++) { + $child = $Node[$i] + if ($child -is [System.Text.Json.Nodes.JsonValue]) { + if ($child.GetValueKind() -eq [System.Text.Json.JsonValueKind]::String -and $Map.ContainsKey($arrPath)) { + $orig = $child.GetValue[string]() + $new = Get-PaxDeidByType -Value $orig -Type $Map[$arrPath] + if ($new -ne $orig) { $Node[$i] = [System.Text.Json.Nodes.JsonValue]::Create($new) } + } + } else { + Invoke-PaxDeidJsonNode -Node $child -Path $arrPath -Map $Map + } + } + } +} + +function Get-PaxDeidJson { + # Deidentify a JSON blob (e.g. the AuditData column) IN FULL: parse, recursively scrub only the + # mapped PII leaves, re-serialize. Returns the value unchanged when off/empty. Returns the + # redaction marker ONLY when the value cannot be parsed as JSON (safety net so a malformed blob + # can never leak). Re-serializes with relaxed escaping to stay close to the source formatting. + param([string]$Value) + if (-not $script:PaxDeidEnabled -or [string]::IsNullOrEmpty($Value)) { return $Value } + $node = $null + try { $node = [System.Text.Json.Nodes.JsonNode]::Parse($Value) } catch { return '[REDACTED-DEIDENTIFY]' } + if ($null -eq $node) { return $Value } + if ($null -eq $script:PaxDeidJsonMapCached) { $script:PaxDeidJsonMapCached = Get-PaxDeidJsonMap } + if ($null -eq $script:PaxDeidJsonOpts) { + $script:PaxDeidJsonOpts = [System.Text.Json.JsonSerializerOptions]::new() + $script:PaxDeidJsonOpts.Encoder = [System.Text.Encodings.Web.JavaScriptEncoder]::UnsafeRelaxedJsonEscaping + } + Invoke-PaxDeidJsonNode -Node $node -Path '' -Map $script:PaxDeidJsonMapCached + return $node.ToJsonString($script:PaxDeidJsonOpts) +} + +function Invoke-PaxRawDeidentify { + <# + .SYNOPSIS + Post-write pass: deidentify a finished RAW output CSV in place. + .DESCRIPTION + Operates on whichever target columns are present in the file's header, so the SAME + function handles both the EntraUsers CSV (47-col) and the Purview audit CSV (153-col + exploded). Each present identity/resource column is hashed with the type-appropriate + Get-PaxDeid* helper; tokens match the Python-deidentified rollup because the salt and + algorithm are shared. Any raw AuditData / CopilotEventData JSON blob column is + deidentified IN PLACE via Get-PaxDeidJson — a recursive, path-aware System.Text.Json + scrub that tokenizes only the PII leaves and leaves every other field (timestamps, + workload, app/tenant ids, schema urls, flags) byte-for-byte identical, so the blob stays + full-fidelity Purview data with only PII altered. (A blob that fails to parse is redacted + to [REDACTED-DEIDENTIFY] as a safety net so PII can never leak.) + No-op when -Deidentify is off. Rewrites atomically via a temp file + Move-Item, preserving + column order. PowerShell only ever scrubs RAW files; rollup outputs are owned by Python. + #> + param([Parameter(Mandatory = $true)][string]$Path) + if (-not $script:PaxDeidEnabled) { return } + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path)) { return } + + # Column name -> deid helper kind. Union of audit (153-col) + EntraUsers (47-col) fields. + # Column names are disjoint across the two schemas, so one map serves both. + $map = @{ + # Purview audit exploded identity + resource fields + 'UserId' = 'Upn' + 'MailboxOwnerUPN' = 'Upn' + 'MailboxGuid' = 'Guid' + 'LogonUserSid' = 'Sid' + 'MailboxOwnerSid' = 'Sid' + 'DeviceDisplayName' = 'Name' + 'SiteUrl' = 'Resource' + 'SourceRelativeUrl' = 'Resource' + 'SourceFileName' = 'File' + 'AccessedResource_SiteUrl' = 'Resource' + 'AccessedResource_Name' = 'File' + # Purview audit exploded — additional identity / network / resource fields + # (flat top-level columns produced by the 153-col explosion; nested JSON PII is + # handled separately by Get-PaxDeidJson on the AuditData blob). + 'ClientIP' = 'Ip' + 'ClientIPAddress' = 'Ip' + 'ActorIpAddress' = 'Ip' + 'UserKey' = 'Guid' + 'TokenObjectId' = 'Guid' + 'SessionId' = 'Guid' + 'ThreadId' = 'Token' + 'ChatId' = 'Token' + 'ConversationId' = 'Token' + 'Site' = 'Resource' + 'MeetingURL' = 'Resource' + 'TeamName' = 'Name' + 'ChannelName' = 'Name' + 'VideoName' = 'File' + 'FormName' = 'File' + # EntraUsers identity fields + 'userPrincipalName' = 'Upn' + 'displayName' = 'Name' + 'mail' = 'Upn' + 'givenName' = 'Name' + 'surname' = 'Name' + 'UserName' = 'Upn' + 'employeeId' = 'Token' + 'onPremisesImmutableId' = 'Token' + 'proxyAddresses_Primary' = 'Proxy' + 'proxyAddresses_All' = 'Proxy' + 'id' = 'Guid' + 'manager_id' = 'Guid' + 'manager_userPrincipalName' = 'Upn' + 'manager_displayName' = 'Name' + 'manager_mail' = 'Upn' + 'ManagerID' = 'Guid' + } + # JSON blob columns: deidentified in place by recursive path-aware scrub (Get-PaxDeidJson), + # NOT flat-hashed — their PII lives in nested keys the column map cannot reach. + $jsonCols = @('AuditData', 'CopilotEventData') + + # Stream the RAW file record-by-record (RFC-4180) and write each deidentified row + # through the existing streaming CSV writer in bounded batches, so the complete + # file is never materialized. Token generation is stateless/deterministic, so no + # per-identity map is retained. Column order is preserved from the header; output + # quoting is byte-identical to the prior in-place rewrite (same writer). + $reader = [System.IO.StreamReader]::new($Path, $true) + $readerDone = $false + try { + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $header) { return } + $columns = @($header | ForEach-Object { [string]$_ }) + $active = @{} + foreach ($c in $columns) { if ($map.ContainsKey($c)) { $active[$c] = $map[$c] } } + $activeJson = @($jsonCols | Where-Object { $columns -contains $_ }) + if ($active.Count -eq 0 -and $activeJson.Count -eq 0) { return } + $colCount = $columns.Count + $activeKeys = @($active.Keys) + $tmp = "$Path.deid.tmp" + $rowCount = 0 + $batch = New-Object System.Collections.Generic.List[object] + try { + try { + Open-CsvWriter -Path $tmp -Columns $columns + while ($true) { + $fields = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $fields) { break } + $row = [ordered]@{} + for ($i = 0; $i -lt $colCount; $i++) { $row[$columns[$i]] = if ($i -lt $fields.Length) { [string]$fields[$i] } else { '' } } + foreach ($col in $activeKeys) { + $val = $row[$col] + if ([string]::IsNullOrEmpty($val)) { continue } + $row[$col] = switch ($active[$col]) { + 'Upn' { Get-PaxDeidUpn -Value $val } + 'Name' { Get-PaxDeidName -Value $val } + 'Guid' { Get-PaxDeidGuid -Value $val } + 'Sid' { Get-PaxDeidSid -Value $val } + 'Token' { Get-PaxDeidToken -Value $val } + 'Resource' { Get-PaxDeidResource -Value $val } + 'File' { Get-PaxDeidFile -Value $val } + 'Proxy' { Get-PaxDeidProxy -Value $val } + 'Ip' { Get-PaxDeidIp -Value $val } + } + } + foreach ($jc in $activeJson) { + if (-not [string]::IsNullOrEmpty($row[$jc])) { $row[$jc] = Get-PaxDeidJson -Value $row[$jc] } + } + $batch.Add([pscustomobject]$row) + $rowCount++ + if ($batch.Count -ge 5000) { Write-CsvRows -Rows $batch -Columns $columns; $batch.Clear() } + } + if ($batch.Count -gt 0) { Write-CsvRows -Rows $batch -Columns $columns } + } + finally { Close-CsvWriter } + # Release the input handle BEFORE the in-place replace so Move-Item can + # overwrite $Path. + $reader.Dispose(); $readerDone = $true + if ($rowCount -eq 0) { + # Header-only source: the prior pass was a no-op (Import-Csv => 0 rows => return). + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + return + } + # Atomic in-place publish (temp + replace), preserving original column order. + Move-Item -LiteralPath $tmp -Destination $Path -Force -ErrorAction Stop + } + catch { + # Fail closed: never leave a partial temp; the original file is untouched + # because the replace happens only on the last line above. + if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } + throw + } + } + finally { if (-not $readerDone) { $reader.Dispose() } } +} + +function Test-PaxFileDeidentified { + <# + .SYNOPSIS + Classify a CSV append target as 'deid' | 'raw' | 'unknown' by sampling its identity column. + .DESCRIPTION + 'deid' = at least one non-empty identity value carries the deidentify marker + (@deidentified.domain). + 'raw' = identity values present, none carry the marker. + 'unknown'= file missing/unreadable, no recognised identity column, or no non-empty + identity values to judge by (caller skips the guard in that case). + Streams at most ~200 rows (pipeline short-circuits) so it is cheap on large files. + #> + param([Parameter(Mandatory = $true)][string]$Path) + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path)) { return 'unknown' } + try { + $sample = @(Import-Csv -LiteralPath $Path | Select-Object -First 200) + } catch { + return 'unknown' + } + if ($sample.Count -eq 0) { return 'unknown' } + $cols = $sample[0].PSObject.Properties.Name + # Identity columns across every deidentified output family (audit raw, CopilotInteraction + # fact (aibv), M365 rollup, Users dim). The aio fact has no UPN column -> 'unknown' there, + # but the paired -AppendUserInfo Users dim (PersonId) carries the signal. + $idCol = @('Audit_UserId', 'UserId', 'PersonId', 'userPrincipalName', 'MailboxOwnerUPN') | + Where-Object { $cols -contains $_ } | Select-Object -First 1 + if (-not $idCol) { return 'unknown' } + $sawValue = $false + foreach ($r in $sample) { + $v = [string]$r.$idCol + if ([string]::IsNullOrEmpty($v)) { continue } + $sawValue = $true + if ($v.TrimEnd().ToLowerInvariant().EndsWith($script:PaxDeidMarkerSuffix)) { return 'deid' } + } + if ($sawValue) { return 'raw' } + return 'unknown' +} + +function Assert-PaxDeidAppendConsistency { + <# + .SYNOPSIS + Hard-stop guard preventing deidentified/raw data from being mixed in an append target. + .DESCRIPTION + Data-accuracy gate for -AppendFile / -AppendUserInfo: + * BLOCK when the target's deidentify state disagrees with this run's -Deidentify state + (mixing hashed + raw identities corrupts user joins and distinct counts). + * BLOCK appending into an already-deidentified target in NON-rollup mode (the whole-file + post-pass would re-hash existing rows -> double-hashing). Rollup append into a + deidentified target is allowed because it MERGES two same-salt outputs. + Throws on violation; the caller renders the message and exits non-zero. + #> + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Targets, + [Parameter(Mandatory = $true)][bool]$IsRollup + ) + $runDeid = [bool]$script:PaxDeidEnabled + foreach ($t in $Targets) { + $path = [string]$t.Path + $label = [string]$t.Label + if ([string]::IsNullOrWhiteSpace($path)) { continue } + $state = Test-PaxFileDeidentified -Path $path + if ($state -eq 'unknown') { continue } + $tgtDeid = ($state -eq 'deid') + if ($tgtDeid -ne $runDeid) { + $desc = if ($tgtDeid) { 'is already DEIDENTIFIED but this run is NOT -Deidentify' } + else { 'is NOT deidentified but this run IS -Deidentify' } + throw ("-Deidentify/append mismatch: the $label target $desc. Mixing deidentified and raw " + + "identities in one file corrupts user joins and distinct counts. Use a separate target " + + "for deidentified output, or re-run with a matching -Deidentify state.`nTarget: $path") + } + if ($tgtDeid -and $runDeid -and -not $IsRollup) { + throw ("-Deidentify cannot append to an already-deidentified file in non-rollup mode: the " + + "$label target is deidentified and a non-rollup append would re-hash its existing rows " + + "(double-hashing). Append on a RAW master and deidentify a fresh copy, or use " + + "-Rollup / -RollupPlusRaw (which merge matching-salt outputs).`nTarget: $path") + } + } +} + +# --- Helper Function: Detect if PAYG billing is configured in tenant --- +function Test-PAYGBillingEnabled { + <# + .SYNOPSIS + Attempts to detect if Microsoft Purview PAYG billing is configured in the tenant. + + .DESCRIPTION + Checks for indicators that PAYG billing is enabled: + - Attempts to query audit records for AIAppInteraction type (PAYG-only) + - If records exist or query succeeds without billing errors, PAYG is likely enabled + - Returns $null if detection is inconclusive + + .OUTPUTS + $true if PAYG billing appears to be enabled + $false if PAYG billing appears to be disabled or not configured + $null if detection is inconclusive (requires actual query attempt) + #> + + # Note: The most reliable way to detect PAYG is to attempt a query for AIAppInteraction + # and check for specific error responses. However, this requires actual data/timeframe. + # Returns $null to indicate "unknown" and lets the post-query detection handle it. + + Write-LogHost "PAYG billing detection: Deferred to post-query validation" -ForegroundColor DarkGray + return $null +} + +# --- Excel Export Validation --- + +# AppendFile with Excel requires ExportWorkbook +if ($AppendFile -and $ExportWorkbook -eq $false) { + # AppendFile is fine for CSV (will be handled later), but needs validation for Excel intent + # This check is only if user explicitly wants Excel append +} + +if ($AppendFile -and -not $ExportWorkbook -and -not $PSBoundParameters.ContainsKey('ExportWorkbook')) { + # User wants AppendFile but didn't specify format - this is OK, will default to CSV append +} + +# Rollup post-processor always requires a single combined Purview CSV. When +# -Rollup / -RollupPlusRaw is bound but -CombineOutput is not, force it on +# here so the export-mode banner below reflects the resolved (combined) shape +# instead of the user's default separate-files layout. The broader +# -Rollup / -RollupPlusRaw validation (mutual exclusion, blockers, processor +# mode detection) still runs later in the rollup-handling block; that block's +# own -CombineOutput auto-enable is a no-op once this one has fired. +if (($Rollup -or $RollupPlusRaw) -and -not $CombineOutput) { + $_rollupSwitchName = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: $_rollupSwitchName auto-enabled -CombineOutput (rollup post-processor requires a single combined Purview CSV)." -ForegroundColor Cyan +} + +# Log export mode +if ($ExportWorkbook) { + # Determine Excel output mode based on -CombineOutput parameter + if ($CombineOutput) { + $excelModeMsg = "Excel export mode: Combined activity tab + separate EntraUsers tab (if requested)" + if ($IncludeAgent365Info -or $OnlyAgent365Info) { $excelModeMsg += " + Agents365 tab" } + Write-Host $excelModeMsg -ForegroundColor Cyan + } else { + # Default for Excel: separated tabs + $excelModeMsg = "Excel export mode: Multi-tab workbook (one tab per activity type)" + if ($IncludeAgent365Info -or $OnlyAgent365Info) { $excelModeMsg += " + Agents365 tab" } + Write-Host $excelModeMsg -ForegroundColor Cyan + } + + if ($AppendFile) { + Write-Host "Append mode: Enabled (will validate existing workbook structure)" -ForegroundColor Cyan + } + Write-Host "" +} else { + # CSV export mode + if ($OnlyUserInfo) { + # OnlyUserInfo mode: No activity files, just Entra user data + $onlyUserMsg = "CSV export mode: Entra user directory and licensing data only (no audit logs)" + if ($IncludeAgent365Info) { $onlyUserMsg += " + separate Agents365 file" } + Write-Host $onlyUserMsg -ForegroundColor Cyan + } elseif ($OnlyAgent365Info) { + # OnlyAgent365Info mode: No activity files, just Agent 365 catalog data + Write-Host "CSV export mode: Microsoft Agent 365 catalog only (no audit logs, no Entra users)" -ForegroundColor Cyan + } else { + # Determine CSV output mode based on -CombineOutput parameter + if ($RAWInputCSV -and -not $CombineOutput.IsPresent) { $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) } + if ($CombineOutput.IsPresent -or $RAWInputCSV) { + # User specified -CombineOutput switch: combine all activity types + $csvModeMsg = "Combined activity file" + if ($IncludeUserInfo) { $csvModeMsg += " + separate EntraUsers file" } + if ($IncludeAgent365Info) { $csvModeMsg += " + separate Agents365 file" } + Write-Host "CSV export mode: $csvModeMsg" -ForegroundColor Cyan + } else { + # Default for live CSV: separate files per activity type + $csvModeMsg = "Separate activity files (one per activity type)" + if ($IncludeUserInfo) { $csvModeMsg += " + EntraUsers file" } + if ($IncludeAgent365Info) { $csvModeMsg += " + Agents365 file" } + Write-Host "CSV export mode: $csvModeMsg" -ForegroundColor Cyan + } + } + Write-Host "" +} + +# OutputPath shape normalization runs AFTER tier inference below (folder-only +# enforcement applies only to Local tier; SharePoint/Fabric URLs are handled +# by the remote-output plumbing). + +# ============================================================================ +# DESTINATION VALIDATION AND TIER INFERENCE +# ---------------------------------------------------------------------------- +# Storage tier is inferred from each -OutputPath* value's form: +# Drive-rooted absolute path -> Local +# https://*.sharepoint* -> SharePoint +# https://*.onelake.dfs.fabric.* -> Fabric +# UNC paths (\\server\share\...) are rejected on every destination switch. +# All destinations supplied to a single run must resolve to the same storage +# tier (no mixed Local/SP/Fabric in one invocation). Any two destinations +# that resolve to identical fully qualified paths are rejected (FullPath +# collision). $OutputPathLog allows the Fabric Files/ form additionally. +# Legacy $script:RemoteOutputMode / $script:RemoteOutputUrl are populated from +# the inferred Purview destination so downstream code that predates the new +# model continues to operate unchanged. +# ============================================================================ +$script:RemoteOutputMode = 'None' # 'None' | 'SharePoint' | 'Fabric' +$script:RemoteOutputUrl = $null +$script:RemoteScratchDir = $null + +# Per-data-type effective destinations and inferred tiers. Populated below. +$script:DestTier = @{} # 'Purview'|'UserInfo'|'Agent365Info'|'Log' -> 'Local'|'SharePoint'|'Fabric' +$script:DestRaw = @{} # original user-supplied value (trimmed) +$script:DestIsBound = @{} # was the switch actually supplied? + +$spUrlPattern = '^https?://[^/]+\.sharepoint(?:-df|-mil)?\.[a-z]{2,3}(?:/.+)?$' +# Fabric Tables/ segment is constrained to identifier-style strings +# (alpha/underscore start, then alphanumeric/underscore) to match the Schemas-mode +# Lakehouse rules and the downstream Delta-write detection regex (search for +# $fabricDeltaMode). Files/ subpaths remain permissive. +# Item segment is EITHER the name form (.Lakehouse, suffix required) OR the +# GUID form (, no suffix) - both are first-class OneLake DFS addressing +# modes (learn.microsoft.com/fabric/onelake/onelake-access-api). The workspace +# segment ([^/]+) already accepts a name or GUID. +$fabricItemSeg = '(?:[^/]+\.Lakehouse|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})' +$fabricRootPattern = "^https://([a-z0-9-]+-)?onelake\.dfs\.fabric\.microsoft\.com/[^/]+/$fabricItemSeg(/Tables(/[A-Za-z_][A-Za-z0-9_]*)?|/Files(/.+)?)?/?`$" +$fabricFilesPattern = "^https://([a-z0-9-]+-)?onelake\.dfs\.fabric\.microsoft\.com/[^/]+/$fabricItemSeg/Files(/.+)?/?`$" + +function script:Get-PathTier { + param([string]$Value, [string]$SwitchName, [switch]$AllowFabricFilesOnly) + if ([string]::IsNullOrWhiteSpace($Value)) { return $null } + $v = $Value.Trim() + if ($v -match '^\\\\') { + Write-Host ("ERROR: -{0} does not accept UNC paths ('{1}')." -f $SwitchName, $v) -ForegroundColor Red + Write-Host " Provide a drive-rooted local path, a SharePoint URL, or a Fabric OneLake URL." -ForegroundColor Yellow + exit 1 + } + if ($v -match '^https?://') { + if ($v -match $spUrlPattern) { return 'SharePoint' } + if ($AllowFabricFilesOnly) { + if ($v -match $fabricFilesPattern) { return 'Fabric' } + Write-Host ("ERROR: -{0} on a Fabric destination must point under Files/ (logs are not tabular)." -f $SwitchName) -ForegroundColor Red + Write-Host (" Provided: {0}" -f $v) -ForegroundColor Yellow + exit 1 + } + if ($v -match $fabricRootPattern) { return 'Fabric' } + Write-Host ("ERROR: -{0} URL is not a recognized SharePoint or Fabric Lakehouse destination." -f $SwitchName) -ForegroundColor Red + Write-Host (" Provided: {0}" -f $v) -ForegroundColor Yellow + Write-Host " Examples:" -ForegroundColor Yellow + Write-Host " https://.sharepoint.com/sites//[/]" -ForegroundColor DarkGray + Write-Host " https://onelake.dfs.fabric.microsoft.com//.Lakehouse" -ForegroundColor DarkGray + Write-Host " https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Tables" -ForegroundColor DarkGray + Write-Host " https://onelake.dfs.fabric.microsoft.com// (GUID form, no .Lakehouse suffix)" -ForegroundColor DarkGray + exit 1 + } + # Local: require drive-rooted absolute path. + if ($v -match '^[A-Za-z]:[\\/]' -or $v -match '^/') { + return 'Local' + } + Write-Host ("ERROR: -{0} must be a drive-rooted absolute path, a SharePoint URL, or a Fabric OneLake URL." -f $SwitchName) -ForegroundColor Red + Write-Host (" Provided: {0}" -f $v) -ForegroundColor Yellow + exit 1 +} + +$destSwitches = @( + @{ Key = 'Purview' ; Name = 'OutputPath' ; Value = $OutputPath ; AllowFabricFilesOnly = $false }, + @{ Key = 'UserInfo' ; Name = 'OutputPathUserInfo' ; Value = $OutputPathUserInfo ; AllowFabricFilesOnly = $false }, + @{ Key = 'Agent365Info' ; Name = 'OutputPathAgent365Info' ; Value = $OutputPathAgent365Info ; AllowFabricFilesOnly = $false }, + @{ Key = 'DefenderUsage'; Name = 'OutputPathDefenderUsage' ; Value = $OutputPathDefenderUsage ; AllowFabricFilesOnly = $false }, + @{ Key = 'Log' ; Name = 'OutputPathLog' ; Value = $OutputPathLog ; AllowFabricFilesOnly = $true } +) +foreach ($ds in $destSwitches) { + $bound = $PSBoundParameters.ContainsKey($ds.Name) + $script:DestIsBound[$ds.Key] = $bound + if ($bound -and -not [string]::IsNullOrWhiteSpace($ds.Value)) { + $tier = script:Get-PathTier -Value $ds.Value -SwitchName $ds.Name -AllowFabricFilesOnly:$ds.AllowFabricFilesOnly + $script:DestTier[$ds.Key] = $tier + $script:DestRaw[$ds.Key] = $ds.Value.Trim() + } +} + +# --- Append-side tier inference --- +# Treat the three -Append* switches as alternate suppliers of the stream destination. +# A relative filename (no scheme, no drive letter) is "deferred" — tier is inherited +# from the corresponding -OutputPath* (or the default scratch dir). A URL or drive- +# rooted full path sets the stream tier directly when the -OutputPath* counterpart +# is unbound (the XOR validator below rejects the both-bound case). +$script:AppendIsBound = @{} +$script:AppendRaw = @{} +$script:AppendIsRemote = @{} +$appendSwitches = @( + @{ Key = 'Purview' ; Name = 'AppendFile' ; Value = $AppendFile ; Bound = $PSBoundParameters.ContainsKey('AppendFile') }, + @{ Key = 'UserInfo' ; Name = 'AppendUserInfo' ; Value = $AppendUserInfo ; Bound = $PSBoundParameters.ContainsKey('AppendUserInfo') }, + @{ Key = 'Agent365Info' ; Name = 'AppendAgent365Info' ; Value = $AppendAgent365Info ; Bound = $PSBoundParameters.ContainsKey('AppendAgent365Info') }, + @{ Key = 'DefenderUsage'; Name = 'AppendDefenderUsage' ; Value = $AppendDefenderUsage ; Bound = $PSBoundParameters.ContainsKey('AppendDefenderUsage') } +) +foreach ($as in $appendSwitches) { + $script:AppendIsBound[$as.Key] = $as.Bound + if (-not $as.Bound -or [string]::IsNullOrWhiteSpace($as.Value)) { continue } + $v = $as.Value.Trim() + $script:AppendRaw[$as.Key] = $v + $looksUrl = $v -match '^https?://' + $looksRooted = $v -match '^[A-Za-z]:[\\/]' -or $v -match '^/' + $looksUnc = $v -match '^\\\\' + if ($looksUnc) { + Write-Host ("ERROR: -{0} does not accept UNC paths ('{1}')." -f $as.Name, $v) -ForegroundColor Red + Write-Host " Provide a relative filename, a drive-rooted local path, a SharePoint URL, or a Fabric OneLake URL." -ForegroundColor Yellow + exit 1 + } + if ($looksUrl -or $looksRooted) { + # Full-form: classify via Get-PathTier (which exits on invalid forms). + $aTier = script:Get-PathTier -Value $v -SwitchName $as.Name + $script:AppendIsRemote[$as.Key] = ($aTier -ne 'Local') + # Promote the stream's tier from the Append side when the OutputPath side is unbound. + if (-not $script:DestTier.ContainsKey($as.Key)) { + $script:DestTier[$as.Key] = $aTier + $script:DestRaw[$as.Key] = $v + } + elseif ($script:DestTier[$as.Key] -ne $aTier) { + # OutputPath* and Append* both bound on different tiers — XOR validator + # below will fire the canonical "both bound" error, but flag the tier + # mismatch here too in case XOR is bypassed by a future code path. + Write-Host ("ERROR: -{0} ({1}) and -{2} ({3}) resolve to different storage tiers." -f $as.Name, $aTier, ($destSwitches | Where-Object Key -EQ $as.Key | Select-Object -First 1 -ExpandProperty Name), $script:DestTier[$as.Key]) -ForegroundColor Red + exit 1 + } + } else { + # Relative filename — tier deferred to the -OutputPath* side (or local default). + $script:AppendIsRemote[$as.Key] = $false + } +} + +# Tier consistency across all bound destinations (excluding -OutputPathLog when it +# targets Fabric Files/ — the log is allowed under Files/ on a Fabric data run). +$tiersForConsistency = @() +foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { + if ($script:DestTier.ContainsKey($k)) { $tiersForConsistency += $script:DestTier[$k] } +} +$tiersForConsistency = $tiersForConsistency | Select-Object -Unique +if ($tiersForConsistency.Count -gt 1) { + Write-Host ("ERROR: Mixed storage tiers across -OutputPath* switches ({0}). All destinations in one run must resolve to the same tier." -f ($tiersForConsistency -join ', ')) -ForegroundColor Red + exit 1 +} + +# FullPath collision: any two bound destinations resolving to identical fully +# qualified paths are rejected. Folder-only inputs that resolve to the same +# folder with different default basenames are not collisions (compared on the +# normalized input as supplied; default basenames are appended downstream). +$normalizedDests = @{} +foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { + if (-not $script:DestRaw.ContainsKey($k)) { continue } + $norm = $script:DestRaw[$k].TrimEnd('/','\').ToLowerInvariant() + # Only flag collisions when the supplied form is a full file path (contains a basename extension). + if ($norm -match '\.[a-z0-9]{2,5}$') { + if ($normalizedDests.ContainsKey($norm)) { + Write-Host ("ERROR: -{0} and -{1} resolve to the same fully qualified path:" -f $normalizedDests[$norm], $k) -ForegroundColor Red + Write-Host (" {0}" -f $script:DestRaw[$k]) -ForegroundColor Yellow + exit 1 + } + $normalizedDests[$norm] = $k + } +} + +# Populate legacy back-compat variables from the Purview destination so downstream +# code paths that pre-date the new model keep working without per-call refactors. +# When Purview is out of scope (-OnlyUserInfo / -OnlyAgent365Info), or when Purview +# is in scope but its destination was supplied via -AppendFile (already absorbed +# into $script:DestTier['Purview'] above), the promotion below still fires. +# Additionally, in only-modes the remote tier may come from -OutputPathUserInfo / +# -AppendUserInfo / -OutputPathAgent365Info / -AppendAgent365Info instead — promote +# from whichever stream is in scope so the remote pre-flight + scratch-dir handling +# downstream activates correctly. +if ($script:DestTier.ContainsKey('Purview')) { + switch ($script:DestTier['Purview']) { + 'SharePoint' { + $script:RemoteOutputMode = 'SharePoint' + $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') + } + 'Fabric' { + $script:RemoteOutputMode = 'Fabric' + $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') + } + } +} +# Fallback promotion: when Purview tier is unset (only-modes), promote from +# the first in-scope non-Purview stream that resolved to a remote tier. +if ($script:RemoteOutputMode -eq 'None') { + foreach ($k in @('UserInfo','Agent365Info')) { + if (-not $script:DestTier.ContainsKey($k)) { continue } + $t = $script:DestTier[$k] + if ($t -eq 'SharePoint' -or $t -eq 'Fabric') { + $script:RemoteOutputMode = $t + $script:RemoteOutputUrl = $script:DestRaw[$k].TrimEnd('/') + break + } + } +} + +# Normalize RemoteOutputUrl + populate $script:DestParentUrl with FOLDER URLs. +# When the source switch was -AppendFile / -AppendUserInfo / -AppendAgent365Info, +# or when -OutputPath* was supplied as a file URL, the original DestRaw values are +# FILE URLs (last segment ends in a recognized data-artifact extension). Downstream +# code — Resolve-SharePointTarget / Resolve-FabricTarget (parse the URL as folder + +# leaf), Get-DisplayPath (appends sibling leaves like .log / _PARTIAL.csv), the +# per-data-type parent-URL routing in the upload sweep ($dtParentUrl / +# $logParentOverride), and Get-RemoteFile-* — all assume folder semantics. Leaving +# a file URL in those code paths would produce '.../foo.csv/foo.log' style nesting +# in display, and a 404 NotFound when re-downloading the append target. +# +# $script:DestRaw stays AS-SUPPLIED so the per-data-type destinations banner can +# still print the file-URL form the user typed in. +# $script:DestParentUrl is the FOLDER URL form used by upload/download routing and +# display-leaf composition. +# $script:RemoteOutputUrl is normalized to a folder URL for the same reason. +# .Lakehouse is a folder marker (Lakehouse root); do NOT treat it as a file. +$script:DestParentUrl = @{} +$script:NormalizeFolderUrl = { + param([string]$u) + if ([string]::IsNullOrWhiteSpace($u)) { return $u } + if ($u -match '/[^/]+\.(csv|log|xlsx|json|parquet|txt)$') { + return ($u -replace '/[^/]+\.[A-Za-z0-9]+$','') + } + return $u +} +if ($script:RemoteOutputUrl) { + $script:RemoteOutputUrl = & $script:NormalizeFolderUrl $script:RemoteOutputUrl +} +foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { + if (-not $script:DestRaw.ContainsKey($k)) { continue } + if (-not $script:DestTier.ContainsKey($k)) { continue } + if ($script:DestTier[$k] -eq 'Local') { continue } + $script:DestParentUrl[$k] = & $script:NormalizeFolderUrl $script:DestRaw[$k] +} + +# Effective-basename helpers. Single source of truth for the leaf used +# by the EntraUsers and Agent 365 streams everywhere display/banner code mentions +# them. Resolution order: +# 1. If -AppendUserInfo / -AppendAgent365Info is bound, inherit the target's +# original leaf — the writer reuses that leaf (so the merged output replaces +# the original remote URL via the upload sweep's Append-leaf match), and +# the banners need to display the same name the customer typed and the +# remote destination will end up holding. +# 2. Else if -OutputPath* was supplied in file-form, use that file's leaf. +# 3. Else default to the timestamped basename. +# NOTE: These helpers must run AFTER $script:AppendRaw / $script:DestRaw are +# populated (done by the destSwitches + appendSwitches blocks above). +$script:GetEffectiveEntraBasename = { + # Returns the basename of the RAW EntraUsers CSV. Under -AppendUserInfo the + # raw is always written to its natural / pristine name (the caller separately + # merges raw -> the AppendUserInfo target via Merge-UsersCsv). If the + # caller-supplied AppendUserInfo leaf happens to collide with the natural raw + # leaf in the same destination folder, the caller is responsible for adding + # a '_raw' suffix to the raw filename (Resolve-PristineRawEntraPath). + # + # Resolution order for the natural raw basename: + # 1. If -OutputPathUserInfo is bound with a file leaf, use that leaf. + # 2. Else default to the timestamped 'EntraUsers_MAClicensing_.csv'. + if ($script:DestIsBound -and $script:DestIsBound['UserInfo'] -and $script:DestRaw.ContainsKey('UserInfo')) { + $leaf = [System.IO.Path]::GetFileName($script:DestRaw['UserInfo']) + if ($leaf -and $leaf -match '\.[A-Za-z0-9]{2,5}$') { return $leaf } + } + return ("EntraUsers_MAClicensing_{0}.csv" -f $global:ScriptRunTimestamp) +} +$script:GetEffectiveAgent365Basename = { + if ($script:AppendIsBound -and $script:AppendIsBound['Agent365Info'] -and $script:AppendRaw.ContainsKey('Agent365Info')) { + return [System.IO.Path]::GetFileName($script:AppendRaw['Agent365Info']) + } + if ($script:DestIsBound -and $script:DestIsBound['Agent365Info'] -and $script:DestRaw.ContainsKey('Agent365Info')) { + $leaf = [System.IO.Path]::GetFileName($script:DestRaw['Agent365Info']) + if ($leaf -and $leaf -match '\.[A-Za-z0-9]{2,5}$') { return $leaf } + } + return ("Agent365_{0}.csv" -f $global:ScriptRunTimestamp) +} + +# Replay/EOM modes are not supported with remote output. +if ($script:RemoteOutputMode -ne 'None') { + $remoteBlockers = @() + if ($UseEOM) { $remoteBlockers += '-UseEOM' } + if ($RAWInputCSV) { $remoteBlockers += '-RAWInputCSV' } + if ($remoteBlockers.Count -gt 0) { + Write-Host ("ERROR: Remote output destinations are not supported with: {0}" -f ($remoteBlockers -join ', ')) -ForegroundColor Red + Write-Host " These modes operate on existing files and have no remote-write story." -ForegroundColor Yellow + exit 1 + } +} + +# ExportWorkbook is restricted to Local and SharePoint tiers. +if ($ExportWorkbook -and $script:DestTier.ContainsKey('Purview') -and $script:DestTier['Purview'] -eq 'Fabric') { + Write-Host "ERROR: -ExportWorkbook is not supported on Fabric (OneLake) destinations." -ForegroundColor Red + Write-Host " Use a Local or SharePoint -OutputPath for workbook output." -ForegroundColor Yellow + exit 1 +} +if ($ExportWorkbook) { + $workbookCompanionConflict = $false + if (-not ($OnlyUserInfo -or $OnlyAgent365Info)) { + if ($script:DestIsBound['UserInfo']) { $workbookCompanionConflict = $true } + if ($script:DestIsBound['Agent365Info']) { $workbookCompanionConflict = $true } + } + if ($workbookCompanionConflict) { + Write-Host "ERROR: -OutputPathUserInfo / -OutputPathAgent365Info cannot be combined with -ExportWorkbook when Purview output is in scope." -ForegroundColor Red + Write-Host " The workbook is a single file containing all tabs - companion CSV destinations would be ignored." -ForegroundColor Yellow + exit 1 + } +} + +# Local-tier OutputPath normalization. When the Purview destination is Local (or +# unbound and therefore defaulted to a local folder), ensure it ends with a path +# separator if it is a folder (no extension) and create the directory if needed. +# Full-path-with-basename Local forms are also accepted; downstream auto-basename +# logic detects the file form and uses the supplied basename verbatim. +if (-not $PSBoundParameters.ContainsKey('OutputPath') -or [string]::IsNullOrWhiteSpace($OutputPath)) { + # When -OutputPath is omitted, infer the run's scratch+log directory from the + # DOMINANT in-scope stream's bound -Append* / -OutputPath* target. Otherwise + # $OutputPath would fall back to $PSScriptRoot — polluting the script's own + # folder with the run log, .pax_incremental shards/seed JSONs, and any rollup + # temp files even though the customer pinned a real data destination elsewhere + # via -AppendFile / -AppendUserInfo / -AppendAgent365Info. + # + # Dominant stream resolution (mirrors XOR validator's scope rules): + # -OnlyUserInfo -> UserInfo (AppendUserInfo / OutputPathUserInfo) + # -OnlyAgent365Info -> Agent365Info (AppendAgent365Info / OutputPathAgent365Info) + # default -> Purview (AppendFile) [XOR guarantees AppendFile + # is bound when OutputPath is not, in scope] + # + # Source of truth is $script:DestRaw — populated by the destSwitches loop above + # from -OutputPath* AND by the appendSwitches loop from rooted/URL -Append* + # values. Relative-filename -Append* forms intentionally do NOT populate DestRaw + # (they defer to the unspecified -OutputPath*), so they fall through to the + # legacy $PSScriptRoot default — correct behavior, since the relative form was + # only ever meaningful with -OutputPath in scope. + # + # Remote URLs are excluded from inference: remote-mode runs stage scratch+log + # locally (the upload sweep handles the move to the remote URL afterward), and + # the pre-existing $PSScriptRoot scratch behavior is preserved unchanged for + # those runs. Cross-tier mismatches are already rejected upstream by the + # tier-consistency validator (L2603-L2613). + $_pvInferKey = if ($OnlyUserInfo) { 'UserInfo' } elseif ($OnlyAgent365Info) { 'Agent365Info' } else { 'Purview' } + $_pvInferParent = $null + if ($script:DestRaw.ContainsKey($_pvInferKey)) { + $_pvInferRaw = $script:DestRaw[$_pvInferKey] + if (-not [string]::IsNullOrWhiteSpace($_pvInferRaw) -and ($_pvInferRaw -notmatch '^https?://')) { + # Folder form -> use as-is; file form -> use parent directory. + $_pvInferCandidate = if (Test-Path -Path $_pvInferRaw -PathType Container) { + $_pvInferRaw + } else { + Split-Path -Parent $_pvInferRaw + } + if ($_pvInferCandidate -and (Test-Path -Path $_pvInferCandidate -PathType Container)) { + $_pvInferParent = $_pvInferCandidate + } + } + } + if ($_pvInferParent) { + $OutputPath = $_pvInferParent + Write-Host ("INFO: Inferred run scratch+log dir from in-scope -{0}: {1}" -f $(if ($OnlyUserInfo) { 'AppendUserInfo/-OutputPathUserInfo' } elseif ($OnlyAgent365Info) { 'AppendAgent365Info/-OutputPathAgent365Info' } else { 'AppendFile' }), $OutputPath) -ForegroundColor DarkGray + } elseif ($PSScriptRoot) { + $OutputPath = $PSScriptRoot + } else { + $OutputPath = (Get-Location).Path + } + if (-not $OutputPath.EndsWith('\') -and -not $OutputPath.EndsWith('/')) { + $OutputPath = $OutputPath + '\' + } + # Do NOT clobber a Purview tier that was already promoted from a remote + # -AppendFile URL (SharePoint/Fabric). The Local default only applies when + # no Append-side promotion has populated DestTier['Purview'] yet — otherwise + # the per-data-type destinations banner mis-tags Purview as [Local]. + if (-not $script:DestTier.ContainsKey('Purview')) { + $script:DestTier['Purview'] = 'Local' + $script:DestRaw['Purview'] = $OutputPath + } +} +if ($script:DestTier.ContainsKey('Purview') -and $script:DestTier['Purview'] -eq 'Local') { + $isFileForm = ($OutputPath -match '\.[a-zA-Z0-9]{2,5}$') -and ($OutputPath -notmatch '[\\/]$') + if (-not $isFileForm) { + if (-not $OutputPath.EndsWith('\') -and -not $OutputPath.EndsWith('/')) { + $OutputPath = $OutputPath + '\' + } + if (-not (Test-Path -Path $OutputPath -PathType Container)) { + try { + New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null + Write-Host "INFO: Created output directory: $OutputPath" -ForegroundColor Green + } + catch { + Write-Host "ERROR: Failed to create output directory: $OutputPath" -ForegroundColor Red + Write-Host "Error: $_" -ForegroundColor Red + exit 1 + } + } + } else { + $parentDir = Split-Path -Parent $OutputPath + if ($parentDir -and -not (Test-Path -Path $parentDir -PathType Container)) { + try { + New-Item -Path $parentDir -ItemType Directory -Force | Out-Null + Write-Host "INFO: Created output directory: $parentDir" -ForegroundColor Green + } + catch { + Write-Host "ERROR: Failed to create output directory: $parentDir" -ForegroundColor Red + Write-Host "Error: $_" -ForegroundColor Red + exit 1 + } + } + } +} + +# Managed-identity-aware host guard. IDENTITY_ENDPOINT is set by Azure runtimes +# (Functions, App Service, Container Apps, Container Instances, Fabric notebook / +# container runtimes, etc.) to advertise that a managed-identity token endpoint +# is available on the host. It does NOT mean managed identity is the only valid +# auth choice. Two shapes are accepted in MI-aware hosts: +# 1. -Auth ManagedIdentity (uses the assigned identity on the host), or +# 2. -Auth AppRegistration with bound credentials (-ClientId plus one of +# -ClientSecret / -ClientCertificateThumbprint / -ClientCertificatePath) - +# legitimate for cross-tenant runs where a service-provider tenant calls +# into a customer tenant from a host that also happens to expose a local +# managed identity. +# Only block when neither shape applies (e.g. the operator forgot +# -Auth ManagedIdentity in a runbook, or invoked an interactive auth mode with +# no credentials in a non-interactive host). +if ($env:IDENTITY_ENDPOINT -and $Auth -ne 'ManagedIdentity') { + $hasAppRegCreds = ($Auth -eq 'AppRegistration') -and ` + $ClientId -and ` + ($ClientSecret -or $ClientCertificateThumbprint -or $ClientCertificatePath) + if (-not $hasAppRegCreds) { + Write-Host "ERROR: Detected a managed-identity environment (IDENTITY_ENDPOINT is set) but -Auth is '$Auth' without credentials." -ForegroundColor Red + Write-Host " Re-run with -Auth ManagedIdentity to use the assigned identity, or supply -ClientId together with -ClientSecret / -ClientCertificateThumbprint / -ClientCertificatePath for -Auth AppRegistration." -ForegroundColor Yellow + exit 1 + } + Write-Host "INFO: Managed identity available in host; proceeding with explicit -Auth AppRegistration as requested." -ForegroundColor DarkGray +} + +# Resolve-DataTypePaths +# --------------------- +# Central per-data-type effective destination lookup. Each downstream writer that +# emits a customer-visible artifact calls this helper for its data-type key and +# receives: +# .Tier - 'Local' | 'SharePoint' | 'Fabric' +# .Raw - the as-supplied input (already validated and trimmed) +# .IsBound - $true if the caller supplied -OutputPath on the CLI +# .EffectiveDir - parent folder/URL the writer should land its file under +# .Basename - default basename to use when the supplied form is folder/URL only +# ('' when the supplied form already includes a basename) +# Data-type keys: 'Purview', 'UserInfo', 'Agent365Info', 'DefenderUsage', 'Log'. +# When a non-Purview data type was not bound on the CLI, it inherits the Purview +# tier/destination so legacy single-destination behavior is preserved. +function script:Get-AISIDResolvedOutputDir { + # Blocker-6: single source of truth for the resolved AISID output BUNDLE folder shared by the + # parameter snapshot, the checkpoint parameter snapshot, the metrics JSON, recovery messages, the + # output roster, and resume validation. Reports the ACTUAL per-run bundle location — the + # 'AISID_' child folder — via the canonical run-folder resolver, NOT the flat + # DefenderUsage parent, and NEVER the internal OS-temp scratch staging path. Local returns + # \AISID_; SharePoint/Fabric returns /AISID_. Returns '' for non-AISID runs. + if ($Dashboard -ne 'AISID') { return '' } + try { + $_run = script:Resolve-AISIDRunFolder + if ($_run -and $_run.DisplayPath) { return [string]$_run.DisplayPath } + } catch {} + return '' +} + +function script:Get-AISIDSnapshotFields { + # Builds the AISID field group embedded in the metrics JSON. Always returns all three keys so + # the metrics schema stays stable across runs; AISID-only values are null/false on a non-AISID + # run, matching the existing metrics-field convention. Pure (all inputs passed in) so the + # parameter-snapshot coverage tests can exercise it directly without running the main script. + param( + [string]$Dashboard, + [string]$ResolvedOutputDir, + [string]$AppendDefenderUsage, + [bool]$DisableAISIDDeltaCache, + [bool]$ExportCompleted = $true + ) + $isAisid = ($Dashboard -eq 'AISID') + return [ordered]@{ + resolvedAISIDOutputDir = if ($isAisid -and $ResolvedOutputDir -and $ExportCompleted) { [string]$ResolvedOutputDir } else { $null } + appendDefenderUsage = if ($isAisid -and $AppendDefenderUsage) { [string]$AppendDefenderUsage } else { $null } + disableAISIDDeltaCache = if ($isAisid) { [bool]$DisableAISIDDeltaCache } else { $false } + } +} + +function script:Resolve-DataTypePaths { + param( + [Parameter(Mandatory)] [ValidateSet('Purview','UserInfo','Agent365Info','DefenderUsage','Log')] [string]$DataType, + [string]$DefaultBasename = '' + ) + $key = $DataType + $isBound = $false + $boundViaAppendOnly = $false + if ($script:DestIsBound -and $script:DestIsBound.ContainsKey($key)) { $isBound = [bool]$script:DestIsBound[$key] } + # DestIsBound is the narrow "-OutputPath was supplied" signal — it does + # NOT cover the case where only the Append-side counterpart (-AppendUserInfo / + # -AppendAgent365Info) is bound. When a rooted/URL Append value is supplied for + # UserInfo or Agent365Info without its OutputPath counterpart, the appendSwitches + # loop at parse-time (mirrored on resume) promotes $script:DestTier[$key] and + # $script:DestRaw[$key] from that Append value, but $script:DestIsBound[$key] + # stays $false. Without the fall-through below, this function would inherit + # tier/raw from Purview — and under -AppendFile, $script:DestRaw['Purview'] + # is the Purview activity FILE path, so the caller ends up joining the + # data-type leaf onto a file path and produces a "\" + # string that no filesystem accepts. Treat an Append-side promotion as "bound" + # for path resolution purposes; the narrow DestIsBound semantics used elsewhere + # (Excel companion conflict check, etc.) remain unchanged. + if (-not $isBound -and $script:AppendIsBound -and $script:AppendIsBound.ContainsKey($key) -and $script:AppendIsBound[$key] -and $script:DestRaw -and $script:DestRaw.ContainsKey($key)) { + $isBound = $true + $boundViaAppendOnly = $true + } + if (-not $isBound) { + # Inherit from Purview tier/raw. + $tier = if ($script:DestTier.ContainsKey('Purview')) { $script:DestTier['Purview'] } else { 'Local' } + $raw = if ($script:DestRaw.ContainsKey('Purview')) { $script:DestRaw['Purview'] } else { $OutputPath } + } else { + $tier = $script:DestTier[$key] + $raw = $script:DestRaw[$key] + } + # Determine whether $raw is folder/URL form vs. file-form (has a basename extension). + $isFileForm = $false + if ($tier -eq 'Local') { + $isFileForm = ($raw -match '\.[a-zA-Z0-9]{2,5}$') -and ($raw -notmatch '[\\/]$') + } elseif ($tier -eq 'SharePoint' -or $tier -eq 'Fabric') { + # Treat trailing-segment-with-extension as file form; otherwise it's a folder/URL. + $last = ($raw.TrimEnd('/').Split('/') | Select-Object -Last 1) + $isFileForm = $last -match '\.[a-zA-Z0-9]{2,5}$' + } + if ($isFileForm) { + $effDir = if ($tier -eq 'Local') { Split-Path -Parent $raw } else { ($raw.TrimEnd('/') -replace '/[^/]+$','') } + # When isBound was promoted ONLY via the Append-side fall-through above, + # $raw is the merge TARGET (-AppendUserInfo / -AppendAgent365Info), not a + # customer-specified raw destination basename. The pristine raw output + # this function's caller will produce must keep its natural name (the + # caller's $DefaultBasename — e.g., 'EntraUsers_MAClicensing_.csv') + # so that it stays distinct from the merge target on disk; the EffectiveDir + # is still inherited because the raw needs to land in the same folder as + # the merge target. Without this carve-out, the caller's collision-check + # (raw-leaf vs. AppendUserInfo-leaf) would fire spuriously and rename + # the raw with a '_raw' suffix that the customer never asked for. + # Agent365's caller already passes the AppendAgent365Info target leaf as + # its $DefaultBasename (via $script:GetEffectiveAgent365Basename), so + # returning $DefaultBasename here is semantically identical for Agent365. + # Callers that explicitly bind -OutputPath set DestIsBound and + # take the normal Split-Path -Leaf $raw branch below — unchanged. + if ($boundViaAppendOnly) { + $basename = $DefaultBasename + } else { + $basename = if ($tier -eq 'Local') { Split-Path -Leaf $raw } else { ($raw.TrimEnd('/').Split('/') | Select-Object -Last 1) } + } + return [pscustomobject]@{ + Tier = $tier; Raw = $raw; IsBound = $isBound + EffectiveDir = $effDir; Basename = $basename + } + } + return [pscustomobject]@{ + Tier = $tier; Raw = $raw; IsBound = $isBound + EffectiveDir = $raw; Basename = $DefaultBasename + } +} + +# Redirect OutputPath to a per-run scratch dir under the OS temp folder when remote mode is active. +# Done BEFORE any downstream code derives paths from $OutputPath. +if ($script:RemoteOutputMode -ne 'None') { + $baseTemp = [System.IO.Path]::GetTempPath() + $script:RemoteScratchDir = Join-Path $baseTemp ("PAX_" + (Get-Date -Format 'yyyyMMdd_HHmmss')) + try { + New-Item -Path $script:RemoteScratchDir -ItemType Directory -Force | Out-Null + } catch { + Write-Host ("ERROR: Failed to create scratch directory '{0}': {1}" -f $script:RemoteScratchDir, $_.Exception.Message) -ForegroundColor Red + exit 1 + } + $OutputPath = $script:RemoteScratchDir + $sep = [System.IO.Path]::DirectorySeparatorChar + if (-not $OutputPath.EndsWith($sep)) { $OutputPath = $OutputPath + $sep } + Write-Host ("INFO: Remote output mode '{0}' active." -f $script:RemoteOutputMode) -ForegroundColor Cyan + Write-Host ("INFO: Local scratch dir : {0}" -f $OutputPath) -ForegroundColor Cyan + Write-Host "INFO: Per-data-type destinations shown in the banner below." -ForegroundColor Cyan +} + +# Validate -AppendFile / -AppendUserInfo / -AppendAgent365Info combinations. +# Each -Append* parameter targets one logical output stream: +# -AppendFile : activity (audit) outputs - CopilotInteraction / M365 / DSPM / etc. +# -AppendUserInfo : EntraUsers / MAC licensing snapshot +# -AppendAgent365Info : Agent 365 catalog +# Only* and Append* targeting the SAME stream compose naturally: Only* scopes the +# run to that stream's data, Append* picks the write mode (merge vs overwrite). +# Only* with -AppendFile is rejected because Only* skips audit log retrieval +# entirely, leaving -AppendFile with no activity data to merge. +# When only an -Append* is given, the matching -Include* is auto-enabled for ergonomics. + +if ($AppendFile -and $OnlyUserInfo) { + Write-Host "ERROR: -AppendFile cannot be used with -OnlyUserInfo" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " -AppendFile targets activity (audit) outputs, but -OnlyUserInfo skips audit" -ForegroundColor Yellow + Write-Host " log retrieval entirely, so there is no activity data to merge. To append the" -ForegroundColor Yellow + Write-Host " EntraUsers snapshot under -OnlyUserInfo, use -AppendUserInfo (the two compose)." -ForegroundColor Yellow + exit 1 +} + +if ($AppendFile -and $OnlyAgent365Info) { + Write-Host "ERROR: -AppendFile cannot be used with -OnlyAgent365Info" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " -AppendFile targets activity (audit) outputs, but -OnlyAgent365Info skips audit" -ForegroundColor Yellow + Write-Host " log retrieval entirely, so there is no activity data to merge. To append the" -ForegroundColor Yellow + Write-Host " Agent 365 catalog under -OnlyAgent365Info, use -AppendAgent365Info (the two compose)." -ForegroundColor Yellow + exit 1 +} + +# Pairing of -AppendFile with -IncludeUserInfo / -IncludeAgent365Info is non-mandatory. +# When -Append* is omitted, the matching dimension file is written non-append for that +# run (the prior target is replaced if it exists). The dedup-on-target join is performed +# only when the matching -Append* switch is also supplied. + +# Auto-imply the matching -Include* when only the -Append* is provided. +if ($AppendUserInfo -and -not $IncludeUserInfo) { $IncludeUserInfo = $true } +if ($AppendAgent365Info -and -not $IncludeAgent365Info) { $IncludeAgent365Info = $true } + +# ============================================================ +# DESTINATION PAIR XOR VALIDATION +# For each output stream, when the stream is in scope, the user must supply +# EXACTLY ONE of (-OutputPath* | -Append*) — never both, never neither. +# When the stream is out of scope, neither member of the pair may be supplied. +# Streams: +# Purview : in scope unless -OnlyUserInfo or -OnlyAgent365Info +# UserInfo : in scope when -IncludeUserInfo or -OnlyUserInfo +# Agent365Info : in scope when -IncludeAgent365Info or -OnlyAgent365Info +# Note: the Purview default-to-$PSScriptRoot fallback fires AFTER this block, +# so we test against $PSBoundParameters here, not the live $OutputPath value. +# Skipped under -Resume: the resume-allowed-list check (later) rejects any +# destination switches on the resume command line, and the checkpoint restore +# rehydrates every per-stream destination from the saved snapshot. +# ============================================================ +if (-not $ResumeSpecified) { + +$pvOutBound = $PSBoundParameters.ContainsKey('OutputPath') +$pvAppBound = $PSBoundParameters.ContainsKey('AppendFile') +$uiOutBound = $PSBoundParameters.ContainsKey('OutputPathUserInfo') +$uiAppBound = $PSBoundParameters.ContainsKey('AppendUserInfo') +$agOutBound = $PSBoundParameters.ContainsKey('OutputPathAgent365Info') +$agAppBound = $PSBoundParameters.ContainsKey('AppendAgent365Info') + +$purviewInScope = (-not $OnlyUserInfo) -and (-not $OnlyAgent365Info) +$userInfoInScope = $IncludeUserInfo -or $OnlyUserInfo +$agentInScope = $IncludeAgent365Info -or $OnlyAgent365Info + +# --- AISID dashboard prerequisites (gate FIRST, one clear message) --- +# -Dashboard AISID requires a run shaped to produce the FULL Purview + Entra + Defender dataset. +# Validate the complete required shape here, before the per-stream checks below, so the user gets a +# SINGLE actionable message listing exactly what to add/remove (rather than hitting per-stream errors +# one at a time). Gracefully exits (no crash, no partial start). Keys off -Dashboard directly. +if ($Dashboard -eq 'AISID') { + $aisidProblems = @() + if (-not $pvOutBound -and -not $pvAppBound) { $aisidProblems += ' - add -OutputPath (or -AppendFile) for the Purview activity stream' } + if (-not ($IncludeUserInfo -or $OnlyUserInfo)) { $aisidProblems += ' - add -IncludeUserInfo (the AISID dashboard joins Defender signals to the Entra user directory)' } + elseif ($IncludeUserInfo -and -not $uiOutBound -and -not $uiAppBound) { $aisidProblems += ' - add -OutputPathUserInfo (or -AppendUserInfo) for the EntraUsers stream' } + if ($OnlyUserInfo) { $aisidProblems += ' - remove -OnlyUserInfo (it suppresses the required Purview + Defender streams)' } + if ($OnlyAgent365Info) { $aisidProblems += ' - remove -OnlyAgent365Info (it suppresses the required Purview + Entra streams)' } + if ($ExcludeCopilotInteraction) { $aisidProblems += ' - remove -ExcludeCopilotInteraction (the AISID dashboard needs Copilot interaction data)' } + if ($IncludeM365Usage) { $aisidProblems += ' - remove -IncludeM365Usage / -Dashboard M365 (a different data source and processor; not part of AISID)' } + if ($RAWInputCSV) { $aisidProblems += ' - remove -RAWInputCSV (AISID needs a live Graph/Defender run, not offline reprocessing)' } + if ($UseEOM) { $aisidProblems += ' - remove -UseEOM (AISID uses the Graph Security API, not Exchange Online Management)' } + if ($ExportWorkbook) { $aisidProblems += ' - remove -ExportWorkbook (incompatible with the rollup run AISID performs)' } + if ($aisidProblems.Count -gt 0) { + Write-Host "ERROR: -Dashboard AISID requires a full Purview + Entra + Defender run. Adjust the following and re-run:" -ForegroundColor Red + $aisidProblems | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } + Write-Host " Example: pwsh -File .\PAX_Purview_Audit_Log_Processor.ps1 -StartDate 2025-10-01 -EndDate 2025-10-31 -Dashboard AISID -IncludeUserInfo -OutputPath C:\Reports\ -OutputPathUserInfo C:\Reports\ -OutputPathDefenderUsage C:\Reports\AISID\" -ForegroundColor Cyan + exit 1 + } +} + +# --- Purview stream --- +if ($purviewInScope) { + if ($pvOutBound -and $pvAppBound) { + Write-Host "ERROR: -OutputPath and -AppendFile cannot both be supplied." -ForegroundColor Red + Write-Host " For the Purview activity stream, provide EXACTLY ONE of the pair:" -ForegroundColor Yellow + Write-Host " -OutputPath : write a new file at the supplied folder/full path" -ForegroundColor Yellow + Write-Host " -AppendFile : append to an existing file (filename or full path)" -ForegroundColor Yellow + exit 1 + } + if (-not $pvOutBound -and -not $pvAppBound) { + Write-Host "ERROR: Purview audit output destination not specified." -ForegroundColor Red + Write-Host " Supply EXACTLY ONE of: -OutputPath OR -AppendFile" -ForegroundColor Yellow + Write-Host " Accepted forms (either switch):" -ForegroundColor Yellow + Write-Host " Local : C:\Reports\ (folder) or C:\Reports\Run.csv (full path)" -ForegroundColor DarkGray + Write-Host " SharePoint : https://.sharepoint.com/sites//[/]" -ForegroundColor DarkGray + Write-Host " Fabric : https://onelake.dfs.fabric.microsoft.com//.Lakehouse[/Tables|/Files...]" -ForegroundColor DarkGray + exit 1 + } +} else { + # Only-modes: Purview not in scope. -AppendFile makes no sense (no Purview file). + # -OutputPath is permitted (acts as scratch dir for remote streams) but flagged. + if ($pvAppBound) { + $onlyName = if ($OnlyUserInfo) { '-OnlyUserInfo' } else { '-OnlyAgent365Info' } + Write-Host ("ERROR: -AppendFile is not valid with {0}." -f $onlyName) -ForegroundColor Red + Write-Host " -AppendFile targets the Purview activity stream, which is out of scope in only-modes." -ForegroundColor Yellow + Write-Host " Use -AppendUserInfo / -AppendAgent365Info to append to the in-scope stream." -ForegroundColor Yellow + exit 1 + } +} + +# --- UserInfo stream --- +if ($userInfoInScope) { + if ($uiOutBound -and $uiAppBound) { + Write-Host "ERROR: -OutputPathUserInfo and -AppendUserInfo cannot both be supplied." -ForegroundColor Red + Write-Host " Provide EXACTLY ONE for the EntraUsers stream." -ForegroundColor Yellow + exit 1 + } + if (-not $uiOutBound -and -not $uiAppBound -and -not $ExportWorkbook) { + Write-Host "ERROR: -IncludeUserInfo / -OnlyUserInfo requires a destination for the EntraUsers stream." -ForegroundColor Red + Write-Host " Supply EXACTLY ONE of: -OutputPathUserInfo OR -AppendUserInfo" -ForegroundColor Yellow + exit 1 + } +} else { + if ($uiOutBound -or $uiAppBound) { + $bn = if ($uiOutBound) { '-OutputPathUserInfo' } else { '-AppendUserInfo' } + Write-Host ("ERROR: {0} requires -IncludeUserInfo or -OnlyUserInfo to be in scope." -f $bn) -ForegroundColor Red + Write-Host " Drop the destination switch, or add -IncludeUserInfo / -OnlyUserInfo." -ForegroundColor Yellow + exit 1 + } +} + +# --- Agent 365 stream --- +if ($agentInScope) { + if ($agOutBound -and $agAppBound) { + Write-Host "ERROR: -OutputPathAgent365Info and -AppendAgent365Info cannot both be supplied." -ForegroundColor Red + Write-Host " Provide EXACTLY ONE for the Agent 365 stream." -ForegroundColor Yellow + exit 1 + } + if (-not $agOutBound -and -not $agAppBound -and -not $ExportWorkbook) { + Write-Host "ERROR: -IncludeAgent365Info / -OnlyAgent365Info requires a destination for the Agent 365 stream." -ForegroundColor Red + Write-Host " Supply EXACTLY ONE of: -OutputPathAgent365Info OR -AppendAgent365Info" -ForegroundColor Yellow + exit 1 + } +} else { + if ($agOutBound -or $agAppBound) { + $bn = if ($agOutBound) { '-OutputPathAgent365Info' } else { '-AppendAgent365Info' } + Write-Host ("ERROR: {0} requires -IncludeAgent365Info or -OnlyAgent365Info to be in scope." -f $bn) -ForegroundColor Red + Write-Host " Drop the destination switch, or add -IncludeAgent365Info / -OnlyAgent365Info." -ForegroundColor Yellow + exit 1 + } +} + +# --- AISID / Defender usage stream --- +# In scope only under -Dashboard AISID. Pair-XOR: -OutputPathDefenderUsage and -AppendDefenderUsage +# cannot both be supplied. Neither-bound is allowed (when no dedicated AISID destination is given the +# AISID output set co-locates with the Purview -OutputPath folder; the dedicated export/append path +# is a later phase). Supplying either switch without -Dashboard AISID is rejected. +$duOutBound = $PSBoundParameters.ContainsKey('OutputPathDefenderUsage') +$duAppBound = $PSBoundParameters.ContainsKey('AppendDefenderUsage') +if ($Dashboard -eq 'AISID') { + if ($duOutBound -and $duAppBound) { + Write-Host "ERROR: -OutputPathDefenderUsage and -AppendDefenderUsage cannot both be supplied." -ForegroundColor Red + Write-Host " Provide EXACTLY ONE for the AISID (Defender) output set." -ForegroundColor Yellow + exit 1 + } +} else { + if ($duOutBound -or $duAppBound) { + $bn = if ($duOutBound) { '-OutputPathDefenderUsage' } else { '-AppendDefenderUsage' } + Write-Host ("ERROR: {0} requires -Dashboard AISID." -f $bn) -ForegroundColor Red + Write-Host " These switches route the AISID (Defender) output set, produced only under -Dashboard AISID." -ForegroundColor Yellow + exit 1 + } +} + +# Folder-only guard: the AISID destination addresses the whole twelve-CSV set, so a file leaf is +# rejected on either switch — supply a folder path / SharePoint folder URL / Fabric OneLake folder URL. +foreach ($duPair in @(@{ N = 'OutputPathDefenderUsage'; V = $OutputPathDefenderUsage }, @{ N = 'AppendDefenderUsage'; V = $AppendDefenderUsage })) { + if ($PSBoundParameters.ContainsKey($duPair.N) -and -not [string]::IsNullOrWhiteSpace($duPair.V)) { + $duNorm = $duPair.V.Trim().TrimEnd('/', '\') + if ($duNorm -match '\.[a-z0-9]{2,5}$') { + Write-Host ("ERROR: -{0} must be a FOLDER, not a file leaf. The AISID output is a set of 9 CSVs." -f $duPair.N) -ForegroundColor Red + Write-Host (" Supplied: {0}" -f $duPair.V) -ForegroundColor Yellow + Write-Host " Provide a folder path, a SharePoint folder URL, or a Fabric OneLake folder URL." -ForegroundColor Yellow + exit 1 + } + } +} + +} # end if (-not $ResumeSpecified) destination-pair XOR validation + +# ============================================================ +# AGENT 365 SWITCH VALIDATION +# Validates -IncludeAgent365Info / -OnlyAgent365Info parameter +# combinations and prints informational banners where the +# Microsoft Graph Agent Package Management API endpoint +# limitations require user awareness. +# ============================================================ + +# Mutually exclusive +if ($IncludeAgent365Info -and $OnlyAgent365Info) { + Write-Host "ERROR: -IncludeAgent365Info and -OnlyAgent365Info are mutually exclusive." -ForegroundColor Red + Write-Host " Use -IncludeAgent365Info to add agents data to a normal Purview run." -ForegroundColor Yellow + Write-Host " Use -OnlyAgent365Info to skip the audit pull and produce only the agents CSV." -ForegroundColor Yellow + exit 1 +} + +# -OnlyAgent365Info conflicts with audit-implying switches +if ($OnlyAgent365Info) { + $conflictingSwitches = @() + if ($IncludeM365Usage) { $conflictingSwitches += '-IncludeM365Usage' } + if ($IncludeCopilotInteraction) { $conflictingSwitches += '-IncludeCopilotInteraction' } + if ($AgentsOnly) { $conflictingSwitches += '-AgentsOnly' } + if ($ExcludeAgents) { $conflictingSwitches += '-ExcludeAgents' } + if ($CombineOutput) { $conflictingSwitches += '-CombineOutput' } + if ($OnlyUserInfo) { $conflictingSwitches += '-OnlyUserInfo' } + if ($AppendFile) { $conflictingSwitches += '-AppendFile' } + if ($conflictingSwitches.Count -gt 0) { + Write-Host "ERROR: -OnlyAgent365Info cannot be combined with: $($conflictingSwitches -join ', ')" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " -OnlyAgent365Info skips the Purview audit pull entirely. Switches that imply" -ForegroundColor Yellow + Write-Host " audit retrieval (or another 'Only' mode) are not compatible with it." -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host " -IncludeUserInfo is allowed (produces EntraUsers CSV alongside Agent 365 CSV)." -ForegroundColor Green + exit 1 + } +} + +# ========================================================================== +# NONINTERACTIVE-HOST DETECTION HELPER +# ========================================================================== +# Used by: +# - the Agent365 + -Auth AppRegistration fail-fast gate (below) +# - Read-Checkpoint's data-loss / selection / re-auth Read-Host prompts +# A host is considered noninteractive when ANY of the following are true: +# * [Environment]::UserInteractive is $false (Windows service, scheduled task, ACA Job). +# * Stdin is redirected ([Console]::IsInputRedirected) — true for pipelines, containers +# that don't allocate a TTY, and most CI runners. +# * $Host.Name is the remoting endpoint ('ServerRemoteHost'). +# * The PAX_NONINTERACTIVE environment variable is set to a truthy value (escape hatch +# so operators can force noninteractive behavior on hosts the detection misses). +# Operators can FORCE interactive behavior via PAX_FORCE_INTERACTIVE=1 if they know the +# host genuinely accepts keyboard input despite a redirected stdin (rare; documented). +function script:Test-IsNonInteractive { + if ($env:PAX_FORCE_INTERACTIVE -in @('1','true','True','TRUE','yes','Yes','YES')) { return $false } + if ($env:PAX_NONINTERACTIVE -in @('1','true','True','TRUE','yes','Yes','YES')) { return $true } + try { if ($Host.Name -eq 'ServerRemoteHost') { return $true } } catch {} + try { if (-not [Environment]::UserInteractive) { return $true } } catch {} + try { if ([Console]::IsInputRedirected) { return $true } } catch {} + return $false +} + +# ========================================================================== +# RUNTIME STATE-CONTRACT HELPER +# ========================================================================== +# Validates that critical hashtables (metrics, partitionStatus, checkpoint payload, +# resume state) contain the required keys at the boundaries where they are +# produced or consumed. Intentionally lightweight: it asserts shape only, never +# value semantics, and never mutates the input. On contract violation it throws +# a precise error of the form: +# "State contract violation: missing required keys: " +# Callers should let the throw propagate; an unguarded violation is a hard bug +# that would otherwise surface as a NullReferenceException later in the run. +function script:Assert-StateContract { + param( + [Parameter(Mandatory)] [string] $ContractName, + [Parameter(Mandatory)] [AllowNull()] $State, + [Parameter(Mandatory)] [string[]] $RequiredKeys + ) + if ($null -eq $State) { + throw "State contract violation: $ContractName is `$null but required keys are: $($RequiredKeys -join ', ')" + } + # Support hashtable, OrderedDictionary, and PSCustomObject inputs uniformly. + $present = @{} + if ($State -is [System.Collections.IDictionary]) { + foreach ($k in $State.Keys) { $present[[string]$k] = $true } + } elseif ($State.PSObject -and $State.PSObject.Properties) { + foreach ($p in $State.PSObject.Properties) { $present[$p.Name] = $true } + } else { + throw "State contract violation: $ContractName has unsupported type $($State.GetType().FullName); expected IDictionary or PSCustomObject." + } + $missing = @() + foreach ($k in $RequiredKeys) { if (-not $present.ContainsKey($k)) { $missing += $k } } + if ($missing.Count -gt 0) { + throw "State contract violation: $ContractName missing required keys: $($missing -join ', ')" + } +} + +# -OnlyAgent365Info is incompatible with replay (-RAWInputCSV), EOM mode, and Resume +# (it skips the audit phase entirely - nothing to checkpoint or resume from). +# -IncludeAgent365Info IS compatible with -Resume: the audit phase resumes from the +# checkpoint and Agent 365 enrichment runs at end of run as it would have originally. +if (($IncludeAgent365Info -or $OnlyAgent365Info)) { + $incompatModeSwitches = @() + if ($RAWInputCSV) { $incompatModeSwitches += '-RAWInputCSV (replay mode)' } + if ($UseEOM) { $incompatModeSwitches += '-UseEOM (Exchange Online Management mode)' } + if ($ResumeSpecified -and $OnlyAgent365Info) { $incompatModeSwitches += '-Resume' } + if ($incompatModeSwitches.Count -gt 0) { + $activeAgentSwitch = if ($OnlyAgent365Info) { '-OnlyAgent365Info' } else { '-IncludeAgent365Info' } + Write-Host "ERROR: $activeAgentSwitch is not supported with: $($incompatModeSwitches -join ', ')" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " Agent 365 enrichment requires a fresh live Microsoft Graph context and runs" -ForegroundColor Yellow + Write-Host " end-to-end as a single pass. It is not compatible with replay/EOM/Resume modes." -ForegroundColor Yellow + exit 1 + } +} + +# Agent 365 + app-only auth (AppRegistration certificate, AppRegistration client secret, +# ManagedIdentity): The Microsoft Graph Agent Package +# Management API exposes an APPLICATION permission (CopilotPackages.Read.All app-role; plus +# Application.Read.All for developer-name resolution) per Microsoft Learn "List Copilot packages" +# (https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api). The app-only token +# already carries these app-roles when they are granted + admin-consented on the app / managed- +# identity service principal, so the Agent 365 phase runs on the EXISTING application context +# with NO interactive delegated sign-in. Because no prompt is involved, app-only + Agent 365 is +# also valid on headless / noninteractive hosts (containers, ACA Jobs, services, CI runners) and +# with -OnlyAgent365Info. A missing app-role, unlicensed tenant, or absent program enrollment is +# surfaced at runtime as a 403 by Test-Agent365FrontierAccess (see the app-only 403 banner there). +# Delegated auth modes (WebLogin / DeviceCode / Credential / Silent) are unchanged and continue to +# consent the Agent 365 scopes at their initial interactive sign-in. +# NOTE: the catalog API is currently published at /beta only; the tenant must hold a Microsoft +# Agent 365 license / program enrollment. No auth-mode is rejected up-front for Agent 365. + +# Validate AppendFile has proper filename format +if ($AppendFile) { + # Check if it's a directory path (ends with slash/backslash or has no extension) + if ($AppendFile -match '[\\/]$') { + Write-Host "ERROR: -AppendFile must specify a filename, not a directory path" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "Valid examples:" -ForegroundColor Green + Write-Host " -AppendFile 'MyReport.xlsx'" -ForegroundColor Green + Write-Host " -AppendFile 'C:\Data\Audit\Report.csv'" -ForegroundColor Green + Write-Host "" -ForegroundColor Yellow + Write-Host "Invalid examples:" -ForegroundColor Red + Write-Host " -AppendFile 'C:\Data\'" -ForegroundColor Red + Write-Host " -AppendFile 'C:\Data\Audit\'" -ForegroundColor Red + exit 1 + } + + # Extract file extension + $appendExt = [System.IO.Path]::GetExtension($AppendFile).ToLower() + + # Validate extension exists + if (-not $appendExt) { + Write-Host "ERROR: -AppendFile must include a file extension (.csv or .xlsx)" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "Valid examples:" -ForegroundColor Green + Write-Host " -AppendFile 'MyReport.xlsx'" -ForegroundColor Green + Write-Host " -AppendFile 'AuditData.csv'" -ForegroundColor Green + exit 1 + } + + # Validate extension matches export mode + if ($ExportWorkbook -and $appendExt -ne '.xlsx') { + Write-Host "ERROR: -AppendFile must use .xlsx extension when -ExportWorkbook is specified" -ForegroundColor Red + Write-Host "You specified: $AppendFile" -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host "Solutions:" -ForegroundColor Green + Write-Host " 1. Change filename to use .xlsx extension" -ForegroundColor Green + Write-Host " 2. Remove -ExportWorkbook to append CSV data instead" -ForegroundColor Green + exit 1 + } + elseif (-not $ExportWorkbook -and $appendExt -ne '.csv') { + Write-Host "ERROR: -AppendFile must use .csv extension for CSV mode" -ForegroundColor Red + Write-Host "You specified: $AppendFile" -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host "Solutions:" -ForegroundColor Green + Write-Host " 1. Change filename to use .csv extension" -ForegroundColor Green + Write-Host " 2. Add -ExportWorkbook to append Excel data instead" -ForegroundColor Green + exit 1 + } +} + +# --- Conflict Detection for -ExcludeCopilotInteraction --- + +$script:ConflictResolved = $false +$script:ConflictChoice = $null + +# Detect conflicts where user wants to both include AND exclude CopilotInteraction: +# 1. Explicit include via -ActivityTypes parameter +# 2. Explicit include via -IncludeCopilotInteraction switch +$explicitInclude = $ActivityTypes -and ($ActivityTypes -contains 'CopilotInteraction') +$explicitIncludeViaSwitch = $IncludeCopilotInteraction + +if ($ExcludeCopilotInteraction -and ($explicitInclude -or $explicitIncludeViaSwitch)) { + if (-not $Force) { + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "CONFLICT DETECTED" -ForegroundColor Red + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "" + + if ($explicitInclude) { + Write-Host "You provided 'CopilotInteraction' in -ActivityTypes but also specified -ExcludeCopilotInteraction switch." -ForegroundColor Yellow + } + elseif ($explicitIncludeViaSwitch) { + Write-Host "You enabled -IncludeCopilotInteraction but also specified -ExcludeCopilotInteraction." -ForegroundColor Yellow + } + + Write-Host "" + Write-Host "Microsoft 365 Copilot data (CopilotInteraction) includes:" -ForegroundColor Cyan + Write-Host " - M365 Copilot (Word, Excel, PowerPoint, Outlook, Teams meetings, etc.)" -ForegroundColor Cyan + Write-Host " - Microsoft 365 Copilot Chat (Office.com)" -ForegroundColor Cyan + Write-Host " - Security Copilot" -ForegroundColor Cyan + Write-Host " - Copilot Studio interactions" -ForegroundColor Cyan + Write-Host " - Billing: FREE (included with E5/Audit Standard)" -ForegroundColor Green + Write-Host "" + Write-Host "Do you want to INCLUDE or EXCLUDE Microsoft 365 Copilot activity type?" -ForegroundColor Yellow + Write-Host " [I] INCLUDE - Proceed with M365 Copilot data enabled (override -ExcludeCopilotInteraction switch)" -ForegroundColor Green + Write-Host " [E] EXCLUDE - Remove CopilotInteraction (honor -ExcludeCopilotInteraction switch)" -ForegroundColor Red + Write-Host "" + + # Strict noninteractive guard: a container/ACA Job/CI + # runner cannot answer I/E. Fail-fast with an explicit, scriptable recovery path + # rather than hanging on stdin. + if (script:Test-IsNonInteractive) { + Write-Host "ERROR: Cannot prompt for INCLUDE/EXCLUDE on a noninteractive host." -ForegroundColor Red + Write-Host " Re-run with EITHER -IncludeCopilotInteraction (to force include) OR" -ForegroundColor Yellow + Write-Host " -ExcludeCopilotInteraction alone (without -IncludeCopilotInteraction) to exclude," -ForegroundColor Yellow + Write-Host " OR pass -Force to honour -ExcludeCopilotInteraction without prompting." -ForegroundColor Yellow + exit 1 + } + + Send-PromptNotification + $userChoice = Read-Host "Enter your choice (I/E)" + + if ($userChoice -eq 'I' -or $userChoice -eq 'i') { + $ExcludeCopilotInteraction = $false + $script:ConflictChoice = 'INCLUDE' + Write-Host "" + Write-Host "Choice: INCLUDE - Proceeding with CopilotInteraction enabled" -ForegroundColor Green + Write-Host "" + } + elseif ($userChoice -eq 'E' -or $userChoice -eq 'e') { + $script:ConflictChoice = 'EXCLUDE' + Write-Host "" + Write-Host "Choice: EXCLUDE - CopilotInteraction will be removed from ActivityTypes" -ForegroundColor Red + Write-Host "" + } + else { + Write-Host "" + Write-Host "ERROR: Invalid choice. Please enter 'I' for INCLUDE or 'E' for EXCLUDE." -ForegroundColor Red + exit 1 + } + + $script:ConflictResolved = $true + } + else { + # Force mode - honor ExcludeCopilotInteraction without prompt + $script:ConflictChoice = 'EXCLUDE (Force mode)' + $script:ConflictResolved = $true + } +} + +# ============================================== +# -OnlyAgent365Info preflight (Y/N, default Y) +# ============================================== +# Without -IncludeAgent365Info, audit-derived columns will be blank in the +# Agent 365 CSV (Date created / Created by). Confirm the user wants to proceed. +# -Force auto-continues without prompting. +if ($OnlyAgent365Info) { + Write-Host "" + Write-Host "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-Host "| -OnlyAgent365Info: Audit-log enrichment will be SKIPPED |" -ForegroundColor Yellow + Write-Host "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-Host "| These Agent 365 columns CANNOT be populated without Purview audit |" -ForegroundColor Yellow + Write-Host "| data and will be left blank in this run: |" -ForegroundColor Yellow + Write-Host "| |" -ForegroundColor Yellow + Write-Host "| - Date created |" -ForegroundColor Yellow + Write-Host "| - Created by |" -ForegroundColor Yellow + Write-Host "| |" -ForegroundColor Yellow + Write-Host "| To populate them, re-run with -IncludeAgent365Info instead. |" -ForegroundColor Yellow + Write-Host "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-Host "" + if ($Force) { + Write-Host " -Force specified: continuing without prompt." -ForegroundColor DarkGray + Write-Host "" + } else { + # Strict noninteractive guard: default to Y on + # noninteractive hosts so scheduled/container runs do not hang on stdin. The + # default of the prompt is already Y, so this preserves operator intent. + if (script:Test-IsNonInteractive) { + Write-Host " Noninteractive host detected: auto-accepting prompt default (Y) for -OnlyAgent365Info." -ForegroundColor DarkGray + Write-Host "" + } else { + $resp = Read-Host " Continue with -OnlyAgent365Info anyway? (Y/N) [default Y]" + if ([string]::IsNullOrWhiteSpace($resp)) { $resp = 'Y' } + if ($resp -notmatch '^(?i:y|yes)$') { + Write-Host " Aborted by user." -ForegroundColor Red + exit 0 + } + Write-Host "" + } + } +} + +# ============================================== +# ImportExcel Module Check (for Excel export) +# ============================================== + +if ($ExportWorkbook) { + Write-Host "Checking ImportExcel module for Excel export..." -ForegroundColor Cyan + + $importExcelModule = Get-Module -ListAvailable -Name ImportExcel | Select-Object -First 1 + if (-not $importExcelModule) { + Write-Host "ImportExcel module not found (required for -ExportWorkbook)." -ForegroundColor Yellow + Write-Host "Installing ImportExcel module..." -ForegroundColor Yellow + Write-Host "" + + try { + Install-Module ImportExcel -Scope CurrentUser -Force -AllowClobber -Repository PSGallery -ErrorAction Stop + Write-Host "ImportExcel module installed successfully!" -ForegroundColor Green + Write-Host "" + + # Re-check for the module + $importExcelModule = Get-Module -ListAvailable -Name ImportExcel | Select-Object -First 1 + if (-not $importExcelModule) { + Write-Host "ERROR: Module installation completed but module not found. Try restarting PowerShell." -ForegroundColor Red + exit 1 + } + } + catch { + Write-Host "ERROR: Failed to install ImportExcel module: $($_.Exception.Message)" -ForegroundColor Red + Write-Host "" + Write-Host "Please install manually using:" -ForegroundColor Yellow + Write-Host " Install-Module ImportExcel -Scope CurrentUser" -ForegroundColor Yellow + Write-Host "" + Write-Host "Falling back to CSV export..." -ForegroundColor Yellow + $script:ExportWorkbook = $false + $script:AppendFile = $false + } + } + else { + Write-Host "ImportExcel module detected: $($importExcelModule.Name) v$($importExcelModule.Version)" -ForegroundColor Green + } + + # Import ImportExcel module + if ($ExportWorkbook) { + try { + Import-Module ImportExcel -ErrorAction Stop + Write-Host "ImportExcel module imported successfully" -ForegroundColor Green + Write-Host "" + } + catch { + Write-Host "ERROR: Failed to import ImportExcel module: $($_.Exception.Message)" -ForegroundColor Red + Write-Host "Falling back to CSV export..." -ForegroundColor Yellow + $script:ExportWorkbook = $false + $script:AppendFile = $false + } + } +} + +# --- Early parameter validation & environment sanity checks --- + +# PowerShell version gate. +# The entire script requires PowerShell 7+ (pwsh.exe). The ONLY exception is -UseEOM +# (which uses Exchange Online Management and runs on Windows PowerShell 5.1 / powershell.exe). +# All other modes — including -RAWInputCSV and -Resume — require pwsh.exe because they +# depend on PS 7-only features (ThreadJob parallelism, ForEach-Object -Parallel, JSON +# parsing performance). +if ($PSVersionTable.PSVersion.Major -lt 7 -and -not $UseEOM) { + Write-Host "" -ForegroundColor Red + Write-Host "═══════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: PowerShell 7+ Required (pwsh.exe)" -ForegroundColor Red + Write-Host "═══════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host " Detected: PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" -ForegroundColor Yellow + Write-Host " Required: PowerShell 7.0 or later (pwsh.exe)" -ForegroundColor Yellow + Write-Host "" + Write-Host " PAX requires PowerShell 7+ for Microsoft Graph API mode, parallel query" -ForegroundColor White + Write-Host " execution, JSON streaming performance, and post-processing features." -ForegroundColor White + Write-Host " Windows PowerShell 5.1 (powershell.exe) is ONLY supported when running with" -ForegroundColor White + Write-Host " the -UseEOM switch (Exchange Online Management mode)." -ForegroundColor White + Write-Host "" + Write-Host " SOLUTIONS:" -ForegroundColor Cyan + Write-Host "" + Write-Host " 1. Recommended: Install PowerShell 7+ and re-run with pwsh.exe:" -ForegroundColor White + Write-Host " https://aka.ms/powershell" -ForegroundColor Gray + Write-Host " pwsh.exe -File .\PAX_Purview_Audit_Log_Processor.ps1 [your parameters]" -ForegroundColor Gray + Write-Host "" + Write-Host " 2. Or run in EOM mode on Windows PowerShell 5.1 by adding -UseEOM:" -ForegroundColor White + Write-Host " .\PAX_Purview_Audit_Log_Processor.ps1 -UseEOM [your other parameters]" -ForegroundColor Gray + Write-Host "" + Write-Host " EOM MODE NOTES:" -ForegroundColor DarkCyan + Write-Host " - Uses Search-UnifiedAuditLog cmdlet (serial processing)" -ForegroundColor Gray + Write-Host " - Requires Exchange Online Management module" -ForegroundColor Gray + Write-Host " - Requires Exchange Admin role or audit log read permissions" -ForegroundColor Gray + Write-Host " - Some features (Entra user enrichment, rollup post-processing) not available" -ForegroundColor Gray + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + exit 1 +} + +if ($ExcludeAgents -and ($AgentId -or $AgentsOnly)) { + Write-Host "ERROR: -ExcludeAgents cannot be used with -AgentId or -AgentsOnly switches." -ForegroundColor Red + Write-Host "These switches are mutually exclusive:" -ForegroundColor Yellow + Write-Host " -AgentId/-AgentsOnly: Filter to ONLY records with agents" -ForegroundColor Yellow + Write-Host " -ExcludeAgents: Filter to ONLY records without agents" -ForegroundColor Yellow + Write-Host "Please use only one filtering approach and re-run." -ForegroundColor Yellow + exit 1 +} + +# ============================================================================ +# Rollup post-processor gating +# ============================================================================ +# Validates -Rollup / -RollupPlusRaw against incompatible switches and +# determines which embedded Python processor will be invoked at end-of-run. +# Runs early so failures are surfaced BEFORE any Graph API calls or auth. +# +# Sets $script:RollupProcessorMode to one of: +# 'None' - rollup not requested +# 'CopilotInteraction' - run Purview_CopilotInteraction_Processor (needs Purview CSV + Entra CSV) +# 'M365Bundle' - run Purview_M365_Usage_Bundle_Explosion_Processor (needs Purview CSV only) +# ============================================================================ +$script:RollupProcessorMode = 'None' +$script:RollupDashboard = 'None' +$script:RollupDashboardProfile = $null + +# Resolves -Dashboard into auto-enabled switches so the existing +# rollup gate + mode decision below operate on a consistent state. Acts only when +# the user explicitly passed -Dashboard; the default 'AIO' is inert so a bare +# -Rollup / -RollupPlusRaw keeps today's behavior exactly. +$dashboardExplicit = $PSBoundParameters.ContainsKey('Dashboard') +$dashboardUC = $Dashboard.ToUpperInvariant() +$dashboardImpliedRollup = $false +if ($dashboardExplicit) { + # C1: AIO/ValueLens (CopilotInteraction) and the M365 usage bundle are different data + # pulls AND different processors — refuse to guess; make the user pick one. + if (($dashboardUC -eq 'AIO' -or $dashboardUC -eq 'VALUELENS') -and $IncludeM365Usage) { + Write-Host "ERROR: -Dashboard $Dashboard and -IncludeM365Usage are incompatible (different data source AND different processor). Pick one." -ForegroundColor Red + Write-Host " -Dashboard AIO|ValueLens : CopilotInteraction rollup (AI-in-One / ValueLens dashboards)." -ForegroundColor Yellow + Write-Host " -Dashboard M365 : M365 Usage Analytics rollup (or simply use -IncludeM365Usage)." -ForegroundColor Yellow + exit 1 + } + # Rule: -Dashboard M365 implies the M365 usage pull; auto-enable it if absent. + if ($dashboardUC -eq 'M365' -and -not $IncludeM365Usage) { + $IncludeM365Usage = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: -Dashboard M365 auto-enabled -IncludeM365Usage (the M365 dashboard consumes the M365 usage bundle)." -ForegroundColor Cyan + } + # Rule: -Dashboard implies a rollup run; default to -Rollup when neither rollup + # switch is present (never -RollupPlusRaw). + if (-not $Rollup -and -not $RollupPlusRaw) { + $Rollup = [System.Management.Automation.SwitchParameter]::new($true) + $dashboardImpliedRollup = $true + if ($dashboardUC -eq 'AISID') { + # AISID's Defender CSVs are not rollup-processed; only the Purview/Entra (Copilot) output is. + # Do NOT emit the "produced by the rollup post-processor" line here (it is misleading for AISID). + Write-Host "INFO: -Dashboard AISID auto-enabled -Rollup for the Purview/Entra (Copilot) output." -ForegroundColor Cyan + } else { + Write-Host "INFO: -Dashboard $Dashboard auto-enabled -Rollup (dashboard output is produced by the rollup post-processor)." -ForegroundColor Cyan + } + } + # -Dashboard AISID: clarify (once, non-fatal) that the Defender-sourced AISID CSVs are exported + # as-is with no rollup processing, even though -Rollup / -RollupPlusRaw is active for Purview/Entra. + # Fires whether -Rollup was auto-enabled above or supplied explicitly by the user. + if ($dashboardUC -eq 'AISID' -and ($Rollup -or $RollupPlusRaw)) { + Write-Host "INFO: -Dashboard AISID: the Purview/Entra output is rollup-processed as usual; the Defender-sourced AISID CSVs are exported as-is (no rollup processor exists for them yet)." -ForegroundColor Cyan + } +} + +if ($Rollup -or $RollupPlusRaw) { + # Mutually exclusive + if ($Rollup -and $RollupPlusRaw) { + Write-Host "ERROR: -Rollup and -RollupPlusRaw are mutually exclusive. Pick one." -ForegroundColor Red + Write-Host " -Rollup : produce ONLY rolled-up CSV(s); raw CSV(s) deleted on success." -ForegroundColor Yellow + Write-Host " -RollupPlusRaw : produce BOTH the raw CSV(s) AND the rolled-up CSV(s)." -ForegroundColor Yellow + exit 1 + } + + $rollupSwitchName = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + $rollupBlockers = @() + if ($UseEOM) { $rollupBlockers += '-UseEOM' } + if ($ExportWorkbook) { $rollupBlockers += '-ExportWorkbook' } + if ($OnlyUserInfo) { $rollupBlockers += '-OnlyUserInfo' } + if ($OnlyAgent365Info) { $rollupBlockers += '-OnlyAgent365Info' } + if ($RAWInputCSV) { $rollupBlockers += '-RAWInputCSV' } + if ($ExcludeCopilotInteraction -and -not $IncludeM365Usage) { $rollupBlockers += '-ExcludeCopilotInteraction' } + + if ($rollupBlockers.Count -gt 0) { + # Name -Dashboard in the message when it is what implied the rollup, so the + # error references a switch the user actually typed. + $rollupTrigger = if ($dashboardImpliedRollup) { "-Dashboard $Dashboard (which implies $rollupSwitchName)" } else { $rollupSwitchName } + Write-Host "ERROR: $rollupTrigger is not supported with the following switch(es): $($rollupBlockers -join ', ')" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host " $rollupSwitchName runs an embedded Python post-processor at the end of the run." -ForegroundColor Yellow + Write-Host " It is only valid for the default CopilotInteraction-only run, an explicit" -ForegroundColor Yellow + Write-Host " -ActivityTypes 'CopilotInteraction' run, or an -IncludeM365Usage run." -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host " Remove the conflicting switch(es) and re-run." -ForegroundColor Yellow + exit 1 + } + + # Determine processor mode based on requested activity types. + # Bind-time logic (mirrors the activity-type finalization that happens later): + # - If -IncludeM365Usage is set, mode is M365Bundle. + # - Else if user did NOT pass -ActivityTypes (default is CopilotInteraction), mode is CopilotInteraction. + # - Else if user passed -ActivityTypes containing exactly the single value 'CopilotInteraction' + # (case-insensitive), mode is CopilotInteraction. + # - Otherwise: hard-fail (rollup is not defined for arbitrary activity-type combinations). + if ($IncludeM365Usage) { + $script:RollupProcessorMode = 'M365Bundle' + $script:RollupDashboard = 'M365' + $script:RollupDashboardProfile = $null + } + else { + $userPassedActivityTypes = $PSBoundParameters.ContainsKey('ActivityTypes') + $normalizedActivity = @() + if ($userPassedActivityTypes -and $ActivityTypes) { + $normalizedActivity = @($ActivityTypes | Where-Object { $_ } | ForEach-Object { $_.Trim() }) + } + $isCopilotOnly = (-not $userPassedActivityTypes) -or + ($normalizedActivity.Count -eq 1 -and $normalizedActivity[0] -ieq 'CopilotInteraction') + + if ($isCopilotOnly) { + $script:RollupProcessorMode = 'CopilotInteraction' + # AIO/AIBV only reach here — an explicit -Dashboard M365 auto-enabled + # -IncludeM365Usage above and took the M365Bundle branch. Default to AIO + # when -Dashboard was not supplied (preserves today's behavior). + # -Dashboard AISID reuses the AIO-shaped CopilotInteraction rollup for its Purview/Entra + # tables, so it maps to the AIO profile here — no AISID-specific rollup mode/profile is + # introduced, and 'AISID' never leaks into the rollup/checkpoint machinery. AISID's own + # Defender CSVs are produced by the (later-phase) Defender pipeline, not this processor. + $script:RollupDashboard = if ($dashboardExplicit) { if ($dashboardUC -eq 'AISID') { 'AIO' } elseif ($dashboardUC -eq 'VALUELENS') { 'AIBV' } else { $dashboardUC } } else { 'AIO' } + $script:RollupDashboardProfile = $script:RollupDashboard.ToLowerInvariant() + } + else { + Write-Host "ERROR: $rollupSwitchName is only valid for CopilotInteraction-only runs or -IncludeM365Usage runs." -ForegroundColor Red + Write-Host " Detected -ActivityTypes: $($normalizedActivity -join ', ')" -ForegroundColor Yellow + Write-Host " Remove $rollupSwitchName, or restrict -ActivityTypes to 'CopilotInteraction', or use -IncludeM365Usage." -ForegroundColor Yellow + exit 1 + } + } + + # CopilotInteraction-mode rollup requires the Entra users CSV for the second processor input. + # Auto-enable -IncludeUserInfo if not already set (single info message; do not prompt). + if ($script:RollupProcessorMode -eq 'CopilotInteraction' -and -not $IncludeUserInfo) { + $IncludeUserInfo = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: $rollupSwitchName (CopilotInteraction mode) auto-enabled -IncludeUserInfo (Entra users CSV is required by the post-processor)." -ForegroundColor Cyan + } + + # Rollup requires a single combined Purview CSV; auto-enable -CombineOutput when missing. + if (-not $CombineOutput) { + $CombineOutput = [System.Management.Automation.SwitchParameter]::new($true) + Write-Host "INFO: $rollupSwitchName auto-enabled -CombineOutput (rollup post-processor requires a single combined Purview CSV)." -ForegroundColor Cyan + } +} + +# --------------------------------------------------------------------------- +# -FillerLabel: org/manager-hierarchy level-filler for the rolled-up AIO/AIBV +# Users output. The hierarchy itself is always built for AIO/AIBV; this only sets +# what goes in level slots deeper than a user. Rollup-only; never the M365 path. +# Resolved here (after the rollup/dashboard state is finalized above) into the two +# script vars threaded to the embedded CopilotInteraction processor. +# --------------------------------------------------------------------------- +$script:HierarchyFillMode = 'none' +$script:HierarchyFillLabel = '' +$fillerLabelBound = $PSBoundParameters.ContainsKey('FillerLabel') +$fillerTextBound = $PSBoundParameters.ContainsKey('FillerLabelText') +if ($fillerTextBound -and -not $fillerLabelBound) { + Write-Host "ERROR: -FillerLabelText is only valid together with -FillerLabel Fixed." -ForegroundColor Red + exit 1 +} +if ($fillerLabelBound) { + if (-not ($Rollup -or $RollupPlusRaw)) { + Write-Host "ERROR: -FillerLabel requires -Rollup or -RollupPlusRaw." -ForegroundColor Red + Write-Host " -FillerLabel only affects the rolled-up AI-in-One / ValueLens Users output." -ForegroundColor Yellow + exit 1 + } + if ($script:RollupProcessorMode -eq 'M365Bundle') { + Write-Host "ERROR: -FillerLabel is not valid with the M365 dashboard (-IncludeM365Usage / -Dashboard M365)." -ForegroundColor Red + Write-Host " The org / manager hierarchy is produced only for the AI-in-One and ValueLens dashboards." -ForegroundColor Yellow + exit 1 + } + $fillMode = ([string]$FillerLabel).Trim() + switch ($fillMode.ToLowerInvariant()) { + 'null' { $script:HierarchyFillMode = 'none' } + 'self' { $script:HierarchyFillMode = 'self' } + 'repeatmanager' { $script:HierarchyFillMode = 'manager' } + 'fixed' { $script:HierarchyFillMode = 'fixed' } + default { + Write-Host "ERROR: -FillerLabel value '$fillMode' is not recognized." -ForegroundColor Red + Write-Host " Use one of: null, Self, RepeatManager, or Fixed (with -FillerLabelText `"`")." -ForegroundColor Yellow + exit 1 + } + } + if ($script:HierarchyFillMode -eq 'fixed') { + if (-not $fillerTextBound -or [string]::IsNullOrWhiteSpace($FillerLabelText)) { + Write-Host "ERROR: -FillerLabel Fixed requires -FillerLabelText `"`" (e.g. -FillerLabel Fixed -FillerLabelText `"Assistive Directs`")." -ForegroundColor Red + exit 1 + } + $script:HierarchyFillLabel = [string]$FillerLabelText + } + elseif ($fillerTextBound) { + Write-Host "ERROR: -FillerLabelText is only valid with -FillerLabel Fixed (not with '$fillMode')." -ForegroundColor Red + exit 1 + } +} + +# ============================================================================ +# EMBEDDED PYTHON POST-PROCESSORS +# ============================================================================ +# Two Python source files are embedded verbatim as single-quoted here-strings +# so the script is fully self-contained — no external .py files are required +# at runtime. When -Rollup or -RollupPlusRaw is used, the relevant body is +# materialized to a temp .py inside .pax_incremental, executed with the +# resolved Python interpreter, and deleted in finally. Single-quoted here- +# strings prevent any PowerShell variable expansion of the Python source. +# ============================================================================ +$Script:EMBEDDED_PROCESSOR_COPILOT_VERSION = '4.2.1' +$Script:EMBEDDED_PROCESSOR_M365_VERSION = '2.6.2' + +# >>> BEGIN-EMBEDDED-COPILOT-PROCESSOR +$Script:EMBEDDED_PROCESSOR_COPILOT = @' +#!/usr/bin/env python3 +""" +Purview CopilotInteraction Processor v4.2.1 +------------------------------------------- +Two-input / two-output preprocessor for the AI Business Value Dashboard +and AI-in-One Rollup PBIPs. + +Output profiles (--profile): + aibv (default) : AI Business Value Dashboard. 50-column fact superset — + 3-value Environment {Cowork, Licensed, Unlicensed}, + all DAX calc-columns pre-computed (Behavior_*, Usage_Mode, + Expertise_Role, Efficiency_Breakdown, Human_Baseline_Min, + Behavior_Plausible, Workflow_Action, Delegation_Event_Key, + Is_Agent_Activity/Web_Grounded_Signal promoted into the + grain for sliceability) + Audit_UserId passthrough. + aio : AI-in-One Dashboard. 36-column fact + 2 trailing append- + reconciliation columns (Message_Id_Raw, ThreadId_Raw) — + 5-value Environment {Autonomous Agent, Cowork, + Agents, Licensed M365 Copilot, Unlicensed Chat}. Reproduces + the AIO output byte-identically in the original 36 + columns (validated); the two raw keys are appended last so + the AIO dashboard is unaffected. ~41% smaller than the aibv + fact. + +Inputs: + --purview (required) + --entra (required) + +Outputs (in --out-dir, default = directory of --purview): + _Interactions_.csv (fact table) + _Users_.csv (dim table) + + These two files are all the AIBV template needs. Pass --with-aggregates + (aibv only) to ALSO write 5 pre-aggregated tables for a future calc-table + offload: + _ActiveDaysSummary_.csv + _UserMonthMetrics_.csv + _LicensedUserRankings_.csv + _UnlicensedUserRankings_.csv + _LicensedUserSummary_.csv + +Grain: + One row per (16-column grain x Message_Id; aibv adds 3 sliceable flag + keys -> 19). DAX measures use DISTINCTCOUNT(Message_Id) which yields exact + parity with the semantic-model definitions at every visual / slicer + combination. Per-resource accumulation is intentionally avoided so counts + are + not inflated (~2.25x) by per (prompt x AccessedResource) iteration. + +INT-surrogated columns (perf): + Message_Id, ThreadId, and UserKey (replaces Audit_UserId) are emitted + as 1-based INTs assigned in input encounter order. Cuts CSV size, + parse time, AND VertiPaq dictionary build time on the three highest- + cardinality GUID columns. UserKey is written to BOTH the fact CSV + and the Users dim CSV (same shared map keyed on normalized UPN), so + the fact↔Users relationship is INT-to-INT. DISTINCTCOUNT semantics + are identical between INT and string surrogates of the same set. + UserMonthKey stays string (cross-processor blast radius). + +Calc cols ported from DAX -> precomputed here for ingestion-time speedup: + Agent_TitleID, Behavior_Source, Value_Outcome, ActivityDate + (= InteractionDate alias). + +Stays in DAX (cross-table dependencies that cannot be precomputed without +shipping Agents 365 / UserMonthMetrics / AgentMetrics into the processor): + Behavior_Enriched_Full (RELATED Agents 365), + User_Stage_Maturity / User_Stage (RELATED UserMonthMetrics), + Usage_Mode, Expertise_Role, Efficiency_Breakdown + (all depend on Behavior_Enriched_Full), + Agent Last Used Date (LOOKUPVALUE AgentMetrics). + +Requirements: + Python 3.9+ + pip install orjson (OPTIONAL - faster JSON parsing; falls back to stdlib json) +""" + +from __future__ import annotations + +import argparse +import csv +import functools +import hashlib +import hmac +import os +import re +import sqlite3 +import sys +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +try: + import orjson + + def json_loads(value: str | bytes) -> Any: + if isinstance(value, str): + value = value.encode("utf-8") + return orjson.loads(value) + + _JSON_ENGINE = "orjson" +except ImportError: + import json as _json + + def json_loads(value: str | bytes) -> Any: + if isinstance(value, bytes): + value = value.decode("utf-8") + return _json.loads(value) + + _JSON_ENGINE = "json (stdlib)" + + +SCRIPT_VERSION = "4.2.1" + +# --------------------------------------------------------------------------- +# Output schemas — TWO PROFILES +# +# --profile aio : reproduces the AIO-faithful output EXACTLY except +# for two provenance columns (Message_Id_Raw, ThreadId_Raw) +# APPENDED LAST for cross-run append reconciliation. +# The original 36 columns are unchanged in name, order, and +# value (5-value Environment vocabulary), so the pre-existing +# AIO output is unchanged; only the two +# trailing raw keys are new. This is the contract the +# AI-in-One dashboard already consumes. +# --profile aibv : the AIBV-faithful superset (50-col fact, 3-value +# Environment, all offloaded calc cols + grain-promoted +# sliceable flags). +# +# Both share one classification CODEBASE; the per-profile vocabulary is +# selected by the `profile` argument threaded through the classifiers. +# --------------------------------------------------------------------------- + +# Common grain prefix (identical in both profiles). +_GRAIN_KEYS_COMMON: tuple[str, ...] = ( + "UserKey", + "InteractionDate", + "AgentId", + "AgentName", + "AppHost", + "Environment", + "License Status", + "Context_Type", + "Behavior_Category", + "Behavior_Enriched", + "AI_Model", + "Is_Sensitive", + "Autonomy_Pattern", + "AppIdentity_AppId", + "AISystemPlugin_Name", + "ThreadId", +) + +# AIO grain = the common 16. +GRAIN_KEYS_AIO: tuple[str, ...] = _GRAIN_KEYS_COMMON + +# AIBV grain = common 16 + 3 promoted per-resource flags (sliceability fix). +GRAIN_KEYS_AIBV: tuple[str, ...] = _GRAIN_KEYS_COMMON + ( + # Promoted into the grain (sliceability fix): computed per-resource and + # bound to AIBV slicers/filters, so they MUST be grain-faithful. Validated + # at 0.000% row inflation on real data. + "Is_Agent_Activity", + "Web_Grounded_Signal", + "Workflow_Action", +) + +# Cross-run append reconciliation keys: the stable raw GUIDs behind the +# INT surrogates Message_Id (message) and ThreadId (thread). Appended as the FINAL +# two columns of EVERY profile so all pre-existing column positions are unchanged. +# The PAX append layer (ConvertTo-FactSeedMaps / Merge-FactCsv) dedups cross-run on +# Message_Id_Raw. Under --deidentify these carry the deterministic deid_guid token +# (same raw GUID -> same token across runs) so append dedup still reconciles. +_RAW_ID_ATTRS: tuple[str, ...] = ( + "Message_Id_Raw", + "ThreadId_Raw", +) + +# AIO non-grain carried attrs end at ActivityDate; +# the trailing _RAW_ID_ATTRS are appended below to form _NONGRAIN_ATTRS_AIO. +_NONGRAIN_ATTRS_AIO_BASE: tuple[str, ...] = ( + "CreationDate", + "WeekStart", + "MonthStart", + "UserMonthKey", + "Has license", + "Resource_Count", + "SensitivityLabelId", + "AccessedResource_Type", + "AccessedResource_Action", + "AccessedResource_SiteUrl", + "AccessedResource_SensitivityLabelId", + "AppIdentity_DisplayName", + "AISystemPlugin_Id", + "ModelTransparencyDetails_ModelName", + "Agent_TitleID", + "Message_isPrompt", + # Calc cols ported from DAX + "Behavior_Source", + "Value_Outcome", + "ActivityDate", +) +# AIO carried attrs = the base set + a stable user-identity column + the +# trailing raw reconciliation keys. +_NONGRAIN_ATTRS_AIO: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + ( + # Stable, deid-consistent user identity for AIO. Mirrors the + # AIBV [Audit_UserId_Normalized] value (deid_upn -> normalize_user_id), so it + # is deterministically de-identified under -Deidentify and never exposes a raw + # UPN. Gives the cross-run append merge key a stable user component in place of + # the per-run UserKey INT surrogate. Placed BEFORE the raw keys so + # Message_Id_Raw / ThreadId_Raw stay the trailing reconciliation columns. + "User_Id_Normalized", +) + _RAW_ID_ATTRS + +# AIBV non-grain carried attrs = AIO base set + AIBV-only offloaded columns, with +# the raw reconciliation keys appended LAST so they remain the trailing two columns +# for this profile too (all pre-existing AIBV column positions unchanged). +_NONGRAIN_ATTRS_AIBV: tuple[str, ...] = _NONGRAIN_ATTRS_AIO_BASE + ( + # M1: UPN passthrough for AIBV joins + DISTINCTCOUNT. + "Audit_UserId", + "Audit_UserId_Normalized", + # AIBV-faithful row-level flags. (Is_Agent_Activity / Web_Grounded_Signal / + # Workflow_Action were promoted into GRAIN_KEYS_AIBV.) `Agent Filter` + # derives from Is_Agent_Activity, so it stays a grain-consistent carried attr. + "Agent Filter", + "Agent Publish Status", + # Downstream classification chain (offloaded; faithful without Agents 365 per F2). + "Behavior_Enriched_Full", + "Usage_Mode", + "Expertise_Role", + "Efficiency_Breakdown", + # ROI baseline pre-join (offloads the Human Equivalent Hours SUMX+RELATED). + "Human_Baseline_Min", + # Remaining row-level calc cols (offloaded). + "Behavior_Plausible", + "Delegation_Event_Key", +) + _RAW_ID_ATTRS + +# Final fact CSV schemas. One row per (grain x Message_Id). Message_Id is +# emitted as a sequential INT surrogate (1-based, assigned in input order). +FACT_HEADER_AIO: list[str] = list(GRAIN_KEYS_AIO) + ["Message_Id"] + list(_NONGRAIN_ATTRS_AIO) +FACT_HEADER_AIBV: list[str] = list(GRAIN_KEYS_AIBV) + ["Message_Id"] + list(_NONGRAIN_ATTRS_AIBV) + + +def schema_for(profile: str) -> tuple[tuple[str, ...], tuple[str, ...], list[str]]: + """Return (grain_keys, nongrain_attrs, fact_header) for the profile.""" + if profile == "aio": + return GRAIN_KEYS_AIO, _NONGRAIN_ATTRS_AIO, FACT_HEADER_AIO + return GRAIN_KEYS_AIBV, _NONGRAIN_ATTRS_AIBV, FACT_HEADER_AIBV + +# Entra column-name aliases used by the existing PBIP M-code. We mirror the +# same renaming so the dim CSV is drop-in compatible with all downstream DAX. +UPN_VARIANTS_NORMALIZED = {"userprincipalname", "upn", "personid"} +DEPARTMENT_VARIANT_NORMALIZED = "department" +# Organization source precedence (most meaningful first). `department` carries the +# human-readable org name in a directory export; a column literally named +# Organization/Organisation is accepted only when no readable department exists, +# because in real exports it is frequently a numeric department identifier. +_DEPARTMENT_SOURCE_PREFERENCE: tuple[str, ...] = ( + DEPARTMENT_VARIANT_NORMALIZED, + "organisation", + "organization", +) +# Name used to retain a displaced Organization-named identifier column. +_DISPLACED_ORG_COLUMN = "Organization_Id" +JOBTITLE_RAW_NAME = "jobTitle" # exact-match rename to "JobTitle" +HAS_LICENSE_VARIANTS = ( + "Has license", + "Has License", + "hasLicense", + "HasLicense", + "Has Copilot License", + "Has Copilot license", + "HasCopilotLicense", + "Has Copilot License Assigned", + "Has Copilot license assigned", + "isUser", +) + +# --------------------------------------------------------------------------- +# Datetime helpers +# --------------------------------------------------------------------------- + +_CREATION_TIME_FORMATS: tuple[str, ...] = ( + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%m/%d/%Y %H:%M:%S", +) + + +def safe_get(obj: Any, key: str) -> Any: + if obj is None: + return None + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def get_array(obj: Any, key: str) -> list[Any]: + value = safe_get(obj, key) + return value if isinstance(value, list) else [] + + +def to_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +def normalize_user_id(value: Any) -> str: + return to_text(value).strip().lower() + + +# --------------------------------------------------------------------------- +# Deidentification (--deidentify): one-way, salted, format-preserving. +# OFF by default; enabled by main() setting the module flag from --deidentify. +# Every PII value becomes a deterministic token so relationships (manager links, +# UserKey/Users joins, distinct-resource counts) are preserved while identities +# are removed. Irreversible (no decode map). The SAME salt + algorithm + formats +# MUST exist verbatim in the PowerShell raw-path deidentifier and the M365 +# processor (PAX deidentify spec) so tokens match across engines. +# --------------------------------------------------------------------------- +_DEIDENTIFY: bool = False +_DEID_SALT = b"PAX-Deidentify-Salt-v1-DO-NOT-CHANGE-7f3c1e9b2d846050a1c4e8b3" +_DEID_DOMAIN = "deidentified.domain" +_deid_cache: dict[str, str] = {} + + +def _deid_hex(value: str, length: int) -> str: + return hmac.new( + _DEID_SALT, value.strip().lower().encode("utf-8"), hashlib.sha256 + ).hexdigest()[:length] + + +def deid_upn(value: str) -> str: + """UPN / email -> <12hex>@deidentified.domain. No-op when off or value empty.""" + if not _DEIDENTIFY or not value: + return value + k = "upn\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + "@" + _DEID_DOMAIN + _deid_cache[k] = v + return v + + +def deid_name(value: str) -> str: + """Person/device display name -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "name\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_guid(value: str) -> str: + """GUID -> deterministic GUID shape xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.""" + if not _DEIDENTIFY or not value: + return value + k = "guid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = f"{h[0:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:32]}" + _deid_cache[k] = v + return v + + +def deid_sid(value: str) -> str: + """SID -> deterministic S-1-5-21---- shape.""" + if not _DEIDENTIFY or not value: + return value + k = "sid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = "S-1-5-21-{0}-{1}-{2}-{3}".format( + int(h[0:8], 16), int(h[8:16], 16), int(h[16:24], 16), int(h[24:32], 16) + ) + _deid_cache[k] = v + return v + + +def deid_token(value: str) -> str: + """Opaque id (employeeId, immutableId) -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "tok\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_resource(value: str) -> str: + """Resource URL -> site_<12hex> (whole-string hash; preserves distinct-count).""" + if not _DEIDENTIFY or not value: + return value + k = "res\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "site_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_file(value: str) -> str: + """File / document name -> file_<12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "file\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "file_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_proxy(value: str) -> str: + """proxyAddresses entry(ies) -> keep smtp:/SMTP: prefix + deidentified email. + Handles ';'-delimited multi-value fields.""" + if not _DEIDENTIFY or not value: + return value + out = [] + for entry in value.split(";"): + if not entry: + out.append(entry) + elif ":" in entry: + prefix, addr = entry.split(":", 1) + out.append(prefix + ":" + deid_upn(addr)) + else: + out.append(deid_upn(entry)) + return ";".join(out) + + +# Non-human/system identities found in Purview audit logs (Teams Sync, SharePoint app, +# SupervisoryReview bots, ServicePrincipals, NT-style accounts, SIDs, bare GUIDs, etc.). +# These have no matching userPrincipalName in EntraUsers and would render as blank +# User/Department rows in downstream visuals. Filter out before any record is emitted. +_UPN_LOCAL_RE = re.compile(r"^[^\s\\@]+$") +_BARE_GUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) + + +def _is_human_upn(uid: str) -> bool: + """True iff uid is a syntactically valid human UPN (local@domain.tld), excluding + well-known service/bot patterns (SupervisoryReview{...}@..., bare GUIDs).""" + if not uid: + return False + s = uid.strip() + if _BARE_GUID_RE.match(s): + return False + if s.lower().startswith("supervisoryreview{"): + return False + if "@" not in s or s.count("@") != 1: + return False + local, domain = s.split("@", 1) + if not _UPN_LOCAL_RE.match(local): + return False + if "." not in domain or not domain or domain.startswith(".") or domain.endswith("."): + return False + return True + + +def parse_creation_time(value: Any) -> datetime | None: + raw = to_text(value).strip() + if not raw: + return None + return _parse_creation_time_cached(raw) + + +@functools.lru_cache(maxsize=None) +def _parse_creation_time_cached(raw: str) -> datetime | None: + # Fast path: ISO 8601 (covers ~100% of Purview audit timestamps). + # datetime.fromisoformat is ~10x faster than strptime and avoids the + # locale lookup that strptime performs on every call. Python 3.11+ + # accepts a trailing "Z"; for 3.10 and earlier we strip it. + try: + if raw.endswith("Z"): + try: + return datetime.fromisoformat(raw) + except ValueError: + return datetime.fromisoformat(raw[:-1]) + return datetime.fromisoformat(raw) + except ValueError: + pass + # Slow path: legacy non-ISO formats kept for backwards compat with + # older / hand-edited audit exports. + for fmt in _CREATION_TIME_FORMATS: + try: + return datetime.strptime(raw, fmt) + except ValueError: + continue + return None + + +def format_creation_date(value: Any) -> str: + parsed = parse_creation_time(value) + if parsed is None: + raw = to_text(value).strip() + if len(raw) >= 10 and raw[4:5] == "-": + return raw[:10] + "T00:00:00.000Z" + return raw + return parsed.replace(tzinfo=timezone.utc).strftime("%Y-%m-%dT00:00:00.000Z") + + +def interaction_date(parsed: datetime | None) -> str: + return parsed.strftime("%Y-%m-%d") if parsed else "" + + +def week_start(parsed: datetime | None) -> str: + if parsed is None: + return "" + return (parsed - timedelta(days=parsed.weekday())).strftime("%Y-%m-%d") + + +def month_start(parsed: datetime | None) -> str: + if parsed is None: + return "" + return parsed.replace(day=1).strftime("%Y-%m-%d") + + +# Cached bundle: given a raw timestamp string, return all 4 derived date +# strings in one shot. Avoids 4x strftime + tzinfo replace per record. The +# distinct raw-timestamp count in a typical dataset is small relative to +# input row count (many records share the same audit timestamp at the +# second granularity), so this collapses ~4N strftime calls to ~K where +# K is the distinct timestamp count. +@functools.lru_cache(maxsize=None) +def _date_strings_for_raw(raw: str) -> tuple[str, str, str, str]: + """ + Returns (creation_date_iso_z, interaction_date, week_start, month_start) + for the given raw timestamp string. Empty string is returned for any + field that cannot be derived (matches non-cached helper semantics). + """ + if not raw: + return ("", "", "", "") + parsed = _parse_creation_time_cached(raw) + if parsed is None: + if len(raw) >= 10 and raw[4:5] == "-": + return (raw[:10] + "T00:00:00.000Z", "", "", "") + return (raw, "", "", "") + # Direct f-string formatting is ~10x faster than strftime (which does + # locale lookup + format-string parsing on every call). Output bytes + # are byte-identical to the prior strftime("%Y-%m-%d") output for any + # year in [1000, 9999] (CreationTime range). + y = parsed.year + m = parsed.month + d = parsed.day + creation = f"{y:04d}-{m:02d}-{d:02d}T00:00:00.000Z" + interaction = f"{y:04d}-{m:02d}-{d:02d}" + # Week start (Monday-based, mirroring strftime((parsed-weekday).strftime)) + ws = parsed - timedelta(days=parsed.weekday()) + week = f"{ws.year:04d}-{ws.month:02d}-{ws.day:02d}" + month = f"{y:04d}-{m:02d}-01" + return (creation, interaction, week, month) + + +def slugify(value: str) -> str: + slug = re.sub(r"[^A-Za-z0-9]+", "-", value.strip()) + return slug.strip("-") + + +# --------------------------------------------------------------------------- +# Audit JSON shaping +# --------------------------------------------------------------------------- + + +def app_identity_values(audit_data: dict[str, Any]) -> tuple[str, str]: + app_identity = safe_get(audit_data, "AppIdentity") + if isinstance(app_identity, str): + return "", app_identity + if isinstance(app_identity, dict): + return ( + to_text(safe_get(app_identity, "AppId")), + to_text(safe_get(app_identity, "DisplayName")), + ) + return "", "" + + +def derive_agent_name(agent_name: Any, app_identity_display: str, app_identity_app_id: str) -> str: + # Match the BEFORE PBIP behavior: AgentName comes straight from the audit JSON. + # Do NOT synthesize from AppIdentity when it's blank — that fabricates distinct + # agent identities (e.g. "Copilot-Studio-Default--") that + # don't exist in the raw data and inflate Active Agents / per-agent rollups. + return to_text(agent_name).strip() + + +def derive_agent_title_id(agent_id: Any) -> str: + agent_id_text = to_text(agent_id).strip() + if not agent_id_text: + return "" + return agent_id_text.rsplit(".", 1)[-1] + + +def first_dict_item(items: list[Any]) -> dict[str, Any]: + for item in items: + if isinstance(item, dict): + return item + return {} + + +def prompt_messages(ced: dict[str, Any]) -> list[dict[str, Any]]: + prompts: list[dict[str, Any]] = [] + for message in get_array(ced, "Messages"): + if isinstance(message, dict) and message.get("isPrompt") is True: + prompts.append(message) + return prompts + + +def resource_rows(ced: dict[str, Any]) -> list[dict[str, Any]]: + resources = [item for item in get_array(ced, "AccessedResources") if isinstance(item, dict)] + return resources if resources else [{}] + + +def is_copilot_interaction(audit_data: dict[str, Any], raw_row: dict[str, Any]) -> bool: + operation = to_text( + safe_get(audit_data, "Operation") + or raw_row.get("Operation") + or raw_row.get("Operations") + ).strip() + return operation == "CopilotInteraction" + + +# --------------------------------------------------------------------------- +# Classification logic (ports of the PBIP DAX calc columns) +# --------------------------------------------------------------------------- + +_LICENSE_TRUTHY = {"YES", "TRUE", "Y", "1"} +_ACTIVE_RES_ACTION_TOKENS = ("send", "draft", "create", "post", "invoke", "write", "patch", "execute") + + +def normalize_has_license(raw: str) -> str: + """Normalize any truthy/falsy variant to canonical 'TRUE' / 'FALSE'. + + Existing PBIP measures filter with literal `[Has license] = "FALSE"`, so + we canonicalize here to guarantee those filters match regardless of how + the upstream Entra/PAX export rendered the value. + """ + val = (raw or "").strip().upper() + if val in _LICENSE_TRUTHY: + return "TRUE" + if val in {"NO", "FALSE", "N", "0"}: + return "FALSE" + return "FALSE" + + +@functools.lru_cache(maxsize=None) +def compute_license_status(has_license_raw: str) -> str: + val = (has_license_raw or "").strip().upper() + return "M365 Copilot Licensed" if val in _LICENSE_TRUTHY else "Unlicensed" + + +@functools.lru_cache(maxsize=None) +def compute_environment(profile: str, has_license_raw: str, agent_name: str, agent_id: str, app_host: str) -> str: + license_val = (has_license_raw or "").strip().upper() + if profile == "aio": + # AIO vocabulary (5-value, keyed off app_host + agent presence). + host = (app_host or "").lower() + has_agent = bool((agent_name or "").strip()) or bool((agent_id or "").strip()) + if host in {"autonomous", "logic app"}: + return "Autonomous Agent" + if "cowork" in host: + return "Cowork" + if has_agent: + return "Agents" + if license_val in _LICENSE_TRUTHY: + return "Licensed M365 Copilot" + return "Unlicensed Chat" + # AIBV vocabulary (verbatim port of current AIBV calc col `Environment`): + # IF(CONTAINSSTRING(LOWER(TRIM(AgentName)),"cowork"),"Cowork", + # IF(isLicensed,"Licensed","Unlicensed")) + if "cowork" in (agent_name or "").strip().lower(): + return "Cowork" + if license_val in _LICENSE_TRUTHY: + return "Licensed" + return "Unlicensed" + + +@functools.lru_cache(maxsize=None) +def compute_is_sensitive(sens_label: str, resource_sens_label: str) -> str: + return "TRUE" if (sens_label or "").strip() or (resource_sens_label or "").strip() else "FALSE" + + +@functools.lru_cache(maxsize=None) +def compute_ai_model(model_name: str) -> str: + m = (model_name or "").upper() + if not m or m == "NULL": + return "Embedded App (no model logged)" + if "DEEP_LEO" in m: + return "GPT-4 (Standard)" + if "REASONING" in m: + return "Reasoning Model (o1/o3)" + if "OFFENSIVE" in m: + return "Safety Filter (blocked)" + if "GPT-41" in m or "GPT-4.1" in m: + return "GPT-4.1 (Next Gen)" + if "O3-MINI" in m or "O3MINI" in m: + return "o3-mini (Reasoning)" + if "O3" in m or "O1" in m: + return "Reasoning Model (o-series)" + if "GPT-5" in m or "GPT5" in m: + return "GPT-5 (Next Gen)" + if "CLAUDE" in m: + return "Claude (Anthropic)" + if "GEMINI" in m: + return "Gemini (Google)" + if "LLAMA" in m or "META" in m: + return "LLaMA (Meta)" + if "PHI" in m: + return "Phi (Microsoft Small Model)" + return model_name or "" + + +def _resource_behavior( + profile: str, res_type: str, res_action: str, site_url: str, is_active: bool +) -> str: + if res_action in {"sendemailv2", "draftemail", "senddraftemail", "updatedraftemail"}: + return "Email Drafting" + if res_type == "emailmessage": + return "Email Drafting" if is_active else "Email Summarising" + if res_action == "mcp_meetingmanagement": + return "Meeting Scheduling" + if res_type in {"event", "teamsmeeting"}: + return "Meeting Prep" + if res_action in {"postmessagetoconversation", "createchat"}: + return "Teams Messaging" + if res_type in {"teamsmessage", "teamschat", "teamschannel"}: + return "Teams Messaging" + if profile == "aio": + # Any flow/connector/http resource -> "Workflow Execution". + if res_type in {"flow", "connector", "http"}: + return "Workflow Execution" + else: + # AIBV: explicit Flow always; connector/http only with an active verb. + if res_type == "flow": + return "Running a Workflow" + if res_type in {"connector", "http"} and is_active: + return "Running a Workflow" + if res_action in {"executedatasetquery", "getitems", "getalltables", "gettableviews"}: + return "Data Querying" + if res_type in {"xlsx", "csv", "xlsm", "xlsb", "xls"}: + return "Excel Assistance" if is_active else "Spreadsheet Review" + if res_type == "peopleinferenceanswer": + return "People Lookup" + if res_type in {"listitem", "aspx"}: + return "Enterprise Searching" + if res_type == "websearchquery": + return "Web Searching" + if res_type == "pdf": + return "PDF Analysis" + if res_type in {"py", "js", "java", "tsx", "jsx", "css", "php", "sh"} and is_active: + return "Code Writing" + if res_type in {"py", "sql", "js", "java", "json", "xml", "html", "yaml", "yml", "txt"}: + return "Code Analysis" + if res_type in {"png", "jpg", "jpeg", "svg", "gif"} and is_active: + return "Image Generation" + if res_type in {"png", "jpg", "jpeg", "gif"}: + return "Image / Media Analysis" + if res_type in {"streamvideo", "mp4", "mov", "webm", "mkv"}: + return "Video Summarising" + if res_type in {"planid", "taskids"}: + return "Task Management" + if res_type == "looppage": + return "Real-time Collaboration" + if res_type == "http://schema.skype.com/hyperlink": + for token in ("github.com", "stackoverflow.com", "npmjs.com", "pypi.org", "docker.com", "kubernetes.io", "leetcode.com"): + if token in site_url: + return "Code Analysis" + for token in ("learning.cloud.microsoft", "coursera.org", "udemy.com"): + if token in site_url: + return "Agent: Coaching" + if "sharepoint.com" in site_url: + return "Enterprise Searching" + return "Web Searching" + if res_type in {"external", "http"}: + return "Web Searching" + if res_type in {"docx", "doc", "rtf"}: + if is_active: + return "Document Drafting" + if res_action == "read": + return "File Retrieval" + return "Document Summarising" + if res_type in {"pptx", "ppt", "potx"}: + if is_active: + return "Presentation Creation" + if res_action == "read": + return "File Retrieval" + return "Presentation Summarising" + if "service-now.com" in site_url or "servicenow.com" in site_url: + return "Agent: IT & Service Desk" + if "dynamics.com" in site_url: + return "Agent: Sales & Customer" + return "" + + +def _context_behavior(profile: str, app_host: str, ctx_type: str, is_active: bool, has_agent: bool) -> str: + if ctx_type == "teamsmeeting": + return "Meeting Prep" + if ctx_type == "streamvideo": + return "Video Summarising" + if ctx_type == "docx": + return "Document Drafting" if (app_host == "word" and is_active) else "Document Summarising" + if ctx_type in {"xlsx", "xlsm", "xlsb", "xls", "csv"}: + return "Spreadsheet Review" + if ctx_type in {"pptx", "pptm"}: + return "Presentation Creation" if (app_host == "powerpoint" and is_active) else "Presentation Summarising" + if ctx_type in {"teamschat", "teamschannel"}: + return "Teams Messaging" + if ctx_type == "aspx": + return "Enterprise Searching" + if app_host in {"outlook", "outlooksidepane"}: + return "Email Drafting" if is_active else "Email Summarising" + if app_host == "excel": + return "Excel Assistance" + if app_host == "word": + return "Document Drafting" if is_active else "Document Summarising" + if app_host == "powerpoint": + return "Presentation Creation" if is_active else "Presentation Summarising" + if app_host == "stream": + return "Video Summarising" + if app_host == "sharepoint": + return "SharePoint Access" + if app_host == "designer": + return "Image Generation" + if app_host == "onenote": + return "Note Taking" + if app_host == "forms": + return "Form / Survey Work" + if app_host == "planner": + return "Task Management" + if app_host in {"loop", "whiteboard", "vivaengage"}: + return "Real-time Collaboration" + if app_host == "copilot studio": + return "Domain-Specific Agent" + if profile == "aio": + # Autonomous OR logic app -> "Workflow Execution". + if app_host in {"autonomous", "logic app"}: + return "Workflow Execution" + else: + # AIBV: autonomous always; logic app only when an agent context is present. + if app_host == "autonomous": + return "Running a Workflow" + if app_host == "logic app" and has_agent: + return "Running a Workflow" + if app_host in {"datawarehousing core", "power bi"}: + return "Data Querying" + return "General Chat" + + +@functools.lru_cache(maxsize=None) +def compute_behavior_category( + profile: str, + app_host: str, + ctx_type: str, + res_type: str, + res_action: str, + site_url: str, + plugin_id: str, + has_agent: bool, +) -> str: + app_host_l = (app_host or "").lower() + ctx_l = (ctx_type or "").lower() + res_t_l = (res_type or "").lower() + res_a_l = (res_action or "").lower() + site_l = (site_url or "").lower() + plugin_l = (plugin_id or "").lower() + is_active = any(tok in res_a_l for tok in _ACTIVE_RES_ACTION_TOKENS) + + from_resource = _resource_behavior(profile, res_t_l, res_a_l, site_l, is_active) + if from_resource: + return from_resource + if plugin_l == "enterprisesearch": + return "Enterprise Searching" + return _context_behavior(profile, app_host_l, ctx_l, is_active, has_agent) + + +_GENERIC_QA_BEHAVIORS = {"General Q&A", "M365 Chat Q&A", "Teams Q&A", "Browser Q&A", "General Chat"} +_AGENT_NAME_RULES: tuple[tuple[tuple[str, ...], str], ...] = ( + (("coach", "mentor", "learning", "career"), "Agent: Coaching"), + (("research", "analyst", "analy"), "Agent: Research & Analysis"), + (("sales", "commercial", "customer", "crm", "revenue"), "Agent: Sales & Customer"), + (("hr", "recruit", "talent", "onboard", "people"), "Agent: HR & People"), + (("policy", "compliance", "legal", "audit", "risk"), "Agent: Compliance & Policy"), + (("service", "support", "help", "ticket", "incident"), "Agent: IT & Service Desk"), + (("summar", "draft", "translat", "editor"), "Agent: Content Generation"), + (("data", "report", "dashboard", "metric"), "Agent: Data & Reporting"), + (("knowledge", "faq", "wiki", "buddy", "guide"), "Agent: Knowledge Base"), + (("idea", "brainstorm", "creative", "design"), "Agent: Ideation & Creative"), +) + + +@functools.lru_cache(maxsize=None) +def compute_behavior_enriched(profile: str, behavior_category: str, agent_name: str, environment: str) -> str: + # AIO enriches agent/autonomous rows; AIBV enriches agents/cowork rows. + # (In practice AIBV `Environment` never returns "Agents", so only Cowork enriches.) + enrich_envs = {"Agents", "Autonomous Agent"} if profile == "aio" else {"Agents", "Cowork"} + if environment not in enrich_envs: + return behavior_category + if behavior_category not in _GENERIC_QA_BEHAVIORS: + return behavior_category + name_l = (agent_name or "").lower() + for tokens, label in _AGENT_NAME_RULES: + if any(t in name_l for t in tokens): + return label + return "Agent: General Purpose" + + +# Autonomy_Pattern — profile-aware. +# AIO: keyed off the 5-value Environment. +# AIBV: SWITCH(Cowork->3, Is_Agent_Activity->2, Licensed->1, else BLANK). +@functools.lru_cache(maxsize=None) +def compute_autonomy_pattern(profile: str, environment: str, is_agent_activity_str: str) -> str: + if profile == "aio": + if environment == "Licensed M365 Copilot": + return "1 - Copilot" + if environment == "Agents": + return "2 - Agent-Assisted" + if environment == "Autonomous Agent": + return "3 - Autonomous" + return "" + if environment == "Cowork": + return "3 - Cowork" + if is_agent_activity_str == "TRUE": + return "2 - Agent-Assisted" + if environment == "Licensed": + return "1 - Copilot" + return "" + + +# Behavior_Source — profile-aware (AIO: "Autonomous Agent" branch; AIBV: "Cowork"). +@functools.lru_cache(maxsize=None) +def compute_behavior_source( + profile: str, + behavior_category: str, + environment: str, + agent_name: str, + plugin_name: str, + app_host: str, +) -> str: + agent = (agent_name or "").strip() + plugin = (plugin_name or "").strip() + app = (app_host or "").strip() + if profile == "aio" and environment == "Autonomous Agent": + source = "Autonomous Agent" + (f": {agent}" if agent else "") + elif profile != "aio" and environment == "Cowork": + source = "Cowork" + (f": {agent}" if agent else "") + elif environment == "Agents" and agent: + source = f"Agent: {agent}" + elif plugin: + source = f"{app} ({plugin})" + elif app: + source = app + else: + source = "Copilot Chat" + return f"{behavior_category} → {source}" + + +# Verbatim port of current AIBV DAX calc col `Value_Outcome`. +_VO_TIME_EMAIL = frozenset({"Email Summarising", "Email Triage", "Email Thread Summary"}) +_VO_TIME_MEET = frozenset({"Meeting Prep", "Video Summarising"}) +_VO_TIME_DOC = frozenset({"Document Summarising", "Presentation Summarising", "Note Taking"}) +_VO_SEARCH = frozenset({ + "Web Searching", "Enterprise Searching", "File Retrieval", "PDF Analysis", + "SharePoint Access", "People Lookup", "Agent: Knowledge Base", +}) +_VO_COMM = frozenset({"Teams Messaging", "Meeting Scheduling"}) +_VO_SHEET = frozenset({"Spreadsheet Review", "Spreadsheet Analysis", "Excel Assistance"}) +_VO_CONTENT = frozenset({ + "Email Drafting", "Document Drafting", "Presentation Creation", + "Image Generation", "Image / Media Analysis", "Image/Media Analysis", + "Agent: Content Generation", "Agent: Ideation & Creative", +}) +_VO_TEAMCOLLAB = frozenset({"Real-time Collaboration", "Form / Survey Work"}) +_VO_DATA = frozenset({"Data Querying", "Agent: Data & Reporting", "Agent: Research & Analysis"}) +_VO_CODE = frozenset({"Code Writing", "Code Analysis", "Code Analysis (URL)"}) +_VO_COACH = frozenset({"Agent: Coaching", "Agent: Coaching (URL)"}) +_VO_DOMAIN = frozenset({"Domain-Specific Agent", "Cross-Org Agent"}) + + +@functools.lru_cache(maxsize=None) +def compute_value_outcome( + profile: str, behavior_enriched: str, environment: str, is_sensitive_str: str +) -> str: + b = behavior_enriched or "" + # Profile-specific workflow signal (AIO: Workflow Execution / Autonomous Agent; + # AIBV: Running a Workflow / Cowork). + workflow_behavior = "Workflow Execution" if profile == "aio" else "Running a Workflow" + workflow_env = "Autonomous Agent" if profile == "aio" else "Cowork" + if b in _VO_TIME_EMAIL: + return "Time Saved (Email)" + if b in _VO_TIME_MEET: + return "Time Saved (Meetings)" + if b in _VO_TIME_DOC: + return "Time Saved (Documents)" + if b in _VO_SEARCH: + return "Search Time Saved" + if b in _VO_COMM: + return "Communication Time Saved" + if b in _VO_SHEET: + return "Spreadsheet Time Saved" + if b in _VO_CONTENT: + return "Content Output" + if b in _VO_TEAMCOLLAB: + return "Team Collaboration" + if b == workflow_behavior or environment == workflow_env: + return "Workflow Automation" + if b == "Task Management": + return "Task Coordination" + if ( + is_sensitive_str == "TRUE" + and environment != "Agents" + and environment != workflow_env + ): + return "Compliance & Risk" + if b in _VO_DATA: + return "Data-Driven Decisions" + if b in _VO_CODE: + return "Coding Capability" + if b in _VO_COACH: + return "Skills Development" + if b == "Agent: Sales & Customer": + return "Revenue Enablement" + if b == "Agent: IT & Service Desk": + return "Service Desk Deflection" + if b == "Agent: Compliance & Policy": + return "Compliance & Risk" + if b == "Agent: HR & People": + return "HR Expertise" + if b in _VO_DOMAIN: + return "Specialist Expertise" + return "General AI Productivity" + + +# --------------------------------------------------------------------------- +# Downstream classification chain (offloaded from AIBV DAX). +# +# Per F2 (see offload plan): in the current AIBV model `Environment` never +# returns "Agents", so `Behavior_Enriched_Full`'s NeedsEnhancement guard +# (Env = "Agents" && BaseEnriched = "Agent: General Purpose") is always FALSE +# and the RELATED('Agents 365'...) lookups never execute. Therefore +# Behavior_Enriched_Full == Behavior_Enriched and the whole chain +# (Usage_Mode / Expertise_Role / Efficiency_Breakdown) is fully computable +# here WITHOUT ingesting Agents 365 — and stays pixel-identical to AIBV. +# --------------------------------------------------------------------------- + + +def compute_behavior_enriched_full(behavior_enriched: str) -> str: + # No Agents 365 ingestion (by design) => NeedsEnhancement is always FALSE + # => Behavior_Enriched_Full is exactly Behavior_Enriched. + return behavior_enriched + + +_UM_PRODUCING = frozenset({ + "Email Drafting", "Document Drafting", "Presentation Creation", "Image Generation", + "Code Writing", "Code Analysis", "Code Analysis (URL)", "Data Querying", + "Spreadsheet Analysis", "Excel Assistance", "Agent: Content Generation", + "Agent: Ideation & Creative", "Agent: Research & Analysis", "Agent: Data & Reporting", + "Agent: Sales & Customer", "Agent: HR & People", "Agent: IT & Service Desk", + "Agent: Compliance & Policy", "Agent: Coaching", "Agent: Coaching (URL)", + "Domain-Specific Agent", "Cross-Org Agent", "Form / Survey Work", + "Real-time Collaboration", "Note Taking", "Teams Messaging", "Meeting Scheduling", + "Task Management", +}) +_UM_CONSUMING = frozenset({ + "Document Summarising", "Email Summarising", "Email Thread Summary", "Email Triage", + "Presentation Summarising", "Video Summarising", "Meeting Prep", + "Image / Media Analysis", "Image/Media Analysis", "Sensitive Content Interaction", +}) +_UM_FINDING = frozenset({ + "Web Searching", "Enterprise Searching", "PDF Analysis", "SharePoint Access", + "File Retrieval", "People Lookup", "Agent: Knowledge Base", "Spreadsheet Review", +}) + + +@functools.lru_cache(maxsize=None) +def compute_usage_mode(behavior_enriched_full: str, environment: str, app_host: str) -> str: + # Verbatim port of AIBV `Usage_Mode` with the Agents-365 term dropped + # (agentTypeA365 = IFERROR(RELATED(...),BLANK()) -> BLANK when A365 absent, + # so its IN {...} test is always FALSE). + behavior = behavior_enriched_full + host = (app_host or "").lower() + is_delegating = ( + environment == "Cowork" + or host == "autonomous" + or behavior == "Running a Workflow" + ) + if is_delegating: + return "5 - Delegating" + if behavior in _UM_PRODUCING: + return "4 - Producing" + if behavior in _UM_CONSUMING: + return "3 - Consuming" + if behavior in _UM_FINDING: + return "2 - Finding" + return "1 - Asking" + + +# Verbatim port of AIBV `Expertise_Role` (ordered IF cascade on Behavior_Enriched_Full). +_EXPERTISE_RULES: tuple[tuple[frozenset[str], str], ...] = ( + (frozenset({"Data Querying", "Agent: Data & Reporting", "Spreadsheet Analysis"}), "Data Analyst"), + (frozenset({"Code Writing", "Code Analysis", "Code Analysis (URL)"}), "Software Engineer"), + (frozenset({"Agent: Research & Analysis"}), "Business Analyst"), + (frozenset({"Agent: Compliance & Policy", "Sensitive Content Interaction"}), "Compliance Specialist"), + (frozenset({"Agent: Sales & Customer"}), "Sales Consultant"), + (frozenset({"Agent: IT & Service Desk"}), "IT Specialist"), + (frozenset({"Agent: HR & People"}), "HR Specialist"), + (frozenset({"Agent: Coaching", "Agent: Coaching (URL)"}), "Coach"), + (frozenset({"Running a Workflow", "Task Management"}), "Automation Engineer"), + (frozenset({"Domain-Specific Agent", "Cross-Org Agent"}), "Domain Expert"), + (frozenset({"Email Drafting"}), "Communications Specialist"), + (frozenset({"Email Triage", "Meeting Scheduling", "Email Summarising", "Email Thread Summary"}), "Executive Assistant"), + (frozenset({"Document Drafting", "Agent: Content Generation", "Note Taking", "Document Summarising"}), "Content Writer"), + (frozenset({"Presentation Creation", "Presentation Summarising"}), "Presentation Designer"), + (frozenset({"Image Generation", "Image/Media Analysis", "Image / Media Analysis", "Agent: Ideation & Creative"}), "Visual Designer"), + (frozenset({"Meeting Prep", "Video Summarising"}), "Meeting Coordinator"), + (frozenset({"Web Searching", "PDF Analysis", "Agent: Knowledge Base"}), "Researcher"), + (frozenset({"Enterprise Searching", "SharePoint Access", "File Retrieval", "People Lookup"}), "Knowledge Navigator"), + (frozenset({"Spreadsheet Review", "Excel Assistance"}), "Spreadsheet Specialist"), + (frozenset({"Real-time Collaboration", "Form / Survey Work", "Form/Survey Work", "Teams Messaging"}), "Collaboration Lead"), +) + + +@functools.lru_cache(maxsize=None) +def compute_expertise_role(behavior_enriched_full: str) -> str: + for members, label in _EXPERTISE_RULES: + if behavior_enriched_full in members: + return label + return "" # AIBV returns BLANK() + + +# Verbatim port of AIBV `Efficiency_Breakdown` (behavior cascade, then Behavior_Category fallback). +_EFF_RULES: tuple[tuple[frozenset[str], str], ...] = ( + (frozenset({"Email Summarising", "Email Triage", "Email Thread Summary", "Email Drafting"}), "Email"), + (frozenset({"Document Summarising", "Note Taking", "Document Drafting", "Agent: Content Generation"}), "Document Assistance"), + (frozenset({"Presentation Summarising", "Presentation Creation"}), "Presentations"), + (frozenset({"Meeting Prep", "Video Summarising", "Meeting Scheduling"}), "Meetings"), + (frozenset({"Web Searching", "Enterprise Searching", "PDF Analysis", "SharePoint Access", "File Retrieval", "People Lookup", "Agent: Knowledge Base", "Agent: Research & Analysis"}), "Search & Research"), + (frozenset({"Spreadsheet Review", "Excel Assistance", "Spreadsheet Analysis", "Data Querying", "Agent: Data & Reporting"}), "Data & Spreadsheets"), + (frozenset({"Image Generation", "Image / Media Analysis", "Image/Media Analysis", "Agent: Ideation & Creative", "Code Writing", "Code Analysis", "Code Analysis (URL)"}), "Creative & Technical"), + (frozenset({"Teams Messaging", "Real-time Collaboration", "Form / Survey Work", "Task Management", "Running a Workflow"}), "Collaboration & Workflows"), + (frozenset({"Agent: Sales & Customer", "Agent: IT & Service Desk", "Agent: HR & People", "Agent: Compliance & Policy", "Agent: Coaching", "Agent: Coaching (URL)", "Domain-Specific Agent", "Cross-Org Agent"}), "Specialist Agents"), +) + + +@functools.lru_cache(maxsize=None) +def compute_efficiency_breakdown(behavior_enriched_full: str, behavior_category: str) -> str: + for members, label in _EFF_RULES: + if behavior_enriched_full in members: + return label + if behavior_category == "Teams Q&A": + return "Teams Chat" + if behavior_category == "M365 Chat Q&A": + return "BizChat Q&A" + if behavior_category == "Browser Q&A": + return "BizChat Q&A" + return "General Q&A" + + +# --------------------------------------------------------------------------- +# Embedded static value map (offloads the ROI baseline join). +# +# `Human Time Estimates` is a static lookup (Behavior -> Human Baseline (min)). +# The AIBV measure `Human Equivalent Hours` does: +# SUMX(FILTER(fact, isPrompt="TRUE"), DIVIDE(RELATED(HTE[Human Baseline]),60,0)) +# which traverses fact -> Behavior Value Map (BVM) -> Human Time Estimates (HTE) +# per row. We pre-join `Human_Baseline_Min` onto every fact row so the PBIT +# measure collapses to SUM(fact[Human_Baseline_Min])/60 (no SUMX, no RELATED). +# +# FIDELITY: HTE is reachable in the model ONLY through the BVM bridge, so a +# baseline is emitted ONLY when Behavior_Enriched_Full is present in BVM (then +# looked up in HTE; BVM is a strict subset of HTE, so the lookup always hits). +# Behaviors not in BVM contribute 0 hours in the current AIBV — we emit "" for +# them, which SUMs to 0 identically. Both maps are transcribed verbatim from +# the AIBV `.pbit` static #table literals (see temp/_offload_test/_parse_maps.py). +# --------------------------------------------------------------------------- + +_HUMAN_BASELINE_MIN: dict[str, int] = { + "Agent: Coaching": 45, + "Agent: Coaching (URL)": 25, + "Agent: Compliance & Policy": 25, + "Agent: Content Generation": 25, + "Agent: Data & Reporting": 35, + "Agent: General Purpose": 15, + "Agent: HR & People": 35, + "Agent: IT & Service Desk": 20, + "Agent: Ideation & Creative": 40, + "Agent: Knowledge Base": 12, + "Agent: Research & Analysis": 45, + "Agent: Sales & Customer": 35, + "Browser Q&A": 10, + "Code Analysis": 30, + "Code Analysis (URL)": 15, + "Code Writing": 45, + "Cross-Org Agent": 30, + "Data Querying": 30, + "Document Drafting": 60, + "Document Summarising": 20, + "Domain-Specific Agent": 25, + "Email Drafting": 8, + "Email Summarising": 4, + "Email Thread Summary": 5, + "Email Triage": 10, + "Enterprise Searching": 18, + "Excel Assistance": 30, + "File Retrieval": 15, + "Form / Survey Work": 25, + "Form/Survey Work": 25, + "General Chat": 10, + "General Q&A": 10, + "Image / Media Analysis": 8, + "Image Generation": 60, + "Image/Media Analysis": 8, + "M365 Chat Q&A": 10, + "Meeting Prep": 15, + "Meeting Scheduling": 12, + "Note Taking": 20, + "PDF Analysis": 35, + "People Lookup": 10, + "Presentation Creation": 90, + "Presentation Summarising": 12, + "Real-time Collaboration": 30, + "Running a Workflow": 15, + "Sensitive Content Interaction": 20, + "SharePoint Access": 12, + "Spreadsheet Analysis": 40, + "Spreadsheet Review": 25, + "Task Management": 20, + "Teams Messaging": 8, + "Teams Q&A": 10, + "Video Summarising": 30, + "Web Searching": 22, +} + +# Behaviors present in the Behavior Value Map bridge (gates HTE reachability). +_BVM_BEHAVIORS: frozenset = frozenset({ + "Agent: Coaching", + "Agent: Compliance & Policy", + "Agent: Content Generation", + "Agent: Data & Reporting", + "Agent: General Purpose", + "Agent: HR & People", + "Agent: IT & Service Desk", + "Agent: Ideation & Creative", + "Agent: Knowledge Base", + "Agent: Research & Analysis", + "Agent: Sales & Customer", + "Code Analysis", + "Code Writing", + "Data Querying", + "Document Drafting", + "Document Summarising", + "Domain-Specific Agent", + "Email Drafting", + "Email Summarising", + "Enterprise Searching", + "Excel Assistance", + "File Retrieval", + "Form / Survey Work", + "General Chat", + "Image / Media Analysis", + "Image Generation", + "Meeting Prep", + "Meeting Scheduling", + "Note Taking", + "PDF Analysis", + "People Lookup", + "Presentation Creation", + "Presentation Summarising", + "Real-time Collaboration", + "Running a Workflow", + "SharePoint Access", + "Spreadsheet Review", + "Task Management", + "Teams Messaging", + "Video Summarising", + "Web Searching", +}) + + +@functools.lru_cache(maxsize=None) +def compute_human_baseline_min(behavior_enriched_full: str) -> str: + # Emit the baseline only when reachable via the BVM bridge (AIBV topology). + if behavior_enriched_full in _BVM_BEHAVIORS: + return str(_HUMAN_BASELINE_MIN[behavior_enriched_full]) + return "" + + +# --------------------------------------------------------------------------- +# Remaining row-level calc cols (offloaded from AIBV DAX). +# --------------------------------------------------------------------------- + +# Verbatim port of AIBV `Behavior_Plausible`. +_UNLICENSED_PLAUSIBLE = frozenset({ + "General Chat", "Web Searching", "PDF Analysis", "Document Summarising", + "Image / Media Analysis", "Image Generation", "Code Analysis", "Translation", +}) +_BP_WORKAROUND_EMAIL = frozenset({"Email Summarising", "Email Drafting"}) +_BP_WORKAROUND_SHEET = frozenset({"Excel Assistance", "Spreadsheet Review", "Data Querying"}) +_BP_WORKAROUND_MEET = frozenset({"Meeting Prep", "Meeting Scheduling"}) +_BP_WORKAROUND_ENT = frozenset({"Enterprise Searching", "People Lookup"}) +_BP_WORKAROUND_WORKFLOW = frozenset({"Running a Workflow", "Task Management"}) + + +@functools.lru_cache(maxsize=None) +def compute_behavior_plausible(license_status: str, behavior_category: str) -> str: + lic = license_status + beh = behavior_category + if lic == "M365 Copilot Licensed" or beh in _UNLICENSED_PLAUSIBLE: + return beh + if beh in _BP_WORKAROUND_EMAIL: + return "Free Chat Workaround (pasting Email)" + if beh in _BP_WORKAROUND_SHEET: + return "Free Chat Workaround (pasting Spreadsheet/Data)" + if beh in _BP_WORKAROUND_MEET: + return "Free Chat Workaround (pasting Meeting info)" + if beh == "Teams Messaging": + return "Free Chat Workaround (pasting Teams content)" + if beh in _BP_WORKAROUND_ENT: + return "Free Chat Workaround (pasting Enterprise data)" + if beh in _BP_WORKAROUND_WORKFLOW: + return "Free Chat Workaround (pasting Workflow)" + if beh == "Real-time Collaboration": + return "Free Chat Workaround (pasting Loop content)" + if beh == "Code Writing": + return "Free Chat Workaround (pasting Code)" + if beh == "Video Summarising": + return "Free Chat Workaround (uploading Video)" + return "Free Chat Workaround (Other)" + + +# Verbatim port of AIBV `Workflow_Action`. +@functools.lru_cache(maxsize=None) +def compute_workflow_action(behavior_enriched_full: str, res_action: str, app_host: str) -> str: + if behavior_enriched_full != "Running a Workflow": + return "" + ra = (res_action or "").lower() + host = (app_host or "").lower() + if "send" in ra or "post" in ra or "notify" in ra: + return "Sending / Notifying" + if "create" in ra or "draft" in ra or "write" in ra or "add" in ra: + return "Creating Content" + if "invoke" in ra or "execute" in ra or "trigger" in ra or "run" in ra: + return "Invoking / Triggering" + if "update" in ra or "patch" in ra or "modify" in ra or "set" in ra: + return "Updating Records" + if "read" in ra or "get" in ra or "list" in ra or "fetch" in ra: + return "Reading Data" + if "delete" in ra or "remove" in ra: + return "Deleting / Removing" + if host == "autonomous": + return "Autonomous Run (no action logged)" + if host == "logic app": + return "Logic App Run (no action logged)" + return "Workflow (other)" + + +def compute_delegation_event_key( + audit_user_id: str, + interaction_date_str: str, + agent_name: str, + workflow_action: str, + app_host: str, +) -> str: + # Verbatim port of AIBV `Delegation_Event_Key`: + # Audit_UserId & "|" & FORMAT(InteractionDate,"yyyy-mm-dd") & "|" & + # COALESCE(AgentName, Workflow_Action, AppHost, "unknown-workflow") + # NOTE: DAX FORMAT "mm" resolves to MONTH here (not minute) because it does + # not follow h/hh — so interaction_date_str ("%Y-%m-%d") matches exactly. + # COALESCE: we treat empty/whitespace as blank (skip it). The blank-vs-empty + # nuance + rollup-vs-fanout grain interaction affect the + # parity (measure: [Delegation Events] = DISTINCTCOUNT, filtered Usage_Mode + # = "5 - Delegating"). + tail = "unknown-workflow" + for candidate in (agent_name, workflow_action, app_host): + if candidate and candidate.strip(): + tail = candidate + break + return f"{audit_user_id}|{interaction_date_str}|{tail}" + + +def compute_user_month_key(audit_user_id: str, month_start_str: str) -> str: + if not audit_user_id or not month_start_str: + return "" + # MonthStart is YYYY-MM-DD; format key as YYYY-MM (mirrors DAX FORMAT(...,"yyyy-MM")) + return f"{audit_user_id}|{month_start_str[:7]}" + + +# Verbatim port of AIBV calc col `Agent Publish Status`. +@functools.lru_cache(maxsize=None) +def compute_agent_publish_status(agent_id: str, agent_name: str) -> str: + has_agent_id = bool((agent_id or "").strip()) + if not has_agent_id: + return "Not an Agent Row" + if "draft as 1p" in (agent_name or "").lower(): + return "Unpublished" + return "Published" + + +# Verbatim port of AIBV calc col `Is_Agent_Activity` (emitted as TRUE/FALSE text, +# mirroring the Is_Sensitive / Message_isPrompt convention; the PBIT types it +# logical). res_type is the per-resource AccessedResource_Type. +@functools.lru_cache(maxsize=None) +def compute_is_agent_activity(agent_name: str, agent_id: str, app_host: str, res_type: str) -> str: + has_agent = bool((agent_name or "").strip()) + has_agent_id = bool((agent_id or "").strip()) + host = (app_host or "").lower() + rt = (res_type or "").lower() + is_autonomous = host in {"autonomous", "logic app"} or rt in {"flow", "connector"} + return "TRUE" if (has_agent or has_agent_id or is_autonomous) else "FALSE" + + +# Verbatim port of AIBV calc col `Web_Grounded_Signal`. +@functools.lru_cache(maxsize=None) +def compute_web_grounded_signal(res_type: str, site_url: str) -> str: + rt = (res_type or "").lower() + su = (site_url or "").lower() + is_internal = "sharepoint.com" in su or ".onmicrosoft.com" in su + if ( + rt == "websearchquery" + or rt in {"external", "http"} + or (rt == "http://schema.skype.com/hyperlink" and not is_internal) + ): + return "Web Grounded" + return "Not Web Grounded" + + +# --------------------------------------------------------------------------- +# Entra loader / Users dim CSV writer +# --------------------------------------------------------------------------- + + +def _normalize_col_name(name: str) -> str: + return re.sub(r"[\s_\-.()\[\]]", "", (name or "").lower()) + + +def detect_has_license_column(headers: list[str]) -> str | None: + for variant in HAS_LICENSE_VARIANTS: + if variant in headers: + return variant + return None + + +def detect_upn_column(headers: list[str]) -> str | None: + for h in headers: + if _normalize_col_name(h) in UPN_VARIANTS_NORMALIZED: + return h + return None + + +def detect_department_column(headers: list[str]) -> str | None: + # Selection is by meaning, not by source-column position: the readable department + # name owns `Organization` whenever one is present, because the dashboards bind + # their Organization slicers to that value. + for wanted in _DEPARTMENT_SOURCE_PREFERENCE: + for h in headers: + if _normalize_col_name(h) == wanted: + return h + return None + + +def detect_displaced_org_columns(headers: list[str], chosen: str | None) -> list[str]: + """Organization-named source columns displaced by the chosen department column. + + When a readable `department` column is promoted to `Organization`, any + pre-existing column already named Organization/Organisation would collide. + Those are preserved under a separate truthful name rather than dropped, + because a numeric department identifier is legitimate data in its own right. + """ + if not chosen: + return [] + displaced: list[str] = [] + for h in headers: + if h == chosen: + continue + if _normalize_col_name(h) in {"organization", "organisation"}: + displaced.append(h) + return displaced + + +def detect_jobtitle_column(headers: list[str]) -> str | None: + for h in headers: + if _normalize_col_name(h) == "jobtitle": + return h + return None + + +def load_licensing_map(licensing_csv: str) -> tuple[dict[str, str], int]: + """Read a standalone licensing CSV (UPN + 'Has License' columns) and return + (normalized_upn -> raw license value, data_row_count). + + Used by the 3-file input mode to merge license status into a users-only + Entra file. UPN + license columns are detected with the same alias lists as + the Entra loader. Duplicate UPNs are last-write-wins (mirrors the M-code + Table.Distinct behavior on the licensed-users path). + """ + with open(licensing_csv, "r", encoding="utf-8-sig", newline="") as fin: + reader = csv.DictReader(fin) + headers = reader.fieldnames or [] + if not headers: + raise ValueError(f"Licensing CSV has no header row: {licensing_csv}") + upn_col = detect_upn_column(headers) + if upn_col is None: + raise ValueError( + f"Licensing CSV has no recognized UPN column " + f"(expected one of {sorted(UPN_VARIANTS_NORMALIZED)}): {licensing_csv}" + ) + lic_col = detect_has_license_column(headers) + if lic_col is None: + raise ValueError( + f"Licensing CSV has no recognized license column " + f"(expected one of {list(HAS_LICENSE_VARIANTS)}): {licensing_csv}" + ) + license_map: dict[str, str] = {} + row_count = 0 + for row in reader: + row_count += 1 + upn_norm = (row.get(upn_col) or "").strip().lower() + if not upn_norm: + continue + license_map[upn_norm] = row.get(lic_col) or "" + return license_map, row_count + + +# --------------------------------------------------------------------------- +# Org / manager hierarchy (Users dim enrichment; AIO + AIBV; always on). +# Built from the Entra manager links (id <-> manager_id, UPN fallback) that PAX +# already pulls. Structure is keyed on the INT UserKey surrogate, so every +# *_UserKey / OrgLevel / HierarchyPath column is identical with or without +# --deidentify; the *_Name columns use the display name as written (hashed when +# --deidentify is on). Filler for level slots deeper than a user is controlled +# by --hierarchy-fill (none|self|manager|fixed); 'fixed' uses --hierarchy-fill-label. +# --------------------------------------------------------------------------- +_HIER_LEVELS = 15 # Level0..Level14 denormalized top-down columns +_HIER_WALK_CAP = 1000 # safety backstop for manager-chain walks (cycle guard also applies) +_HIER_FILL_MODE = "none" # none | self | manager | fixed (set in main() from --hierarchy-fill) +_HIER_FILL_LABEL = "" # literal label for 'fixed' (set in main() from --hierarchy-fill-label) + + +def _hier_columns() -> list[str]: + cols = [ + "Manager_UserKey", "OrgLevel", "HierarchyPath", "TopOfChain_UserKey", + "IsManager", "DirectReports", "TotalReports", + ] + for i in range(_HIER_LEVELS): + cols.append(f"Level{i}_UserKey") + cols.append(f"Level{i}_Name") + return cols + + +_HIER_COLUMNS: list[str] = _hier_columns() + + +def _hier_filler(uk: int, mgr, name_by_uk: dict, mode: str, label: str) -> tuple[str, str]: + """(UserKey, Name) to place in a level slot DEEPER than the user's own level.""" + if mode == "self": + return str(uk), name_by_uk.get(uk, "") + if mode == "manager": + ref = mgr if mgr is not None else uk + return str(ref), name_by_uk.get(ref, "") + if mode == "fixed": + return "", label + return "", "" # none + + +def _build_org_hierarchy(uk_by_id, uk_by_upn, mgr_ptr, name_by_uk) -> dict: + """Return {UserKey -> {hier_col: value}}. + + uk_by_id : normalized Entra id -> UserKey + uk_by_upn : normalized UPN -> UserKey + mgr_ptr : UserKey -> (manager_id_norm, manager_upn_norm) + name_by_uk: UserKey -> display name (as written; deid'd when applicable) + """ + # Immediate manager UserKey for each user (id link first, UPN fallback). + direct_mgr: dict = {} + for uk, (mid, mupn) in mgr_ptr.items(): + m = uk_by_id.get(mid) if mid else None + if m is None and mupn: + m = uk_by_upn.get(mupn) + if m == uk: + m = None # ignore self-management + direct_mgr[uk] = m + + direct_reports: dict = {} + for uk, m in direct_mgr.items(): + if m is not None: + direct_reports[m] = direct_reports.get(m, 0) + 1 + + all_uks = set(uk_by_upn.values()) | set(name_by_uk.keys()) | set(direct_mgr.keys()) + total_reports: dict = {} + result: dict = {} + mode = _HIER_FILL_MODE + label = _HIER_FILL_LABEL + + for uk in all_uks: + chain = [] + seen = set() + cur = uk + while cur is not None and cur not in seen and len(chain) < _HIER_WALK_CAP: + seen.add(cur) + chain.append(cur) + cur = direct_mgr.get(cur) + # every ancestor of uk gains one report (cycle-safe via `seen`) + for anc in chain[1:]: + total_reports[anc] = total_reports.get(anc, 0) + 1 + chain.reverse() # top .. user + depth = len(chain) - 1 + top = chain[0] + mgr = direct_mgr.get(uk) + rec = { + "Manager_UserKey": str(mgr) if mgr is not None else "", + "OrgLevel": str(depth), + "HierarchyPath": "/".join(str(x) for x in chain), + "TopOfChain_UserKey": str(top), + } + n = len(chain) + for i in range(_HIER_LEVELS): + if i < n: + node = chain[i] + rec[f"Level{i}_UserKey"] = str(node) + rec[f"Level{i}_Name"] = name_by_uk.get(node, "") + else: + fk, fn = _hier_filler(uk, mgr, name_by_uk, mode, label) + rec[f"Level{i}_UserKey"] = fk + rec[f"Level{i}_Name"] = fn + result[uk] = rec + + for uk in all_uks: + dr = direct_reports.get(uk, 0) + result[uk]["DirectReports"] = str(dr) + result[uk]["IsManager"] = "TRUE" if dr > 0 else "FALSE" + result[uk]["TotalReports"] = str(total_reports.get(uk, 0)) + + return result + + +class _SqliteRowView: + """Re-iterable, disk-backed drop-in replacement for ``list(csv.DictReader)``. + + Stages every source row into a temporary SQLite table (source sequence + + JSON payload) in bounded batches during construction, then serves ordered + re-iteration by reopening a cursor on each pass. The complete row set is + never held in memory. It behaves like a list for the only two operations the + callers use — ``len()`` and (repeatable) iteration — and each yielded item is + the original ``csv.DictReader`` dict reconstructed identically, so downstream + calculations, ordering, and output are unchanged. + """ + + def __init__(self, reader, batch_size: int = 2000): + import json + fd, self._path = tempfile.mkstemp(prefix="pax_rowview_", suffix=".sqlite") + os.close(fd) + self._conn = sqlite3.connect(self._path) + self._conn.execute("PRAGMA journal_mode=OFF") + self._conn.execute("PRAGMA synchronous=OFF") + self._conn.execute("CREATE TABLE rows (seq INTEGER PRIMARY KEY, payload TEXT NOT NULL)") + seq = 0 + batch: list = [] + try: + self._conn.execute("BEGIN") + for row in reader: + batch.append((seq, json.dumps(row, ensure_ascii=False))) + seq += 1 + if len(batch) >= batch_size: + self._conn.executemany("INSERT INTO rows (seq, payload) VALUES (?, ?)", batch) + batch.clear() + if batch: + self._conn.executemany("INSERT INTO rows (seq, payload) VALUES (?, ?)", batch) + self._conn.execute("COMMIT") + except Exception: + try: + self._conn.execute("ROLLBACK") + except Exception: + pass + self.close() + raise + self._count = seq + + def __len__(self) -> int: + return self._count + + def __iter__(self): + import json + cur = self._conn.cursor() + try: + cur.execute("SELECT payload FROM rows ORDER BY seq") + for (payload,) in cur: + yield json.loads(payload) + finally: + cur.close() + + def close(self) -> None: + conn = getattr(self, "_conn", None) + if conn is not None: + try: + conn.close() + except Exception: + pass + self._conn = None + path = getattr(self, "_path", None) + if path: + try: + if os.path.exists(path): + os.remove(path) + except Exception: + pass + + def __del__(self): + self.close() + + +# Exact-case Users columns the AIO semantic model exposes as source columns. Power Query M is +# case-sensitive, so these names must match exactly. PowerShell CSV readers, by contrast, treat +# headers case-INSENSITIVELY and reject a file carrying both `displayName` and `DisplayName`, so +# these two are RENAMED to their canonical form rather than duplicated. `mail` and `Email` differ +# by more than case and may safely coexist, so `mail` is retained and `Email` added alongside it. +# ValueLens consumes the directory-native names and is deliberately unaffected. +_AIO_CANONICAL_RENAMES: tuple[tuple[str, str], ...] = ( + ("displayName", "DisplayName"), + ("country", "Country"), +) +_AIO_EMAIL_SOURCE = "mail" +_AIO_EMAIL_CANONICAL = "Email" + + +# UserKey allocation. A new key is always taken from a counter positioned STRICTLY +# ABOVE every key already reserved by a prior append target, so a key that is no +# longer present in the current directory export can never be handed to a different +# identity. With no reserved keys the counter starts at 1 and allocates 1, 2, 3, ... +# in encounter order. +_NEXT_USER_KEY = 1 + + +def reset_user_key_allocator(user_key_map: dict[str, int]) -> None: + """Position the allocator above every already-reserved key.""" + global _NEXT_USER_KEY + highest = 0 + for reserved in user_key_map.values(): + if reserved > highest: + highest = reserved + _NEXT_USER_KEY = highest + 1 + + +def mint_user_key(user_key_map: dict[str, int], normalized_identity: str) -> int: + """Return the key reserved for an identity, allocating the next free key once.""" + global _NEXT_USER_KEY + reserved = user_key_map.get(normalized_identity) + if reserved is not None: + return reserved + assigned = _NEXT_USER_KEY + _NEXT_USER_KEY = assigned + 1 + user_key_map[normalized_identity] = assigned + return assigned + + +def load_entra_and_write_users( + entra_csv: str, + users_out_csv: str, + user_key_map: dict[str, int], + licensing_csv: str | None = None, + quiet: bool = False, + profile: str = "aibv", +) -> dict[str, dict[str, str]]: + """ + Read the Entra CSV, write the Users dim CSV (with PBIP-compatible renames + + precomputed License Status + UserKey INT surrogate), and return a dict + keyed on PersonId_Normalized -> {"Has license": ..., "License Status": ...} + for fact-row lookup. + + Mutates `user_key_map` (normalized_upn -> int) in place — every Entra row + with a non-empty PersonId_Normalized is assigned a UserKey (1-based, in + Entra-file order). The same map is reused by the fact path so any audit + user already in Entra resolves to the same INT. + + Mirrors the rename/normalization logic in the existing PBIP M-code: + userPrincipalName/upn/personid -> PersonId + department -> Organization + jobTitle -> JobTitle + Has license variants -> "Has license" + adds PersonId_Normalized (lower+trim of PersonId) + adds License Status (precomputed) + adds TotalEmployees (row count, repeated per row) + + 3-file input mode: when `licensing_csv` is provided, `entra_csv` is treated + as a users-only file and the per-user license value is merged in from the + separate licensing CSV (keyed on normalized UPN). When `licensing_csv` is + None the function behaves exactly as before (license read from the combined + Entra row), so the legacy 2-file output is byte-identical. + """ + license_map: dict[str, str] | None = None + licensing_rows = 0 + if licensing_csv: + license_map, licensing_rows = load_licensing_map(licensing_csv) + + with open(entra_csv, "r", encoding="utf-8-sig", newline="") as fin: + # Sniff via a generous quote-aware reader; encoding="utf-8-sig" eats BOM if present. + reader = csv.DictReader(fin) + original_headers = reader.fieldnames or [] + if not original_headers: + raise ValueError(f"Entra CSV has no header row: {entra_csv}") + + upn_col = detect_upn_column(original_headers) + dept_col = detect_department_column(original_headers) + has_license_col = detect_has_license_column(original_headers) + jobtitle_col = detect_jobtitle_column(original_headers) + + # Build rename map: source_header -> target_header + rename_map: dict[str, str] = {} + if upn_col and upn_col != "PersonId": + rename_map[upn_col] = "PersonId" + if dept_col and dept_col != "Organization": + rename_map[dept_col] = "Organization" + # Retain any other Organization-named source column (typically a numeric + # department identifier) under a separate name so promoting the readable + # department neither collides with it nor discards it. + for displaced in detect_displaced_org_columns(original_headers, dept_col): + alt = _DISPLACED_ORG_COLUMN + suffix = 2 + while alt in original_headers or alt in rename_map.values(): + alt = f"{_DISPLACED_ORG_COLUMN}_{suffix}" + suffix += 1 + rename_map[displaced] = alt + if jobtitle_col and jobtitle_col != "JobTitle": + rename_map[jobtitle_col] = "JobTitle" + if has_license_col and has_license_col != "Has license": + rename_map[has_license_col] = "Has license" + + # AIO canonical identity headers. Renamed (not duplicated) so the emitted CSV never + # carries two headers differing only by case, which PowerShell CSV readers reject. + # When the export already supplies the exact canonical header, the case-only variant is + # dropped from the OUTPUT but its source name is remembered: a blank canonical value + # still falls back to the variant's value rather than emitting a blank field. + aio_fallback_source_by_canonical: dict[str, str] = {} + if profile == "aio": + for _src, _canon in _AIO_CANONICAL_RENAMES: + _srcs = [h for h in original_headers if h == _src] + _canons = [h for h in original_headers if h == _canon] + if _canons: + for h in _srcs: + rename_map[h] = None + aio_fallback_source_by_canonical[_canon] = h + elif _srcs: + rename_map[_srcs[0]] = _canon + + # Final header list for users CSV — preserve original order, apply renames, + # then append injected columns. UserKey is the INT surrogate that joins + # to the fact table. + renamed_headers = [rename_map.get(h, h) for h in original_headers] + # A None target means the source column is intentionally dropped (AIO case-only variant + # superseded by an already canonical header). + renamed_headers = [h for h in renamed_headers if h is not None] + injected = ["UserKey", "PersonId_Normalized", "License Status", "TotalEmployees"] + if "Has license" not in renamed_headers: + renamed_headers.append("Has license") + for inj in injected: + if inj not in renamed_headers: + renamed_headers.append(inj) + # AIO exact-case alias headers (aio profile only, additive). ValueLens output shape + # is deliberately unchanged. + if profile == "aio": + if _AIO_EMAIL_CANONICAL not in renamed_headers: + renamed_headers.append(_AIO_EMAIL_CANONICAL) + # Org/manager hierarchy columns (always appended; AIO/AIBV Users dim). + for hc in _HIER_COLUMNS: + if hc not in renamed_headers: + renamed_headers.append(hc) + + rows = _SqliteRowView(reader) + + total_rows = len(rows) + user_lookup: dict[str, dict[str, str]] = {} + + # --- Org/manager hierarchy pre-pass: assign UserKeys in Entra-file order and + # build the link maps from the SAME rename + deid the write loop applies, then + # resolve the hierarchy. Always on for the AIO/AIBV Users dim. UserKeys assigned + # here are reused by the write loop (identical to the prior lazy assignment). --- + uk_by_id: dict[str, int] = {} + uk_by_upn: dict[str, int] = {} + mgr_ptr: dict[int, tuple[str, str]] = {} + name_by_uk: dict[int, str] = {} + for src_row in rows: + pid = src_row.get(upn_col, "") if (upn_col and upn_col != "PersonId") else src_row.get("PersonId", "") + pid = "" if pid is None else str(pid) + if _DEIDENTIFY: + pid = deid_upn(pid) + pid_norm = pid.strip().lower() + if not pid_norm: + continue + uk = mint_user_key(user_key_map, pid_norm) + uk_by_upn[pid_norm] = uk + rid = src_row.get("id", "") + rid = "" if rid is None else str(rid) + if _DEIDENTIFY: + rid = deid_guid(rid) + rid_norm = rid.strip().lower() + if rid_norm: + uk_by_id[rid_norm] = uk + mid = src_row.get("manager_id", "") + mid = "" if mid is None else str(mid) + if _DEIDENTIFY: + mid = deid_guid(mid) + mupn = src_row.get("manager_userPrincipalName", "") + mupn = "" if mupn is None else str(mupn) + if _DEIDENTIFY: + mupn = deid_upn(mupn) + mgr_ptr[uk] = (mid.strip().lower(), mupn.strip().lower()) + dn = src_row.get("displayName", "") + dn = "" if dn is None else str(dn) + if _DEIDENTIFY: + dn = deid_name(dn) + name_by_uk[uk] = dn + hier_by_uk = _build_org_hierarchy(uk_by_id, uk_by_upn, mgr_ptr, name_by_uk) + + out_dir = Path(users_out_csv).parent + out_dir.mkdir(parents=True, exist_ok=True) + + pax_licensed = 0 + pax_unlicensed = 0 + no_license_col = 0 + matched_in_licensing = 0 + seen_normalized_keys: set[str] = set() + + with open(users_out_csv, "w", encoding="utf-8", newline="") as fout: + writer = csv.DictWriter(fout, fieldnames=renamed_headers, lineterminator="\n") + writer.writeheader() + + for src_row in rows: + # Apply renames + ensure all renamed_headers keys exist in the out row. + out_row: dict[str, str] = {h: "" for h in renamed_headers} + for src_h, value in src_row.items(): + tgt_h = rename_map.get(src_h, src_h) + if tgt_h in out_row: + out_row[tgt_h] = "" if value is None else str(value) + + # AIO canonical selection: prefer a nonblank exact canonical value, otherwise fall + # back to the dropped case-only variant's source value. Done BEFORE the + # de-identification block so the selected value is the one that gets transformed. + # Lookups are explicit and case-sensitive - no case-insensitive dictionary access. + if profile == "aio": + for _canon, _srcname in aio_fallback_source_by_canonical.items(): + if not out_row.get(_canon, ""): + _fb = src_row.get(_srcname, "") + out_row[_canon] = "" if _fb is None else str(_fb) + + # Deidentification (no-op unless --deidentify): transform identity columns + # IN PLACE before PersonId_Normalized / UserKey derive from PersonId, so the + # fact<->Users join and manager links stay consistent across the hash. + if _DEIDENTIFY: + if "PersonId" in out_row: + out_row["PersonId"] = deid_upn(out_row["PersonId"]) + if "displayName" in out_row: + out_row["displayName"] = deid_name(out_row["displayName"]) + # AIO emits the canonical header instead of the case-only variant, so the same + # transform must be applied to it (dict keys are case-sensitive in Python). + if "DisplayName" in out_row: + out_row["DisplayName"] = deid_name(out_row["DisplayName"]) + if "Email" in out_row: + out_row["Email"] = deid_upn(out_row["Email"]) + if "mail" in out_row: + out_row["mail"] = deid_upn(out_row["mail"]) + if "givenName" in out_row: + out_row["givenName"] = deid_name(out_row["givenName"]) + if "surname" in out_row: + out_row["surname"] = deid_name(out_row["surname"]) + if "UserName" in out_row: + out_row["UserName"] = deid_upn(out_row["UserName"]) + if "employeeId" in out_row: + out_row["employeeId"] = deid_token(out_row["employeeId"]) + if "onPremisesImmutableId" in out_row: + out_row["onPremisesImmutableId"] = deid_token(out_row["onPremisesImmutableId"]) + if "proxyAddresses_Primary" in out_row: + out_row["proxyAddresses_Primary"] = deid_proxy(out_row["proxyAddresses_Primary"]) + if "proxyAddresses_All" in out_row: + out_row["proxyAddresses_All"] = deid_proxy(out_row["proxyAddresses_All"]) + if "id" in out_row: + out_row["id"] = deid_guid(out_row["id"]) + if "manager_id" in out_row: + out_row["manager_id"] = deid_guid(out_row["manager_id"]) + if "manager_userPrincipalName" in out_row: + out_row["manager_userPrincipalName"] = deid_upn(out_row["manager_userPrincipalName"]) + if "manager_displayName" in out_row: + out_row["manager_displayName"] = deid_name(out_row["manager_displayName"]) + if "manager_mail" in out_row: + out_row["manager_mail"] = deid_upn(out_row["manager_mail"]) + if "ManagerID" in out_row: + out_row["ManagerID"] = deid_guid(out_row["ManagerID"]) + + # PersonId_Normalized + person_id = out_row.get("PersonId", "") + person_id_norm = person_id.strip().lower() if person_id else "" + out_row["PersonId_Normalized"] = person_id_norm + + # UserKey (INT surrogate; assigned in Entra-file order) + if person_id_norm: + user_key = mint_user_key(user_key_map, person_id_norm) + out_row["UserKey"] = str(user_key) + else: + out_row["UserKey"] = "" + + # License Status (mirrors PBIP DAX exactly). + # We also normalize Has license to canonical TRUE/FALSE so existing + # measures that filter `[Has license] = "FALSE"` match regardless + # of source casing. + # + # 3-file input mode: when a separate licensing file is supplied, the + # per-user license value comes from that file (keyed on normalized + # UPN). When it is NOT supplied, the value is read from the + # (combined) Entra row exactly as before -> byte-identical 2-file + # output. + if license_map is not None: + has_license_raw = license_map.get(person_id_norm, "") + else: + has_license_raw = out_row.get("Has license", "") + normalized_has_license = normalize_has_license(has_license_raw) + if license_map is not None: + if person_id_norm and person_id_norm in license_map: + matched_in_licensing += 1 + if (has_license_raw or "").strip().upper() in _LICENSE_TRUTHY: + pax_licensed += 1 + else: + pax_unlicensed += 1 + elif has_license_col is None: + no_license_col += 1 + elif (has_license_raw or "").strip().upper() in _LICENSE_TRUTHY: + pax_licensed += 1 + else: + pax_unlicensed += 1 + out_row["Has license"] = normalized_has_license + out_row["License Status"] = compute_license_status(normalized_has_license) + + # TotalEmployees (matches M-code: row count repeated per row) + out_row["TotalEmployees"] = str(total_rows) + + # Org/manager hierarchy columns (always emitted; blank if no UserKey). + uk_str = out_row.get("UserKey", "") + if uk_str: + hrec = hier_by_uk.get(int(uk_str)) + if hrec: + for hc in _HIER_COLUMNS: + out_row[hc] = hrec.get(hc, "") + + # AIO exact-case aliases. Copied AFTER the de-identification transforms above so a + # deidentified run carries the transformed value and never the original. An alias + # already supplied by the directory export is preserved rather than overwritten + # with a blank source value. + if profile == "aio": + if not out_row.get(_AIO_EMAIL_CANONICAL, ""): + out_row[_AIO_EMAIL_CANONICAL] = out_row.get(_AIO_EMAIL_SOURCE, "") + + writer.writerow(out_row) + + # Build fact-lookup dict (dedupe on normalized key, last-wins matches + # M-code Table.Distinct behavior on the licensed-users path). + if person_id_norm: + user_lookup[person_id_norm] = { + "Has license": normalized_has_license, + "License Status": out_row["License Status"], + } + seen_normalized_keys.add(person_id_norm) + + if not quiet: + print(f" Entra rows: {total_rows:,}") + print(f" Unique users (norm): {len(seen_normalized_keys):,}") + if license_map is not None: + print(f" Licensing file: {licensing_csv}") + print(f" Licensing rows: {licensing_rows:,}") + print(f" Matched to users: {matched_in_licensing:,}") + print(f" Licensed: {pax_licensed:,}") + print(f" Unlicensed: {pax_unlicensed:,}") + elif has_license_col: + print(f" License col detected: '{has_license_col}'") + print(f" Licensed (PAX): {pax_licensed:,}") + print(f" Unlicensed (PAX): {pax_unlicensed:,}") + else: + print(f" License col detected: NO RECOGNIZED LICENSE COLUMN FOUND IN ENTRA CSV") + print(f" Fallback: every user will be tagged 'Unlicensed' until a recognized column is present.") + + rows.close() + return user_lookup + + +# --------------------------------------------------------------------------- +# Fact row explosion + output +# --------------------------------------------------------------------------- + + +def explode_record( + audit_data: dict[str, Any], + user_lookup: dict[str, dict[str, str]], + user_key_map: dict[str, int], + thread_key_map: dict[str, int], + profile: str, +) -> list[dict[str, Any]]: + creation_time_raw = audit_data.get("CreationTime") + creation_time_raw_str = to_text(creation_time_raw).strip() + # Cached bundle: 4 derived date strings in one shot, keyed on the raw + # timestamp string (~K distinct values across N records). + creation_date_str, interaction_date_str, week_start_str, month_start_str = ( + _date_strings_for_raw(creation_time_raw_str) + ) + app_identity_app_id, app_identity_display = app_identity_values(audit_data) + agent_id = to_text(audit_data.get("AgentId")) + agent_name = derive_agent_name(audit_data.get("AgentName"), app_identity_display, app_identity_app_id) + + ced = audit_data.get("CopilotEventData") + if not isinstance(ced, dict): + return [] + + prompts = prompt_messages(ced) + if not prompts: + return [] + + resources = resource_rows(ced) + real_resource_count = sum(1 for item in get_array(ced, "AccessedResources") if isinstance(item, dict)) + resource_count_value = real_resource_count if real_resource_count > 0 else 1 + first_context = first_dict_item(get_array(ced, "Contexts")) + first_plugin = first_dict_item(get_array(ced, "AISystemPlugin")) + first_model = first_dict_item(get_array(ced, "ModelTransparencyDetails")) + + audit_user_id_raw = to_text(audit_data.get("UserId")) + if not _is_human_upn(audit_user_id_raw): + return [] + # Deidentify (no-op unless --deidentify) AFTER the human-UPN filter so the filter + # sees the original; every downstream UserKey/Audit_UserId/join derives from the + # hashed value, keeping it consistent with the (also-hashed) Users dim. + audit_user_id_raw = deid_upn(audit_user_id_raw) + audit_user_id_norm = normalize_user_id(audit_user_id_raw) + # UserKey INT surrogate. If this audit user wasn't in Entra, mint a new + # INT and stash so subsequent rows for the same user reuse it. The + # caller tracks unmatched-vs-Entra via the lookup membership check. + if audit_user_id_norm: + user_key = mint_user_key(user_key_map, audit_user_id_norm) + else: + user_key = "" + # ThreadId INT surrogate. deid_guid is a no-op unless --deidentify; under + # --deidentify it returns a deterministic, format-preserving token (same raw + # ThreadId -> same token across runs) so the INT-surrogate keying, the + # ThreadId_Raw output column, and cross-run append dedup stay consistent. + thread_id_raw = deid_guid(to_text(ced.get("ThreadId"))) + if thread_id_raw: + thread_key = thread_key_map.get(thread_id_raw) + if thread_key is None: + thread_key = len(thread_key_map) + 1 + thread_key_map[thread_id_raw] = thread_key + else: + thread_key = "" + app_host_str = to_text(ced.get("AppHost")) + sens_label_str = to_text(ced.get("SensitivityLabelId")) + ctx_type_str = to_text(first_context.get("Type")) if first_context else "" + plugin_id_str = to_text(first_plugin.get("Id")) if first_plugin else "" + model_name_str = to_text(first_model.get("ModelName")) if first_model else "" + + # User-level lookups (constant per record) + user_rec = user_lookup.get(audit_user_id_norm) or {} + has_license_raw = user_rec.get("Has license", "") + license_status = user_rec.get("License Status") or compute_license_status(has_license_raw) + environment = compute_environment(profile, has_license_raw, agent_name, agent_id, app_host_str) + ai_model = compute_ai_model(model_name_str) + user_month_key = compute_user_month_key(audit_user_id_raw, month_start_str) + + is_aibv = profile != "aio" + + # Per-record constants hoisted out of the (prompt x resource) inner loop. + # All grain values are pre-stringified via to_text() exactly once so the + # rollup loop can use the tuple directly as the dict key. + user_key_text = to_text(user_key) + thread_key_text = to_text(thread_key) + agent_title_id = derive_agent_title_id(agent_id) + aisystem_plugin_name_str = to_text(first_plugin.get("Name")) if first_plugin else "" + in_entra = (audit_user_id_norm in user_lookup) if audit_user_id_norm else True + # AIBV-only per-record constants. + agent_publish_status = compute_agent_publish_status(agent_id, agent_name) if is_aibv else "" + # has_agent gate for the Behavior_Category "logic app" branch (AIBV). + has_agent_ctx = bool(agent_name.strip()) or bool(agent_id.strip()) + + # Stable portion of the nongrain dict (everything that does NOT depend on + # the per-resource fields). Built once per record; copied per emitted row + # and updated with the resource-varying keys. Common keys first; AIBV-only + # keys appended only for the aibv profile so the AIO output stays the exact + # Column set. + base_nongrain: dict[str, Any] = { + "CreationDate": creation_date_str, + "WeekStart": week_start_str, + "MonthStart": month_start_str, + "UserMonthKey": user_month_key, + "Has license": has_license_raw, + "Resource_Count": resource_count_value, + "SensitivityLabelId": sens_label_str, + # AccessedResource_* injected per-resource below. + "AccessedResource_Type": "", + "AccessedResource_Action": "", + "AccessedResource_SiteUrl": "", + "AccessedResource_SensitivityLabelId": "", + "AppIdentity_DisplayName": app_identity_display, + "AISystemPlugin_Id": plugin_id_str, + "ModelTransparencyDetails_ModelName": model_name_str, + "Agent_TitleID": agent_title_id, + "Message_isPrompt": "TRUE", + # Behavior_Source / Value_Outcome injected per-resource below. + "Behavior_Source": "", + "Value_Outcome": "", + "ActivityDate": interaction_date_str, + # Stable, deid-consistent user identity (AIO parity with the + # AIBV [Audit_UserId_Normalized] value). Emitted only for the AIO profile + # (AIBV's header carries Audit_UserId_Normalized instead), so for AIBV this + # key is a harmless extra that its fact-header selection ignores. + "User_Id_Normalized": audit_user_id_norm, + # Cross-run append reconciliation key (trailing). Constant per record; + # Message_Id_Raw (per message) is injected in the emit loop below. + "ThreadId_Raw": thread_id_raw, + } + if is_aibv: + base_nongrain.update({ + # M1: UPN passthrough (raw mirrors AIBV [Audit_UserId]; normalized for joins). + "Audit_UserId": audit_user_id_raw, + "Audit_UserId_Normalized": audit_user_id_norm, + # Agent Filter injected per-resource; Agent Publish Status constant per record. + "Agent Filter": "", + "Agent Publish Status": agent_publish_status, + # Downstream chain + ROI baseline + remaining calc cols (per-resource). + "Behavior_Enriched_Full": "", + "Usage_Mode": "", + "Expertise_Role": "", + "Efficiency_Breakdown": "", + "Human_Baseline_Min": "", + "Behavior_Plausible": "", + "Delegation_Event_Key": "", + }) + + # Output schema: list of tuples + # (grain_tuple, message_id_str, nongrain_dict, in_entra, audit_user_id_norm) + # consumed directly by run_processor's rollup loop. The grain arity differs + # by profile (AIO 16 keys; AIBV 19 — the 3 promoted sliceable flags). + rows: list[tuple[tuple[str, ...], str, dict[str, Any], bool, str]] = [] + for message in prompts: + # deid_guid is a no-op unless --deidentify (then deterministic + format- + # preserving), so message_id doubles as the raw Message_Id_Raw dedup key + # AND the stable mid_to_int surrogate key that aligns with --seed-mid-map + # across runs. + message_id = deid_guid(to_text(message.get("Id"))) + for resource in resources: + res_type_str = to_text(resource.get("Type")) + res_action_str = to_text(resource.get("Action")) + res_site_str = to_text(resource.get("SiteUrl")) + res_sens_label_str = to_text(resource.get("SensitivityLabelId")) + behavior_category = compute_behavior_category( + profile, app_host_str, ctx_type_str, res_type_str, res_action_str, + res_site_str, plugin_id_str, has_agent_ctx, + ) + behavior_enriched = compute_behavior_enriched( + profile, behavior_category, agent_name, environment + ) + is_sensitive_str = compute_is_sensitive(sens_label_str, res_sens_label_str) + behavior_source = compute_behavior_source( + profile, behavior_category, environment, agent_name, + aisystem_plugin_name_str, app_host_str, + ) + value_outcome = compute_value_outcome( + profile, behavior_enriched, environment, is_sensitive_str, + ) + + nongrain = dict(base_nongrain) + nongrain["Message_Id_Raw"] = message_id + nongrain["AccessedResource_Type"] = res_type_str + nongrain["AccessedResource_Action"] = res_action_str + nongrain["AccessedResource_SiteUrl"] = deid_resource(res_site_str) + nongrain["AccessedResource_SensitivityLabelId"] = res_sens_label_str + nongrain["Behavior_Source"] = behavior_source + nongrain["Value_Outcome"] = value_outcome + + common_grain = ( + user_key_text, + interaction_date_str, + agent_id, + agent_name, + app_host_str, + environment, + license_status, + ctx_type_str, + behavior_category, + behavior_enriched, + ai_model, + is_sensitive_str, + ) + + if is_aibv: + # AIBV-faithful per-resource flags (3 are grain keys). + is_agent_activity_str = compute_is_agent_activity( + agent_name, agent_id, app_host_str, res_type_str + ) + web_grounded_str = compute_web_grounded_signal(res_type_str, res_site_str) + # Autonomy_Pattern depends on per-resource Is_Agent_Activity (AIBV). + autonomy_pattern = compute_autonomy_pattern( + profile, environment, is_agent_activity_str + ) + # Downstream chain (faithful without Agents 365 per F2). + behavior_enriched_full = compute_behavior_enriched_full(behavior_enriched) + usage_mode = compute_usage_mode(behavior_enriched_full, environment, app_host_str) + expertise_role = compute_expertise_role(behavior_enriched_full) + efficiency_breakdown = compute_efficiency_breakdown( + behavior_enriched_full, behavior_category + ) + human_baseline_min = compute_human_baseline_min(behavior_enriched_full) + behavior_plausible = compute_behavior_plausible(license_status, behavior_category) + workflow_action = compute_workflow_action( + behavior_enriched_full, res_action_str, app_host_str + ) + delegation_event_key = compute_delegation_event_key( + audit_user_id_raw, interaction_date_str, agent_name, + workflow_action, app_host_str, + ) + grain_tuple = common_grain + ( + autonomy_pattern, + app_identity_app_id, + aisystem_plugin_name_str, + thread_key_text, + is_agent_activity_str, + web_grounded_str, + workflow_action, + ) + nongrain["Agent Filter"] = "Agents" if is_agent_activity_str == "TRUE" else "" + nongrain["Behavior_Enriched_Full"] = behavior_enriched_full + nongrain["Usage_Mode"] = usage_mode + nongrain["Expertise_Role"] = expertise_role + nongrain["Efficiency_Breakdown"] = efficiency_breakdown + nongrain["Human_Baseline_Min"] = human_baseline_min + nongrain["Behavior_Plausible"] = behavior_plausible + nongrain["Delegation_Event_Key"] = delegation_event_key + else: + # AIO: autonomy keyed purely off Environment; 16-key grain. + autonomy_pattern = compute_autonomy_pattern(profile, environment, "") + grain_tuple = common_grain + ( + autonomy_pattern, + app_identity_app_id, + aisystem_plugin_name_str, + thread_key_text, + ) + + rows.append((grain_tuple, message_id, nongrain, in_entra, audit_user_id_norm)) + + return rows + + +# --------------------------------------------------------------------------- +# Pre-aggregated tables (AIBV only) — offload the DAX calculated tables. +# +# Each of these replaces a DAX calc table that SUMMARIZEs the ENTIRE fact on +# every refresh. We compute them ONCE here from the same rollup that produces +# the fact CSV, so they equal exactly what the existing DAX would compute when +# evaluated over the rolled-up fact (internally consistent — a reviewer can +# keep the DAX calc tables over the rollup fact and get the same numbers). +# +# Grain-key index map (AIBV profile): see GRAIN_KEYS_AIBV. +# [1]=InteractionDate [3]=AgentName [6]=License Status +# The remaining inputs come from the nongrain dict +# (MonthStart, WeekStart, Audit_UserId, Behavior_Enriched_Full, Usage_Mode). +# --------------------------------------------------------------------------- + +_VALUEFOCUS_MODES = frozenset({"4 - Producing", "5 - Delegating"}) + + +def _percentile_inc(sorted_vals: list[float], p: float) -> float: + """PERCENTILE.INC / PERCENTILEX.INC — linear interpolation, p in [0, 1].""" + n = len(sorted_vals) + if n == 0: + return 0.0 + if n == 1: + return sorted_vals[0] + rank = p * (n - 1) + lo = int(rank) # floor (rank is non-negative) + if lo + 1 >= n: + return sorted_vals[lo] + frac = rank - lo + return sorted_vals[lo] + (sorted_vals[lo + 1] - sorted_vals[lo]) * frac + + +def _usage_rank(avg_ppw: float, p90: float, p75: float, p50: float, p25: float) -> str: + if avg_ppw == 0: + return "0. No Usage" + if avg_ppw >= p90: + return "5. Top 10% Users" + if avg_ppw >= p75: + return "4. 75-90% Users" + if avg_ppw >= p50: + return "3. 50-75% Users" + if avg_ppw >= p25: + return "2. 25-50% Users" + return "1. Bottom 25% Users" + + +def _user_stage(active_days: int, behavior_count: int, value_focus_share: float, has_agent: bool) -> str: + if active_days >= 15 or (active_days >= 10 and value_focus_share >= 0.30 and has_agent): + return "4 - Power" + if active_days >= 8 and behavior_count >= 5: + return "3 - Habitual" + if active_days >= 3 and behavior_count >= 3: + return "2 - Developing" + return "1 - Beginner" + + +def _activity_segment(avg_days: float) -> str: + if avg_days == 0: + return "0. No Activity" + if avg_days <= 5: + return "1. 1-5 Chat Days/Month - 'Infrequent'" + if avg_days <= 10: + return "2. 6-10 Chat Days/Month - 'Moderate'" + if avg_days <= 19: + return "3. 11-19 Chat Days/Month - 'Frequent'" + return "4. 20+ Chat Days/Month - 'Daily'" + + +def _fmt_float(x: float) -> str: + """Shortest round-trippable float, integers without trailing '.0'.""" + if x == int(x): + return str(int(x)) + return repr(x) + + +def compute_and_write_aggregates( + rollup: dict[tuple[Any, ...], dict[str, Any]], + agg_paths: dict[str, str], + quiet: bool = False, +) -> dict[str, int]: + """Build the 5 AIBV pre-aggregated tables from the rollup and write them. + + Returns {table_name: row_count}. `agg_paths` keys: + active_days, user_month_metrics, licensed_rankings, + unlicensed_rankings, licensed_summary. + """ + # Per (Audit_UserId, MonthStart) accumulators. + um: dict[tuple[str, str], dict[str, Any]] = {} + # Per Audit_UserId accumulators (for the rankings). + ua: dict[str, dict[str, Any]] = {} + + for grain_key, nongrain in rollup.items(): + gk = grain_key[0] # rollup key is ((grain_tuple), mid_int) + mid = grain_key[1] + interaction_date = gk[1] + agent_name = gk[3] + license_status = gk[6] + uid = nongrain["Audit_UserId"] + month = nongrain["MonthStart"] + week = nongrain["WeekStart"] + bef = nongrain["Behavior_Enriched_Full"] + usage_mode = nongrain["Usage_Mode"] + + mk = (uid, month) + a = um.get(mk) + if a is None: + a = um[mk] = { + "idates": set(), "mids": set(), "behaviors": set(), + "has_agent": False, "rows": 0, "valuefocus": 0, "license": license_status, + } + a["idates"].add(interaction_date) + a["mids"].add(mid) + a["behaviors"].add(bef) + if agent_name.strip(): + a["has_agent"] = True + a["rows"] += 1 + if usage_mode in _VALUEFOCUS_MODES: + a["valuefocus"] += 1 + if license_status < a["license"]: # MIN(License Status), lexicographic (matches DAX MIN) + a["license"] = license_status + + u = ua.get(uid) + if u is None: + u = ua[uid] = {"rows": 0, "weeks": set(), "license": license_status} + u["rows"] += 1 + u["weeks"].add(week) + if license_status < u["license"]: + u["license"] = license_status + + # ---- ActiveDaysSummary (filter ChatActiveDays > 0; always true here) ---- + ads_rows: list[tuple[str, str, int, int, str]] = [] + for (uid, month), a in um.items(): + chat_active_days = len(a["idates"]) + if chat_active_days <= 0: + continue + ads_rows.append((uid, month, chat_active_days, len(a["mids"]), a["license"])) + ads_rows.sort(key=lambda r: (r[0], r[1])) + + # ---- UserMonthMetrics ---- + umm_rows: list[tuple] = [] + for (uid, month), a in um.items(): + active_days = len(a["idates"]) + behavior_count = len(a["behaviors"]) + value_focus_share = (a["valuefocus"] / a["rows"]) if a["rows"] else 0.0 + has_agent = a["has_agent"] + user_month_key = f"{uid}|{month[:7]}" if (uid and month) else "" + stage = _user_stage(active_days, behavior_count, value_focus_share, has_agent) + umm_rows.append(( + uid, month, behavior_count, "True" if has_agent else "False", + active_days, user_month_key, stage, value_focus_share, + )) + umm_rows.sort(key=lambda r: (r[0], r[1])) + + # ---- Rankings (per user, partitioned by license) ---- + def _build_rankings(target_license: str) -> list[tuple]: + summary = [] # (uid, total_prompts, total_weeks, avg_ppw) + for uid, u in ua.items(): + if u["license"] != target_license: + continue + total_prompts = u["rows"] + total_weeks = len(u["weeks"]) + avg_ppw = (total_prompts / total_weeks) if total_weeks else 0.0 + summary.append((uid, total_prompts, total_weeks, avg_ppw)) + avgs = sorted(s[3] for s in summary) + p90 = _percentile_inc(avgs, 0.90) + p75 = _percentile_inc(avgs, 0.75) + p50 = _percentile_inc(avgs, 0.50) + p25 = _percentile_inc(avgs, 0.25) + out = [] + for uid, tp, tw, avg in summary: + out.append((uid, _usage_rank(avg, p90, p75, p50, p25), tp, tw, avg)) + out.sort(key=lambda r: r[0]) + return out + + licensed_rank_rows = _build_rankings("M365 Copilot Licensed") + unlicensed_rank_rows = _build_rankings("Unlicensed") + + # ---- Licensed Chat User Summary (from ActiveDaysSummary, licensed only) ---- + lsum: dict[str, dict[str, int]] = {} + for uid, month, chat_active_days, prompt_count, lic in ads_rows: + if lic != "M365 Copilot Licensed": + continue + s = lsum.get(uid) + if s is None: + s = lsum[uid] = {"days": 0, "months": 0, "prompts": 0} + s["days"] += chat_active_days + s["months"] += 1 # every ADS row already has ChatActiveDays > 0 + s["prompts"] += prompt_count + summary_rows: list[tuple] = [] + for uid, s in lsum.items(): + total_days = s["days"] + total_months = s["months"] + total_prompts = s["prompts"] + avg_days = (total_days / total_months) if total_months else 0.0 + summary_rows.append(( + uid, _activity_segment(avg_days), total_days, total_months, + total_prompts, avg_days, + )) + summary_rows.sort(key=lambda r: r[0]) + + # ---- write all 5 ---- + def _write(path: str, header: list[str], rows: list[tuple], float_cols: set[int]) -> int: + with open(path, "w", encoding="utf-8", newline="") as f: + w = csv.writer(f, lineterminator="\n") + w.writerow(header) + for r in rows: + w.writerow([_fmt_float(v) if i in float_cols else v for i, v in enumerate(r)]) + return len(rows) + + counts = {} + counts["active_days"] = _write( + agg_paths["active_days"], + ["Audit_UserId", "MonthStart", "ChatActiveDays", "PromptCount", "LicenseStatus"], + ads_rows, set(), + ) + counts["user_month_metrics"] = _write( + agg_paths["user_month_metrics"], + ["Audit_UserId", "MonthStart", "BehaviorCount", "HasAgent", "ActiveDays", + "UserMonthKey", "UserStage", "ValueFocusShare"], + umm_rows, {7}, + ) + counts["licensed_rankings"] = _write( + agg_paths["licensed_rankings"], + ["Audit_UserId", "Usage Rank", "TotalPrompts", "TotalWeeks", "AvgPromptsPerWeek"], + licensed_rank_rows, {4}, + ) + counts["unlicensed_rankings"] = _write( + agg_paths["unlicensed_rankings"], + ["Audit_UserId", "Usage Rank", "TotalPrompts", "TotalWeeks", "AvgPromptsPerWeek"], + unlicensed_rank_rows, {4}, + ) + counts["licensed_summary"] = _write( + agg_paths["licensed_summary"], + ["Audit_UserId", "Activity Segment", "TotalActiveDays", "TotalMonths", + "TotalPrompts", "AvgActiveDaysPerMonth"], + summary_rows, {5}, + ) + + if not quiet: + print(" Pre-aggregated tables (ValueLens):") + print(f" ActiveDaysSummary: {counts['active_days']:,} rows") + print(f" UserMonthMetrics: {counts['user_month_metrics']:,} rows") + print(f" Licensed User Rankings: {counts['licensed_rankings']:,} rows") + print(f" Unlicensed User Rankings: {counts['unlicensed_rankings']:,} rows") + print(f" Licensed User Summary: {counts['licensed_summary']:,} rows") + + return counts + + +def run_processor( + purview_csv: str, + entra_csv: str, + fact_out_csv: str, + users_out_csv: str, + profile: str = "aibv", + agg_paths: dict[str, str] | None = None, + quiet: bool = False, + licensing_csv: str | None = None, + seed_mid_map_path: str | None = None, + seed_thread_map_path: str | None = None, + seed_userkey_map_path: str | None = None, +) -> dict[str, Any]: + start_time = time.perf_counter() + stats: dict[str, Any] = { + "input_records": 0, + "skipped_non_copilot": 0, + "output_rows": 0, + "errors": 0, + "unmatched_users": 0, + } + + profile_label = "ValueLens" if profile == "aibv" else "AI-in-One" + + if not quiet: + print(f"Purview CopilotInteraction Processor v{SCRIPT_VERSION}") + print(f" Profile: {profile_label}") + print(f" JSON engine: {_JSON_ENGINE}") + print(f" Purview input: {purview_csv}") + print(f" Entra input: {entra_csv}") + if licensing_csv: + print(f" Licensing: {licensing_csv}") + print(f" Purview output: {fact_out_csv}") + print(f" Entra output: {users_out_csv}") + print() + print("Loading Entra users + writing Users dim CSV...") + + # Shared INT-surrogate maps. UserKey is populated first by the Entra + # loader (so Entra-known users get the lowest INTs / lowest dictionary + # offsets in VertiPaq); the fact path then reuses + extends the map. + # mid_to_int is declared here (alongside the other two) so all three can be + # pre-seeded from the PAX append-target maps before the Entra load + fact loop. + user_key_map: dict[str, int] = {} + thread_key_map: dict[str, int] = {} + mid_to_int: dict[str, int] = {} + + # PAX append support: pre-seed the + # three INT-surrogate maps from JSON snapshots of the append-target's existing + # surrogates so retained rows keep stable Message_Id / ThreadKey / UserKey + # values across runs. No-op when the seed paths are None (the standalone + # 2/3-file runs and any non-append PAX run). + def _load_int_seed(path: str, target: dict[str, int]) -> None: + with open(path, "r", encoding="utf-8") as f: + data = json_loads(f.read()) + if not isinstance(data, dict): + return + for k, v in data.items(): + try: + target[str(k)] = int(v) + except (TypeError, ValueError): + continue + + if seed_userkey_map_path: + _load_int_seed(seed_userkey_map_path, user_key_map) + if seed_thread_map_path: + _load_int_seed(seed_thread_map_path, thread_key_map) + if seed_mid_map_path: + _load_int_seed(seed_mid_map_path, mid_to_int) + reset_user_key_allocator(user_key_map) + + user_lookup = load_entra_and_write_users( + entra_csv, users_out_csv, user_key_map, licensing_csv=licensing_csv, quiet=quiet, profile=profile + ) + + if not quiet: + print() + print("Flattening CopilotInteraction records...") + + # One row per (grain x distinct Message_Id). Per-resource accumulation + # is intentionally avoided here so counts are not inflated ~2.25x by + # per (prompt x AccessedResource) iteration. Downstream measures use + # DISTINCTCOUNT(Message_Id) for exact parity with the semantic-model + # definitions. + # + # Message_Id is INT-surrogated (1-based, encounter order) for CSV size + # and parse-time win on the highest-cardinality column. + # + # Key: (grain_tuple, message_id_int) + # Value: dict of non-grain attrs (last-write-wins on a per-resource + # basis for AccessedResource_* / SensitivityLabelId — same + # semantic as the prior dict-overwrite behavior). + rollup: dict[tuple[Any, ...], dict[str, Any]] = {} + unmatched: set[str] = set() + + with open(purview_csv, "r", encoding="utf-8-sig", newline="") as fin: + reader = csv.DictReader(fin) + + for raw_row in reader: + stats["input_records"] += 1 + + audit_raw = raw_row.get("AuditData", "") or "" + try: + audit_data = json_loads(audit_raw) if audit_raw.strip() else {} + except Exception: + stats["errors"] += 1 + continue + + if not isinstance(audit_data, dict): + stats["errors"] += 1 + continue + + if not is_copilot_interaction(audit_data, raw_row): + stats["skipped_non_copilot"] += 1 + continue + + try: + rows = explode_record(audit_data, user_lookup, user_key_map, thread_key_map, profile) + except Exception: + stats["errors"] += 1 + continue + + for grain_key, message_id_str, nongrain, in_entra, audit_user_norm in rows: + stats["output_rows"] += 1 + if not in_entra and audit_user_norm: + unmatched.add(audit_user_norm) + mid_int = mid_to_int.get(message_id_str) + if mid_int is None: + mid_int = len(mid_to_int) + 1 + mid_to_int[message_id_str] = mid_int + rollup[(grain_key, mid_int)] = nongrain + + if not quiet: + print(f" Input records: {stats['input_records']:,}") + print(f" Skipped (non-Copilot): {stats['skipped_non_copilot']:,}") + print(f" Raw prompt rows: {stats['output_rows']:,}") + print(f" Errors: {stats['errors']:,}") + print() + print("Writing rolled-up fact CSV...") + + # Profile-specific output schema (AIO = 36-col; AIBV = 50-col superset). + grain_keys, nongrain_attrs_sel, fact_header = schema_for(profile) + with open(fact_out_csv, "w", encoding="utf-8", newline="") as fout: + writer = csv.writer(fout, lineterminator="\n") + writer.writerow(fact_header) + # Pre-compute the index of Message_Id within FACT_HEADER so we can + # splice the INT surrogate into a list-of-attrs in one shot. The + # list-based csv.writer.writerow path is materially faster than + # DictWriter (skips dict-to-list translation + per-row genexpr). + nongrain_attrs = nongrain_attrs_sel # local rebind + grain_len = len(grain_keys) + for (grain_key, mid_int), attrs in rollup.items(): + # fact_header = grain_keys + ("Message_Id",) + nongrain_attrs + row_out = list(grain_key) + row_out.append(mid_int) + row_out.extend(attrs[k] for k in nongrain_attrs) + writer.writerow(row_out) + + stats["output_rows_rollup"] = len(rollup) + stats["distinct_message_ids"] = len(mid_to_int) + stats["distinct_thread_ids"] = len(thread_key_map) + stats["distinct_user_keys"] = len(user_key_map) + stats["unmatched_users"] = len(unmatched) + + # Pre-aggregated tables (AIBV profile only). These offload the DAX + # calculated tables (ActiveDaysSummary / UserMonthMetrics / rankings / + # summary), each of which otherwise SUMMARIZEs the whole fact on refresh. + if profile != "aio" and agg_paths: + if not quiet: + print() + print("Writing pre-aggregated tables...") + compute_and_write_aggregates(rollup, agg_paths, quiet=quiet) + elapsed = time.perf_counter() - start_time + if not quiet: + reduction_pct = (1 - len(rollup) / stats["output_rows"]) * 100 if stats["output_rows"] else 0 + print(f" Rollup rows: {len(rollup):,} ({reduction_pct:.1f}% reduction)") + print(f" Distinct Message_Ids: {len(mid_to_int):,}") + print(f" Distinct ThreadIds: {len(thread_key_map):,}") + fact_user_keys = {k[0][0] for k in rollup.keys()} + print(f" Users in dimension: {len(user_key_map):,}") + print(f" Users represented in fact: {len(fact_user_keys):,}") + print(f" Unmatched users: {stats['unmatched_users']:,}") + print(f" Elapsed: {elapsed:.2f}s") + + return stats + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + f"Purview CopilotInteraction Processor v{SCRIPT_VERSION} - " + "Two/three-input, two-output preprocessor that produces a rolled-up " + "Interactions fact CSV (~85% row reduction via PromptCount grain) " + "and a Users dim CSV for the ValueLens Dashboard PBIP." + ) + ) + parser.add_argument( + "--purview", + required=True, + help="Path to the raw Purview audit log CSV (must contain AuditData column).", + ) + parser.add_argument( + "--entra", + required=True, + help=( + "Path to the Entra users CSV (UPN + org columns). The license column " + "is optional: supply it separately via --licensing (recommended for " + "standalone use), or omit --licensing to use a single combined " + "users+licensing file (license column auto-detected)." + ), + ) + parser.add_argument( + "--licensing", + default=None, + help=( + "Path to a separate licensing CSV (UPN + 'Has License' columns), e.g. " + "the Microsoft Admin Center Copilot user export. When provided, " + "--entra is treated as a users-only file and license status is merged " + "in from this file (the 3-file workflow). Omit to use a single " + "combined Entra file (license column auto-detected). Applies to both " + "ValueLens and AI-in-One profiles." + ), + ) + parser.add_argument( + "--combined-entra", + action="store_true", + default=False, + help=( + "Legacy 2-file mode: assert that --entra is a single combined " + "users+licensing file (as produced by the PAX script). Mutually " + "exclusive with --licensing. Optional - even without this flag a " + "combined file still works (the license column is auto-detected)." + ), + ) + parser.add_argument( + "--out-dir", + "-o", + default=None, + help="Directory for output files. Default: same directory as the Purview file.", + ) + parser.add_argument( + "--profile", + "-p", + choices=("aibv", "aio"), + default="aibv", + help=( + "Output profile. ValueLens Dashboard (default; internally selected by PAX) " + "superset (50-col fact, 3-value Environment). 'aio' = AI-in-One " + "Dashboard (36-col fact, 5-value Environment) — reproduces the " + "AIO output exactly." + ), + ) + parser.add_argument( + "--quiet", + "-q", + action="store_true", + default=False, + help="Suppress progress output.", + ) + parser.add_argument( + "--with-aggregates", + action="store_true", + default=False, + help=( + "Also write the ValueLens pre-aggregated tables (ActiveDaysSummary, " + "UserMonthMetrics, Licensed/Unlicensed user rankings, Licensed user " + "summary). OFF by default — the ValueLens template only needs the two core " + "rollup files (Interactions + Users). No effect for --profile aio." + ), + ) + parser.add_argument( + # Deprecated no-op: aggregates are now OFF by default, so this flag does + # nothing. Kept so older command lines don't error. + "--no-aggregates", + action="store_true", + default=False, + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--deidentify", + action="store_true", + default=False, + help=( + "One-way hash all identifying values (UPNs, names, manager fields, " + "Entra/mailbox GUIDs and SIDs, resource URLs) for anonymous reporting. " + "Deterministic and format-preserving, so manager links and " + "UserKey/Users joins are kept; irreversible (no decode map)." + ), + ) + parser.add_argument( + # PAX append support. + "--seed-mid-map", + default=None, + help=( + "Optional JSON file mapping {Message_Id_Raw -> existing INT surrogate} " + "extracted from the target Fact CSV. Pre-seeds mid_to_int so cross-run " + "appends preserve Message_Id INTs and dedup source rows on Message_Id_Raw." + ), + ) + parser.add_argument( + "--seed-thread-map", + default=None, + help=( + "Optional JSON file mapping {ThreadId_Raw -> existing INT surrogate} " + "extracted from the target Fact CSV. Pre-seeds thread_key_map so cross-run " + "appends preserve ThreadId INTs." + ), + ) + parser.add_argument( + "--seed-userkey-map", + default=None, + help=( + "Optional JSON file mapping {PersonId_Normalized -> existing UserKey INT} " + "extracted from the merged Users CSV. Pre-seeds user_key_map so Entra users " + "carried forward from prior runs keep their UserKey across the append." + ), + ) + parser.add_argument( + "--hierarchy-fill", + choices=("none", "self", "manager", "fixed"), + default="none", + help=( + "Filler for org-hierarchy level slots deeper than a user's own level " + "(Users dim, AIO/ValueLens). 'none' (default) leaves them blank; 'self' " + "repeats the user; 'manager' repeats the user's manager; 'fixed' uses " + "--hierarchy-fill-label. The hierarchy columns are always emitted." + ), + ) + parser.add_argument( + "--hierarchy-fill-label", + default="", + help="Literal label used when '--hierarchy-fill fixed' is selected.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {SCRIPT_VERSION}", + ) + + args = parser.parse_args() + + global _DEIDENTIFY + _DEIDENTIFY = bool(args.deidentify) + + global _HIER_FILL_MODE, _HIER_FILL_LABEL + _HIER_FILL_MODE = args.hierarchy_fill + _HIER_FILL_LABEL = args.hierarchy_fill_label or "" + if _HIER_FILL_MODE == "fixed" and not _HIER_FILL_LABEL: + print( + 'ERROR: --hierarchy-fill fixed requires --hierarchy-fill-label "".', + file=sys.stderr, + ) + sys.exit(1) + if _HIER_FILL_MODE != "fixed" and _HIER_FILL_LABEL: + print( + "ERROR: --hierarchy-fill-label is only valid with --hierarchy-fill fixed.", + file=sys.stderr, + ) + sys.exit(1) + + if args.licensing and args.combined_entra: + print( + "ERROR: --licensing and --combined-entra are mutually exclusive. Use " + "--licensing for the 3-file (separate licensing) workflow, or " + "--combined-entra (or neither) for a single combined Entra file.", + file=sys.stderr, + ) + sys.exit(1) + + purview_path = os.path.abspath(args.purview) + entra_path = os.path.abspath(args.entra) + for label, p in (("Purview", purview_path), ("Entra", entra_path)): + if not os.path.isfile(p): + print(f"ERROR: {label} input file not found: {p}", file=sys.stderr) + sys.exit(1) + + licensing_path = os.path.abspath(args.licensing) if args.licensing else None + if licensing_path and not os.path.isfile(licensing_path): + print(f"ERROR: Licensing input file not found: {licensing_path}", file=sys.stderr) + sys.exit(1) + + out_dir = Path(os.path.abspath(args.out_dir)) if args.out_dir else Path(purview_path).parent + out_dir.mkdir(parents=True, exist_ok=True) + + purview_stem = Path(purview_path).stem + entra_stem = Path(entra_path).stem + run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + # PAX embed (the standalone timestamps these two): + # the core rollup outputs use NON-timestamped names so PAX's post-run + # Merge-FactCsv / Merge-UsersCsv resolve them by exact name. The run timestamp + # is already baked into the input filenames (Purview_Audit_*_.csv, + # EntraUsers_MAClicensing_.csv), so it is not duplicated on the output. + fact_out = str(out_dir / f"{purview_stem}_Interactions.csv") + users_out = str(out_dir / f"{entra_stem}_Users.csv") + + # Pre-aggregated table paths (AIBV only, opt-in via --with-aggregates). + # Default: not written — the AIBV template consumes only the Interactions + # + Users rollups. The aggregates remain available for the future + # calc-table offload (point the 5 DAX calc tables at these CSVs). + agg_paths: dict[str, str] | None = None + if args.profile != "aio" and args.with_aggregates: + agg_paths = { + "active_days": str(out_dir / f"{purview_stem}_ActiveDaysSummary_{run_ts}.csv"), + "user_month_metrics": str(out_dir / f"{purview_stem}_UserMonthMetrics_{run_ts}.csv"), + "licensed_rankings": str(out_dir / f"{purview_stem}_LicensedUserRankings_{run_ts}.csv"), + "unlicensed_rankings": str(out_dir / f"{purview_stem}_UnlicensedUserRankings_{run_ts}.csv"), + "licensed_summary": str(out_dir / f"{purview_stem}_LicensedUserSummary_{run_ts}.csv"), + } + + stats = run_processor( + purview_csv=purview_path, + entra_csv=entra_path, + fact_out_csv=fact_out, + users_out_csv=users_out, + profile=args.profile, + agg_paths=agg_paths, + quiet=args.quiet, + licensing_csv=licensing_path, + seed_mid_map_path=args.seed_mid_map, + seed_thread_map_path=args.seed_thread_map, + seed_userkey_map_path=args.seed_userkey_map, + ) + sys.exit(1 if stats["errors"] > 0 else 0) + + +if __name__ == "__main__": + main() +'@ +# <<< END-EMBEDDED-COPILOT-PROCESSOR + +# >>> BEGIN-EMBEDDED-M365-PROCESSOR +$Script:EMBEDDED_PROCESSOR_M365 = @' +#!/usr/bin/env python3 +""" +Purview M365 Usage Bundle Explosion Processor v2.6.2 +===================================================== +Two-mode processor for Purview audit log CSV exports: + + ROLLUP MODE (default): Aggregates exploded events into rolled-up rows keyed by + (UserId, CreationDate, Operation, Workload, SourceFileExtension, AppHost, + AgentId, AgentName, ContextType) + with EventCount, MIN(CreationTime), MAX(CreationTime), IsAgentInteraction. + Targets 80%+ row reduction for Power BI ingestion. + Streaming — no exploded rows held in memory. + + After the rollup CSV is written, a second pass streams through it to produce + two additional analytics files (unless --no-userstats is specified): + - UserStats: One row per user with 66 columns of pre-computed metrics + (Copilot/M365 event counts, tier classifications, priority + scores, usage ranks, active-day counts, activity segments). + - SessionCohort: One row per (UserId, App) pair with a session-count bucket + (1-5, 6-10, 11-20, 21-40, 41-60, 61-80, 81+). + + These files allow Power Query to join pre-computed results instead of + recalculating expensive DAX/M expressions, cutting dashboard load times. + + EVENT-LEVEL MODE (--mode event-level): v1-compatible 153-column explosion output. + For debugging and reconciliation. + UserStats and SessionCohort files are NOT generated in this mode. + +Requirements: + Python 3.9+ + pip install orjson (OPTIONAL - 5-10x faster JSON parsing; falls back to stdlib json) + +Usage: + # (A) Single PAX / PowerShell export: + python Purview_M365_Usage_Bundle_Explosion_Processor.py --pax + + # (B) Manual 4-pull export from Purview Audit: + python Purview_M365_Usage_Bundle_Explosion_Processor.py \ + --teams --outlook --files --copilot + + Common optional flags: + --output-dir Where to write outputs (default: input folder) + --skip-precompute Skip UserStats + SessionCohort + --reconcile Sample-based correctness check + --prompt-filter Prompt|Response|Both|Null + --debug-events v1-compatible 153-column event-level CSV + --quiet Suppress progress output + +Output files (rollup mode — all share the same timestamp): + _Rollup_.csv 13 columns — aggregated events + agent fields + _UserStats_.csv 66 columns — per-user metrics + _SessionCohort_.csv 3 columns — (UserId, App, Bucket) + _SessionStats_.csv 8 columns — (UserId, Date, AppHost, + SessionCount, PromptCount, + AgentPromptCount, + ResponseCount, AgentSessionCount) + matches AI in One DISTINCTCOUNT(ThreadId); + AgentPromptCount = prompts on agent-flagged + threads + ( = input file's stem for single input, or '_Combined' for multi-input. + Rename the output file or use --output-dir if you want a tenant-specific name.) + +Output file (event-level mode): + _Exploded_.csv 153 columns — one row per event + +Arguments: + --input, -i Path to Purview audit log CSV (required). + --output-dir, -o Directory for output files (default: input file's directory). + --mode, -m Processing mode: rollup (default) or event-level. + --reconcile Run sample-based reconciliation after rollup processing. + --prompt-filter Filter Copilot messages: Prompt|Response|Both|Null. + --no-userstats Skip UserStats and SessionCohort generation (rollup only). + --quiet, -q Suppress progress output (only errors are printed). + --version Show version and exit. + +Examples: + # Default rollup (13-column output + UserStats + SessionCohort) + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv + + # Combine the validated 4-pull bundle (Teams + Outlook + Files + Copilot) in one run + python Purview_M365_Usage_Bundle_Explosion_Processor.py \ + -i Teams_Export.csv Outlook_Export.csv Files_Export.csv Copilot_Export.csv \ + --combined-stem ZavaCorp_2025_11 + + # Rollup with output in a different directory + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --output-dir ./output + + # Rollup only — skip UserStats and SessionCohort generation + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --no-userstats + + # v1-compatible event-level explosion (153-column output) + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --mode event-level + + # Rollup with sample-based reconciliation check + python Purview_M365_Usage_Bundle_Explosion_Processor.py -i Purview_Export.csv --reconcile + +Validated 4-pull strategy (Purview Audit → Activities filter, type+click each chip): + Teams (7d): MessageSent, MessageRead, ChatCreated, TeamsSessionStarted, + MeetingParticipantDetail + Outlook (30d): MailItemsAccessed, Send, MailboxLogin + Files (60d): FileAccessed, FileModified, FileDownloaded, FileUploaded + Copilot (30d): CopilotInteraction, AIAppInteraction (filter by record type) + +Author: Microsoft Copilot Growth ROI Advisory Team (copilot-roi-advisory-team-gh@microsoft.com) +""" + +from __future__ import annotations + +import argparse +import bisect +import csv +import hashlib +import hmac +import os +import random +import re +import sys +import time +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor, as_completed, wait, FIRST_COMPLETED +from datetime import datetime, timezone, date, timedelta +from pathlib import Path +from typing import Any + +# ─── Fast JSON: prefer orjson, fall back to stdlib ─────────────────────────── +try: + import orjson + + def json_loads(s: str | bytes) -> Any: + if isinstance(s, str): + s = s.encode("utf-8") + return orjson.loads(s) + + def json_dumps_compact(obj: Any) -> str: + return orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS).decode("utf-8") + + _JSON_ENGINE = "orjson" +except ImportError: + import json as _json + + def json_loads(s: str | bytes) -> Any: # type: ignore[misc] + if isinstance(s, bytes): + s = s.decode("utf-8") + return _json.loads(s) + + def json_dumps_compact(obj: Any) -> str: # type: ignore[misc] + return _json.dumps(obj, separators=(",", ":"), default=str) + + _JSON_ENGINE = "json (stdlib)" + +# ═════════════════════════════════════════════════════════════════════════════ +# CONSTANTS +# ═════════════════════════════════════════════════════════════════════════════ + +SCRIPT_VERSION = "2.6.2" + +EXPLOSION_PER_RECORD_ROW_CAP = 1000 +STREAMING_CHUNK_SIZE = 5000 + +# Unified 153-column header matching Power BI M code schema exactly. +# Order matches the #"Changed Type" step in M365Usage.tmdl. +# AuditData is intentionally excluded — raw JSON is never written to output. +M365_UNIFIED_HEADER: list[str] = [ + "RecordId", "CreationDate", "RecordType", "Operation", "UserId", + "AssociatedAdminUnits", "AssociatedAdminUnitsNames", + "@odata.type", "CreationTime", "Id", "OrganizationId", + "ResultStatus", "UserKey", "UserType", "Version", "Workload", + "ClientIP", "ObjectId", "AzureActiveDirectoryEventType", + "ActorContextId", "ActorIpAddress", "InterSystemsId", "IntraSystemId", + "SupportTicketId", "TargetContextId", "ApplicationId", + "DeviceProperties.OS", "DeviceProperties.BrowserType", + "ErrorNumber", + "SiteUrl", "SourceRelativeUrl", "SourceFileName", "SourceFileExtension", + "ListId", "ListItemUniqueId", "WebId", "ApplicationDisplayName", "EventSource", + "ItemType", "SiteSensitivityLabelId", "GeoLocation", "IsManagedDevice", + "DeviceDisplayName", "ListBaseType", "ListServerTemplate", + "AuthenticationType", "Site", "DoNotDistributeEvent", "HighPriorityMediaProcessing", + "BrowserName", "BrowserVersion", "CorrelationId", "Platform", "UserAgent", + "ActorInfoString", "AppId", "AuthType", "ClientAppId", "ClientIPAddress", + "ClientInfoString", "ExternalAccess", "InternalLogonType", "LogonType", + "LogonUserSid", "MailboxGuid", "MailboxOwnerSid", "MailboxOwnerUPN", + "OrganizationName", "OriginatingServer", "SessionId", + "TokenObjectId", "TokenTenantId", "TokenType", "SaveToSentItems", + "OperationCount", "FileSizeBytes", + "MeetingId", "MeetingType", "EventSignature", "EventData", + "Permission", "SensitivityLabelId", "SharingLinkScope", + "TargetUserOrGroupType", "TargetUserOrGroupName", + "MeetingURL", "ChatId", "MessageId", "MessageSizeInBytes", "MessageType", + "FormId", "FormName", "VideoId", "VideoName", "ChannelId", "ViewDuration", + "ClientRegion", "CopilotLogVersion", "TargetId", + "TeamName", "TeamGuid", "ResponseId", "IsAnonymous", "DeviceType", + "ChannelName", "ChannelGuid", "ChannelType", "AppName", "EnvironmentName", + "PlanId", "PlanName", "TaskId", "TaskName", "PercentComplete", + "CrossMailboxOperation", + "RecordTypeNum", "ResultStatus_Audit", + "ModelId", "ModelProvider", "ModelFamily", + "TokensTotal", "TokensInput", "TokensOutput", "DurationMs", "OutcomeStatus", + "ConversationId", "TurnNumber", "RetryCount", "ClientVersion", "ClientPlatform", + "AgentId", "AgentName", "AgentVersion", "AgentCategory", "ApplicationName", + "AppHost", "ThreadId", + "Context_Id", "Context_Type", + "Message_Id", "Message_isPrompt", + "AccessedResource_Action", "AccessedResource_PolicyDetails", "AccessedResource_SiteUrl", + "AISystemPlugin_Id", "AISystemPlugin_Name", + "ModelTransparencyDetails_ModelName", "MessageIds", + "AccessedResource_Name", "AccessedResource_SensitivityLabel", + "AccessedResource_ResourceType", "SensitivityLabel", "Context_Item", +] + +# Rollup output header (13 columns) — matches M365Usage.tmdl fingerprint: +# required keys + EventCount + temporal MIN/MAX + agent telemetry. +ROLLUP_HEADER: list[str] = [ + "UserId", "CreationDate", "Operation", "Workload", + "SourceFileExtension", "AppHost", + "EventCount", "ItemsAccessedCount", "CreationTime", "MaxCreationTime", + "AgentId", "AgentName", "ContextType", "IsAgentInteraction", +] + +# Reconciliation sample size +RECONCILE_SAMPLE_SIZE = 10_000 + +# ── Operation canonicalization ────────────────────────────────────── +# Legacy/wrong names that have appeared in older exports or older DAX models. +# Renamed at intake so historical data merges cleanly with current canonical pulls. +OP_RENAME: dict[str, str] = { + "FileViewed": "FileAccessed", + "MeetingParticipantJoined": "MeetingParticipantDetail", + "ConnectedAIAppInteraction": "AIAppInteraction", +} + +# ── UserStats classification sets (match Power Query logic exactly) ────────── +WORD_EXTS: set[str] = {"docx", "doc", "dotx"} +EXCEL_EXTS: set[str] = {"xlsx", "xls", "xlsm", "csv"} +PPT_EXTS: set[str] = {"pptx", "ppt", "ppsx"} +OFFICE_EXTS: set[str] = WORD_EXTS | EXCEL_EXTS | PPT_EXTS + +# Canonical 14 ops required by the CLO TMDL DAX measures, validated against MS Learn. +FILE_OPS: set[str] = { + "FileAccessed", # canonical (was FileViewed in legacy) + "FileModified", + "FileDownloaded", + "FileUploaded", +} +OUTLOOK_OPS: set[str] = {"Send", "MailItemsAccessed", "MailboxLogin"} # active-DAY + COUNT +TEAMS_OPS: set[str] = { + "MessageSent", # Msgs Sent + "MessageRead", "ChatCreated", # Msgs Read (Graph-API tenants emit ChatCreated) + "MeetingParticipantDetail", # canonical (was MeetingParticipantJoined) + "TeamsSessionStarted", # Meetings/calls fallback +} +COPILOT_OPS: set[str] = {"CopilotInteraction", "AIAppInteraction"} # AIAppInteraction = agents/connected apps + +# AppHost values that indicate an agent / connected-app interaction. +AGENT_APPHOSTS: set[str] = {"agent", "copilotstudio", "declarativeagent", "customengineagent"} + +# ── DAX-aligned op/ext sets (for the CE/LP precomputed columns) ── +# These mirror the exact filters in the PBIT measures Word/Excel/PowerPoint/Outlook/ +# Teams Activity *V2, Copilot All Apps Total, and CE Copilot Percentile. +# Important: ops are matched AFTER OP_RENAME canonicalization, so the legacy names +# ("FileViewed", "MeetingParticipantJoined") are listed under their canonical aliases. +DAX_FILE_OPS: set[str] = { + "FileAccessed", # canonical of legacy FileViewed (DAX checks both) + "FilePreviewed", + "FileModified", + "FileDownloaded", + "FileUploaded", +} +DAX_OUTLOOK_OPS: set[str] = {"Send", "MailItemsAccessed"} # MailboxLogin intentionally excluded (matches DAX) +DAX_TEAMS_OPS: set[str] = { + "MessageSent", "MessageRead", "MessagesListed", "ChatRetrieved", + "MeetingParticipantDetail", # canonical of MeetingParticipantJoined + "MeetingStarted", "MeetingEnded", "TeamsSessionStarted", +} + +USERSTATS_HEADER: list[str] = [ + "UserId", + "CopilotEC", "M365EC", "ExCopEC", "ExM365EC", + "IsCopilotUser", "CopilotTierColumn", "M365TierColumn", + "PriorityScatterColumn", "ExcelPriority", + "CopilotUsageRankColumn", "M365UsageRankColumn", + "TeamsActiveDays", "OutlookActiveDays", "WordActiveDays", + "ExcelActiveDays", "PowerPointActiveDays", + "TeamsActivityCount", "OutlookActivityCount", "OfficeFilesActivityCount", + "TeamsActivitySegment", "OutlookActivitySegment", "WordActivitySegment", + "ExcelActivitySegment", "PowerPointActivitySegment", + "OfficeFilesActivitySegment", "OverallM365ActivitySegment", + # ── Precomputed raw activity counts + CE percentile ranks per window. + # Windows: _L30 = trailing 30 days ending at max(CreationDate); _L60 = trailing 60; + # _Full = entire data range. Filters match the corresponding DAX measures exactly + # (post-canonicalization). CE ranks are integer 0-100; blank when raw is 0. + "TeamsRaw_L30", "TeamsRaw_L60", "TeamsRaw_Full", + "OutlookRaw_L30", "OutlookRaw_L60", "OutlookRaw_Full", + "WordRaw_L30", "WordRaw_L60", "WordRaw_Full", + "ExcelRaw_L30", "ExcelRaw_L60", "ExcelRaw_Full", + "PowerPointRaw_L30", "PowerPointRaw_L60", "PowerPointRaw_Full", + "CopilotChatRaw_L30", "CopilotChatRaw_L60", "CopilotChatRaw_Full", + "CERank_Teams_L30", "CERank_Teams_L60", "CERank_Teams_Full", + "CERank_Outlook_L30", "CERank_Outlook_L60", "CERank_Outlook_Full", + "CERank_Word_L30", "CERank_Word_L60", "CERank_Word_Full", + "CERank_Excel_L30", "CERank_Excel_L60", "CERank_Excel_Full", + "CERank_PowerPoint_L30", "CERank_PowerPoint_L60", "CERank_PowerPoint_Full", + "CERank_M365AllApps_L30", "CERank_M365AllApps_L60", "CERank_M365AllApps_Full", + "CECopilotPercentile_L30", "CECopilotPercentile_L60", "CECopilotPercentile_Full", +] + +# Percentile window codes used in column names. Order matters for writer. +RANK_WINDOWS: tuple[str, ...] = ("L30", "L60", "Full") + +SESSIONCOHORT_HEADER: list[str] = ["UserId", "AppColumn", "SessionCohort"] + +# SessionStats — AI in One parity. Per (UserId, CreationDate, AppHost) we count +# DISTINCT ThreadIds (matches Microsoft AI in One `Sessions` measure), plus prompt / +# response counts and an agent-only thread count. License filtering happens downstream +# in DAX via the EntraUsers relationship; this CSV stays license-agnostic. +# AgentPromptCount — exact chat vs agent split at message-tally time +# (uses the same is_agent flag the rollup already determines per record). +SESSIONSTATS_HEADER: list[str] = [ + "UserId", "CreationDate", "AppHost", + "SessionCount", "PromptCount", "AgentPromptCount", + "ResponseCount", "AgentSessionCount", +] + +# Date formats accepted for CreationDate normalization (broadest to narrowest) +_CREATION_DATE_FORMATS: tuple[str, ...] = ( + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%m/%d/%Y %H:%M:%S", + "%Y-%m-%d", + "%m/%d/%Y", +) + +# GroupKey type: adds agent_id, agent_name, context_type so multi-agent users +# don't collapse rows together. IsAgentInteraction is derived on write from AgentId. +# (user_id_lower, creation_date_normalized, operation, workload, sfe_lower, app_host, +# agent_id, agent_name, context_type) +GroupKey = tuple[str, str, str, str, str, str, str, str, str] + + +class RollupAccum: + """Lightweight accumulator for one rollup group — avoids dataclass import overhead.""" + __slots__ = ( + "event_count", "items_accessed_count", + "min_creation_time", "max_creation_time", + "original_user_id", + "is_agent_interaction", + ) + + def __init__( + self, + event_count: int, + items_accessed: int, + min_ct: str, + max_ct: str, + original_uid: str, + is_agent: bool = False, + ) -> None: + self.event_count = event_count + self.items_accessed_count = items_accessed + self.min_creation_time = min_ct + self.max_creation_time = max_ct + self.original_user_id = original_uid # first-seen casing for output + self.is_agent_interaction = is_agent + + +# SessionStats group key: (uid_lower, creation_date, app_host). +SessionKey = tuple[str, str, str] + + +class SessionAccum: + """Per-(user, date, app_host) Copilot session accumulator. + + Mirrors the AI in One `Sessions` measure: DISTINCTCOUNT(ThreadId) where at least + one message in the thread is a user prompt (isPrompt=True). Threads with only + AI responses (no user prompt) are excluded — same as the AI in One filter. + """ + __slots__ = ( + "thread_ids", "agent_thread_ids", + "prompt_count", "agent_prompt_count", "response_count", + "original_user_id", + ) + + def __init__(self, original_uid: str) -> None: + self.thread_ids: set[str] = set() + self.agent_thread_ids: set[str] = set() + self.prompt_count: int = 0 + self.agent_prompt_count: int = 0 # prompts on agent-flagged threads + self.response_count: int = 0 + self.original_user_id = original_uid + + +def normalize_creation_date(raw: str) -> str: + """Parse any Purview date format → 'YYYY-MM-DDT00:00:00.000Z' (midnight UTC).""" + if not raw or not isinstance(raw, str): + return "" + raw = raw.strip() + if not raw: + return "" + for fmt in _CREATION_DATE_FORMATS: + try: + dt = datetime.strptime(raw, fmt) + return dt.strftime("%Y-%m-%d") + "T00:00:00.000Z" + except ValueError: + continue + # Fallback: try extracting date portion from ISO-like string + if len(raw) >= 10 and raw[4:5] == "-": + return raw[:10] + "T00:00:00.000Z" + return raw # unparseable — pass through + + +def _norm_key_str(val: Any) -> str: + """Normalize a string value for use as a rollup key: strip whitespace, empty if None.""" + if val is None: + return "" + if not isinstance(val, str): + val = str(val) + val = val.strip() + if val.lower() in ("null", "none"): + return "" + return val + + +# ═════════════════════════════════════════════════════════════════════════════ +# UTILITY FUNCTIONS +# ═════════════════════════════════════════════════════════════════════════════ + +def safe_get(obj: Any, key: str) -> Any: + """Safely retrieve a property from a dict-like object.""" + if obj is None: + return None + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def select_first_non_null(values: list[Any]) -> Any: + """Return the first non-None, non-empty-string value.""" + for v in values: + if v is not None and v != "": + return v + return None + + +def to_num(val: Any) -> float | None: + """Convert to number, return None on failure.""" + if val is None: + return None + if isinstance(val, (int, float)): + return float(val) + if isinstance(val, str): + val = val.strip() + if not val: + return None + try: + return float(val) + except (ValueError, TypeError): + return None + return None + + +def format_date_purview(val: Any) -> str: + """Format a date value to ISO 8601 UTC string.""" + if val is None: + return "" + if isinstance(val, str): + val = val.strip() + if not val: + return "" + # Try common Purview date formats + for fmt in ( + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y %I:%M:%S %p", + "%m/%d/%Y %H:%M:%S", + ): + try: + dt = datetime.strptime(val, fmt) + return dt.replace(tzinfo=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + except ValueError: + continue + return val # Return as-is if no format matches + return str(val) + + +def to_json_if_object(val: Any) -> str: + """Serialize non-scalars to compact JSON, pass scalars through as strings.""" + if val is None: + return "" + if isinstance(val, (str, int, float, bool)): + return str(val) + try: + return json_dumps_compact(val) + except Exception: + return "" + + +def bool_tf(val: Any) -> str: + """Convert bool-like value to 'TRUE'/'FALSE' string.""" + if val is None: + return "" + if isinstance(val, bool): + return "TRUE" if val else "FALSE" + if isinstance(val, str): + low = val.strip().lower() + if low in ("true", "1", "yes"): + return "TRUE" + if low in ("false", "0", "no"): + return "FALSE" + return str(val) + + +def get_array_fast(obj: Any, key: str) -> list: + """Extract an array property, always returning a list.""" + if obj is None: + return [] + val = safe_get(obj, key) + if val is None: + return [] + if isinstance(val, list): + return val + if isinstance(val, (str, int, float, bool)): + return [] + try: + return list(val) + except (TypeError, ValueError): + return [] + + +# ═════════════════════════════════════════════════════════════════════════════ +# AGENT CATEGORIZATION +# ═════════════════════════════════════════════════════════════════════════════ + +def categorize_agent(agent_id: Any) -> str: + """Categorize agent based on AgentId pattern.""" + if not agent_id or not isinstance(agent_id, str): + return "" + if agent_id.startswith("CopilotStudio.Declarative."): + return "Declarative Agent" + if agent_id.startswith("CopilotStudio.CustomEngine."): + return "Custom Engine Agent" + if agent_id.startswith("P_"): + return "Declarative Agent (Purview)" + return "Other Agent" + + +# ═════════════════════════════════════════════════════════════════════════════ +# USERSTATS CLASSIFICATION & COMPUTATION HELPERS +# ═════════════════════════════════════════════════════════════════════════════ + +def is_copilot(op: str, wl: str) -> bool: + """True if the row represents a Copilot or agent / connected-app event.""" + return wl == "Copilot" or op in COPILOT_OPS + +def is_excel_file_op(ext: str, op: str) -> bool: + """True if the row is a file operation on an Excel-family extension.""" + return (ext or "").lower() in EXCEL_EXTS and op in FILE_OPS + + +def app_column(ext: str, op: str, wl: str) -> str: + """Classify a row into an application column for session cohort grouping.""" + e = (ext or "").lower() + if e in WORD_EXTS and op in FILE_OPS: + return "Word" + if e in EXCEL_EXTS and op in FILE_OPS: + return "Excel" + if e in PPT_EXTS and op in FILE_OPS: + return "PowerPoint" + if wl == "Exchange" and op in OUTLOOK_OPS: + return "Outlook" + if wl == "MicrosoftTeams" and op in TEAMS_OPS: + return "Teams" + if wl == "Copilot" or op == "CopilotInteraction": + return "Copilot" + return "M365 All Apps" + + +def percentile_inc(values: list[float], p: float) -> float: + """Inclusive linear interpolation — matches PQ List.Percentile and numpy 'linear'.""" + if not values: + return 0.0 + sorted_vals = sorted(values) + n = len(sorted_vals) + idx = p * (n - 1) + lo, hi = int(idx), min(int(idx) + 1, n - 1) + return sorted_vals[lo] + (idx - lo) * (sorted_vals[hi] - sorted_vals[lo]) + + +def tier_fn(cnt: float, p90: float, p75: float, p50: float, zero_is_bottom: bool) -> str: + """Assign a percentile-tier label.""" + if zero_is_bottom and cnt == 0: + return "Bottom 50%" + if cnt >= p90: + return "Top 10%" + if cnt >= p75: + return "10-25%" + if cnt >= p50: + return "25-50%" + return "Bottom 50%" + + +def priority_fn(m365_tier: str, cop_tier: str) -> str: + """Map (M365 tier, Copilot tier) pair to a priority label.""" + top2 = {"Top 10%", "10-25%"} + if m365_tier in top2 and cop_tier in top2: + return "Promoter" + if m365_tier == "Top 10%" and cop_tier == "25-50%": + return "High" + if m365_tier == "Top 10%" and cop_tier == "Bottom 50%": + return "Critical" + if m365_tier == "10-25%" and cop_tier == "25-50%": + return "Medium" + if m365_tier == "10-25%" and cop_tier == "Bottom 50%": + return "High" + if m365_tier in {"25-50%", "Bottom 50%"} and cop_tier in top2: + return "Promoter" + if m365_tier == "25-50%" and cop_tier == "25-50%": + return "Medium" + if m365_tier == "25-50%" and cop_tier == "Bottom 50%": + return "Medium" + return "Low" + + +def seg_fn(days: int, window_days: int) -> str: + """Map active-day count to a per-week engagement-segment label. + + Normalizes to active days per week so labels mean the same thing regardless + of pull length: <1, 1-2, 3-4, 5+ days/week. Window_days is the calendar + span (max date - min date + 1) of the rolled-up data; 0 yields No Usage. + """ + if window_days <= 0 or days <= 0: + return "0. No Usage" + rate = days * 7.0 / window_days + if rate < 1.0: + return "1. <1 Day/Week (Light)" + if rate < 3.0: + return "2. 1-2 Days/Week (Moderate)" + if rate < 5.0: + return "3. 3-4 Days/Week (Frequent)" + return "4. 5+ Days/Week (Daily)" + + +def compute_ranks(values_by_uid: dict[str, float]) -> dict[str, int]: + """ + 0-based ascending rank via stable sort. Ties get different sequential + indices (matches PQ Table.Sort + Table.AddIndexColumn behaviour). + """ + sorted_uids = sorted(values_by_uid.items(), key=lambda x: x[1]) + return {uid: i for i, (uid, _) in enumerate(sorted_uids)} + + +# ═════════════════════════════════════════════════════════════════════════════ +# ROLLUP KEY EXTRACTION (lightweight — no row dicts built) +# ═════════════════════════════════════════════════════════════════════════════ + +def _compute_copilot_event_count( + ced: dict, + operation: str, + prompt_filter: str | None, +) -> int: + """ + Compute the number of exploded rows a Copilot record would produce, + using the same array-length logic as v1 (explode_copilot_record), + but WITHOUT materializing any row dicts. + Returns 0 if prompt_filter eliminates all messages (record skipped). + """ + messages = get_array_fast(ced, "Messages") + contexts = get_array_fast(ced, "Contexts") + resources = get_array_fast(ced, "AccessedResources") + plugins_raw = get_array_fast(ced, "AISystemPlugin") + model_det_raw = get_array_fast(ced, "ModelTransparencyDetails") + sensitivity_labels = get_array_fast(ced, "SensitivityLabels") + + # Prompt filtering (same logic as v1 lines 571-581) + if prompt_filter: + pf_lower = prompt_filter.lower() + if pf_lower == "null": + messages = [m for m in messages if safe_get(m, "isPrompt") is None] + elif pf_lower == "both": + messages = [m for m in messages if safe_get(m, "isPrompt") is not None] + elif pf_lower == "prompt": + messages = [m for m in messages if safe_get(m, "isPrompt") is True] + elif pf_lower == "response": + messages = [m for m in messages if safe_get(m, "isPrompt") is False] + if not messages: + return 0 # record filtered out entirely + + # Context items max for CopilotInteraction + context_items_max: int = 0 + if operation == "CopilotInteraction" and contexts: + for ctx in contexts: + if ctx: + items = get_array_fast(ctx, "Items") + if items and len(items) > context_items_max: + context_items_max = len(items) + + if prompt_filter: + row_count = max(1, len(messages)) + else: + array_counts = [ + 1, len(messages), len(contexts), len(resources), + len(sensitivity_labels), len(plugins_raw), len(model_det_raw), + ] + if context_items_max > 0: + array_counts.append(context_items_max) + row_count = max(array_counts) + + return min(max(row_count, 1), EXPLOSION_PER_RECORD_ROW_CAP) + + +def _count_mail_items_accessed(audit_data: dict) -> int: + """Items represented by one MailItemsAccessed event. + Sums len(Folders[].FolderItems[]) when present; falls back to 1 (Bind-style).""" + folders = audit_data.get("Folders") + if isinstance(folders, list): + total = 0 + for fld in folders: + if not isinstance(fld, dict): + continue + fi = fld.get("FolderItems") + if isinstance(fi, list): + total += len(fi) + if total > 0: + return total + return 1 + + +# Non-human/system identities found in Purview audit logs (Teams Sync, SharePoint app, +# SupervisoryReview bots, ServicePrincipals, NT-style accounts, SIDs, bare GUIDs, etc.). +# These have no matching userPrincipalName in EntraUsers and would render as blank +# User/Department rows in license-recommendation visuals. Filter out at the rollup stage. +_UPN_LOCAL_RE = re.compile(r"^[^\s\\@]+$") +_BARE_GUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I) + + +def _is_human_upn(uid: str) -> bool: + """True iff uid is a syntactically valid human UPN (local@domain.tld), excluding + well-known service/bot patterns (SupervisoryReview{...}@..., bare GUIDs).""" + if not uid: + return False + s = uid.strip() + if _BARE_GUID_RE.match(s): + return False + if s.lower().startswith("supervisoryreview{"): + return False + if "@" not in s or s.count("@") != 1: + return False + local, domain = s.split("@", 1) + if not _UPN_LOCAL_RE.match(local): + return False + if "." not in domain or not domain or domain.startswith(".") or domain.endswith("."): + return False + return True + + +# --------------------------------------------------------------------------- +# Deidentification (--deidentify): one-way, salted, format-preserving. +# OFF by default; enabled by main() setting the module flag from --deidentify +# (and propagated to explosion worker processes via _deid_init_worker). Every +# PII value becomes a deterministic token so relationships (user joins, +# distinct-resource counts) are preserved while identities are removed. +# Irreversible (no decode map). The SAME salt + algorithm + formats MUST exist +# verbatim in the PowerShell raw-path deidentifier and the CopilotInteraction +# processor (PAX deidentify spec) so tokens match across engines. +# --------------------------------------------------------------------------- +_DEIDENTIFY: bool = False +_DEID_SALT = b"PAX-Deidentify-Salt-v1-DO-NOT-CHANGE-7f3c1e9b2d846050a1c4e8b3" +_DEID_DOMAIN = "deidentified.domain" +_deid_cache: dict[str, str] = {} + + +def _deid_init_worker(flag: bool) -> None: + """ProcessPoolExecutor initializer: propagate the deidentify flag to workers.""" + global _DEIDENTIFY + _DEIDENTIFY = flag + + +def _deid_hex(value: str, length: int) -> str: + return hmac.new( + _DEID_SALT, value.strip().lower().encode("utf-8"), hashlib.sha256 + ).hexdigest()[:length] + + +def deid_upn(value: str) -> str: + """UPN / email -> <12hex>@deidentified.domain. No-op when off or value empty.""" + if not _DEIDENTIFY or not value: + return value + k = "upn\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + "@" + _DEID_DOMAIN + _deid_cache[k] = v + return v + + +def deid_name(value: str) -> str: + """Person/device display name -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "name\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_guid(value: str) -> str: + """GUID -> deterministic GUID shape xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.""" + if not _DEIDENTIFY or not value: + return value + k = "guid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = f"{h[0:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:32]}" + _deid_cache[k] = v + return v + + +def deid_sid(value: str) -> str: + """SID -> deterministic S-1-5-21---- shape.""" + if not _DEIDENTIFY or not value: + return value + k = "sid\x00" + value + v = _deid_cache.get(k) + if v is None: + h = _deid_hex(value, 32) + v = "S-1-5-21-{0}-{1}-{2}-{3}".format( + int(h[0:8], 16), int(h[8:16], 16), int(h[16:24], 16), int(h[24:32], 16) + ) + _deid_cache[k] = v + return v + + +def deid_token(value: str) -> str: + """Opaque id (employeeId, immutableId) -> <12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "tok\x00" + value + v = _deid_cache.get(k) + if v is None: + v = _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_resource(value: str) -> str: + """Resource URL -> site_<12hex> (whole-string hash; preserves distinct-count).""" + if not _DEIDENTIFY or not value: + return value + k = "res\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "site_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_file(value: str) -> str: + """File / document name -> file_<12hex>.""" + if not _DEIDENTIFY or not value: + return value + k = "file\x00" + value + v = _deid_cache.get(k) + if v is None: + v = "file_" + _deid_hex(value, 12) + _deid_cache[k] = v + return v + + +def deid_proxy(value: str) -> str: + """proxyAddresses entry(ies) -> keep smtp:/SMTP: prefix + deidentified email. + Handles ';'-delimited multi-value fields.""" + if not _DEIDENTIFY or not value: + return value + out = [] + for entry in value.split(";"): + if not entry: + out.append(entry) + elif ":" in entry: + prefix, addr = entry.split(":", 1) + out.append(prefix + ":" + deid_upn(addr)) + else: + out.append(deid_upn(entry)) + return ";".join(out) + + +def _extract_rollup_keys( + record: dict, + audit_data: dict, + ced: dict | None, + prompt_filter: str | None = None, +) -> tuple[GroupKey, int, int, str, str, bool] | None: + """ + Extract rollup group key + event count + items-accessed count + creation_time + + original UserId + is_agent_interaction flag. + + Returns None if the record is filtered out (e.g. prompt_filter eliminates all messages). + Returns: + (group_key, event_count, items_accessed_count, creation_time_iso, + original_user_id, is_agent_interaction) + where group_key uses lowercased UserId for case-insensitive grouping and + includes (agent_id, agent_name, context_type) so multi-agent users don't collapse. + """ + # UserId: original casing preserved for output; lowered for grouping key + raw_uid = _norm_key_str(safe_get(audit_data, "UserId") or record.get("UserId", "")) + # Filter non-human/system identities (Teams Sync, ServicePrincipals, SIDs, bots, etc.) + if not _is_human_upn(raw_uid): + return None + uid_lower = raw_uid.lower() + + # CreationDate: from CSV, normalized to midnight + creation_date = normalize_creation_date(record.get("CreationDate", "")) + + # Operation: from audit_data → CSV fallback, preserve case, then canonicalize via OP_RENAME + operation = _norm_key_str( + safe_get(audit_data, "Operation") or record.get("Operation", "") or record.get("Operations", "") + ) + operation = OP_RENAME.get(operation, operation) + + # Workload: from audit_data, preserve case + workload = _norm_key_str(safe_get(audit_data, "Workload")) + + # SourceFileExtension: lowercased for DAX LOWER() compatibility + sfe = _norm_key_str(safe_get(audit_data, "SourceFileExtension")).lower() + + # AppHost: from CED for Copilot, empty otherwise, preserve case + if ced: + app_host = _norm_key_str( + safe_get(ced, "AppHost") or safe_get(audit_data, "AppHost") + ) + else: + app_host = _norm_key_str(safe_get(audit_data, "AppHost")) + + # Agent telemetry: from CopilotEventData when present, fall back to top-level AuditData fields + agent_id = _norm_key_str( + (safe_get(ced, "AgentId") if ced else None) + or safe_get(audit_data, "AgentId") + ) + agent_name = _norm_key_str( + (safe_get(ced, "AgentName") if ced else None) + or safe_get(audit_data, "AgentName") + ) + context_type = "" + if ced: + contexts = get_array_fast(ced, "Contexts") + if contexts: + # First context wins for the key; multi-context records still collapse + # cleanly because event_count already reflects context-array length. + first_ctx = contexts[0] + if isinstance(first_ctx, dict): + context_type = _norm_key_str(safe_get(first_ctx, "Type")) + + # CreationTime: from audit_data, ISO formatted for lexicographic MIN/MAX + creation_time = format_date_purview(safe_get(audit_data, "CreationTime")) + + # Event count + if ced: + event_count = _compute_copilot_event_count(ced, operation, prompt_filter) + if event_count == 0: + return None # filtered out + else: + event_count = 1 # Non-Copilot: always 1:1 + + # Items accessed count: only meaningful for MailItemsAccessed (Exchange). + items_accessed_count = 0 + if operation == "MailItemsAccessed": + items_accessed_count = _count_mail_items_accessed(audit_data) + + # IsAgentInteraction: TRUE iff AgentId present, AppHost is an agent surface, + # or Operation is an agent op (AIAppInteraction). + is_agent_interaction = bool( + agent_id + or app_host.lower() in AGENT_APPHOSTS + or operation == "AIAppInteraction" + ) + + group_key: GroupKey = ( + uid_lower, creation_date, operation, workload, sfe, app_host, + agent_id, agent_name, context_type, + ) + return ( + group_key, + event_count, + items_accessed_count, + creation_time, + raw_uid, + is_agent_interaction, + ) + + +# ═════════════════════════════════════════════════════════════════════════════ +# PATH A: NON-COPILOT M365 EXTRACTION +# ═════════════════════════════════════════════════════════════════════════════ + + +def _get_nv_prop(nv_list: Any, prop_name: str) -> Any: + """Extract a value from a Name/Value pair list by Name — matches M code's GetNVProp.""" + if not nv_list or not isinstance(nv_list, list): + return None + for item in nv_list: + if isinstance(item, dict) and safe_get(item, "Name") == prop_name: + return safe_get(item, "Value") + return None + + +def _build_unified_row(record: dict, audit_data: dict) -> dict: + """ + Build a complete row dict with all 153 M code columns populated. + Extracts fields from the CSV record and AuditData JSON. + DeviceProperties uses NV-pivot for .OS and .BrowserType only (matches M code). + RecordTypeNum and ResultStatus_Audit are computed aliases. + """ + # CSV-level fields + record_id = ( + record.get("RecordId") + or record.get("Identity") + or record.get("Id") + or safe_get(audit_data, "Id") + or "" + ) + # Read from singular first, fall back to plural for backwards-compatible input + op_val = safe_get(audit_data, "Operation") or record.get("Operation") or record.get("Operations", "") + uid_val = safe_get(audit_data, "UserId") or record.get("UserId") or record.get("UserIds", "") + record_type = record.get("RecordType", "") + result_status = safe_get(audit_data, "ResultStatus") or "" + + # CreationTime formatting + creation_time_raw = safe_get(audit_data, "CreationTime") + creation_time = format_date_purview(creation_time_raw) if creation_time_raw else "" + + # DeviceProperties NV-pivot (matches M code's GetNVProp — only .OS and .BrowserType) + dev_props = safe_get(audit_data, "DeviceProperties") + dp_os = _get_nv_prop(dev_props, "OS") or "" + dp_browser = _get_nv_prop(dev_props, "BrowserType") or "" + + # Computed alias: RecordTypeNum = int(RecordType) + try: + record_type_num = int(record_type) if record_type else "" + except (ValueError, TypeError): + record_type_num = "" + + # ApplicationId with fallback chain + app_id_resolved = select_first_non_null([ + safe_get(audit_data, "ApplicationId"), + safe_get(audit_data, "AppId"), + safe_get(audit_data, "ClientAppId"), + ]) or "" + + # AgentCategory is computed + agent_id_val = safe_get(audit_data, "AgentId") or "" + agent_category = categorize_agent(agent_id_val) if agent_id_val else "" + + row = { + "RecordId": record_id, + "CreationDate": record.get("CreationDate", ""), + "RecordType": record_type, + "Operation": op_val, + "UserId": uid_val, + "AssociatedAdminUnits": record.get("AssociatedAdminUnits", "") or safe_get(audit_data, "AssociatedAdminUnits") or "", + "AssociatedAdminUnitsNames": record.get("AssociatedAdminUnitsNames", "") or safe_get(audit_data, "AssociatedAdminUnitsNames") or "", + "@odata.type": safe_get(audit_data, "@odata.type") or "", + "CreationTime": creation_time, + "Id": safe_get(audit_data, "Id") or "", + "OrganizationId": safe_get(audit_data, "OrganizationId") or "", + "ResultStatus": result_status, + "UserKey": safe_get(audit_data, "UserKey") or "", + "UserType": safe_get(audit_data, "UserType") or "", + "Version": safe_get(audit_data, "Version") or "", + "Workload": safe_get(audit_data, "Workload") or "", + "ClientIP": safe_get(audit_data, "ClientIP") or "", + "ObjectId": safe_get(audit_data, "ObjectId") or "", + "AzureActiveDirectoryEventType": safe_get(audit_data, "AzureActiveDirectoryEventType") or "", + "ActorContextId": safe_get(audit_data, "ActorContextId") or "", + "ActorIpAddress": safe_get(audit_data, "ActorIpAddress") or "", + "InterSystemsId": safe_get(audit_data, "InterSystemsId") or "", + "IntraSystemId": safe_get(audit_data, "IntraSystemId") or "", + "SupportTicketId": safe_get(audit_data, "SupportTicketId") or "", + "TargetContextId": safe_get(audit_data, "TargetContextId") or "", + "ApplicationId": app_id_resolved, + "DeviceProperties.OS": dp_os, + "DeviceProperties.BrowserType": dp_browser, + "ErrorNumber": safe_get(audit_data, "ErrorNumber") or "", + "SiteUrl": safe_get(audit_data, "SiteUrl") or "", + "SourceRelativeUrl": safe_get(audit_data, "SourceRelativeUrl") or "", + "SourceFileName": safe_get(audit_data, "SourceFileName") or "", + "SourceFileExtension": safe_get(audit_data, "SourceFileExtension") or "", + "ListId": safe_get(audit_data, "ListId") or "", + "ListItemUniqueId": safe_get(audit_data, "ListItemUniqueId") or "", + "WebId": safe_get(audit_data, "WebId") or "", + "ApplicationDisplayName": safe_get(audit_data, "ApplicationDisplayName") or "", + "EventSource": safe_get(audit_data, "EventSource") or "", + "ItemType": safe_get(audit_data, "ItemType") or "", + "SiteSensitivityLabelId": safe_get(audit_data, "SiteSensitivityLabelId") or "", + "GeoLocation": safe_get(audit_data, "GeoLocation") or "", + "IsManagedDevice": safe_get(audit_data, "IsManagedDevice") or "", + "DeviceDisplayName": safe_get(audit_data, "DeviceDisplayName") or "", + "ListBaseType": safe_get(audit_data, "ListBaseType") or "", + "ListServerTemplate": safe_get(audit_data, "ListServerTemplate") or "", + "AuthenticationType": safe_get(audit_data, "AuthenticationType") or "", + "Site": safe_get(audit_data, "Site") or "", + "DoNotDistributeEvent": safe_get(audit_data, "DoNotDistributeEvent") or "", + "HighPriorityMediaProcessing": safe_get(audit_data, "HighPriorityMediaProcessing") or "", + "BrowserName": safe_get(audit_data, "BrowserName") or "", + "BrowserVersion": safe_get(audit_data, "BrowserVersion") or "", + "CorrelationId": safe_get(audit_data, "CorrelationId") or "", + "Platform": safe_get(audit_data, "Platform") or "", + "UserAgent": safe_get(audit_data, "UserAgent") or "", + "ActorInfoString": safe_get(audit_data, "ActorInfoString") or "", + "AppId": safe_get(audit_data, "AppId") or "", + "AuthType": safe_get(audit_data, "AuthType") or "", + "ClientAppId": safe_get(audit_data, "ClientAppId") or "", + "ClientIPAddress": safe_get(audit_data, "ClientIPAddress") or "", + "ClientInfoString": safe_get(audit_data, "ClientInfoString") or "", + "ExternalAccess": safe_get(audit_data, "ExternalAccess") or "", + "InternalLogonType": safe_get(audit_data, "InternalLogonType") or "", + "LogonType": safe_get(audit_data, "LogonType") or "", + "LogonUserSid": safe_get(audit_data, "LogonUserSid") or "", + "MailboxGuid": safe_get(audit_data, "MailboxGuid") or "", + "MailboxOwnerSid": safe_get(audit_data, "MailboxOwnerSid") or "", + "MailboxOwnerUPN": safe_get(audit_data, "MailboxOwnerUPN") or "", + "OrganizationName": safe_get(audit_data, "OrganizationName") or "", + "OriginatingServer": safe_get(audit_data, "OriginatingServer") or "", + "SessionId": safe_get(audit_data, "SessionId") or "", + "TokenObjectId": safe_get(audit_data, "TokenObjectId") or "", + "TokenTenantId": safe_get(audit_data, "TokenTenantId") or "", + "TokenType": safe_get(audit_data, "TokenType") or "", + "SaveToSentItems": safe_get(audit_data, "SaveToSentItems") or "", + "OperationCount": safe_get(audit_data, "OperationCount") or "", + "FileSizeBytes": safe_get(audit_data, "FileSizeBytes") or "", + # Teams / Meetings / Chat + "MeetingId": safe_get(audit_data, "MeetingId") or "", + "MeetingType": safe_get(audit_data, "MeetingType") or "", + "EventSignature": safe_get(audit_data, "EventSignature") or "", + "EventData": safe_get(audit_data, "EventData") or "", + "Permission": safe_get(audit_data, "Permission") or "", + "SensitivityLabelId": safe_get(audit_data, "SensitivityLabelId") or "", + "SharingLinkScope": safe_get(audit_data, "SharingLinkScope") or "", + "TargetUserOrGroupType": safe_get(audit_data, "TargetUserOrGroupType") or "", + "TargetUserOrGroupName": safe_get(audit_data, "TargetUserOrGroupName") or "", + "MeetingURL": safe_get(audit_data, "MeetingURL") or "", + "ChatId": safe_get(audit_data, "ChatId") or "", + "MessageId": safe_get(audit_data, "MessageId") or "", + "MessageSizeInBytes": safe_get(audit_data, "MessageSizeInBytes") or "", + "MessageType": safe_get(audit_data, "MessageType") or "", + # Forms + "FormId": safe_get(audit_data, "FormId") or "", + "FormName": safe_get(audit_data, "FormName") or "", + # Video / Stream + "VideoId": safe_get(audit_data, "VideoId") or "", + "VideoName": safe_get(audit_data, "VideoName") or "", + "ChannelId": safe_get(audit_data, "ChannelId") or "", + "ViewDuration": safe_get(audit_data, "ViewDuration") or "", + "ClientRegion": safe_get(audit_data, "ClientRegion") or "", + "CopilotLogVersion": safe_get(audit_data, "CopilotLogVersion") or "", + "TargetId": safe_get(audit_data, "TargetId") or "", + # Teams details + "TeamName": safe_get(audit_data, "TeamName") or "", + "TeamGuid": safe_get(audit_data, "TeamGuid") or "", + "ResponseId": safe_get(audit_data, "ResponseId") or "", + "IsAnonymous": safe_get(audit_data, "IsAnonymous") or "", + "DeviceType": safe_get(audit_data, "DeviceType") or "", + "ChannelName": safe_get(audit_data, "ChannelName") or "", + "ChannelGuid": safe_get(audit_data, "ChannelGuid") or "", + "ChannelType": safe_get(audit_data, "ChannelType") or "", + "AppName": safe_get(audit_data, "AppName") or "", + "EnvironmentName": safe_get(audit_data, "EnvironmentName") or "", + # Planner + "PlanId": safe_get(audit_data, "PlanId") or "", + "PlanName": safe_get(audit_data, "PlanName") or "", + "TaskId": safe_get(audit_data, "TaskId") or "", + "TaskName": safe_get(audit_data, "TaskName") or "", + "PercentComplete": safe_get(audit_data, "PercentComplete") or "", + "CrossMailboxOperation": safe_get(audit_data, "CrossMailboxOperation") or "", + # Computed aliases + "RecordTypeNum": record_type_num, + "ResultStatus_Audit": result_status, + # Copilot model/token fields (populated from CED for Copilot records; from root for non-Copilot) + "ModelId": safe_get(audit_data, "ModelId") or "", + "ModelProvider": safe_get(audit_data, "ModelProvider") or "", + "ModelFamily": safe_get(audit_data, "ModelFamily") or "", + "TokensTotal": safe_get(audit_data, "TokensTotal") or "", + "TokensInput": safe_get(audit_data, "TokensInput") or "", + "TokensOutput": safe_get(audit_data, "TokensOutput") or "", + "DurationMs": safe_get(audit_data, "DurationMs") or "", + "OutcomeStatus": safe_get(audit_data, "OutcomeStatus") or "", + "ConversationId": safe_get(audit_data, "ConversationId") or "", + "TurnNumber": safe_get(audit_data, "TurnNumber") or "", + "RetryCount": safe_get(audit_data, "RetryCount") or "", + "ClientVersion": safe_get(audit_data, "ClientVersion") or "", + "ClientPlatform": safe_get(audit_data, "ClientPlatform") or "", + "AgentId": agent_id_val, + "AgentName": safe_get(audit_data, "AgentName") or "", + "AgentVersion": safe_get(audit_data, "AgentVersion") or "", + "AgentCategory": agent_category, + "ApplicationName": safe_get(audit_data, "ApplicationName") or "", + "SensitivityLabel": safe_get(audit_data, "SensitivityLabel") or "", + # CED sub-fields — empty for non-Copilot records, populated by Copilot path + "AppHost": "", + "ThreadId": "", + "Context_Id": "", + "Context_Type": "", + "Message_Id": "", + "Message_isPrompt": "", + "AccessedResource_Action": "", + "AccessedResource_PolicyDetails": "", + "AccessedResource_SiteUrl": "", + "AISystemPlugin_Id": "", + "AISystemPlugin_Name": "", + "ModelTransparencyDetails_ModelName": "", + "MessageIds": "", + "AccessedResource_Name": "", + "AccessedResource_SensitivityLabel": "", + "AccessedResource_ResourceType": "", + "Context_Item": "", + } + if _DEIDENTIFY: + # Exploded (153-col) identity + resource fields. AccessedResource_SiteUrl/Name on + # Copilot rows are hashed in explode_copilot_record (set there after this returns). + row["UserId"] = deid_upn(row["UserId"]) + row["MailboxOwnerUPN"] = deid_upn(row["MailboxOwnerUPN"]) + row["MailboxGuid"] = deid_guid(row["MailboxGuid"]) + row["LogonUserSid"] = deid_sid(row["LogonUserSid"]) + row["MailboxOwnerSid"] = deid_sid(row["MailboxOwnerSid"]) + row["DeviceDisplayName"] = deid_name(row["DeviceDisplayName"]) + row["SiteUrl"] = deid_resource(row["SiteUrl"]) + row["SourceRelativeUrl"] = deid_resource(row["SourceRelativeUrl"]) + row["SourceFileName"] = deid_file(row["SourceFileName"]) + return row + + +def explode_m365_record(record: dict, audit_data: dict) -> list[dict]: + """ + Extract a non-Copilot M365 record (Path A). + Produces exactly 1 row per record with all 153 M code columns. + No array explosion — M code does not explode non-Copilot arrays. + """ + return [_build_unified_row(record, audit_data)] + + +# ═════════════════════════════════════════════════════════════════════════════ +# PATH B: COPILOT EXPLOSION +# ═════════════════════════════════════════════════════════════════════════════ + +def explode_copilot_record( + record: dict, + audit_data: dict, + ced: dict, + prompt_filter: str | None = None, +) -> list[dict]: + """ + Explode a Copilot record (Path B). + Starts from the unified 153-column base row, then overrides CED-specific fields. + Extracts Messages, Contexts, AccessedResources, AISystemPlugin, + ModelTransparencyDetails, SensitivityLabels and builds N parallel-indexed rows. + """ + # Extract array fields from CopilotEventData + messages = get_array_fast(ced, "Messages") + contexts = get_array_fast(ced, "Contexts") + resources = get_array_fast(ced, "AccessedResources") + plugins_raw = get_array_fast(ced, "AISystemPlugin") + model_det_raw = get_array_fast(ced, "ModelTransparencyDetails") + message_ids = get_array_fast(ced, "MessageIds") + sensitivity_labels = get_array_fast(ced, "SensitivityLabels") + + # Prompt filtering + if prompt_filter: + filtered: list = [] + pf_lower = prompt_filter.lower() + if pf_lower == "null": + filtered = [m for m in messages if safe_get(m, "isPrompt") is None] + elif pf_lower == "both": + filtered = [m for m in messages if safe_get(m, "isPrompt") is not None] + elif pf_lower == "prompt": + filtered = [m for m in messages if safe_get(m, "isPrompt") is True] + elif pf_lower == "response": + filtered = [m for m in messages if safe_get(m, "isPrompt") is False] + messages = filtered + if not messages: + return [] + + # Detect activity type for 2-level explosion + activity_type = safe_get(audit_data, "Operation") or "" + + # Context items max for CopilotInteraction + context_items_max: int = 0 + if activity_type == "CopilotInteraction" and contexts: + for ctx in contexts: + if ctx: + items = get_array_fast(ctx, "Items") + if items and len(items) > context_items_max: + context_items_max = len(items) + + # Calculate row count + if prompt_filter: + row_count = max(1, len(messages)) + else: + array_counts = [ + 1, len(messages), len(contexts), len(resources), + len(sensitivity_labels), len(plugins_raw), len(model_det_raw), + ] + if context_items_max > 0: + array_counts.append(context_items_max) + row_count = max(array_counts) + + row_count = min(row_count, EXPLOSION_PER_RECORD_ROW_CAP) + if row_count < 1: + row_count = 1 + + # ── Build unified base row with all 153 M code columns ─────────────── + base = _build_unified_row(record, audit_data) + + # ── Override CED-specific scalar fields with deep CED extraction ───── + # AppHost: prefer CED → audit_data → Workload + base["AppHost"] = select_first_non_null([ + safe_get(ced, "AppHost"), + safe_get(audit_data, "AppHost"), + safe_get(audit_data, "Workload"), + ]) or "" + + base["ThreadId"] = safe_get(ced, "ThreadId") or "" + + # AgentVersion: prefer audit_data → CED fallbacks + base["AgentVersion"] = select_first_non_null([ + safe_get(audit_data, "AgentVersion"), + safe_get(ced, "AgentVersion"), + safe_get(ced, "Version"), + ]) or "" + + # ApplicationName: prefer audit_data → CED fallbacks + base["ApplicationName"] = select_first_non_null([ + safe_get(audit_data, "ApplicationName"), + safe_get(ced, "HostAppName"), + safe_get(ced, "ClientAppName"), + ]) or "" + + # Model fields from CED with fallbacks + base["ModelId"] = select_first_non_null([ + safe_get(ced, "ModelId"), safe_get(ced, "ModelID"), safe_get(audit_data, "ModelId"), + ]) or "" + base["ModelProvider"] = select_first_non_null([ + safe_get(ced, "ModelProvider"), safe_get(ced, "Provider"), safe_get(ced, "ModelVendor"), + ]) or "" + base["ModelFamily"] = select_first_non_null([ + safe_get(ced, "ModelFamily"), safe_get(ced, "ModelType"), + ]) or "" + + # Token usage from CED + usage_node = select_first_non_null([ + safe_get(ced, "Usage"), safe_get(ced, "TokenUsage"), + safe_get(ced, "Tokens"), safe_get(audit_data, "Usage"), + ]) + tokens_total: Any = None + tokens_input: Any = None + tokens_output: Any = None + if usage_node and isinstance(usage_node, dict): + tokens_total = to_num(select_first_non_null([ + safe_get(usage_node, "Total"), safe_get(usage_node, "TotalTokens"), + safe_get(usage_node, "TokensTotal"), + ])) + tokens_input = to_num(select_first_non_null([ + safe_get(usage_node, "Input"), safe_get(usage_node, "Prompt"), + safe_get(usage_node, "InputTokens"), safe_get(usage_node, "TokensInput"), + ])) + tokens_output = to_num(select_first_non_null([ + safe_get(usage_node, "Output"), safe_get(usage_node, "Completion"), + safe_get(usage_node, "OutputTokens"), safe_get(usage_node, "TokensOutput"), + ])) + if not tokens_total and (tokens_input or tokens_output): + try: + tokens_total = (tokens_input or 0) + (tokens_output or 0) + except Exception: + pass + base["TokensTotal"] = tokens_total if tokens_total is not None else "" + base["TokensInput"] = tokens_input if tokens_input is not None else "" + base["TokensOutput"] = tokens_output if tokens_output is not None else "" + + # Duration, outcome, conversation from CED + duration_ms = to_num(select_first_non_null([ + safe_get(ced, "DurationMs"), safe_get(ced, "ElapsedMs"), + safe_get(ced, "ProcessingTimeMs"), safe_get(ced, "LatencyMs"), + ])) + base["DurationMs"] = duration_ms if duration_ms is not None else "" + + outcome_status: Any = select_first_non_null([ + safe_get(ced, "OutcomeStatus"), safe_get(ced, "Outcome"), + safe_get(ced, "Result"), safe_get(ced, "Status"), + ]) + if isinstance(outcome_status, bool): + outcome_status = "Success" if outcome_status else "Failure" + base["OutcomeStatus"] = outcome_status or "" + + base["ConversationId"] = select_first_non_null([ + safe_get(ced, "ConversationId"), safe_get(ced, "ConversationID"), + safe_get(ced, "SessionId"), + ]) or "" + + turn_number = to_num(select_first_non_null([ + safe_get(ced, "TurnNumber"), safe_get(ced, "TurnIndex"), + safe_get(ced, "MessageIndex"), + ])) + base["TurnNumber"] = turn_number if turn_number is not None else "" + + retry_count = to_num(select_first_non_null([ + safe_get(ced, "RetryCount"), safe_get(ced, "Retries"), + ])) + base["RetryCount"] = retry_count if retry_count is not None else "" + + base["ClientVersion"] = select_first_non_null([ + safe_get(ced, "ClientVersion"), safe_get(ced, "Version"), safe_get(ced, "Build"), + ]) or "" + base["ClientPlatform"] = select_first_non_null([ + safe_get(ced, "ClientPlatform"), safe_get(ced, "Platform"), safe_get(ced, "OS"), + ]) or "" + + # MessageIds: semicolon-joined (matches M code's Text.Combine) + base["MessageIds"] = ";".join(str(m) for m in message_ids) if message_ids else "" + + # ── Build rows with indexed array access ───────────────────────────── + rows: list[dict] = [] + for i in range(row_count): + row = dict(base) # shallow copy of all 153 columns + + # Indexed array access — Contexts + if i < len(contexts) and contexts[i]: + row["Context_Id"] = safe_get(contexts[i], "Id") or "" + row["Context_Type"] = safe_get(contexts[i], "Type") or "" + else: + row["Context_Id"] = "" + row["Context_Type"] = "" + + # Messages + if i < len(messages): + msg = messages[i] + if isinstance(msg, dict): + row["Message_Id"] = safe_get(msg, "Id") or "" + row["Message_isPrompt"] = bool_tf(safe_get(msg, "isPrompt")) + else: + row["Message_Id"] = str(msg) if msg is not None else "" + row["Message_isPrompt"] = "" + else: + row["Message_Id"] = "" + row["Message_isPrompt"] = "" + + # AccessedResources + if i < len(resources) and resources[i]: + res = resources[i] + row["AccessedResource_Action"] = safe_get(res, "Action") or "" + row["AccessedResource_PolicyDetails"] = to_json_if_object(safe_get(res, "PolicyDetails")) + row["AccessedResource_SiteUrl"] = deid_resource(safe_get(res, "SiteUrl") or "") + row["AccessedResource_Name"] = deid_file(safe_get(res, "Name") or "") + row["AccessedResource_SensitivityLabel"] = safe_get(res, "SensitivityLabel") or "" + row["AccessedResource_ResourceType"] = safe_get(res, "ResourceType") or "" + else: + row["AccessedResource_Action"] = "" + row["AccessedResource_PolicyDetails"] = "" + row["AccessedResource_SiteUrl"] = "" + row["AccessedResource_Name"] = "" + row["AccessedResource_SensitivityLabel"] = "" + row["AccessedResource_ResourceType"] = "" + + # AISystemPlugin + if i < len(plugins_raw) and plugins_raw[i]: + row["AISystemPlugin_Id"] = safe_get(plugins_raw[i], "Id") or "" + row["AISystemPlugin_Name"] = safe_get(plugins_raw[i], "Name") or "" + else: + row["AISystemPlugin_Id"] = "" + row["AISystemPlugin_Name"] = "" + + # ModelTransparencyDetails + if i < len(model_det_raw) and model_det_raw[i]: + row["ModelTransparencyDetails_ModelName"] = safe_get(model_det_raw[i], "ModelName") or "" + else: + row["ModelTransparencyDetails_ModelName"] = "" + + # SensitivityLabel (from CED SensitivityLabels array) + if i < len(sensitivity_labels): + row["SensitivityLabel"] = str(sensitivity_labels[i]) if sensitivity_labels[i] is not None else "" + + # Context_Item — full mode: one item per row across all contexts + if activity_type == "CopilotInteraction": + found_item = None + for ctx in contexts: + if ctx: + items = get_array_fast(ctx, "Items") + if items and i < len(items): + found_item = items[i] + break + row["Context_Item"] = to_json_if_object(found_item) if found_item else "" + else: + row["Context_Item"] = "" + + rows.append(row) + + return rows + + +# ═════════════════════════════════════════════════════════════════════════════ +# ROUTER: Dispatch to Path A or Path B +# ═════════════════════════════════════════════════════════════════════════════ + +def explode_record( + record: dict, + prompt_filter: str | None = None, +) -> list[dict]: + """ + Parse AuditData and route to appropriate explosion path. + Returns list of flattened row dicts, or empty list on error. + """ + audit_data_raw = record.get("AuditData", "") + if not audit_data_raw or not isinstance(audit_data_raw, str) or not audit_data_raw.strip(): + return [] + + try: + audit_data = json_loads(audit_data_raw) + except Exception: + return [] + + if not isinstance(audit_data, dict): + return [] + + ced = safe_get(audit_data, "CopilotEventData") + if ced and isinstance(ced, dict): + return explode_copilot_record(record, audit_data, ced, prompt_filter=prompt_filter) + else: + return explode_m365_record(record, audit_data) + + +# ═════════════════════════════════════════════════════════════════════════════ +# HEADER — Fixed schema, no dynamic discovery needed +# ═════════════════════════════════════════════════════════════════════════════ +# Output columns are exactly M365_UNIFIED_HEADER (153 columns in M code order). +# No schema discovery pass is needed because both Path A and Path B produce +# row dicts that contain exactly these keys. + + +# ═════════════════════════════════════════════════════════════════════════════ +# CHUNK PROCESSOR (unit of parallel work) +# ═════════════════════════════════════════════════════════════════════════════ + +def _process_chunk(args: tuple) -> tuple[list[dict], int, int]: + """ + Process a chunk of CSV rows → exploded row dicts. + Returns (exploded_rows, input_count, error_count). + """ + chunk, prompt_filter = args + results: list[dict] = [] + errors = 0 + + for record in chunk: + try: + rows = explode_record(record, prompt_filter=prompt_filter) + results.extend(rows) + except Exception: + errors += 1 + + return results, len(chunk), errors + + +# ═════════════════════════════════════════════════════════════════════════════ +# MAIN EXPLOSION ORCHESTRATOR +# ═════════════════════════════════════════════════════════════════════════════ + +def run_explosion( + input_csv: str, + output_csv: str, + prompt_filter: str | None = None, + workers: int = 0, + chunk_size: int = STREAMING_CHUNK_SIZE, + quiet: bool = False, +) -> dict[str, Any]: + """ + Main entry point: reads input CSV, explodes all records, writes output CSV. + Uses multiprocessing for large files, single-process for small ones. + + Returns a stats dict with counts and timing. + """ + if not os.path.isfile(input_csv): + print(f"ERROR: Input file not found: {input_csv}", file=sys.stderr) + sys.exit(1) + + if workers <= 0: + workers = min(os.cpu_count() or 1, 8) + + t_start = time.perf_counter() + stats = { + "input_records": 0, + "output_rows": 0, + "errors": 0, + "chunks_processed": 0, + } + + if not quiet: + print(f"Purview M365 Usage Bundle Explosion Processor v{SCRIPT_VERSION}") + print(f" JSON engine: {_JSON_ENGINE}") + print(f" Input: {input_csv}") + print(f" Output: {output_csv}") + print(f" Prompt filter: {prompt_filter or 'None'}") + print(f" Workers: {workers}") + print(f" Chunk size: {chunk_size}") + print() + + # ── Phase 1: Fixed schema ───────────────────────────────────────────── + final_header = list(M365_UNIFIED_HEADER) # 153 columns in M code order + if not quiet: + print(f"Phase 1: Using fixed {len(final_header)}-column M code schema") + + # ── Phase 2: Process chunks ────────────────────────────────────────── + if not quiet: + print("Phase 2: Processing records...") + + # Streamed explode-and-write: the output header is fixed (final_header), + # so exploded rows never need to be pooled for column discovery. Input is + # read in bounded chunks and each chunk's exploded rows are written to the + # output CSV as soon as the chunk completes, then released. Neither the full + # input nor the full exploded row set is held in memory; peak usage stays a + # small window of in-flight chunks regardless of input size or explosion ratio. + if not quiet: + print("Phase 3: Writing output CSV...") + + os.makedirs(os.path.dirname(os.path.abspath(output_csv)), exist_ok=True) + output_rows = 0 + total_input = 0 + + def _read_chunks(reader): + current: list[dict] = [] + for row in reader: + current.append(row) + if len(current) >= chunk_size: + yield current + current = [] + if current: + yield current + + with open(input_csv, "r", encoding="utf-8-sig", newline="") as f_in, \ + open(output_csv, "w", encoding="utf-8", newline="") as f_out: + reader = csv.DictReader(f_in) + writer = csv.DictWriter(f_out, fieldnames=final_header, extrasaction="ignore", lineterminator="\n") + writer.writeheader() + + use_parallel = workers > 1 + + if use_parallel: + with ProcessPoolExecutor(max_workers=workers, initializer=_deid_init_worker, initargs=(_DEIDENTIFY,)) as executor: + inflight: set = set() + max_inflight = max(1, workers * 2) + chunk_iter = _read_chunks(reader) + exhausted = False + while not exhausted or inflight: + while not exhausted and len(inflight) < max_inflight: + try: + chunk = next(chunk_iter) + except StopIteration: + exhausted = True + break + total_input += len(chunk) + inflight.add(executor.submit(_process_chunk, (chunk, prompt_filter))) + if not inflight: + break + done, inflight = wait(inflight, return_when=FIRST_COMPLETED) + for future in done: + try: + exploded, _in_count, err_count = future.result() + if exploded: + writer.writerows(exploded) + output_rows += len(exploded) + stats["errors"] += err_count + stats["chunks_processed"] += 1 + if not quiet and stats["chunks_processed"] % 5 == 0: + print(f" Chunks completed: {stats['chunks_processed']}") + except Exception as exc: + stats["errors"] += 1 + if not quiet: + print(f" Chunk failed: {exc}", file=sys.stderr) + else: + for chunk in _read_chunks(reader): + total_input += len(chunk) + exploded, _in_count, err_count = _process_chunk((chunk, prompt_filter)) + if exploded: + writer.writerows(exploded) + output_rows += len(exploded) + stats["errors"] += err_count + stats["chunks_processed"] += 1 + if not quiet and stats["chunks_processed"] % 5 == 0: + print(f" Chunks completed: {stats['chunks_processed']}") + + stats["input_records"] = total_input + stats["output_rows"] = output_rows + + if not quiet: + print(f" Loaded {total_input:,} input records") + + t_elapsed = time.perf_counter() - t_start + + # ── Summary ────────────────────────────────────────────────────────── + if not quiet: + print() + print("=== EXPLOSION SUMMARY ===") + print(f" Input records: {stats['input_records']:,}") + print(f" Output rows: {stats['output_rows']:,}") + if stats["output_rows"] > stats["input_records"] and stats["input_records"] > 0: + ratio = round(stats["output_rows"] / stats["input_records"], 2) + extra = stats["output_rows"] - stats["input_records"] + print(f" Expansion: {ratio}x ({extra:,} additional rows from array explosion)") + elif stats["output_rows"] == stats["input_records"]: + print(" Expansion: 1:1 (no arrays exploded)") + elif stats["input_records"] > 0: + filtered = stats["input_records"] - stats["output_rows"] + print(f" Reduction: {filtered:,} records filtered out") + if stats["errors"] > 0: + print(f" Errors: {stats['errors']:,} record(s) failed to process") + print(f" Columns: {len(final_header):,}") + print(f" Elapsed: {t_elapsed:.2f}s") + if stats["output_rows"] > 0 and t_elapsed > 0: + print(f" Throughput: {stats['output_rows'] / t_elapsed:,.0f} rows/sec") + print(f" Output file: {output_csv}") + print() + + return stats + + +# ═════════════════════════════════════════════════════════════════════════════ +# ROLLUP ORCHESTRATOR (streaming — no exploded rows in memory) +# ═════════════════════════════════════════════════════════════════════════════ + +def run_rollup( + input_csv: str | list[str], + output_csv: str, + prompt_filter: str | None = None, + quiet: bool = False, + session_stats_csv: str | None = None, +) -> dict[str, Any]: + """ + Streaming rollup: read one or more CSVs row-by-row → parse AuditData → extract + 9 group keys + CreationTime + agent flag → accumulate into + dict[GroupKey, RollupAccum] → write 13-column CSV. + + `input_csv` accepts a single path (PAX/PowerShell single-file export) or a + list of paths (manual 4-pull export from Purview Audit). Output schema is + identical either way — the same PBIT template ingests both modes. + + When `session_stats_csv` is provided, a parallel pass over CopilotEventData + accumulates per-(UserId, CreationDate, AppHost) DISTINCTCOUNT(ThreadId) + + prompt/response counts and writes an 8-column SessionStats CSV. This matches + the AI in One `Sessions` measure unit (one thread = one session). + + No exploded row dicts are ever stored in memory. + """ + if isinstance(input_csv, (str, Path)): + input_paths: list[str] = [str(input_csv)] + else: + input_paths = [str(p) for p in input_csv] + + for p in input_paths: + if not os.path.isfile(p): + print(f"ERROR: Input file not found: {p}", file=sys.stderr) + sys.exit(1) + + t_start = time.perf_counter() + rollup: dict[GroupKey, RollupAccum] = {} + sessions: dict[SessionKey, SessionAccum] = {} if session_stats_csv else {} + track_sessions: bool = bool(session_stats_csv) + stats: dict[str, Any] = { + "input_records": 0, + "virtual_exploded_event_count": 0, + "output_rows": 0, + "parse_errors": 0, + "session_rows": 0, + "session_threads": 0, + "session_prompts": 0, + } + + if not quiet: + print(f"Purview M365 Usage Bundle Explosion Processor v{SCRIPT_VERSION} [ROLLUP MODE]") + print(f" JSON engine: {_JSON_ENGINE}") + if len(input_paths) == 1: + print(f" Input: {input_paths[0]}") + else: + print(f" Inputs ({len(input_paths)}):") + for p in input_paths: + print(f" {p}") + print(f" Output: {output_csv}") + if session_stats_csv: + print(f" Session stats: {session_stats_csv}") + print(f" Prompt filter: {prompt_filter or 'None'}") + print() + print("Processing records (streaming rollup)...") + + # ── Streaming read + accumulate (across one OR many input files) ───── + for input_csv_path in input_paths: + with open(input_csv_path, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for record in reader: + stats["input_records"] += 1 + + # Progress indicator + if not quiet and stats["input_records"] % 500_000 == 0: + print(f" {stats['input_records']:>12,} records processed, " + f"{len(rollup):,} groups...") + + # Parse AuditData JSON + audit_data_raw = record.get("AuditData", "") + if not audit_data_raw or not isinstance(audit_data_raw, str) or not audit_data_raw.strip(): + stats["parse_errors"] += 1 + continue + try: + audit_data = json_loads(audit_data_raw) + except Exception: + stats["parse_errors"] += 1 + continue + if not isinstance(audit_data, dict): + stats["parse_errors"] += 1 + continue + + ced = safe_get(audit_data, "CopilotEventData") + if ced and not isinstance(ced, dict): + ced = None + + # Extract rollup keys (lightweight — no row dict built) + result = _extract_rollup_keys(record, audit_data, ced, prompt_filter) + if result is None: + continue # filtered out by prompt_filter or non-human UPN + + (group_key, event_count, items_accessed_count, + creation_time, original_uid, is_agent) = result + stats["virtual_exploded_event_count"] += event_count + + # Accumulate into rollup dict + if group_key in rollup: + acc = rollup[group_key] + acc.event_count += event_count + acc.items_accessed_count += items_accessed_count + if is_agent: + acc.is_agent_interaction = True + if creation_time: + if not acc.min_creation_time or creation_time < acc.min_creation_time: + acc.min_creation_time = creation_time + if not acc.max_creation_time or creation_time > acc.max_creation_time: + acc.max_creation_time = creation_time + else: + rollup[group_key] = RollupAccum( + event_count=event_count, + items_accessed=items_accessed_count, + min_ct=creation_time, + max_ct=creation_time, + original_uid=original_uid, + is_agent=is_agent, + ) + + # ── SessionStats accumulation (AI in One parity) ─── + # Only records with a CopilotEventData payload contribute. Threads + # without at least one user prompt are excluded (matches AI in One + # `Message_isPrompt = TRUE` filter). + if track_sessions and ced: + msgs = get_array_fast(ced, "Messages") + prompts_here = 0 + responses_here = 0 + for m in msgs: + ip = safe_get(m, "isPrompt") + if ip is True: + prompts_here += 1 + elif ip is False: + responses_here += 1 + thread_id = _norm_key_str(safe_get(ced, "ThreadId")) + # group_key layout: (uid_lower, creation_date, op, wl, sfe, + # app_host, agent_id, agent_name, context_type) + skey: SessionKey = (group_key[0], group_key[1], group_key[5]) + sacc = sessions.get(skey) + if sacc is None: + sacc = SessionAccum(original_uid=original_uid) + sessions[skey] = sacc + sacc.prompt_count += prompts_here + sacc.response_count += responses_here + if is_agent: + sacc.agent_prompt_count += prompts_here # exact chat/agent split + if thread_id and prompts_here > 0: + sacc.thread_ids.add(thread_id) + if is_agent: + sacc.agent_thread_ids.add(thread_id) + stats["session_prompts"] += prompts_here + + # ── Write rollup output CSV ────────────────────────────────────────── + stats["output_rows"] = len(rollup) + + if not quiet: + print(f" {stats['input_records']:>12,} records processed (done)") + print(f" Writing {stats['output_rows']:,} rollup rows...") + + os.makedirs(os.path.dirname(os.path.abspath(output_csv)), exist_ok=True) + with open(output_csv, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(ROLLUP_HEADER) + for (uid_lower, cdate, op, wl, sfe, ah, agent_id, agent_name, ctx_type), acc in rollup.items(): + writer.writerow([ + deid_upn(acc.original_user_id), # output original casing, NOT lowered key + cdate, + op, + wl, + sfe, + ah, + acc.event_count, + acc.items_accessed_count, + acc.min_creation_time, # CreationTime = MIN + acc.max_creation_time, # MaxCreationTime = MAX + agent_id, + agent_name, + ctx_type, + "TRUE" if acc.is_agent_interaction else "FALSE", + ]) + + # ── SessionStats CSV (AI in One parity) ───────────────────── + if track_sessions: + os.makedirs(os.path.dirname(os.path.abspath(session_stats_csv)), exist_ok=True) + with open(session_stats_csv, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(SESSIONSTATS_HEADER) + for (uid_lower, cdate, ah), sacc in sessions.items(): + session_count = len(sacc.thread_ids) + if session_count == 0 and sacc.prompt_count == 0: + continue # no signal — skip + writer.writerow([ + deid_upn(sacc.original_user_id), + cdate, + ah, + session_count, + sacc.prompt_count, + sacc.agent_prompt_count, + sacc.response_count, + len(sacc.agent_thread_ids), + ]) + stats["session_rows"] += 1 + stats["session_threads"] += session_count + + t_elapsed = time.perf_counter() - t_start + + # ── Summary report ─────────────────────────────────────────────────── + if not quiet: + pct = 0.0 + if stats["virtual_exploded_event_count"] > 0: + pct = (1 - stats["output_rows"] / stats["virtual_exploded_event_count"]) * 100 + print() + print("=== ROLLUP SUMMARY ===") + print(f" Input records: {stats['input_records']:>14,}") + print(f" Virtual exploded events: {stats['virtual_exploded_event_count']:>14,}") + print(f" Rollup output rows: {stats['output_rows']:>14,}") + print(f" Row reduction: {pct:>13.1f}%" + f" ({stats['virtual_exploded_event_count']:,} -> {stats['output_rows']:,})") + if stats["parse_errors"] > 0: + print(f" Parse errors: {stats['parse_errors']:>14,}") + print(f" Columns: {len(ROLLUP_HEADER):>14}") + print(f" Elapsed: {t_elapsed:>13.2f}s") + if stats["input_records"] > 0 and t_elapsed > 0: + print(f" Throughput: {stats['input_records'] / t_elapsed:>12,.0f} input records/sec") + print(f" Output file: {output_csv}") + if track_sessions: + print() + print("=== SESSIONSTATS SUMMARY (AI in One parity) ===") + print(f" SessionStats rows: {stats['session_rows']:>14,}") + print(f" Distinct Copilot sessions: {stats['session_threads']:>14,}") + print(f" User prompts counted: {stats['session_prompts']:>14,}") + print(f" Output file: {session_stats_csv}") + print() + + return stats + + +# ═════════════════════════════════════════════════════════════════════════════ +# RECONCILIATION (sample-based validation of rollup correctness) +# ═════════════════════════════════════════════════════════════════════════════ + +def run_reconcile( + input_csv: str, + prompt_filter: str | None = None, + sample_size: int = RECONCILE_SAMPLE_SIZE, + quiet: bool = False, +) -> bool: + """ + Sample-based reconciliation: read a sample of records, run both rollup-key + extraction and full event-level explosion, compare total and filtered counts. + + Returns True if all checks pass, False otherwise. + """ + if not os.path.isfile(input_csv): + print(f"ERROR: Input file not found: {input_csv}", file=sys.stderr) + return False + + if not quiet: + print(f"\n=== RECONCILIATION CHECK (sample {sample_size:,} records) ===\n") + + # ── Read sample ────────────────────────────────────────────────────── + all_records: list[dict] = [] + with open(input_csv, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for record in reader: + all_records.append(record) + + if len(all_records) > sample_size: + sample = random.sample(all_records, sample_size) + else: + sample = all_records + sample_size = len(sample) + + if not quiet: + print(f" Total input records: {len(all_records):,}") + print(f" Sample size: {sample_size:,}") + + # ── Run event-level explosion on sample ────────────────────────────── + event_rows: list[dict] = [] + event_errors = 0 + for record in sample: + try: + rows = explode_record(record, prompt_filter=prompt_filter) + # Apply same non-human UPN filter as rollup path so totals reconcile + rows = [r for r in rows if _is_human_upn(r.get("UserId", ""))] + event_rows.extend(rows) + except Exception: + event_errors += 1 + + # ── Run rollup-key extraction on same sample ───────────────────────── + rollup_sample: dict[GroupKey, RollupAccum] = {} + rollup_errors = 0 + for record in sample: + audit_data_raw = record.get("AuditData", "") + if not audit_data_raw or not isinstance(audit_data_raw, str) or not audit_data_raw.strip(): + rollup_errors += 1 + continue + try: + audit_data = json_loads(audit_data_raw) + except Exception: + rollup_errors += 1 + continue + if not isinstance(audit_data, dict): + rollup_errors += 1 + continue + + ced = safe_get(audit_data, "CopilotEventData") + if ced and not isinstance(ced, dict): + ced = None + + result = _extract_rollup_keys(record, audit_data, ced, prompt_filter) + if result is None: + continue + (group_key, event_count, items_accessed_count, + creation_time, original_uid, is_agent) = result + + if group_key in rollup_sample: + acc = rollup_sample[group_key] + acc.event_count += event_count + acc.items_accessed_count += items_accessed_count + if is_agent: + acc.is_agent_interaction = True + if creation_time: + if not acc.min_creation_time or creation_time < acc.min_creation_time: + acc.min_creation_time = creation_time + if not acc.max_creation_time or creation_time > acc.max_creation_time: + acc.max_creation_time = creation_time + else: + rollup_sample[group_key] = RollupAccum( + event_count=event_count, + items_accessed=items_accessed_count, + min_ct=creation_time, + max_ct=creation_time, + original_uid=original_uid, + is_agent=is_agent, + ) + + # ── Compare totals ─────────────────────────────────────────────────── + rollup_total = sum(acc.event_count for acc in rollup_sample.values()) + event_total = len(event_rows) + all_pass = True + + def _check(label: str, rollup_val: Any, event_val: Any) -> bool: + nonlocal all_pass + match = rollup_val == event_val + symbol = "PASS" if match else "FAIL" + if not quiet: + print(f" {label}") + print(f" Rollup: {rollup_val} Event-level: {event_val} [{symbol}]") + if not match: + all_pass = False + return match + + _check("Total event count (SUM(EventCount) vs COUNTROWS)", + rollup_total, event_total) + + # ── Filter-specific checks ─────────────────────────────────────────── + # Helper: count event-level rows matching a filter + def _ev_count(**filters: str | set) -> int: + count = 0 + for row in event_rows: + match = True + for col, val in filters.items(): + row_val = row.get(col, "") + if isinstance(val, set): + if row_val.lower() not in val: + match = False + break + else: + if row_val != val: + match = False + break + if match: + count += 1 + return count + + # Helper: sum EventCount from rollup for matching groups + def _ru_count(**filters: str | set) -> int: + total = 0 + for (uid_l, cdate, op, wl, sfe, ah, agent_id, agent_name, ctx_type), acc in rollup_sample.items(): + match = True + key_map = {"Operation": op, "Workload": wl, + "SourceFileExtension": sfe, "AppHost": ah, + "AgentId": agent_id, "AgentName": agent_name, + "ContextType": ctx_type} + for col, val in filters.items(): + key_val = key_map.get(col, "") + if isinstance(val, set): + if key_val.lower() not in val: + match = False + break + else: + if key_val != val: + match = False + break + if match: + total += acc.event_count + return total + + # Check 1: Teams MessageSent + _check("Teams MessageSent (Workload=MicrosoftTeams, Operation=MessageSent)", + _ru_count(Workload="MicrosoftTeams", Operation="MessageSent"), + _ev_count(Workload="MicrosoftTeams", Operation="MessageSent")) + + # Check 2: Exchange Send + _check("Exchange Send (Workload=Exchange, Operation=Send)", + _ru_count(Workload="Exchange", Operation="Send"), + _ev_count(Workload="Exchange", Operation="Send")) + + # Check 3: CopilotInteraction + AppHost=Teams + _check("Copilot Teams (Operation=CopilotInteraction, AppHost=Teams)", + _ru_count(Operation="CopilotInteraction", AppHost="Teams"), + _ev_count(Operation="CopilotInteraction", AppHost="Teams")) + + # Check 4: Excel FileAccessed + _check("Excel FileAccessed (Operation=FileAccessed, SourceFileExtension in xlsx/xls/xlsm/csv)", + _ru_count(Operation="FileAccessed", SourceFileExtension={"xlsx", "xls", "xlsm", "csv"}), + _ev_count(Operation="FileAccessed", SourceFileExtension={"xlsx", "xls", "xlsm", "csv"})) + + # ── Temporal checks ────────────────────────────────────────────────── + rollup_min_ct = min((acc.min_creation_time for acc in rollup_sample.values() if acc.min_creation_time), default="") + rollup_max_ct = max((acc.max_creation_time for acc in rollup_sample.values() if acc.max_creation_time), default="") + event_times = [r.get("CreationTime", "") for r in event_rows if r.get("CreationTime")] + event_min_ct = min(event_times) if event_times else "" + event_max_ct = max(event_times) if event_times else "" + + _check("MIN(CreationTime)", rollup_min_ct, event_min_ct) + _check("MAX(CreationTime)", rollup_max_ct, event_max_ct) + + if not quiet: + print() + reduction_pct = 0.0 + if event_total > 0: + reduction_pct = (1 - len(rollup_sample) / event_total) * 100 + print(f" Event-level rows: {event_total:,} -> Rollup groups: {len(rollup_sample):,}" + f" ({reduction_pct:.1f}% reduction)") + print(f" Overall: {'ALL CHECKS PASSED' if all_pass else 'SOME CHECKS FAILED'}") + print() + + return all_pass + + +# ═════════════════════════════════════════════════════════════════════════════ +# USERSTATS & SESSION COHORT WRITER +# ═════════════════════════════════════════════════════════════════════════════ + +def write_userstats_files( + aggregated_csv_path: str | Path, + userstats_csv_path: str | Path, + session_csv_path: str | Path, + quiet: bool, + session_stats_csv_path: str | Path | None = None, +) -> tuple[int, int]: + """ + Read the just-written aggregated rollup CSV and produce two additional files: + *_UserStats.csv — one row per unique UserId with pre-computed metrics + *_SessionCohort.csv — one row per (UserId, AppColumn) with session cohort label + + When `session_stats_csv_path` is provided, the CECopilotPercentile_* + columns are computed from per-user PromptCount (human interactions) instead of + raw audit-event counts. This matches the AI in One semantics and prevents + service-principal / plugin-chain inflation from skewing the CE Quadrant. + + Returns (user_count, session_cohort_row_count). + """ + agg_path = Path(aggregated_csv_path) + if not agg_path.is_file(): + if not quiet: + print(f"[UserStats] WARNING: Aggregated CSV not found: {agg_path} — skipping.", + file=sys.stderr) + return 0, 0 + + userstats_path = Path(userstats_csv_path) + session_path = Path(session_csv_path) + + t_start = time.perf_counter() + + # ── Per-user accumulators ──────────────────────────────────────────── + uid_original: dict[str, str] = {} # uid_lower → first-seen casing + cop_ec: dict[str, int] = defaultdict(int) + m365_ec: dict[str, int] = defaultdict(int) + ex_cop_ec: dict[str, int] = defaultdict(int) + ex_m365_ec: dict[str, int] = defaultdict(int) + + t_days: dict[str, set[str]] = defaultdict(set) + o_days: dict[str, set[str]] = defaultdict(set) + w_days: dict[str, set[str]] = defaultdict(set) + x_days: dict[str, set[str]] = defaultdict(set) + p_days: dict[str, set[str]] = defaultdict(set) + + t_ec: dict[str, int] = defaultdict(int) + o_ec: dict[str, int] = defaultdict(int) + off_ec: dict[str, int] = defaultdict(int) + + # ── DAX-aligned per-user raw activity counts, computed per window. + # Three windows (L30, L60, Full) feed both the LP Weighted measures and the + # CE percentile ranks. Each is a dict keyed by window code → {uid: count}. + def _wbuckets() -> dict[str, dict[str, int]]: + return {w: defaultdict(int) for w in RANK_WINDOWS} + teams_raw = _wbuckets() + outlook_raw = _wbuckets() + word_raw = _wbuckets() + excel_raw = _wbuckets() + ppt_raw = _wbuckets() + copilot_chat_raw = _wbuckets() # Operation = "CopilotInteraction" (LP) + ce_copilot_raw = _wbuckets() # broad: Workload="Copilot" OR Op contains "CopilotInteraction" (CE) + + session_ops: dict[tuple[str, str], set[str]] = defaultdict(set) + + # Track every distinct CreationDate seen in the rollup so we can + # derive the data-window span (max - min + 1 calendar days) and + # normalize the engagement segmentation to active-days-per-week. + all_dates: set[str] = set() + + # ── Pass 1 — determine the trailing-window cutoffs ────────── + # Scan CreationDate only to find the most-recent date in the rollup. Cutoffs + # are inclusive lower bounds; a row qualifies for window W iff date_key >= cutoff[W]. + # The "Full" window has no cutoff and always qualifies. + _d_max_str = "" + with open(agg_path, "r", encoding="utf-8-sig", newline="") as _f: + _r = csv.DictReader(_f) + for _row in _r: + _dk = (_row.get("CreationDate", "") or "")[:10] + if _dk and _dk > _d_max_str: + _d_max_str = _dk + if _d_max_str: + try: + _d_max = date.fromisoformat(_d_max_str) + cutoff_l30 = (_d_max - timedelta(days=29)).isoformat() + cutoff_l60 = (_d_max - timedelta(days=59)).isoformat() + except ValueError: + # Bad date — fall back to "everything qualifies" + cutoff_l30 = "" + cutoff_l60 = "" + else: + # No data — sentinel that nothing qualifies for L30/L60 (Full still does) + cutoff_l30 = "9999-12-31" + cutoff_l60 = "9999-12-31" + if not quiet: + print(f"[UserStats] Window cutoffs: L30 >= {cutoff_l30 or '(all)'}, " + f"L60 >= {cutoff_l60 or '(all)'}, max date = {_d_max_str or '(none)'}") + + # ── Stream through aggregated CSV ──────────────────────────────────── + row_count = 0 + with open(agg_path, "r", encoding="utf-8-sig", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + row_count += 1 + + user_id = row.get("UserId", "") + uid_lower = user_id.lower() + if uid_lower not in uid_original: + uid_original[uid_lower] = user_id + + date_key = row.get("CreationDate", "")[:10] # YYYY-MM-DD + op = row.get("Operation", "") + wl = row.get("Workload", "") + ext = (row.get("SourceFileExtension", "") or "").lower() + app_host = (row.get("AppHost", "") or "").lower() + + if date_key: + all_dates.add(date_key) + + try: + event_count = int(row.get("EventCount", "1") or "1") + except (ValueError, TypeError): + event_count = 1 + + copilot = is_copilot(op, wl) + excel_file = is_excel_file_op(ext, op) + + # Core event counts + if copilot: + cop_ec[uid_lower] += event_count + else: + m365_ec[uid_lower] += event_count + + # ExCopEC: Copilot interactions in Excel (via AppHost); ExM365EC: Excel file ops by non-Copilot + if copilot and app_host == "excel": + ex_cop_ec[uid_lower] += 1 # row count, not EventCount + if excel_file and not copilot: + ex_m365_ec[uid_lower] += 1 # row count, not EventCount + + # Active days (distinct CreationDate values) + if wl == "MicrosoftTeams" and op in TEAMS_OPS: + t_days[uid_lower].add(date_key) + if wl == "Exchange" and op in OUTLOOK_OPS: + o_days[uid_lower].add(date_key) + if ext in WORD_EXTS and op in FILE_OPS: + w_days[uid_lower].add(date_key) + if ext in EXCEL_EXTS and op in FILE_OPS: + x_days[uid_lower].add(date_key) + if ext in PPT_EXTS and op in FILE_OPS: + p_days[uid_lower].add(date_key) + + # Activity event counts + if wl == "MicrosoftTeams" and op in TEAMS_OPS: + t_ec[uid_lower] += event_count + if wl == "Exchange" and op in OUTLOOK_OPS: + o_ec[uid_lower] += event_count + if ext in OFFICE_EXTS and op in FILE_OPS: + off_ec[uid_lower] += event_count + + # ── DAX-aligned raw counts, accumulated per window. + # Helper closure: write to Full always; to L60/L30 only if the row's + # date_key satisfies the trailing-window cutoff. + def _bump(buckets: dict[str, dict[str, int]], n: int) -> None: + buckets["Full"][uid_lower] += n + if date_key >= cutoff_l60: + buckets["L60"][uid_lower] += n + if date_key >= cutoff_l30: + buckets["L30"][uid_lower] += n + + if wl == "MicrosoftTeams" and op in DAX_TEAMS_OPS: + _bump(teams_raw, event_count) + if wl == "Exchange" and op in DAX_OUTLOOK_OPS: + _bump(outlook_raw, event_count) + if op in DAX_FILE_OPS: + if ext in WORD_EXTS: + _bump(word_raw, event_count) + elif ext in EXCEL_EXTS: + _bump(excel_raw, event_count) + elif ext in PPT_EXTS: + _bump(ppt_raw, event_count) + if op == "CopilotInteraction": + _bump(copilot_chat_raw, event_count) + # CE Copilot Percentile filter: Workload="Copilot" OR Operation contains "CopilotInteraction" + if wl == "Copilot" or "CopilotInteraction" in op: + _bump(ce_copilot_raw, event_count) + + # Session cohort: distinct active dates per (user, app) + app = app_column(ext, op, wl) + if app != "M365 All Apps": + session_ops[(uid_lower, app)].add(date_key) + + if row_count == 0: + if not quiet: + print("[UserStats] Aggregated CSV has 0 rows — skipping.") + return 0, 0 + + # ── Data-window span ──────────────────────────────────────────────── + # Calendar-day span between the earliest and latest CreationDate in the + # rollup, inclusive. Used to normalize per-app engagement segments to + # active-days-per-week, so labels mean the same thing whether the pull + # covers 8 days, 30 days, or 6 months. + if all_dates: + try: + d_min = min(all_dates) + d_max = max(all_dates) + window_days = ( + date.fromisoformat(d_max) - date.fromisoformat(d_min) + ).days + 1 + except ValueError: + window_days = max(len(all_dates), 1) + else: + window_days = 1 + if not quiet: + print(f"[UserStats] Data window: {window_days} calendar day(s) " + f"({d_min if all_dates else '?'} -> {d_max if all_dates else '?'})") + + # ── Percentile thresholds ──────────────────────────────────────────── + all_uids = sorted(uid_original.keys()) + total_users = len(all_uids) + + cop_vals = [cop_ec.get(u, 0) for u in all_uids] + m365_vals = [m365_ec.get(u, 0) for u in all_uids] + ex_cop_vals = [ex_cop_ec.get(u, 0) for u in all_uids] + ex_m365_vals = [ex_m365_ec.get(u, 0) for u in all_uids] + + cop_p90 = percentile_inc(cop_vals, 0.90) + cop_p75 = percentile_inc(cop_vals, 0.75) + cop_p50 = percentile_inc(cop_vals, 0.50) + + m365_p90 = percentile_inc(m365_vals, 0.90) + m365_p75 = percentile_inc(m365_vals, 0.75) + m365_p50 = percentile_inc(m365_vals, 0.50) + + exc_p90 = percentile_inc(ex_cop_vals, 0.90) + exc_p75 = percentile_inc(ex_cop_vals, 0.75) + exc_p50 = percentile_inc(ex_cop_vals, 0.50) + + exm_p90 = percentile_inc(ex_m365_vals, 0.90) + exm_p75 = percentile_inc(ex_m365_vals, 0.75) + exm_p50 = percentile_inc(ex_m365_vals, 0.50) + + # ── Ranks ──────────────────────────────────────────────────────────── + # Copilot rank: computed only among Copilot users so the range maps to [0, 1] + # within that group. Non-Copilot users are hardcoded to 0.0 downstream. + copilot_uids = [u for u in all_uids if cop_ec.get(u, 0) > 0] + copilot_user_count = len(copilot_uids) + cop_rank = compute_ranks({u: cop_ec.get(u, 0) for u in copilot_uids}) + m365_rank = compute_ranks({u: m365_ec.get(u, 0) for u in all_uids}) + + # ── CE percentile ranks per window (integer 0–100, match DAX exactly) ── + # DAX formula: ROUND( COUNTROWS(users with score <= mine) / COUNTROWS(users with score > 0) * 100 , 0) + # Users with score 0 / no activity → BLANK (we emit empty string). + def _ce_rank_pct(scores: dict[str, int]) -> dict[str, str]: + """Return DAX-exact CE percentile rank per user, as a string ('' for BLANK).""" + positives = sorted(v for v in scores.values() if v > 0) + total = len(positives) + out: dict[str, str] = {} + if total == 0: + return {u: "" for u in scores} + for u, v in scores.items(): + if v <= 0: + out[u] = "" + else: + below = bisect.bisect_right(positives, v) + out[u] = str(round(below / total * 100)) + return out + + # M365 All Apps raw is derived per window (sum of 5 app raws). LP has no M365-AllApps + # measure, so we only compute the rank — not stored as a column. + m365_all_apps_raw = { + w: { + u: teams_raw[w].get(u, 0) + outlook_raw[w].get(u, 0) + word_raw[w].get(u, 0) + + excel_raw[w].get(u, 0) + ppt_raw[w].get(u, 0) + for u in all_uids + } + for w in RANK_WINDOWS + } + ce_rank_teams = {w: _ce_rank_pct({u: teams_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_outlook = {w: _ce_rank_pct({u: outlook_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_word = {w: _ce_rank_pct({u: word_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_excel = {w: _ce_rank_pct({u: excel_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_ppt = {w: _ce_rank_pct({u: ppt_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + ce_rank_all = {w: _ce_rank_pct(m365_all_apps_raw[w]) for w in RANK_WINDOWS} + + # ── CE Copilot Percentile based on PROMPT COUNT (human interactions) ── + # Read the SessionStats CSV (if produced by run_rollup) and tally PromptCount per + # user per window. This is the AI in One semantics: one count per `isPrompt=TRUE` + # message — resistant to AI-response fanout, plugin chains, retries, and most + # service-principal noise. Falls back to audit-event tally if SessionStats is + # missing (older script invocations). + prompt_raw = _wbuckets() + if session_stats_csv_path: + _ss = Path(session_stats_csv_path) + if _ss.is_file(): + with open(_ss, "r", encoding="utf-8-sig", newline="") as _f: + _r = csv.DictReader(_f) + for _row in _r: + _uid = (_row.get("UserId") or "").strip().lower() + if not _uid: + continue + _date_key = (_row.get("CreationDate") or "")[:10] + try: + _pc = int(_row.get("PromptCount") or 0) + except ValueError: + _pc = 0 + if _pc <= 0: + continue + prompt_raw["Full"][_uid] += _pc + if cutoff_l60 and _date_key >= cutoff_l60: + prompt_raw["L60"][_uid] += _pc + if cutoff_l30 and _date_key >= cutoff_l30: + prompt_raw["L30"][_uid] += _pc + if not quiet: + _tot = sum(prompt_raw["Full"].values()) + print(f"[UserStats] CE Copilot Percentile source: PromptCount " + f"({_tot:,} prompts across {len(prompt_raw['Full']):,} users)") + else: + if not quiet: + print(f"[UserStats] WARNING: SessionStats CSV not found: {_ss} — " + f"falling back to audit-event count for CE Copilot Percentile.", + file=sys.stderr) + prompt_raw = ce_copilot_raw # fallback to legacy event-based percentile + else: + prompt_raw = ce_copilot_raw # legacy mode (script invoked without SessionStats) + + ce_copilot_pct = {w: _ce_rank_pct({u: prompt_raw[w].get(u, 0) for u in all_uids}) for w in RANK_WINDOWS} + + # ── Write *_UserStats.csv ──────────────────────────────────────────── + with open(userstats_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(USERSTATS_HEADER) + + for uid in all_uids: + c_ec = cop_ec.get(uid, 0) + m_ec = m365_ec.get(uid, 0) + exc_ec = ex_cop_ec.get(uid, 0) + exm_ec = ex_m365_ec.get(uid, 0) + + is_cop_user = "Copilot User" if c_ec > 0 else "Non-Copilot User" + cop_tier = tier_fn(c_ec, cop_p90, cop_p75, cop_p50, zero_is_bottom=True) + m365_tier = tier_fn(m_ec, m365_p90, m365_p75, m365_p50, zero_is_bottom=False) + priority = priority_fn(m365_tier, cop_tier) + + ex_m365_tier = tier_fn(exm_ec, exm_p90, exm_p75, exm_p50, zero_is_bottom=False) + ex_cop_tier = tier_fn(exc_ec, exc_p90, exc_p75, exc_p50, zero_is_bottom=False) + excel_pri = priority_fn(ex_m365_tier, ex_cop_tier) + + cop_rank_val = 0.0 if c_ec == 0 else cop_rank[uid] / max(copilot_user_count, 1) + m365_rank_val = m365_rank[uid] / total_users + + td = len(t_days.get(uid, set())) + od = len(o_days.get(uid, set())) + wd = len(w_days.get(uid, set())) + xd = len(x_days.get(uid, set())) + pd_ = len(p_days.get(uid, set())) + + t_act = t_ec.get(uid, 0) + o_act = o_ec.get(uid, 0) + off_act = off_ec.get(uid, 0) + + t_seg = "0. No Usage" if td == 0 else seg_fn(td, window_days) + o_seg = "0. No Usage" if od == 0 else seg_fn(od, window_days) + w_seg = "0. No Usage" if wd == 0 else seg_fn(wd, window_days) + x_seg = "0. No Usage" if xd == 0 else seg_fn(xd, window_days) + p_seg = "0. No Usage" if pd_ == 0 else seg_fn(pd_, window_days) + + office_days = wd + xd + pd_ + off_seg = "0. No Usage" if office_days == 0 else seg_fn(office_days, window_days) + + overall_days = len( + t_days.get(uid, set()) | o_days.get(uid, set()) | + w_days.get(uid, set()) | x_days.get(uid, set()) | + p_days.get(uid, set()) + ) + overall_seg = "0. No Usage" if overall_days == 0 else seg_fn(overall_days, window_days) + + writer.writerow([ + uid_original[uid], + c_ec, m_ec, exc_ec, exm_ec, + is_cop_user, cop_tier, m365_tier, + priority, excel_pri, + f"{cop_rank_val:.6f}", f"{m365_rank_val:.6f}", + td, od, wd, xd, pd_, + t_act, o_act, off_act, + t_seg, o_seg, w_seg, x_seg, p_seg, + off_seg, overall_seg, + # Precomputed raw + CE rank columns per window (order must + # match USERSTATS_HEADER: 6 raws × 3 windows, then 7 ranks × 3 windows) + teams_raw["L30"].get(uid, 0), teams_raw["L60"].get(uid, 0), teams_raw["Full"].get(uid, 0), + outlook_raw["L30"].get(uid, 0), outlook_raw["L60"].get(uid, 0), outlook_raw["Full"].get(uid, 0), + word_raw["L30"].get(uid, 0), word_raw["L60"].get(uid, 0), word_raw["Full"].get(uid, 0), + excel_raw["L30"].get(uid, 0), excel_raw["L60"].get(uid, 0), excel_raw["Full"].get(uid, 0), + ppt_raw["L30"].get(uid, 0), ppt_raw["L60"].get(uid, 0), ppt_raw["Full"].get(uid, 0), + copilot_chat_raw["L30"].get(uid, 0), copilot_chat_raw["L60"].get(uid, 0), copilot_chat_raw["Full"].get(uid, 0), + ce_rank_teams["L30"][uid], ce_rank_teams["L60"][uid], ce_rank_teams["Full"][uid], + ce_rank_outlook["L30"][uid], ce_rank_outlook["L60"][uid], ce_rank_outlook["Full"][uid], + ce_rank_word["L30"][uid], ce_rank_word["L60"][uid], ce_rank_word["Full"][uid], + ce_rank_excel["L30"][uid], ce_rank_excel["L60"][uid], ce_rank_excel["Full"][uid], + ce_rank_ppt["L30"][uid], ce_rank_ppt["L60"][uid], ce_rank_ppt["Full"][uid], + ce_rank_all["L30"][uid], ce_rank_all["L60"][uid], ce_rank_all["Full"][uid], + ce_copilot_pct["L30"][uid], ce_copilot_pct["L60"][uid], ce_copilot_pct["Full"][uid], + ]) + + # ── Write *_SessionCohort.csv ──────────────────────────────────────── + session_count = 0 + with open(session_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, lineterminator="\n") + writer.writerow(SESSIONCOHORT_HEADER) + + for (uid, app), ops in sorted(session_ops.items()): + n = len(ops) + if n == 0: + continue + if n <= 5: + cohort = "1-5 sessions" + elif n <= 10: + cohort = "6-10 sessions" + elif n <= 20: + cohort = "11-20 sessions" + elif n <= 40: + cohort = "21-40 sessions" + elif n <= 60: + cohort = "41-60 sessions" + elif n <= 80: + cohort = "61-80 sessions" + else: + cohort = "81+ sessions" + writer.writerow([uid_original[uid], app, cohort]) + session_count += 1 + + t_elapsed = time.perf_counter() - t_start + + if not quiet: + print(f"[UserStats] {total_users:,} users \u2192 {userstats_path.name} " + f"({len(USERSTATS_HEADER)} columns)") + print(f"[SessionCohort] {session_count:,} (user, app) pairs \u2192 {session_path.name}") + print(f"[UserStats] Elapsed: {t_elapsed:.2f}s") + + return total_users, session_count + + +# ═════════════════════════════════════════════════════════════════════════════ +# CLI ENTRY POINT +# ═════════════════════════════════════════════════════════════════════════════ + +def main() -> None: + parser = argparse.ArgumentParser( + prog="purview_m365_processor", + description=( + f"Purview M365 Usage Bundle Processor v{SCRIPT_VERSION}\n" + "Pre-computes the M365 Usage rollup + UserStats + SessionCohort CSVs\n" + "consumed by the Power BI template. Accepts either layout:\n" + " (A) ONE PAX / PowerShell export ............ use --pax\n" + " (B) FOUR manual Purview Audit exports ...... use --teams --outlook --files --copilot\n" + "Output schema is IDENTICAL for both layouts — same PBIT template ingests either." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +EXAMPLES +======== + +(A) Single PAX export (PAX tool or PowerShell Search-UnifiedAuditLog): + + python %(prog)s --pax Purview_Export.csv + + python %(prog)s --pax Purview_Export.csv --output-dir ./output + + +(B) Manual 4-pull export from Purview Audit (validated chip strategy): + + Teams (7d): MessageSent, MessageRead, ChatCreated, TeamsSessionStarted, + MeetingParticipantDetail + Outlook (30d): MailItemsAccessed, Send, MailboxLogin + Files (60d): FileAccessed, FileModified, FileDownloaded, FileUploaded + Copilot (30d): CopilotInteraction, AIAppInteraction (filter by record type) + + python %(prog)s ^ + --teams Teams_Export.csv ^ + --outlook Outlook_Export.csv ^ + --files Files_Export.csv ^ + --copilot Copilot_Export.csv ^ + --output-dir ./output + + +OUTPUT (rollup mode, both layouts produce the same three files): + + _Rollup_.csv 13 cols -> M365Usage table + _UserStats_.csv 40 cols -> UserStats table + _SessionCohort_.csv 3 cols -> SessionCohort table + + defaults to the input file's name (single input) or '_Combined' + (multi-input). Rename the file or use --output-dir if you want a tenant-named folder. + + +ADVANCED +======== + --reconcile Sample-based correctness check vs full event-level explosion. + --debug-events v1-compatible 153-column event-level CSV (single input only). + --skip-precompute Skip UserStats and SessionCohort generation. + --prompt-filter Copilot message filter: Prompt | Response | Both | Null. + --input/-i Power-user / scripted fallback for one or more CSVs. +""", + ) + + # ── Input layout (mutually exclusive, exactly one required) ────────── + layout = parser.add_argument_group( + "INPUT LAYOUT (choose ONE shape that matches how you exported the data)" + ) + layout.add_argument( + "--pax", + metavar="CSV", + help="(A) Single CSV from PAX or PowerShell Search-UnifiedAuditLog.", + ) + layout.add_argument( + "--teams", + metavar="CSV", + help="(B) Teams workload pull from Purview Audit.", + ) + layout.add_argument( + "--outlook", + metavar="CSV", + help="(B) Outlook / Exchange workload pull from Purview Audit.", + ) + layout.add_argument( + "--files", + metavar="CSV", + help="(B) Files (SharePoint + OneDrive) workload pull from Purview Audit.", + ) + layout.add_argument( + "--copilot", + metavar="CSV", + help="(B) Copilot record-type pull (CopilotInteraction + AIAppInteraction).", + ) + layout.add_argument( + "--input", "-i", + nargs="+", + metavar="CSV", + help="Power-user fallback: one or more CSV paths (any combination).", + ) + + # ── Output naming & location ───────────────────────────────────────── + output = parser.add_argument_group("OUTPUT") + output.add_argument( + "--output-dir", "-o", + metavar="DIR", + default=None, + help="Directory for output files. Default: same folder as the (first) input.", + ) + + # ── Optional behaviour flags ───────────────────────────────────────── + advanced = parser.add_argument_group("ADVANCED") + advanced.add_argument( + "--skip-precompute", + action="store_true", + default=False, + help="Skip *_UserStats.csv and *_SessionCohort.csv (only the Rollup is written).", + ) + advanced.add_argument( + "--no-session-stats", + action="store_true", + default=False, + help="Skip *_SessionStats.csv (the AI in One DISTINCTCOUNT(ThreadId) output).", + ) + advanced.add_argument( + "--debug-events", + action="store_true", + default=False, + help="Emit v1-compatible 153-column event-level CSV instead of the rollup (single input only).", + ) + advanced.add_argument( + "--reconcile", + action="store_true", + default=False, + help="Run sample-based reconciliation against the first input.", + ) + advanced.add_argument( + "--prompt-filter", + choices=["Prompt", "Response", "Both", "Null"], + default=None, + help="Filter Copilot messages by isPrompt value.", + ) + advanced.add_argument( + "--quiet", "-q", + action="store_true", + default=False, + help="Suppress progress output (only errors are printed).", + ) + # Hidden legacy alias (kept for older scripts that referenced --no-userstats). + advanced.add_argument( + "--no-userstats", + dest="skip_precompute", + action="store_true", + help=argparse.SUPPRESS, + ) + advanced.add_argument( + "--deidentify", + action="store_true", + default=False, + help=( + "One-way hash all identifying values (UserId, mailbox UPN/GUID/SIDs, " + "device name, resource URLs/file names) for anonymous reporting. " + "Deterministic and format-preserving; irreversible (no decode map)." + ), + ) + advanced.add_argument( + "--rebuild-sidecars-from-rollup", + metavar="ROLLUP_CSV", + default=None, + help=( + "Regenerate UserStats and SessionCohort sidecars from an existing rollup CSV " + "(no Purview input required). Sidecars are written to --output-dir (default: " + "the rollup's parent directory) using the rollup's base stem." + ), + ) + advanced.add_argument( + "--session-stats-for-rebuild", + metavar="SESSIONSTATS_CSV", + default=None, + help=( + "Optional companion to --rebuild-sidecars-from-rollup: when supplied, the " + "CECopilotPercentile_* columns are computed from this SessionStats CSV's " + "PromptCount (AI in One semantics). If omitted, the sidecar rebuild falls " + "back to the audit-event count (legacy behaviour)." + ), + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {SCRIPT_VERSION}", + ) + + args = parser.parse_args() + + global _DEIDENTIFY + _DEIDENTIFY = bool(args.deidentify) + + # ── Standalone sidecar regeneration mode ───────────────────────────── + # When --rebuild-sidecars-from-rollup is supplied, ignore every other + # input/dispatch flag and rebuild UserStats + SessionCohort sidecars + # from the given rollup CSV. Used by the PAX append-merge workflow + # after the PowerShell side unions the current run's rollup with a + # customer-supplied target. + if args.rebuild_sidecars_from_rollup: + rollup_in = os.path.abspath(args.rebuild_sidecars_from_rollup) + if not os.path.isfile(rollup_in): + print(f"ERROR: Rollup CSV not found: {rollup_in}", file=sys.stderr) + sys.exit(1) + session_stats_in: str | None = None + if args.session_stats_for_rebuild: + session_stats_in = os.path.abspath(args.session_stats_for_rebuild) + if not os.path.isfile(session_stats_in): + print( + f"ERROR: SessionStats CSV not found: {session_stats_in}", + file=sys.stderr, + ) + sys.exit(1) + out_dir = ( + Path(os.path.abspath(args.output_dir)) + if args.output_dir + else Path(rollup_in).parent + ) + os.makedirs(out_dir, exist_ok=True) + rollup_stem = Path(rollup_in).stem + m = re.match(r"^(.*?)(?:_Rollup(?:_\d{8}_\d{6})?)$", rollup_stem) + base_stem = m.group(1) if m else rollup_stem + run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + userstats_path = str(out_dir / f"{base_stem}_UserStats_{run_ts}.csv") + session_path = str(out_dir / f"{base_stem}_SessionCohort_{run_ts}.csv") + write_userstats_files( + rollup_in, userstats_path, session_path, args.quiet, + session_stats_csv_path=session_stats_in, + ) + sys.exit(0) + + # ── Resolve the input layout into a flat list ──────────────────────── + pax_inputs: list[str] = [args.pax] if args.pax else [] + workload_inputs: list[tuple[str, str]] = [] # [(label, path), ...] preserves order + for label in ("teams", "outlook", "files", "copilot"): + path = getattr(args, label) + if path: + workload_inputs.append((label, path)) + legacy_inputs: list[str] = list(args.input) if args.input else [] + + if pax_inputs and workload_inputs: + parser.error("--pax cannot be combined with --teams/--outlook/--files/--copilot. " + "Pick the shape that matches your export.") + if (pax_inputs or workload_inputs) and legacy_inputs: + parser.error("--input/-i cannot be combined with --pax or the workload flags.") + + if pax_inputs: + input_paths = [os.path.abspath(pax_inputs[0])] + layout_label = "pax" + elif workload_inputs: + input_paths = [os.path.abspath(p) for _, p in workload_inputs] + layout_label = "manual_4pull" + elif legacy_inputs: + input_paths = [os.path.abspath(p) for p in legacy_inputs] + layout_label = "legacy_input" + else: + parser.error( + "No input given. Use ONE of:\n" + " --pax (single PAX export)\n" + " --teams --outlook --files --copilot (manual 4-pull export)\n" + " --input/-i [ ...] (power-user fallback)" + ) + + for p in input_paths: + if not os.path.isfile(p): + print(f"ERROR: Input file not found: {p}", file=sys.stderr) + sys.exit(1) + + # ── Determine output directory & filenames ─────────────────────────── + first_stem = Path(input_paths[0]).stem + stem = first_stem if len(input_paths) == 1 else f"{first_stem}_Combined" + + output_dir = Path(os.path.abspath(args.output_dir)) if args.output_dir else Path(input_paths[0]).parent + os.makedirs(output_dir, exist_ok=True) + + run_ts = datetime.now().strftime("%Y%m%d_%H%M%S") + event_level = args.debug_events + + if not event_level: + rollup_path = str(output_dir / f"{stem}_Rollup_{run_ts}.csv") + userstats_path = str(output_dir / f"{stem}_UserStats_{run_ts}.csv") + session_path = str(output_dir / f"{stem}_SessionCohort_{run_ts}.csv") + session_stats_path: str | None = ( + None if args.no_session_stats + else str(output_dir / f"{stem}_SessionStats_{run_ts}.csv") + ) + else: + if len(input_paths) > 1: + print("ERROR: --debug-events accepts only one input CSV.", file=sys.stderr) + sys.exit(1) + rollup_path = str(output_dir / f"{stem}_Exploded_{run_ts}.csv") + session_stats_path = None + + # ── Dispatch ───────────────────────────────────────────────────────── + if not event_level: + stats = run_rollup( + input_csv=input_paths if len(input_paths) > 1 else input_paths[0], + output_csv=rollup_path, + prompt_filter=args.prompt_filter, + quiet=args.quiet, + session_stats_csv=session_stats_path, + ) + exit_code = 1 if stats["parse_errors"] > stats["input_records"] * 0.1 else 0 + + if not args.skip_precompute: + write_userstats_files( + rollup_path, userstats_path, session_path, args.quiet, + session_stats_csv_path=session_stats_path, + ) + else: + stats = run_explosion( + input_csv=input_paths[0], + output_csv=rollup_path, + prompt_filter=args.prompt_filter, + quiet=args.quiet, + ) + exit_code = 1 if stats["errors"] > 0 else 0 + + if args.reconcile: + reconcile_passed = run_reconcile( + input_csv=input_paths[0], + prompt_filter=args.prompt_filter, + quiet=args.quiet, + ) + if not reconcile_passed: + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() +'@ +# <<< END-EMBEDDED-M365-PROCESSOR + +# ============================================================================ +# ROLLUP POST-PROCESSOR HELPER FUNCTIONS +# ============================================================================ +# Resolve-PythonExe -> locates a Python 3.10+ interpreter (auto-installs via winget, +# falls back to the python.org installer if winget unavailable). +# Install-OrjsonIfMissing -> installs the optional 'orjson' package; warn-and-continue +# on failure (both processors auto-fallback to stdlib json). +# Invoke-EmbeddedProcessor -> materializes the embedded .py to .pax_incremental, +# runs it with the provided argument list, captures stderr, +# and deletes the temp file in finally. +# All three functions log via Write-LogHost / Write-Log (defined earlier in the script). +# These functions are only invoked when -Rollup or -RollupPlusRaw is used. +# Minimum required Python version is 3.10 (the embedded processors use PEP 604 unions). +# ============================================================================ + +$Script:ROLLUP_PYTHON_MIN_MAJOR = 3 +$Script:ROLLUP_PYTHON_MIN_MINOR = 10 + +function Resolve-PythonExe { + [CmdletBinding()] + param([switch]$AllowAutoInstall) + + # Returns: hashtable @{ Path=; Args=; Version='3.13.1' } + # Throws on hard failure (no usable interpreter even after install attempts). + + $candidates = @( + @{ Cmd = 'python'; LauncherArgs = @() } + @{ Cmd = 'py'; LauncherArgs = @('-3.13') } + @{ Cmd = 'py'; LauncherArgs = @('-3.12') } + @{ Cmd = 'py'; LauncherArgs = @('-3.11') } + @{ Cmd = 'py'; LauncherArgs = @('-3.10') } + @{ Cmd = 'python3'; LauncherArgs = @() } + ) + + foreach ($c in $candidates) { + try { + $cmdInfo = Get-Command -Name $c.Cmd -ErrorAction SilentlyContinue + if (-not $cmdInfo) { continue } + $probeArgs = @($c.LauncherArgs) + @('-c', 'import sys; print("{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro))') + $verOutput = & $c.Cmd @probeArgs 2>$null + if ($LASTEXITCODE -ne 0 -or -not $verOutput) { continue } + $verText = ($verOutput | Select-Object -First 1).Trim() + if ($verText -notmatch '^(\d+)\.(\d+)\.(\d+)') { continue } + $maj = [int]$Matches[1] + $min = [int]$Matches[2] + if ($maj -lt $Script:ROLLUP_PYTHON_MIN_MAJOR) { continue } + if ($maj -eq $Script:ROLLUP_PYTHON_MIN_MAJOR -and $min -lt $Script:ROLLUP_PYTHON_MIN_MINOR) { continue } + return @{ Path = $c.Cmd; Args = $c.LauncherArgs; Version = $verText } + } + catch { continue } + } + + if (-not $AllowAutoInstall) { + throw "No Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+ interpreter found on PATH." + } + + # Auto-install path: try winget first, then python.org installer. + Write-LogHost "Rollup: no Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+ found. Attempting auto-install..." -ForegroundColor Yellow + + $winget = Get-Command -Name 'winget' -ErrorAction SilentlyContinue + $installed = $false + if ($winget) { + Write-LogHost "Rollup: installing Python 3.13 via winget (Python.Python.3.13)..." -ForegroundColor Cyan + try { + $wingetArgs = @('install', '--id', 'Python.Python.3.13', '-e', '--accept-source-agreements', '--accept-package-agreements', '--scope', 'user', '--silent') + & winget @wingetArgs | Out-Host + if ($LASTEXITCODE -eq 0) { $installed = $true } + } + catch { + Write-LogHost "Rollup: winget install failed: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + else { + Write-LogHost "Rollup: winget is not available on this host; falling back to python.org installer." -ForegroundColor Yellow + } + + if (-not $installed) { + # python.org per-user silent install + try { + $tmpDir = Join-Path ([IO.Path]::GetTempPath()) ("PAX_PythonInstaller_" + [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $tmpDir -Force | Out-Null + $installerUrl = 'https://www.python.org/ftp/python/3.13.1/python-3.13.1-amd64.exe' + $installerPath = Join-Path $tmpDir 'python-3.13.1-amd64.exe' + Write-LogHost "Rollup: downloading $installerUrl ..." -ForegroundColor Cyan + Invoke-WebRequest -Uri $installerUrl -OutFile $installerPath -UseBasicParsing + Write-LogHost "Rollup: running silent per-user install (PrependPath=1)..." -ForegroundColor Cyan + $proc = Start-Process -FilePath $installerPath -ArgumentList @('/quiet', 'InstallAllUsers=0', 'PrependPath=1', 'Include_launcher=1', 'Include_pip=1') -Wait -PassThru + if ($proc.ExitCode -eq 0) { $installed = $true } + else { Write-LogHost "Rollup: python.org installer exited with code $($proc.ExitCode)." -ForegroundColor Yellow } + } + catch { + Write-LogHost "Rollup: python.org install failed: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + if (-not $installed) { + throw "Rollup: failed to auto-install Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+. Install Python manually from https://www.python.org/downloads/ and re-run." + } + + # Refresh PATH from process + machine + user scopes so new install is visible without restart. + $paths = @( + [Environment]::GetEnvironmentVariable('Path', 'Machine'), + [Environment]::GetEnvironmentVariable('Path', 'User'), + $env:Path + ) | Where-Object { $_ } + $env:Path = ($paths -join ';') + + # Recurse once with auto-install disabled to prevent loops. + return Resolve-PythonExe -AllowAutoInstall:$false +} + +function Write-PaxRollupStage { + <# + .SYNOPSIS + Emit one sanitized rollup stage record (entry, heartbeat, completion, skip, or failure). + + .DESCRIPTION + Rollup preparation runs several phases that can each take a long time on a large + append target, and without stage records a run looks idle while it is working. Each + record names the stage, its status, and the seconds elapsed in that stage, followed + by caller-supplied AGGREGATE counters only. + + Nothing identifying is accepted or written. Counter values are emitted only when + they are numeric or boolean; any other value is replaced by a placeholder so a key, + row, field, package identifier, path, URL, user principal name, tenant or account + identifier, or credential can never reach the record. Emission is routed through the + host log helper when it exists and is inert when it does not, so the function is safe + in isolated unit extraction. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$Stage, + [ValidateSet('entry', 'heartbeat', 'complete', 'skipped', 'failed')][string]$Status = 'entry', + [hashtable]$Counters + ) + if ($null -eq $script:PaxRollupStageStarted) { $script:PaxRollupStageStarted = @{} } + $nowUtc = [System.DateTime]::UtcNow + if ($Status -eq 'entry' -or -not $script:PaxRollupStageStarted.ContainsKey($Stage)) { + $script:PaxRollupStageStarted[$Stage] = $nowUtc + } + $elapsed = [double](($nowUtc - [datetime]$script:PaxRollupStageStarted[$Stage]).TotalSeconds) + $parts = [System.Collections.Generic.List[string]]::new() + if ($Counters) { + foreach ($name in @($Counters.Keys | Sort-Object)) { + $v = $Counters[$name] + if ($v -is [bool]) { [void]$parts.Add(('{0}={1}' -f $name, $v)); continue } + if ($v -is [int] -or $v -is [int64] -or $v -is [int16] -or $v -is [uint32] -or $v -is [uint64] -or $v -is [byte]) { + [void]$parts.Add(('{0}={1:N0}' -f $name, [int64]$v)); continue + } + if ($v -is [double] -or $v -is [single] -or $v -is [decimal]) { + [void]$parts.Add(('{0}={1:N2}' -f $name, [double]$v)); continue + } + [void]$parts.Add(('{0}=' -f $name)) + } + } + $detail = if ($parts.Count -gt 0) { ' ' + ($parts.ToArray() -join ' ') } else { '' } + $color = if ($Status -eq 'failed') { 'Yellow' } else { 'DarkGray' } + $msg = ('Rollup stage [{0}] {1}: elapsed={2:N1}s{3}' -f $Stage, $Status, $elapsed, $detail) + $lh = Microsoft.PowerShell.Core\Get-Command -Name 'Write-LogHost' -CommandType Function -ErrorAction SilentlyContinue + if ($lh) { try { & $lh -Message $msg -ForegroundColor $color } catch { } } +} + +function Write-PaxRollupReaderMode { + <# + .SYNOPSIS + Record exactly once which CSV record reader backend serves the bounded rollup pipelines. + + .DESCRIPTION + The record reader chooses its backend once per process. This helper reports that single + decision one time so a run states plainly whether the compiled reader or the built-in + parser is active, without repeating the fact on every stage. Only the mode is recorded. + #> + [CmdletBinding()] + param() + if ($script:PaxRollupReaderModeLogged) { return } + $script:PaxRollupReaderModeLogged = $true + $compiled = $false + try { + if ($script:PaxCsvBackendOverride -ne 'Fallback') { + $compiled = [bool](script:Initialize-PaxCsvCompiledReader) + } + } + catch { $compiled = $false } + Write-PaxRollupStage -Stage 'csv-reader' -Status 'complete' -Counters @{ compiledReader = $compiled } +} + +function Invoke-PaxBoundedProcess { + <# + .SYNOPSIS + Run a child process under a wall-clock bound and never wait on it indefinitely. + + .DESCRIPTION + Optional package handling contacts an external tool that can stall without ever + exiting. Waiting on such a child with no bound would hold the whole run. This helper + starts the child with redirected output, waits at most the supplied number of + seconds, and on expiry terminates ONLY the process it started (and that process's + own descendants), then reports the outcome. + + The returned record carries the outcome, the exit code when the child exited on its + own, and the captured output. Nothing is thrown for a non-zero exit or a timeout, so + callers decide whether the outcome matters. Only inbox types are used. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $FilePath, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $ArgumentList, + [int] $TimeoutSeconds = 120 + ) + + $result = [pscustomobject]@{ + Outcome = 'Failed' + ExitCode = $null + StdOut = '' + StdErr = '' + Message = '' + Seconds = [double]0 + } + + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $FilePath + foreach ($a in @($ArgumentList)) { [void]$psi.ArgumentList.Add([string]$a) } + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + + $proc = $null + $sw = [System.Diagnostics.Stopwatch]::StartNew() + try { + $proc = [System.Diagnostics.Process]::Start($psi) + if ($null -eq $proc) { + $result.Message = 'process could not be started' + return $result + } + # Read both streams asynchronously so a full pipe buffer can never deadlock the wait. + $outTask = $proc.StandardOutput.ReadToEndAsync() + $errTask = $proc.StandardError.ReadToEndAsync() + $bound = [int]([math]::Max(1, $TimeoutSeconds)) * 1000 + if ($proc.WaitForExit($bound)) { + [void]$outTask.Wait(5000) + [void]$errTask.Wait(5000) + try { $result.StdOut = [string]$outTask.Result } catch { $result.StdOut = '' } + try { $result.StdErr = [string]$errTask.Result } catch { $result.StdErr = '' } + $result.ExitCode = [int]$proc.ExitCode + $result.Outcome = if ($result.ExitCode -eq 0) { 'Succeeded' } else { 'ExitedNonZero' } + } + else { + $result.Outcome = 'TimedOut' + $result.Message = ("no exit within {0}s" -f $TimeoutSeconds) + # Terminate ONLY this cycle-started child and its own descendants. + try { $proc.Kill($true) } catch { try { $proc.Kill() } catch { } } + try { [void]$proc.WaitForExit(5000) } catch { } + } + } + catch { + $result.Outcome = 'Failed' + $result.Message = $_.Exception.Message + } + finally { + $sw.Stop() + $result.Seconds = [double]$sw.Elapsed.TotalSeconds + if ($proc) { try { $proc.Dispose() } catch { } } + } + return $result +} + +function Install-OrjsonIfMissing { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs, + [int] $ProbeTimeoutSeconds = 60, + [int] $InstallTimeoutSeconds = 180 + ) + + # Best-effort. Both embedded processors fall back to stdlib json on import failure, so + # every outcome here is reported and the run continues. Both the probe and the install + # are bounded so neither can hold the run open, and a child that outlives its bound is + # terminated before the run moves on. + Write-PaxRollupStage -Stage 'json-package' -Status 'entry' + + $showArgs = @($LauncherArgs) + @('-m', 'pip', '--disable-pip-version-check', 'show', 'orjson') + $probe = Invoke-PaxBoundedProcess -FilePath $PythonExe -ArgumentList $showArgs -TimeoutSeconds $ProbeTimeoutSeconds + if ($probe.Outcome -eq 'Succeeded') { + Write-PaxRollupStage -Stage 'json-package' -Status 'complete' -Counters @{ alreadyPresent = $true; installAttempted = $false } + return $true + } + if ($probe.Outcome -eq 'TimedOut') { + Write-LogHost "Rollup: optional JSON package check did not finish within its time limit and was stopped. Continuing with the built-in JSON reader." -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ probeTimedOut = $true; installAttempted = $false } + return $false + } + if ($probe.Outcome -eq 'Failed') { + Write-LogHost "Rollup: optional JSON package check could not run. Continuing with the built-in JSON reader." -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ probeFailed = $true; installAttempted = $false } + return $false + } + + Write-LogHost "Rollup: 'orjson' not installed; installing for faster JSON parsing (~5-10x). Falls back to stdlib json on failure." -ForegroundColor Cyan + $installArgs = @($LauncherArgs) + @('-m', 'pip', '--disable-pip-version-check', 'install', '--quiet', '--user', 'orjson') + $install = Invoke-PaxBoundedProcess -FilePath $PythonExe -ArgumentList $installArgs -TimeoutSeconds $InstallTimeoutSeconds + if ($install.StdOut) { foreach ($ln in ($install.StdOut -split "`r?`n")) { if ($ln.Trim()) { Write-Log $ln } } } + if ($install.StdErr) { foreach ($ln in ($install.StdErr -split "`r?`n")) { if ($ln.Trim()) { Write-Log $ln } } } + + if ($install.Outcome -eq 'Succeeded') { + Write-PaxRollupStage -Stage 'json-package' -Status 'complete' -Counters @{ alreadyPresent = $false; installAttempted = $true; installed = $true } + return $true + } + if ($install.Outcome -eq 'TimedOut') { + Write-LogHost "Rollup: 'orjson' install did not finish within its time limit and was stopped. Continuing with stdlib json." -ForegroundColor Yellow + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ installTimedOut = $true; installAttempted = $true; installed = $false } + return $false + } + if ($install.Outcome -eq 'ExitedNonZero') { + Write-LogHost "Rollup: 'orjson' install returned exit code $($install.ExitCode). Continuing with stdlib json." -ForegroundColor Yellow + } + else { + Write-LogHost "Rollup: 'orjson' install could not run: $($install.Message). Continuing with stdlib json." -ForegroundColor Yellow + } + Write-PaxRollupStage -Stage 'json-package' -Status 'failed' -Counters @{ installAttempted = $true; installed = $false } + return $false +} + +function Install-DeltalakeIfMissing { + <# + .SYNOPSIS + Ensures the Python 'deltalake' package is installed for Fabric Lakehouse Delta-table writes. + + .DESCRIPTION + Invoked only when the run will write to a Fabric Lakehouse target (Tables/ namespace). + Performs a quiet per-user install on first use. On failure, returns $false with a clear + actionable message — the caller must abort the Delta write path rather than continue. + + Unlike Install-OrjsonIfMissing, the deltalake package is NOT optional: the Fabric Tables + write path cannot fall back to a pure-Python implementation. Callers must check the + return value and abort the run cleanly if the install fails. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + + $showArgs = @($LauncherArgs) + @('-m', 'pip', 'show', 'deltalake') + & $PythonExe @showArgs 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { return $true } + + Write-LogHost "Fabric: 'deltalake' Python package not installed; installing (required for Lakehouse Tables/ writes)." -ForegroundColor Cyan + $installArgs = @($LauncherArgs) + @('-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', '--user', 'deltalake>=0.15') + try { + & $PythonExe @installArgs 2>&1 | ForEach-Object { Write-Log $_ } + if ($LASTEXITCODE -eq 0) { + Write-LogHost "Fabric: 'deltalake' installed successfully." -ForegroundColor Green + return $true + } + Write-LogHost "Fabric: 'deltalake' install returned exit code $LASTEXITCODE. Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red + return $false + } + catch { + Write-LogHost "Fabric: 'deltalake' install threw: $($_.Exception.Message). Install manually with 'pip install deltalake' and re-run." -ForegroundColor Red + return $false + } +} + +function ConvertTo-UsersSeedMap { + <# + .SYNOPSIS + Extract {PersonId_Normalized -> UserKey (INT)} from a target Users CSV and write it + as a compact JSON file the embedded CopilotInteraction processor consumes via + `--seed-userkey-map`. + + .DESCRIPTION + Pre-flight helper for -AppendUserInfo. Read the existing target Users CSV produced by a + prior run, build a dict of PersonId_Normalized -> UserKey, and write JSON. Skips rows + with empty PersonId_Normalized or non-integer UserKey. First occurrence of any + PersonId_Normalized wins (target is the source of truth). + + .OUTPUTS + Int count of entries written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUsersCsv, + [Parameter(Mandatory)] [string] $OutJsonPath, + # Fixed in-memory buffer budget for the bounded sort (bytes). Governs only + # WHEN buffered pairs spill; never caps how many rows are processed. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default. It is forwarded verbatim to + # the bounded sorts, which pass the callback an AGGREGATE snapshot only and swallow + # any exception it raises. Supplying it cannot change the seed JSON bytes, the key + # set, the ordering, or the success of the build. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 + ) + if (-not (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf)) { + throw "ConvertTo-UsersSeedMap: target Users CSV not found: '$TargetUsersCsv'" + } + + # Bounded seed builder (shared shape with ConvertTo-FactSeedMaps): stream the + # target once, spill (rawKey, sequence, int-value) pairs, fold to distinct + # first-occurrence keys via the frozen external-sort machinery, and stream the + # compact JSON one single-property object at a time so escaping / integer + # formatting match the prior ConvertTo-Json output byte-for-byte. The complete + # target is never materialized. + $buildSeedJson = { + param([string]$PairsCsvPath, [string]$OutTempJsonPath, [string]$WorkDir, [string]$Tag, [int64]$Budget, [scriptblock]$Progress, [double]$ProgressInterval) + $u8 = [System.Text.UTF8Encoding]::new($false) + $byKeyPath = Join-Path $WorkDir ($Tag + '_bykey.csv') + $firstPath = Join-Path $WorkDir ($Tag + '_first.csv') + $bySeqPath = Join-Path $WorkDir ($Tag + '_byseq.csv') + [void](script:Invoke-PaxExternalSort -InputPath $PairsCsvPath -OutputPath $byKeyPath -KeyColumn @('RawKey', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + $cmp = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $r1 = [System.IO.StreamReader]::new($byKeyPath, $true) + $w1 = [System.IO.StreamWriter]::new($firstPath, $false, $u8) + $distinct = [int64]0 + try { + $null = script:Read-PaxCsvRecord -Reader $r1 + script:Write-PaxCsvRecord -Writer $w1 -Fields @('RawKey', 'SeqPad', 'Val') + $haveLast = $false; $lastKey = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r1 + if ($null -eq $row) { break } + $k = if ($row.Length -ge 1) { [string]$row[0] } else { '' } + if ((-not $haveLast) -or ($cmp.Compare($k, $lastKey) -ne 0)) { + script:Write-PaxCsvRecord -Writer $w1 -Fields $row + $distinct++ + $lastKey = $k; $haveLast = $true + } + } + } + finally { $w1.Flush(); $w1.Dispose(); $r1.Dispose() } + [void](script:Invoke-PaxExternalSort -InputPath $firstPath -OutputPath $bySeqPath -KeyColumn @('SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + $r2 = [System.IO.StreamReader]::new($bySeqPath, $true) + $w2 = [System.IO.StreamWriter]::new($OutTempJsonPath, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $r2 + $w2.Write('{') + $wrote = $false + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r2 + if ($null -eq $row) { break } + $rawKey = [string]$row[0] + $valStr = if ($row.Length -ge 3) { [string]$row[2] } else { '0' } + $obj = [ordered]@{} + $obj[$rawKey] = [int]::Parse($valStr, [System.Globalization.CultureInfo]::InvariantCulture) + $oneLine = $obj | ConvertTo-Json -Compress -Depth 2 + $inner = $oneLine.Substring(1, $oneLine.Length - 2) + if ($wrote) { $w2.Write(',') } + $w2.Write($inner) + $wrote = $true + } + $w2.Write('}') + } + finally { $w2.Flush(); $w2.Dispose(); $r2.Dispose() } + return $distinct + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_usersseed_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $outTemp = $OutJsonPath + '.seedtmp' + $count = 0 + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $pairs = Join-Path $workDir 'pairs.csv' + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + $reader = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + $pw = [System.IO.StreamWriter]::new($pairs, $false, $utf8NoBom) + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('RawKey', 'SeqPad', 'Val') + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -ne $header) { + # Match the prior Import-Csv fail-closed behavior on a duplicated member. + $seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($h in $header) { if (-not $seen.Add([string]$h)) { throw "ConvertTo-UsersSeedMap: duplicate header column '$h' in target Users CSV." } } + $pidIdx = -1; $ukIdx = -1 + for ($i = 0; $i -lt $header.Length; $i++) { + $name = [string]$header[$i] + if ([string]::Equals($name, 'PersonId_Normalized', [System.StringComparison]::OrdinalIgnoreCase)) { $pidIdx = $i } + elseif ([string]::Equals($name, 'UserKey', [System.StringComparison]::OrdinalIgnoreCase)) { $ukIdx = $i } + } + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $seq = [int64]0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $row) { break } + $pidRaw = if ($pidIdx -ge 0 -and $pidIdx -lt $row.Length) { [string]$row[$pidIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($pidRaw)) { + $ukStr = if ($ukIdx -ge 0 -and $ukIdx -lt $row.Length) { [string]$row[$ukIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($ukStr)) { + $ukInt = 0 + if ([int]::TryParse($ukStr, [ref]$ukInt)) { + script:Write-PaxCsvRecord -Writer $pw -Fields @($pidRaw, $seq.ToString('D19', $inv), $ukInt.ToString($inv)) + } + } + } + $seq++ + } + } + } + finally { $pw.Flush(); $pw.Dispose(); $reader.Dispose() } + $count = [int](& $buildSeedJson $pairs $outTemp $workDir 'seed' $MemoryBudgetBytes $ProgressCallback $ProgressIntervalSeconds) + try { $null = (Get-Content -LiteralPath $outTemp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "ConvertTo-UsersSeedMap: seed JSON validation failed for '$outTemp': $($_.Exception.Message)" } + # Aggregate-only stage record. Routed through a host lookup so the function stays + # inert when it is extracted and exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + if ($__paxStage) { + $__seedBytes = try { [int64](Get-Item -LiteralPath $outTemp).Length } catch { [int64]0 } + try { & $__paxStage -Stage 'users-seed-validate' -Status 'complete' -Counters @{ distinctKeys = [int64]$count; seedBytes = $__seedBytes } } catch { } + } + $outDir = Split-Path -Parent $OutJsonPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + Move-Item -LiteralPath $outTemp -Destination $OutJsonPath -Force -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $outTemp) { try { Remove-Item -LiteralPath $outTemp -Force -ErrorAction SilentlyContinue } catch {} } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + return $count +} + +function ConvertTo-PaxUserKeyContinuity { + <# + .SYNOPSIS + Build one combined {identity -> UserKey} continuity map from whichever existing + append targets are present and emit the paired continuity seed and audit-only + identity list. + + .DESCRIPTION + A Users target on its own describes only the identities that were present in a + directory export. An identity that appears solely in audit activity lives only in + the Fact target, so a map built from the Users target alone reserves none of those + keys and a later run can hand one of them to a different person. + + Either target may be absent. Continuity is then derived from the one that is + present, so a run that appends only the activity data still keeps every key it + already handed out. + + Both targets are streamed once through the bounded record reader and the bounded + external-sort machinery; neither file is held in memory. Identity comparison is + case-insensitive and culture-independent, matching the rest of the append path. + + The prior mapping is validated BEFORE anything is published. The build fails closed + when one identity carries more than one key, when one key is claimed by more than + one identity, when a key is absent or is not a positive whole number where + continuity would be assumed, when a required identity or key column is missing, + when either file cannot be read as well-formed CSV, or when the paired outputs + cannot both be completed. Only aggregate counts are reported; no identity and no + key value ever reaches a message. + + .OUTPUTS + Hashtable: SeededKeys, UsersIdentities, AuditOnlyIdentities, MaxReservedUserKey. + #> + [CmdletBinding()] + param( + [Parameter()] [string] $TargetUsersCsv, + [Parameter()] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $OutJsonPath, + [Parameter(Mandatory)] [string] $OutAuditOnlyCsvPath, + # Fixed in-memory buffer budget for the bounded sorts (bytes). Governs only + # WHEN buffered pairs spill; never caps how many rows are processed. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default and forwarded verbatim to the + # bounded sorts, which pass an AGGREGATE snapshot and swallow any exception it + # raises. Supplying it cannot change any output byte. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 + ) + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + $cmpId = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + + # First matching column position for any of the supplied names (BOM strip / trim, + # case-insensitive), or -1. + $findCol = { + param($header, [string[]]$names) + for ($i = 0; $i -lt $header.Length; $i++) { + $h = [string]$header[$i] + if ($i -eq 0) { $h = $h -replace '^\uFEFF', '' } + $h = $h.Trim() + foreach ($n in $names) { + if ([string]::Equals($h, $n, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } + } + } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_ukcont_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $seedTemp = $OutJsonPath + '.conttmp' + $auditTemp = $OutAuditOnlyCsvPath + '.conttmp' + $seededKeys = 0; $usersIdentities = 0; $auditOnlyIdentities = 0; $maxReserved = 0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $pairs = Join-Path $workDir 'pairs.csv' + $pw = [System.IO.StreamWriter]::new($pairs, $false, $u8) + $seq = [int64]0 + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('RawKey', 'KeyPad', 'SeqPad', 'Val', 'Side') + + # --- Side 0: the existing Users target, in file order. --- + if ($TargetUsersCsv -and (Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf)) { + $ur = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + try { + $uh = script:Read-PaxCsvRecord -Reader $ur + if ($null -ne $uh) { + $uPid = & $findCol $uh @('PersonId_Normalized') + $uKey = & $findCol $uh @('UserKey') + if ($uPid -lt 0 -or $uKey -lt 0) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Users target does not carry both the normalized identity column and the UserKey column; refusing to assume key continuity.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ur + if ($null -eq $row) { break } + $idRaw = if ($uPid -lt $row.Length) { ([string]$row[$uPid]).Trim() } else { '' } + if (-not [string]::IsNullOrWhiteSpace($idRaw)) { + $kStr = if ($uKey -lt $row.Length) { ([string]$row[$uKey]).Trim() } else { '' } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Users target holds an identity whose UserKey is absent or is not a positive whole number; refusing to assume key continuity.' + } + if ($kInt -gt $maxReserved) { $maxReserved = $kInt } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), $seq.ToString('D19', $inv), $kInt.ToString($inv), '0') + $seq++ + } + } + } + } + finally { $ur.Dispose() } + } + + # --- Side 1: the existing Fact target, in file order. --- + if ($TargetFactCsv -and (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { + $fr = [System.IO.StreamReader]::new($TargetFactCsv, $true) + try { + $fh = script:Read-PaxCsvRecord -Reader $fr + if ($null -ne $fh) { + $fId = & $findCol $fh @('User_Id_Normalized', 'Audit_UserId_Normalized') + $fKey = & $findCol $fh @('UserKey') + if ($fId -lt 0 -or $fKey -lt 0) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Fact target does not carry both the normalized identity column and the UserKey column; refusing to assume key continuity.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $row) { break } + $idRaw = if ($fId -lt $row.Length) { ([string]$row[$fId]).Trim() } else { '' } + if (-not [string]::IsNullOrWhiteSpace($idRaw)) { + $kStr = if ($fKey -lt $row.Length) { ([string]$row[$fKey]).Trim() } else { '' } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'ConvertTo-PaxUserKeyContinuity: the existing Fact target holds an identity whose UserKey is absent or is not a positive whole number; refusing to assume key continuity.' + } + if ($kInt -gt $maxReserved) { $maxReserved = $kInt } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), $seq.ToString('D19', $inv), $kInt.ToString($inv), '1') + $seq++ + } + } + } + } + finally { $fr.Dispose() } + } + } + finally { $pw.Flush(); $pw.Dispose() } + + # --- Identity view: one group per identity, first appearance first. --- + $byIdentity = Join-Path $workDir 'by_identity.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byIdentity -KeyColumn @('RawKey', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $identities = Join-Path $workDir 'identities.csv' + $splitIdentities = 0 + $ir = [System.IO.StreamReader]::new($byIdentity, $true) + $iw = [System.IO.StreamWriter]::new($identities, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $ir + script:Write-PaxCsvRecord -Writer $iw -Fields @('RawKey', 'SeqPad', 'Val', 'HasUsers') + $haveGroup = $false; $gRaw = ''; $gSeq = ''; $gVal = ''; $gHasUsers = '0'; $gSplit = $false + $rRaw = ''; $rSeq = ''; $rVal = ''; $rSide = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ir + $isEnd = ($null -eq $row) + if (-not $isEnd) { + $rRaw = [string]$row[0] + $rSeq = [string]$row[2] + $rVal = [string]$row[3] + $rSide = [string]$row[4] + } + if ($haveGroup -and ($isEnd -or ($cmpId.Compare($rRaw, $gRaw) -ne 0))) { + if ($gSplit) { $splitIdentities++ } + script:Write-PaxCsvRecord -Writer $iw -Fields @($gRaw, $gSeq, $gVal, $gHasUsers) + if ($gHasUsers -eq '1') { $usersIdentities++ } else { $auditOnlyIdentities++ } + $haveGroup = $false + } + if ($isEnd) { break } + if (-not $haveGroup) { + $haveGroup = $true; $gRaw = $rRaw; $gSeq = $rSeq; $gVal = $rVal; $gHasUsers = '0'; $gSplit = $false + } + if ($rVal -ne $gVal) { $gSplit = $true } + if ($rSide -eq '0') { $gHasUsers = '1' } + } + } + finally { $iw.Flush(); $iw.Dispose(); $ir.Dispose() } + + # --- Key view: one group per reserved key. --- + $byKey = Join-Path $workDir 'by_key.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byKey -KeyColumn @('KeyPad', 'SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $sharedKeys = 0 + $kr = [System.IO.StreamReader]::new($byKey, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $kr + $haveK = $false; $kPad = ''; $kFirstId = ''; $kShared = $false + $rPad = ''; $rId = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $kr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId = [string]$row[0]; $rPad = [string]$row[1] } + if ($haveK -and ($isEnd -or ($rPad -ne $kPad))) { + if ($kShared) { $sharedKeys++ } + $haveK = $false + } + if ($isEnd) { break } + if (-not $haveK) { $haveK = $true; $kPad = $rPad; $kFirstId = $rId; $kShared = $false } + if ($cmpId.Compare($rId, $kFirstId) -ne 0) { $kShared = $true } + } + } + finally { $kr.Dispose() } + + if ($splitIdentities -gt 0 -or $sharedKeys -gt 0) { + Write-PaxRollupStage -Stage 'users-continuity' -Status 'failed' -Counters @{ splitIdentities = [int64]$splitIdentities; sharedUserKeys = [int64]$sharedKeys } + throw ('ConvertTo-PaxUserKeyContinuity: the existing append targets disagree about user keys ({0} identity group(s) carry more than one key; {1} key(s) are claimed by more than one identity). Refusing to build a continuity map that would corrupt the append.' -f $splitIdentities, $sharedKeys) + } + + # --- Emit the paired outputs in first-appearance order. --- + $ordered = Join-Path $workDir 'ordered.csv' + [void](script:Invoke-PaxExternalSort -InputPath $identities -OutputPath $ordered -KeyColumn @('SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $or = [System.IO.StreamReader]::new($ordered, $true) + $jw = [System.IO.StreamWriter]::new($seedTemp, $false, $u8) + $aw = [System.IO.StreamWriter]::new($auditTemp, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $or + script:Write-PaxCsvRecord -Writer $aw -Fields @('PersonId_Normalized', 'UserKey') + $jw.Write('{') + $wrote = $false + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $or + if ($null -eq $row) { break } + $rawKey = [string]$row[0] + $valStr = [string]$row[2] + $hasUsers = [string]$row[3] + $obj = [ordered]@{} + $obj[$rawKey] = [int]::Parse($valStr, $inv) + $oneLine = $obj | ConvertTo-Json -Compress -Depth 2 + $innerJson = $oneLine.Substring(1, $oneLine.Length - 2) + if ($wrote) { $jw.Write(',') } + $jw.Write($innerJson) + $wrote = $true + $seededKeys++ + if ($hasUsers -ne '1') { script:Write-PaxCsvRecord -Writer $aw -Fields @($rawKey, $valStr) } + } + $jw.Write('}') + } + finally { $jw.Flush(); $jw.Dispose(); $aw.Flush(); $aw.Dispose(); $or.Dispose() } + + try { $null = (Get-Content -LiteralPath $seedTemp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "ConvertTo-PaxUserKeyContinuity: continuity seed validation failed: $($_.Exception.Message)" } + if (-not (Test-Path -LiteralPath $auditTemp -PathType Leaf)) { + throw 'ConvertTo-PaxUserKeyContinuity: the audit-only identity list was not produced; refusing to publish an incomplete pair.' + } + foreach ($outPath in @($OutJsonPath, $OutAuditOnlyCsvPath)) { + $outDir = Split-Path -Parent $outPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + } + Move-Item -LiteralPath $seedTemp -Destination $OutJsonPath -Force -ErrorAction Stop + try { Move-Item -LiteralPath $auditTemp -Destination $OutAuditOnlyCsvPath -Force -ErrorAction Stop } + catch { + if (Test-Path -LiteralPath $OutJsonPath) { try { Remove-Item -LiteralPath $OutJsonPath -Force -ErrorAction SilentlyContinue } catch { } } + throw + } + Write-PaxRollupStage -Stage 'users-continuity' -Status 'complete' -Counters @{ reservedKeys = [int64]$seededKeys; directoryIdentities = [int64]$usersIdentities; auditOnlyIdentities = [int64]$auditOnlyIdentities; highestReservedKey = [int64]$maxReserved } + } + catch { + foreach ($tempPath in @($seedTemp, $auditTemp)) { + if (Test-Path -LiteralPath $tempPath) { try { Remove-Item -LiteralPath $tempPath -Force -ErrorAction SilentlyContinue } catch { } } + } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch { } } + } + + return @{ + SeededKeys = [int]$seededKeys + UsersIdentities = [int]$usersIdentities + AuditOnlyIdentities = [int]$auditOnlyIdentities + MaxReservedUserKey = [int]$maxReserved + } +} + +function Add-PaxAuditOnlyUserRows { + <# + .SYNOPSIS + Write a copy of the current-run Users CSV that also carries one row for every + audit-only identity, so every Fact user resolves to a Users row. + + .DESCRIPTION + An identity that only ever appears in audit activity has no directory row, so the + current run emits no Users row for it and the Fact rows that carry its key have + nothing to join to. This helper adds one row per such identity carrying ONLY the + normalized identity and the key already reserved for it. Every descriptive, + licensing, organizational, manager, hierarchy and profile value is left blank; + nothing is invented. + + The missing identities are taken from two sources: the prior audit-only list, and + the candidate activity data for this run. The second source covers a person seen + for the very first time in this run's activity, who has no prior row anywhere and + would otherwise leave the key their Fact rows carry unresolved. + + A real directory row always wins: an audit-only identity that IS present in the + current run is skipped entirely, and a later run that obtains a real row folds it + in through the union merge while keeping the same key. + + Both inputs are streamed through the bounded machinery. Publication is refused when + the audit-only list repeats an identity or repeats a key, when one identity carries + more than one key in the candidate activity data, or when a reserved key disagrees + with the key that identity carries in the candidate activity data. + + .OUTPUTS + Int count of rows added. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $CurrentUsersCsv, + [Parameter(Mandatory)] [string] $AuditOnlyCsv, + [Parameter(Mandatory)] [string] $OutputPath, + [Parameter()] [string] $CandidateFactCsv, + [int64] $MemoryBudgetBytes = 33554432 + ) + if (-not (Test-Path -LiteralPath $CurrentUsersCsv -PathType Leaf)) { + throw "Add-PaxAuditOnlyUserRows: current Users CSV not found: '$CurrentUsersCsv'" + } + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + $cmpId = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $findCol = { + param($header, [string]$name) + for ($i = 0; $i -lt $header.Length; $i++) { + $h = [string]$header[$i] + if ($i -eq 0) { $h = $h -replace '^\uFEFF', '' } + if ([string]::Equals($h.Trim(), $name, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } + } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_ukplace_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $tmpOut = $OutputPath + '.placing' + $added = 0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + + # --- Header of the current-run Users CSV (bounded: header record only). --- + $hdr = $null + $hr = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { $hdr = script:Read-PaxCsvRecord -Reader $hr } finally { $hr.Dispose() } + if ($null -eq $hdr) { throw 'Add-PaxAuditOnlyUserRows: the current-run Users CSV has no header record.' } + $colPidNorm = & $findCol $hdr 'PersonId_Normalized' + $colUserKey = & $findCol $hdr 'UserKey' + $colPersonId = & $findCol $hdr 'PersonId' + if ($colPidNorm -lt 0 -or $colUserKey -lt 0) { + throw 'Add-PaxAuditOnlyUserRows: the current-run Users CSV does not carry both the normalized identity column and the UserKey column; refusing to add rows.' + } + + # --- Pair both identity sources into one bounded work file. --- + $pairs = Join-Path $workDir 'pairs.csv' + $pw = [System.IO.StreamWriter]::new($pairs, $false, $u8) + $seq = [int64]0 + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('Id', 'Side', 'SeqPad', 'Val') + $cr = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $cr + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr + if ($null -eq $row) { break } + $idRaw = if ($colPidNorm -lt $row.Length) { ([string]$row[$colPidNorm]).Trim() } else { '' } + if (-not [string]::IsNullOrWhiteSpace($idRaw)) { + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, '0', $seq.ToString('D19', $inv), '') + $seq++ + } + } + } + finally { $cr.Dispose() } + + if (Test-Path -LiteralPath $AuditOnlyCsv -PathType Leaf) { + $ar = [System.IO.StreamReader]::new($AuditOnlyCsv, $true) + try { + $ah = script:Read-PaxCsvRecord -Reader $ar + if ($null -ne $ah) { + $aId = & $findCol $ah 'PersonId_Normalized' + $aKey = & $findCol $ah 'UserKey' + if ($aId -lt 0 -or $aKey -lt 0) { + throw 'Add-PaxAuditOnlyUserRows: the audit-only identity list is missing a required column; refusing to add rows.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ar + if ($null -eq $row) { break } + $idRaw = if ($aId -lt $row.Length) { ([string]$row[$aId]).Trim() } else { '' } + $kStr = if ($aKey -lt $row.Length) { ([string]$row[$aKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($idRaw)) { continue } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'Add-PaxAuditOnlyUserRows: the audit-only identity list carries a key that is not a positive whole number; refusing to add rows.' + } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, '1', $seq.ToString('D19', $inv), $kInt.ToString($inv)) + $seq++ + } + } + } + finally { $ar.Dispose() } + } + + # --- Side 2: the candidate activity data, in file order. An identity first --- + # --- seen in THIS run has no prior audit-only row, so its key is taken here. --- + if ($CandidateFactCsv -and (Test-Path -LiteralPath $CandidateFactCsv -PathType Leaf)) { + $fr = [System.IO.StreamReader]::new($CandidateFactCsv, $true) + try { + $fh = script:Read-PaxCsvRecord -Reader $fr + if ($null -ne $fh) { + $fId = & $findCol $fh 'User_Id_Normalized' + if ($fId -lt 0) { $fId = & $findCol $fh 'Audit_UserId_Normalized' } + $fKey = & $findCol $fh 'UserKey' + if ($fId -lt 0 -or $fKey -lt 0) { + throw 'Add-PaxAuditOnlyUserRows: the candidate Fact CSV does not carry both the normalized identity column and the UserKey column; refusing to add rows.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $row) { break } + $idRaw = if ($fId -lt $row.Length) { ([string]$row[$fId]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($idRaw)) { continue } + $kStr = if ($fKey -lt $row.Length) { ([string]$row[$fKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($kStr)) { continue } + $kInt = 0 + if ((-not [int]::TryParse($kStr, [ref]$kInt)) -or $kInt -lt 1) { + throw 'Add-PaxAuditOnlyUserRows: the candidate Fact CSV carries a key that is not a positive whole number; refusing to add rows.' + } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, '2', $seq.ToString('D19', $inv), $kInt.ToString($inv)) + $seq++ + } + } + } + finally { $fr.Dispose() } + } + } + finally { $pw.Flush(); $pw.Dispose() } + + # --- Group per identity; keep only identities with no current-run row. --- + $sorted = Join-Path $workDir 'sorted.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $sorted -KeyColumn @('Id', 'Side', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + + $additions = Join-Path $workDir 'additions.csv' + $sr = [System.IO.StreamReader]::new($sorted, $true) + $aw = [System.IO.StreamWriter]::new($additions, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $sr + script:Write-PaxCsvRecord -Writer $aw -Fields @('Id', 'KeyPad', 'Val') + $haveGroup = $false; $gId = ''; $gHasCurrent = $false; $gVal = ''; $gAuditRows = 0; $gFactRows = 0; $gFactVal = ''; $gFactSplit = $false + $rId = ''; $rSide = ''; $rVal = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $sr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId = [string]$row[0]; $rSide = [string]$row[1]; $rVal = [string]$row[3] } + if ($haveGroup -and ($isEnd -or ($cmpId.Compare($rId, $gId) -ne 0))) { + if ($gAuditRows -gt 1) { + throw 'Add-PaxAuditOnlyUserRows: the audit-only identity list repeats an identity; refusing to add rows.' + } + if ($gFactSplit) { + throw 'Add-PaxAuditOnlyUserRows: one identity carries more than one key in the candidate activity data; refusing to add rows.' + } + if ($gAuditRows -eq 1 -and $gFactRows -gt 0 -and $gFactVal -ne $gVal) { + throw 'Add-PaxAuditOnlyUserRows: an identity''s reserved key disagrees with the key it carries in the candidate activity data; refusing to add rows.' + } + if ((-not $gHasCurrent) -and $gAuditRows -eq 1) { + $kv = [int]::Parse($gVal, $inv) + script:Write-PaxCsvRecord -Writer $aw -Fields @($gId, $kv.ToString('D19', $inv), $gVal) + $added++ + } + elseif ((-not $gHasCurrent) -and $gAuditRows -eq 0 -and $gFactRows -gt 0) { + $kv = [int]::Parse($gFactVal, $inv) + script:Write-PaxCsvRecord -Writer $aw -Fields @($gId, $kv.ToString('D19', $inv), $gFactVal) + $added++ + } + $haveGroup = $false + } + if ($isEnd) { break } + if (-not $haveGroup) { $haveGroup = $true; $gId = $rId; $gHasCurrent = $false; $gVal = ''; $gAuditRows = 0; $gFactRows = 0; $gFactVal = ''; $gFactSplit = $false } + if ($rSide -eq '0') { $gHasCurrent = $true } + elseif ($rSide -eq '1') { $gAuditRows++; $gVal = $rVal } + else { + $gFactRows++ + if ($gFactRows -eq 1) { $gFactVal = $rVal } + elseif ($rVal -ne $gFactVal) { $gFactSplit = $true } + } + } + } + finally { $aw.Flush(); $aw.Dispose(); $sr.Dispose() } + + # --- Refuse a repeated key across the rows about to be added. --- + if ($added -gt 0) { + $byKey = Join-Path $workDir 'additions_bykey.csv' + [void](script:Invoke-PaxExternalSort -InputPath $additions -OutputPath $byKey -KeyColumn @('KeyPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + $dr = [System.IO.StreamReader]::new($byKey, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $dr + $prevPad = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $dr + if ($null -eq $row) { break } + $pad = [string]$row[1] + if ($null -ne $prevPad -and $pad -eq $prevPad) { + throw 'Add-PaxAuditOnlyUserRows: two audit-only identities claim the same key; refusing to add rows.' + } + $prevPad = $pad + } + } + finally { $dr.Dispose() } + } + + # --- Publish: the current rows verbatim, then the added rows. --- + $width = $hdr.Length + $ow = [System.IO.StreamWriter]::new($tmpOut, $false, $u8) + try { + $cr2 = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr2 + if ($null -eq $row) { break } + script:Write-PaxCsvRecord -Writer $ow -Fields $row + } + } + finally { $cr2.Dispose() } + if ($added -gt 0) { + $ar2 = [System.IO.StreamReader]::new($additions, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $ar2 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ar2 + if ($null -eq $row) { break } + $fields = New-Object 'string[]' $width + for ($j = 0; $j -lt $width; $j++) { $fields[$j] = '' } + $fields[$colPidNorm] = [string]$row[0] + $fields[$colUserKey] = [string]$row[2] + if ($colPersonId -ge 0) { $fields[$colPersonId] = [string]$row[0] } + script:Write-PaxCsvRecord -Writer $ow -Fields $fields + } + } + finally { $ar2.Dispose() } + } + } + finally { $ow.Flush(); $ow.Dispose() } + + Move-Item -LiteralPath $tmpOut -Destination $OutputPath -Force -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $tmpOut) { try { Remove-Item -LiteralPath $tmpOut -Force -ErrorAction SilentlyContinue } catch { } } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch { } } + } + + return [int]$added +} + +function Test-PaxUserKeyReferentialIntegrity { + <# + .SYNOPSIS + Validate a candidate Fact file and a candidate Users file TOGETHER, before either + one is allowed to replace an existing append target. + + .DESCRIPTION + Runs the joins the dashboard will run. Every check streams through the bounded + machinery, so file size does not govern memory. Only aggregate counts are + reported; no identity and no key value ever reaches a message. + + Checks: no key is claimed by more than one identity; no identity carries more than + one key; every non-blank Fact key resolves to exactly one Users row; the resolved + Users identity matches the Fact identity on that row. + + The Users candidate is optional. With no Users candidate there is nothing to + resolve against, so the Fact candidate is validated for identity-to-key uniqueness + and for keys shared between two identities rather than for orphan resolution. + + .OUTPUTS + Hashtable: Passed, ConflictingUserKeys, SplitIdentities, OrphanFactKeys, + MismatchedFactRows, UsersRows, FactRows. + #> + [CmdletBinding()] + param( + [Parameter()] [string] $CandidateUsersCsv, + [Parameter()] [string] $CandidateFactCsv, + [int64] $MemoryBudgetBytes = 33554432 + ) + $hasUsersCandidate = -not [string]::IsNullOrWhiteSpace($CandidateUsersCsv) + if ($hasUsersCandidate -and -not (Test-Path -LiteralPath $CandidateUsersCsv -PathType Leaf)) { + throw "Test-PaxUserKeyReferentialIntegrity: candidate Users CSV not found: '$CandidateUsersCsv'" + } + if (-not $hasUsersCandidate -and -not ($CandidateFactCsv -and (Test-Path -LiteralPath $CandidateFactCsv -PathType Leaf))) { + throw 'Test-PaxUserKeyReferentialIntegrity: neither candidate file was supplied; there is nothing to validate.' + } + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + $cmpId = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $findCol = { + param($header, [string[]]$names) + for ($i = 0; $i -lt $header.Length; $i++) { + $h = [string]$header[$i] + if ($i -eq 0) { $h = $h -replace '^\uFEFF', '' } + $h = $h.Trim() + foreach ($n in $names) { + if ([string]::Equals($h, $n, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } + } + } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_ukgate_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $conflictingKeys = 0; $splitIdentities = 0; $orphanFactKeys = 0; $mismatchedFactRows = 0 + $usersRows = [int64]0; $factRows = [int64]0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $pairs = Join-Path $workDir 'pairs.csv' + $pw = [System.IO.StreamWriter]::new($pairs, $false, $u8) + $seq = [int64]0 + try { + script:Write-PaxCsvRecord -Writer $pw -Fields @('Id', 'KeyPad', 'Side', 'SeqPad') + + if ($hasUsersCandidate) { + $ur = [System.IO.StreamReader]::new($CandidateUsersCsv, $true) + try { + $uh = script:Read-PaxCsvRecord -Reader $ur + if ($null -eq $uh) { throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Users CSV has no header record.' } + $uId = & $findCol $uh @('PersonId_Normalized') + $uKey = & $findCol $uh @('UserKey') + if ($uId -lt 0 -or $uKey -lt 0) { + throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Users CSV does not carry both the normalized identity column and the UserKey column.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $ur + if ($null -eq $row) { break } + $usersRows++ + $idRaw = if ($uId -lt $row.Length) { ([string]$row[$uId]).Trim() } else { '' } + $kStr = if ($uKey -lt $row.Length) { ([string]$row[$uKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($idRaw) -or [string]::IsNullOrWhiteSpace($kStr)) { continue } + $kInt = 0 + if (-not [int]::TryParse($kStr, [ref]$kInt)) { continue } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), '0', $seq.ToString('D19', $inv)) + $seq++ + } + } + finally { $ur.Dispose() } + } + + if ($CandidateFactCsv -and (Test-Path -LiteralPath $CandidateFactCsv -PathType Leaf)) { + $fr = [System.IO.StreamReader]::new($CandidateFactCsv, $true) + try { + $fh = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $fh) { throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Fact CSV has no header record.' } + $fId = & $findCol $fh @('User_Id_Normalized', 'Audit_UserId_Normalized') + $fKey = & $findCol $fh @('UserKey') + if ($fId -lt 0 -or $fKey -lt 0) { + throw 'Test-PaxUserKeyReferentialIntegrity: the candidate Fact CSV does not carry both the normalized identity column and the UserKey column.' + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $fr + if ($null -eq $row) { break } + $factRows++ + $kStr = if ($fKey -lt $row.Length) { ([string]$row[$fKey]).Trim() } else { '' } + if ([string]::IsNullOrWhiteSpace($kStr)) { continue } + $idRaw = if ($fId -lt $row.Length) { ([string]$row[$fId]).Trim() } else { '' } + $kInt = 0 + if (-not [int]::TryParse($kStr, [ref]$kInt)) { continue } + script:Write-PaxCsvRecord -Writer $pw -Fields @($idRaw, $kInt.ToString('D19', $inv), '1', $seq.ToString('D19', $inv)) + $seq++ + } + } + finally { $fr.Dispose() } + } + } + finally { $pw.Flush(); $pw.Dispose() } + + # --- Key view: exactly one Users row per key, and every Fact row agrees. --- + $byKey = Join-Path $workDir 'by_key.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byKey -KeyColumn @('KeyPad', 'Side', 'SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + $kr = [System.IO.StreamReader]::new($byKey, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $kr + $haveK = $false; $kPad = ''; $kUsersRows = 0; $kUsersId = ''; $kFactMismatch = 0; $kFactRows = 0; $kFirstFactId = ''; $kFactIdConflict = $false + $rId = ''; $rPad = ''; $rSide = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $kr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId = [string]$row[0]; $rPad = [string]$row[1]; $rSide = [string]$row[2] } + if ($haveK -and ($isEnd -or ($rPad -ne $kPad))) { + if ($kUsersRows -gt 1) { $conflictingKeys++ } + elseif ($kUsersRows -eq 0 -and $kFactRows -gt 0) { + # With no Users candidate there is nothing to resolve against, so the + # only thing a key group can prove is that one key means one person. + if ($hasUsersCandidate) { $orphanFactKeys++ } + elseif ($kFactIdConflict) { $conflictingKeys++ } + } + $mismatchedFactRows += $kFactMismatch + $haveK = $false + } + if ($isEnd) { break } + if (-not $haveK) { $haveK = $true; $kPad = $rPad; $kUsersRows = 0; $kUsersId = ''; $kFactMismatch = 0; $kFactRows = 0; $kFirstFactId = ''; $kFactIdConflict = $false } + if ($rSide -eq '0') { + $kUsersRows++ + if ($kUsersRows -eq 1) { $kUsersId = $rId } + } + else { + $kFactRows++ + if ($kUsersRows -ge 1 -and ($cmpId.Compare($rId, $kUsersId) -ne 0)) { $kFactMismatch++ } + if ($kUsersRows -eq 0) { + if ($kFactRows -eq 1) { $kFirstFactId = $rId } + elseif ($cmpId.Compare($rId, $kFirstFactId) -ne 0) { $kFactIdConflict = $true } + } + } + } + } + finally { $kr.Dispose() } + + # --- Identity view: exactly one key per identity across the Users candidate. --- + $byId = Join-Path $workDir 'by_id.csv' + [void](script:Invoke-PaxExternalSort -InputPath $pairs -OutputPath $byId -KeyColumn @('Id', 'Side', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + $dr = [System.IO.StreamReader]::new($byId, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $dr + $haveI = $false; $iId = ''; $iFirstPad = ''; $iSplit = $false + $rId2 = ''; $rPad2 = ''; $rSide2 = '' + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $dr + $isEnd = ($null -eq $row) + if (-not $isEnd) { $rId2 = [string]$row[0]; $rPad2 = [string]$row[1]; $rSide2 = [string]$row[2] } + if ($haveI -and ($isEnd -or ($cmpId.Compare($rId2, $iId) -ne 0))) { + if ($iSplit) { $splitIdentities++ } + $haveI = $false + } + if ($isEnd) { break } + if (-not $haveI) { $haveI = $true; $iId = $rId2; $iFirstPad = $rPad2; $iSplit = $false } + if ($rSide2 -eq '0' -or $rSide2 -eq '1') { + if ($rPad2 -ne $iFirstPad) { $iSplit = $true } + } + } + } + finally { $dr.Dispose() } + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch { } } + } + + $passed = (($conflictingKeys -eq 0) -and ($splitIdentities -eq 0) -and ($orphanFactKeys -eq 0) -and ($mismatchedFactRows -eq 0)) + return @{ + Passed = [bool]$passed + ConflictingUserKeys = [int]$conflictingKeys + SplitIdentities = [int]$splitIdentities + OrphanFactKeys = [int]$orphanFactKeys + MismatchedFactRows = [int]$mismatchedFactRows + UsersRows = [int64]$usersRows + FactRows = [int64]$factRows + } +} + +function Invoke-PaxFactSeedAccelerator { + <# + .SYNOPSIS + Prepare and publish the pair of Fact continuity seed maps through the embedded + disk-backed seed builder. + + .DESCRIPTION + Continuity seed preparation reads the whole append target once and folds it to + distinct first-occurrence keys. On a large target that fold dominates the run, so + this helper hands the fold to a small embedded builder that streams the CSV and + accumulates keys in a temporary, disk-backed database with a fixed cache bound. + Peak memory therefore does not grow with the target, and there is no row or key + limit. + + The builder writes TEMPORARY seed files only and reports a machine-readable result + file. This function reads that file rather than console text, and it publishes a + seed only after BOTH temporary seeds exist, are non-empty, parse as JSON objects, + and report non-negative counts. Any prior published seeds and the append target are + left byte-for-byte unchanged unless that full validation passes. + + Three outcomes are distinguished. 'Success' means both seeds are published. + 'CompatibilityFallbackRequired' means the target holds a key form the builder is + not proven to serialize identically; nothing is published, every temporary artifact + is removed, and the caller runs the accepted implementation instead. 'Failed' means + seed preparation must be abandoned so continuity is never silently changed. + + Only aggregate numeric progress reaches the log: rows read, key counts, and elapsed + seconds. No key, row, field, file name, path, URL, identifier, or credential is + emitted, and the temporary builder script is removed in a finally block. + + .OUTPUTS + PSCustomObject with .Status, .MidCount, .ThreadCount, .RowsRead, .ElapsedSeconds, + .ExitCode and .Reason. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $OutMidJsonPath, + [Parameter(Mandatory)] [string] $OutThreadJsonPath, + [Parameter(Mandatory)] [string] $PythonExe, + [string[]] $LauncherArgs = @(), + [Parameter(Mandatory)] [string] $ScriptDir, + [double] $HeartbeatIntervalSeconds = 0 + ) + + $pySource = @' +"""PAX Fact Continuity Seed Builder. + +Streams a target Fact CSV once and folds it to the two continuity seed maps that +keep Message_Id and thread surrogates stable across runs. Keys accumulate in a +temporary, disk-backed database with a fixed cache bound, so peak memory does not +grow with the target and no row or key limit applies. + +Only temporary outputs and one machine-readable result file are written; the +caller validates and publishes the pair. +""" + +import csv +import json +import os +import sqlite3 +import sys +import time + +STATUS_SUCCESS = 'success' +STATUS_COMPAT = 'compatibility-fallback-required' +STATUS_FAILED = 'failed' + +EXIT_SUCCESS = 0 +EXIT_FAILED = 1 +EXIT_COMPAT = 3 + +MIN_SUPPORTED = 0x20 +MAX_SUPPORTED = 0x7E + +INT32_MIN = -2147483648 +INT32_MAX = 2147483647 + +BATCH_ROWS = 5000 +FIELD_LIMIT = 16777216 +CACHE_KIB = 2048 +HEARTBEAT_ROW_STRIDE = 1000 + +_FOLD_TABLE = {c: c + 32 for c in range(0x41, 0x5B)} + + +def fold(text): + """Fold ASCII A-Z to a-z and leave every other character untouched.""" + return text.translate(_FOLD_TABLE) + + +def is_supported(text): + """Report whether every character lies in the range this builder is proven + to serialize identically to the accepted implementation. A key outside that + range is never normalized, rejected, or altered here; the whole build defers + to the accepted implementation instead.""" + for ch in text: + code = ord(ch) + if code < MIN_SUPPORTED or code > MAX_SUPPORTED: + return False + return True + + +def parse_surrogate(text): + """Accept an optional sign followed by ASCII digits, within signed 32-bit + range. Every other form is a skip, matching the accepted implementation.""" + if text is None: + return None + trimmed = text.strip() + if not trimmed: + return None + start = 0 + negative = False + if trimmed[0] in '+-': + negative = trimmed[0] == '-' + start = 1 + digits = trimmed[start:] + if not digits: + return None + for ch in digits: + if ch < '0' or ch > '9': + return None + value = int(digits) + if negative: + value = -value + if value < INT32_MIN or value > INT32_MAX: + return None + return value + + +def find_column(header, name): + """Resolve a column index case-insensitively, keeping the last match so the + resolution order matches the accepted implementation.""" + wanted = fold(name) + found = -1 + for index, cell in enumerate(header): + if fold(cell) == wanted: + found = index + return found + + +class Heartbeat(object): + """Aggregate-only progress, never more often than the requested interval.""" + + def __init__(self, interval, started): + self.interval = interval + self.started = started + self.last = started + + def emit(self, rows, mid_keys, thread_keys, now, force=False): + if self.interval <= 0: + return + if not force and (now - self.last) < self.interval: + return + self.last = now + sys.stdout.write( + 'PAXSEED heartbeat rowsRead=%d midKeys=%d threadKeys=%d elapsedSeconds=%.1f\n' + % (rows, mid_keys, thread_keys, now - self.started)) + sys.stdout.flush() + + +def remove_quietly(path): + try: + if path and os.path.exists(path): + os.remove(path) + except OSError: + pass + + +_RECORD_BREAKS = ',\r\n' + + +class QuoteAudit(object): + """Detect the one malformed quoting form the strict reader accepts but the + accepted reader rejects: a quote inside an unquoted field. A quoted field may + span physical lines, so the in-quotes state carries across them; a line that + holds no quote can neither open nor close one, which keeps the scan off the + hot path for ordinary rows.""" + + def __init__(self): + self.in_quotes = False + self.divergent = False + + def scan(self, line): + if not self.in_quotes and '"' not in line: + return + # 0 fresh, 1 unquoted, 2 inside quotes, 3 after a closing quote. + state = 2 if self.in_quotes else 0 + for ch in line: + if state == 0: + if ch == '"': + state = 2 + elif ch not in _RECORD_BREAKS: + state = 1 + elif state == 1: + if ch == '"': + self.divergent = True + return + elif ch in _RECORD_BREAKS: + state = 0 + elif state == 2: + if ch == '"': + state = 3 + else: + if ch == '"': + state = 2 + elif ch in _RECORD_BREAKS: + state = 0 + self.in_quotes = state == 2 + + +def audit_lines(handle, audit): + """Feed the reader while auditing the same characters, so a divergent record + is known as soon as the reader produces it and always before publication.""" + for line in handle: + audit.scan(line) + yield line + + +def write_seed(connection, table, path): + """Stream one compact seed map in first-occurrence order, UTF-8 without a + byte-order mark. An empty map is the literal two-character object.""" + count = 0 + with open(path, 'w', encoding='utf-8', newline='') as handle: + handle.write('{') + first = True + cursor = connection.execute( + 'SELECT raw_key, surrogate_value FROM %s ORDER BY source_sequence' % table) + for raw_key, surrogate in cursor: + if not first: + handle.write(',') + handle.write(json.dumps(raw_key, ensure_ascii=True)) + handle.write(':') + handle.write(str(int(surrogate))) + first = False + count += 1 + handle.write('}') + return count + + +def build(target_csv, mid_temp, thread_temp, work_dir, interval): + started = time.monotonic() + beat = Heartbeat(interval, started) + db_path = os.path.join(work_dir, 'pax_fact_seed.sqlite') + remove_quietly(db_path) + connection = sqlite3.connect(db_path, isolation_level=None) + try: + connection.execute('PRAGMA journal_mode = OFF') + connection.execute('PRAGMA synchronous = OFF') + connection.execute('PRAGMA temp_store = FILE') + connection.execute('PRAGMA cache_size = -%d' % CACHE_KIB) + for table in ('mid_seed', 'thread_seed'): + connection.execute( + 'CREATE TABLE %s (' + 'normalized_key TEXT PRIMARY KEY, ' + 'raw_key TEXT NOT NULL, ' + 'source_sequence INTEGER NOT NULL, ' + 'surrogate_value INTEGER NOT NULL)' % table) + + rows_read = 0 + mid_pairs = 0 + thread_pairs = 0 + with open(target_csv, 'r', encoding='utf-8-sig', newline='') as handle: + # Strict parsing rejects content after a closing quote and an unterminated + # quoted field at end of input, matching the accepted reader. The audit + # closes the remaining divergence: a quote inside an unquoted field, which + # the accepted reader rejects and the strict reader would accept. + audit = QuoteAudit() + reader = csv.reader(audit_lines(handle, audit), strict=True) + try: + header = next(reader) + except StopIteration: + header = None + + if header is not None: + if header: + header[0] = header[0].lstrip('\ufeff') + for cell in header: + if not is_supported(cell) or '"' in cell: + return (STATUS_COMPAT, 'unsupported-header-character', + 0, 0, 0, time.monotonic() - started) + seen = set() + for cell in header: + folded = fold(cell) + if folded in seen: + return (STATUS_FAILED, 'duplicate-header', + 0, 0, 0, time.monotonic() - started) + seen.add(folded) + + mid_raw_index = find_column(header, 'Message_Id_Raw') + mid_value_index = find_column(header, 'Message_Id') + thread_raw_index = find_column(header, 'ThreadId_Raw') + thread_key_index = find_column(header, 'ThreadKey') + thread_id_index = find_column(header, 'ThreadId') + if thread_key_index >= 0: + thread_value_index = thread_key_index + else: + thread_value_index = thread_id_index + + mid_batch = [] + thread_batch = [] + connection.execute('BEGIN') + for row in reader: + # Every character of this record has already been audited. + if audit.divergent: + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'unquoted-field-quote', + 0, 0, rows_read, time.monotonic() - started) + if 0 <= mid_raw_index < len(row): + raw_mid = row[mid_raw_index] + else: + raw_mid = '' + if raw_mid.strip(): + if not is_supported(raw_mid): + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'unsupported-key-character', + 0, 0, rows_read, time.monotonic() - started) + if 0 <= mid_value_index < len(row): + text = row[mid_value_index] + else: + text = '' + # A quote character inside a consumed value cannot be told apart + # from a quote the accepted reader rejects, so adjudication is + # left to that reader. + if '"' in raw_mid or '"' in text: + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'ambiguous-quoting', + 0, 0, rows_read, time.monotonic() - started) + value = parse_surrogate(text) + if value is not None: + mid_batch.append((fold(raw_mid), raw_mid, rows_read, value)) + mid_pairs += 1 + if 0 <= thread_raw_index < len(row): + raw_thread = row[thread_raw_index] + else: + raw_thread = '' + if raw_thread.strip(): + if not is_supported(raw_thread): + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'unsupported-key-character', + 0, 0, rows_read, time.monotonic() - started) + if 0 <= thread_value_index < len(row): + text = row[thread_value_index] + else: + text = '' + if '"' in raw_thread or '"' in text: + connection.execute('ROLLBACK') + return (STATUS_COMPAT, 'ambiguous-quoting', + 0, 0, rows_read, time.monotonic() - started) + value = parse_surrogate(text) + if value is not None: + thread_batch.append((fold(raw_thread), raw_thread, rows_read, value)) + thread_pairs += 1 + rows_read += 1 + if len(mid_batch) >= BATCH_ROWS: + connection.executemany( + 'INSERT OR IGNORE INTO mid_seed VALUES (?, ?, ?, ?)', mid_batch) + del mid_batch[:] + if len(thread_batch) >= BATCH_ROWS: + connection.executemany( + 'INSERT OR IGNORE INTO thread_seed VALUES (?, ?, ?, ?)', thread_batch) + del thread_batch[:] + if (rows_read % HEARTBEAT_ROW_STRIDE) == 0: + beat.emit(rows_read, mid_pairs, thread_pairs, time.monotonic()) + if mid_batch: + connection.executemany( + 'INSERT OR IGNORE INTO mid_seed VALUES (?, ?, ?, ?)', mid_batch) + del mid_batch[:] + if thread_batch: + connection.executemany( + 'INSERT OR IGNORE INTO thread_seed VALUES (?, ?, ?, ?)', thread_batch) + del thread_batch[:] + connection.execute('COMMIT') + + connection.execute('CREATE INDEX ix_mid_seq ON mid_seed(source_sequence)') + connection.execute('CREATE INDEX ix_thread_seq ON thread_seed(source_sequence)') + mid_count = write_seed(connection, 'mid_seed', mid_temp) + thread_count = write_seed(connection, 'thread_seed', thread_temp) + elapsed = time.monotonic() - started + beat.emit(rows_read, mid_count, thread_count, time.monotonic(), True) + return (STATUS_SUCCESS, '', mid_count, thread_count, rows_read, elapsed) + finally: + try: + connection.close() + finally: + remove_quietly(db_path) + remove_quietly(db_path + '-journal') + remove_quietly(db_path + '-wal') + remove_quietly(db_path + '-shm') + + +def main(argv): + if len(argv) != 7: + return EXIT_FAILED + target_csv = argv[1] + mid_temp = argv[2] + thread_temp = argv[3] + work_dir = argv[4] + try: + interval = float(argv[5]) + except ValueError: + interval = 0.0 + result_path = argv[6] + + status = STATUS_FAILED + reason = 'unexpected-error' + mid_count = 0 + thread_count = 0 + rows_read = 0 + elapsed = 0.0 + try: + csv.field_size_limit(FIELD_LIMIT) + status, reason, mid_count, thread_count, rows_read, elapsed = build( + target_csv, mid_temp, thread_temp, work_dir, interval) + except csv.Error: + status, reason = STATUS_FAILED, 'csv-error' + except sqlite3.Error: + status, reason = STATUS_FAILED, 'sqlite-error' + except UnicodeDecodeError: + status, reason = STATUS_FAILED, 'encoding-error' + except OSError: + status, reason = STATUS_FAILED, 'io-error' + except Exception: + status, reason = STATUS_FAILED, 'unexpected-error' + + if status != STATUS_SUCCESS: + mid_count = 0 + thread_count = 0 + remove_quietly(mid_temp) + remove_quietly(thread_temp) + + payload = { + 'status': status, + 'reason': reason, + 'rowsRead': rows_read, + 'midKeyCount': mid_count, + 'threadKeyCount': thread_count, + 'elapsedSeconds': round(elapsed, 3), + } + try: + with open(result_path, 'w', encoding='utf-8', newline='') as handle: + json.dump(payload, handle) + except OSError: + return EXIT_FAILED + + if status == STATUS_SUCCESS: + return EXIT_SUCCESS + if status == STATUS_COMPAT: + return EXIT_COMPAT + return EXIT_FAILED + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) +'@ + + if (-not (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { + throw "Invoke-PaxFactSeedAccelerator: target Fact CSV not found: '$TargetFactCsv'" + } + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_factseedpy_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $midTemp = $OutMidJsonPath + '.pyseedtmp' + $threadTemp = $OutThreadJsonPath + '.pyseedtmp' + $resultPath = Join-Path $workDir 'seed_result.json' + $tempPyPath = Join-Path $ScriptDir ('PAX_FactContinuitySeedBuilder_{0}.py' -f ([System.Guid]::NewGuid().ToString('N'))) + # Stage records are routed through a host lookup so the function stays inert when it + # is extracted and exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + + $status = 'Failed' + $reason = 'not-started' + $rowsRead = [int64]0 + $midCount = 0 + $threadCount = 0 + $elapsedSeconds = [double]0 + $exitCode = -1 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + if (-not (Test-Path -LiteralPath $ScriptDir -PathType Container)) { + [void][System.IO.Directory]::CreateDirectory($ScriptDir) + } + [System.IO.File]::WriteAllText($tempPyPath, $pySource, [System.Text.UTF8Encoding]::new($false)) + + $builderArgs = @($LauncherArgs) + @( + $tempPyPath + $TargetFactCsv + $midTemp + $threadTemp + $workDir + ([double]$HeartbeatIntervalSeconds).ToString($inv) + $resultPath + ) + $prevPyIoEnc = $env:PYTHONIOENCODING + try { + $env:PYTHONIOENCODING = 'utf-8' + # Only a line matching this exact aggregate shape is forwarded; every other + # byte the builder writes is dropped, so no key, row, path, or identifier can + # reach the log through this channel. + & $PythonExe @builderArgs 2>&1 | ForEach-Object { + $line = [string]$_ + if ($line -match '^PAXSEED heartbeat rowsRead=(\d+) midKeys=(\d+) threadKeys=(\d+) elapsedSeconds=(\d+(?:\.\d+)?)$') { + if ($__paxStage) { + try { + & $__paxStage -Stage 'fact-seed' -Status 'heartbeat' -Counters @{ + rowsRead = [int64]$Matches[1] + midKeys = [int64]$Matches[2] + threadKeys = [int64]$Matches[3] + elapsedSeconds = [double]::Parse($Matches[4], $inv) + } + } + catch { } + } + } + } + $exitCode = [int]$LASTEXITCODE + } + finally { + if ($null -eq $prevPyIoEnc) { Remove-Item Env:PYTHONIOENCODING -ErrorAction SilentlyContinue } + else { $env:PYTHONIOENCODING = $prevPyIoEnc } + } + + # Machine-readable outcome only; console text is never parsed. + $payload = $null + if (Test-Path -LiteralPath $resultPath -PathType Leaf) { + try { $payload = (Get-Content -LiteralPath $resultPath -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { $payload = $null } + } + if ($null -eq $payload) { + $status = 'Failed' + $reason = 'missing-result' + } + else { + $rowsRead = [int64]$payload.rowsRead + $elapsedSeconds = [double]$payload.elapsedSeconds + $reason = [string]$payload.reason + switch ([string]$payload.status) { + 'success' { + $status = if ($exitCode -eq 0) { 'Success' } else { 'Failed' } + if ($status -eq 'Failed') { $reason = 'exit-code-mismatch' } + $midCount = [int]$payload.midKeyCount + $threadCount = [int]$payload.threadKeyCount + } + 'compatibility-fallback-required' { $status = 'CompatibilityFallbackRequired' } + default { $status = 'Failed' } + } + } + + if ($status -eq 'Success') { + # Paired publication: verify BOTH temporary seeds before either is moved. + foreach ($tp in @($midTemp, $threadTemp)) { + if (-not (Test-Path -LiteralPath $tp -PathType Leaf)) { + $status = 'Failed'; $reason = 'missing-temp-seed'; break + } + if ([int64](Get-Item -LiteralPath $tp).Length -le 0) { + $status = 'Failed'; $reason = 'empty-temp-seed'; break + } + $parsed = $null + try { $parsed = (Get-Content -LiteralPath $tp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { $parsed = $null } + if ($null -eq $parsed -or -not ($parsed -is [System.Management.Automation.PSCustomObject])) { + $status = 'Failed'; $reason = 'invalid-temp-seed'; break + } + } + } + if ($status -eq 'Success' -and (($midCount -lt 0) -or ($threadCount -lt 0))) { + $status = 'Failed'; $reason = 'negative-count' + } + + if ($status -eq 'Success') { + if ($__paxStage) { + try { + & $__paxStage -Stage 'fact-seed-validate' -Status 'complete' -Counters @{ + midKeys = [int64]$midCount + threadKeys = [int64]$threadCount + midBytes = [int64](Get-Item -LiteralPath $midTemp).Length + threadBytes = [int64](Get-Item -LiteralPath $threadTemp).Length + } + } + catch { } + } + foreach ($pair in @(@($midTemp, $OutMidJsonPath), @($threadTemp, $OutThreadJsonPath))) { + $outDir = Split-Path -Parent $pair[1] + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force -ErrorAction Stop | Out-Null + } + } + # Publish as a pair. If the second move fails the first is rolled back, so a + # result never leaves one seed published against a stale partner. + $midRollback = $null + $midExisted = Test-Path -LiteralPath $OutMidJsonPath -PathType Leaf + if ($midExisted) { + $midRollback = $OutMidJsonPath + '.pyseedbak' + Copy-Item -LiteralPath $OutMidJsonPath -Destination $midRollback -Force -ErrorAction Stop + } + try { + Move-Item -LiteralPath $midTemp -Destination $OutMidJsonPath -Force -ErrorAction Stop + Move-Item -LiteralPath $threadTemp -Destination $OutThreadJsonPath -Force -ErrorAction Stop + } + catch { + if ($midRollback -and (Test-Path -LiteralPath $midRollback -PathType Leaf)) { + try { Move-Item -LiteralPath $midRollback -Destination $OutMidJsonPath -Force -ErrorAction Stop } catch { } + } + elseif ((-not $midExisted) -and (Test-Path -LiteralPath $OutMidJsonPath -PathType Leaf)) { + try { Remove-Item -LiteralPath $OutMidJsonPath -Force -ErrorAction SilentlyContinue } catch { } + } + throw + } + finally { + if ($midRollback -and (Test-Path -LiteralPath $midRollback)) { try { Remove-Item -LiteralPath $midRollback -Force -ErrorAction SilentlyContinue } catch { } } + } + } + } + catch { + $status = 'Failed' + $reason = 'accelerator-error' + } + finally { + # The temporary builder script, the temporary database, and any unpublished + # temporary seed are always removed. Published seeds are never touched here. + if (Test-Path -LiteralPath $tempPyPath) { try { Remove-Item -LiteralPath $tempPyPath -Force -ErrorAction SilentlyContinue } catch {} } + foreach ($tp in @($midTemp, $threadTemp, ($OutMidJsonPath + '.pyseedbak'))) { + if (Test-Path -LiteralPath $tp) { try { Remove-Item -LiteralPath $tp -Force -ErrorAction SilentlyContinue } catch {} } + } + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + + if ($status -ne 'Success') { $midCount = 0; $threadCount = 0 } + return [PSCustomObject]@{ + Status = $status + MidCount = $midCount + ThreadCount = $threadCount + RowsRead = $rowsRead + ElapsedSeconds = $elapsedSeconds + ExitCode = $exitCode + Reason = $reason + } +} + +function ConvertTo-FactSeedMaps { + <# + .SYNOPSIS + Extract {Message_Id_Raw -> Message_Id (INT)} and {ThreadId_Raw -> ThreadKey (INT)} + from a target CopilotInteraction Fact CSV and write each as a compact JSON file + the embedded processor consumes via `--seed-mid-map` / `--seed-thread-map`. + + .DESCRIPTION + Pre-flight helper for -AppendFile (CopilotInteraction rollup mode). Reads the + existing target Fact CSV produced by a prior run and builds two seed maps keyed + by the raw GUID columns. First case-insensitive occurrence wins; rows with a + blank raw GUID or a non-integer surrogate are skipped. + + The target is never materialized in full. Each row streams exactly once through + the RFC-4180 reader; valid (raw-key, sequence, integer) pairs for each map are + spilled to bounded temporary CSVs and folded to distinct first-occurrence keys + using only the frozen external-sort machinery, then the compact seed JSON is + streamed one property at a time. Peak memory is governed by a fixed byte budget + rather than by the target size, and no row, key, or JSON property set is held + whole in memory. + + The seed JSON bytes are identical to the prior in-memory implementation: each + property is produced by ConvertTo-Json of a single-property ordered object with + its outer braces removed, so key escaping and integer formatting are unchanged, + an empty map is the literal '{}', and property order is first-occurrence order. + Neither seed file is published until both are complete and validated; on any + failure the function raises a terminating error, leaves any prior seed outputs + byte-for-byte unchanged, publishes no partial output, and removes its temporary + work. There is no in-memory fallback. + + .OUTPUTS + PSCustomObject with .MidCount and .ThreadCount integer fields. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $OutMidJsonPath, + [Parameter(Mandatory)] [string] $OutThreadJsonPath, + # Fixed in-memory buffer budget for the bounded sorts (bytes). It governs only + # WHEN buffered pairs spill to run files; it never caps how many rows or keys + # are processed. A small value is used by tests to force multi-run spilling. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default. It is forwarded verbatim to + # the bounded sorts, which pass the callback an AGGREGATE snapshot only and swallow + # any exception it raises. Supplying it cannot change either seed JSON's bytes, the + # key sets, the ordering, or the paired fail-closed publication. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 + ) + if (-not (Test-Path -LiteralPath $TargetFactCsv -PathType Leaf)) { + throw "ConvertTo-FactSeedMaps: target Fact CSV not found: '$TargetFactCsv'" + } + + # Build one compact seed JSON at $OutTempJsonPath from a normalized pairs CSV + # (header RawKey,SeqPad,Val; one row per source row). Distinct keys are folded + # case-insensitively (first original occurrence wins, first casing + first value + # kept) by two bounded external sorts and a streaming first-per-key pass; the + # JSON is streamed one single-property object at a time. Returns the distinct + # key count. Defined inline so the whole streaming contract lives in this one + # extractable function. + $buildSeedJson = { + param([string]$PairsCsvPath, [string]$OutTempJsonPath, [string]$WorkDir, [string]$Tag, [int64]$Budget, [scriptblock]$Progress, [double]$ProgressInterval) + $u8 = [System.Text.UTF8Encoding]::new($false) + $byKeyPath = Join-Path $WorkDir ($Tag + '_bykey.csv') + $firstPath = Join-Path $WorkDir ($Tag + '_first.csv') + $bySeqPath = Join-Path $WorkDir ($Tag + '_byseq.csv') + # 1) Sort by (RawKey, SeqPad) case-insensitively so equal keys are adjacent and + # ascending by original sequence within each key group. + [void](script:Invoke-PaxExternalSort -InputPath $PairsCsvPath -OutputPath $byKeyPath -KeyColumn @('RawKey', 'SeqPad') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + # 2) Emit the first row of each case-insensitive key group (lowest sequence), + # keeping that first row's original key casing and value. + $cmp = script:Get-PaxComparer -Name 'OrdinalIgnoreCase' + $r1 = [System.IO.StreamReader]::new($byKeyPath, $true) + $w1 = [System.IO.StreamWriter]::new($firstPath, $false, $u8) + $distinct = [int64]0 + try { + $null = script:Read-PaxCsvRecord -Reader $r1 + script:Write-PaxCsvRecord -Writer $w1 -Fields @('RawKey', 'SeqPad', 'Val') + $haveLast = $false; $lastKey = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r1 + if ($null -eq $row) { break } + $k = if ($row.Length -ge 1) { [string]$row[0] } else { '' } + if ((-not $haveLast) -or ($cmp.Compare($k, $lastKey) -ne 0)) { + script:Write-PaxCsvRecord -Writer $w1 -Fields $row + $distinct++ + $lastKey = $k; $haveLast = $true + } + } + } + finally { $w1.Flush(); $w1.Dispose(); $r1.Dispose() } + # 3) Restore first-occurrence order (ascending original sequence). + [void](script:Invoke-PaxExternalSort -InputPath $firstPath -OutputPath $bySeqPath -KeyColumn @('SeqPad') -Comparer 'Ordinal' -MemoryBudgetBytes $Budget -FanIn 8 -WorkRoot $WorkDir -ProgressCallback $Progress -ProgressIntervalSeconds $ProgressInterval) + # 4) Stream the compact JSON object one property at a time. Each property's + # exact bytes come from ConvertTo-Json of a single-property ordered object + # with its outer braces removed, so escaping and integer formatting match the + # prior implementation byte-for-byte. An empty map emits the literal '{}'. + $r2 = [System.IO.StreamReader]::new($bySeqPath, $true) + $w2 = [System.IO.StreamWriter]::new($OutTempJsonPath, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $r2 + $w2.Write('{') + $wrote = $false + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $r2 + if ($null -eq $row) { break } + $rawKey = [string]$row[0] + $valStr = if ($row.Length -ge 3) { [string]$row[2] } else { '0' } + $obj = [ordered]@{} + $obj[$rawKey] = [int]::Parse($valStr, [System.Globalization.CultureInfo]::InvariantCulture) + $oneLine = $obj | ConvertTo-Json -Compress -Depth 2 + $inner = $oneLine.Substring(1, $oneLine.Length - 2) + if ($wrote) { $w2.Write(',') } + $w2.Write($inner) + $wrote = $true + } + $w2.Write('}') + } + finally { $w2.Flush(); $w2.Dispose(); $r2.Dispose() } + return $distinct + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_factseed_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $midTemp = $OutMidJsonPath + '.seedtmp' + $threadTemp = $OutThreadJsonPath + '.seedtmp' + $midCount = 0 + $threadCount = 0 + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $midPairs = Join-Path $workDir 'mid_pairs.csv' + $threadPairs = Join-Path $workDir 'thread_pairs.csv' + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + + # --- Single streaming pass over the target: emit normalized (RawKey,SeqPad,Val) + # pair rows for each map. The complete target is never materialized. --- + $reader = [System.IO.StreamReader]::new($TargetFactCsv, $true) + $mw = [System.IO.StreamWriter]::new($midPairs, $false, $utf8NoBom) + $tw = [System.IO.StreamWriter]::new($threadPairs, $false, $utf8NoBom) + try { + script:Write-PaxCsvRecord -Writer $mw -Fields @('RawKey', 'SeqPad', 'Val') + script:Write-PaxCsvRecord -Writer $tw -Fields @('RawKey', 'SeqPad', 'Val') + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -ne $header) { + # Reject duplicate header columns (case-insensitive), matching the prior + # reader's fail-closed behavior on a duplicated member. + $seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($h in $header) { + if (-not $seen.Add([string]$h)) { throw "ConvertTo-FactSeedMaps: duplicate header column '$h' in target Fact CSV." } + } + $midRawIdx = -1; $midValIdx = -1; $threadRawIdx = -1; $threadKeyIdx = -1; $threadIdIdx = -1 + for ($i = 0; $i -lt $header.Length; $i++) { + $name = [string]$header[$i] + if ([string]::Equals($name, 'Message_Id_Raw', [System.StringComparison]::OrdinalIgnoreCase)) { $midRawIdx = $i } + elseif ([string]::Equals($name, 'Message_Id', [System.StringComparison]::OrdinalIgnoreCase)) { $midValIdx = $i } + elseif ([string]::Equals($name, 'ThreadId_Raw', [System.StringComparison]::OrdinalIgnoreCase)) { $threadRawIdx = $i } + elseif ([string]::Equals($name, 'ThreadKey', [System.StringComparison]::OrdinalIgnoreCase)) { $threadKeyIdx = $i } + elseif ([string]::Equals($name, 'ThreadId', [System.StringComparison]::OrdinalIgnoreCase)) { $threadIdIdx = $i } + } + # ThreadKey overrides ThreadId as the thread surrogate when present. + $threadValIdx = if ($threadKeyIdx -ge 0) { $threadKeyIdx } else { $threadIdIdx } + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $seq = [int64]0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $row) { break } + $seqPad = $seq.ToString('D19', $inv) + $midRaw = if ($midRawIdx -ge 0 -and $midRawIdx -lt $row.Length) { [string]$row[$midRawIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($midRaw)) { + $midValStr = if ($midValIdx -ge 0 -and $midValIdx -lt $row.Length) { [string]$row[$midValIdx] } else { '' } + $midInt = 0 + if ([int]::TryParse($midValStr, [ref]$midInt)) { + script:Write-PaxCsvRecord -Writer $mw -Fields @($midRaw, $seqPad, $midInt.ToString($inv)) + } + } + $threadRaw = if ($threadRawIdx -ge 0 -and $threadRawIdx -lt $row.Length) { [string]$row[$threadRawIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($threadRaw)) { + $threadValStr = if ($threadValIdx -ge 0 -and $threadValIdx -lt $row.Length) { [string]$row[$threadValIdx] } else { '' } + $threadInt = 0 + if ([int]::TryParse($threadValStr, [ref]$threadInt)) { + script:Write-PaxCsvRecord -Writer $tw -Fields @($threadRaw, $seqPad, $threadInt.ToString($inv)) + } + } + $seq++ + } + } + } + finally { + $mw.Flush(); $mw.Dispose(); $tw.Flush(); $tw.Dispose(); $reader.Dispose() + } + + # --- Build each seed JSON to a temp sibling using only bounded machinery. --- + $midCount = [int](& $buildSeedJson $midPairs $midTemp $workDir 'mid' $MemoryBudgetBytes $ProgressCallback $ProgressIntervalSeconds) + $threadCount = [int](& $buildSeedJson $threadPairs $threadTemp $workDir 'thread' $MemoryBudgetBytes $ProgressCallback $ProgressIntervalSeconds) + + # --- Validate both completed temp files parse before either is published. --- + foreach ($tp in @($midTemp, $threadTemp)) { + try { $null = (Get-Content -LiteralPath $tp -Raw -Encoding UTF8) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "ConvertTo-FactSeedMaps: seed JSON validation failed for '$tp': $($_.Exception.Message)" } + } + + # Aggregate-only stage record for the validated pair, emitted before publication. + # Routed through a host lookup so the function stays inert when it is extracted and + # exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + if ($__paxStage) { + $__midBytes = try { [int64](Get-Item -LiteralPath $midTemp).Length } catch { [int64]0 } + $__thrBytes = try { [int64](Get-Item -LiteralPath $threadTemp).Length } catch { [int64]0 } + try { + & $__paxStage -Stage 'fact-seed-validate' -Status 'complete' -Counters @{ + midKeys = [int64]$midCount + threadKeys = [int64]$threadCount + midBytes = $__midBytes + threadBytes = $__thrBytes + } + } catch { } + } + + # --- Publish only after both seeds are complete and valid. Prior published + # seeds are never touched before this point. --- + foreach ($pair in @(@($midTemp, $OutMidJsonPath), @($threadTemp, $OutThreadJsonPath))) { + $outDir = Split-Path -Parent $pair[1] + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + } + Move-Item -LiteralPath $midTemp -Destination $OutMidJsonPath -Force -ErrorAction Stop + Move-Item -LiteralPath $threadTemp -Destination $OutThreadJsonPath -Force -ErrorAction Stop + } + catch { + # Fail closed: remove any unpublished temp siblings so no partial seed is left + # behind, then re-raise. Prior published seed files are never modified before + # the paired publication step above. + foreach ($tp in @($midTemp, $threadTemp)) { + if (Test-Path -LiteralPath $tp) { try { Remove-Item -LiteralPath $tp -Force -ErrorAction SilentlyContinue } catch {} } + } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + return [PSCustomObject]@{ + MidCount = $midCount + ThreadCount = $threadCount + } +} + +function ConvertTo-RecordIdExclusion { + <# + .SYNOPSIS + Extract the RecordId column from a target exploded/event-level CSV and write a + newline-delimited file the embedded M365Bundle processor consumes via + `--exclude-record-ids` for cross-run dedup. + + .DESCRIPTION + Pre-flight helper for -AppendFile (M365Bundle event-level mode / -RollupPlusRaw + raw-CSV append). Streams the target CSV without loading the full row set, finds + the RecordId column index from the header, and writes one RecordId per line. + Blank / duplicate RecordIds are skipped. + + .OUTPUTS + Int count of unique RecordIds written. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetCsv, + [Parameter(Mandatory)] [string] $OutListPath + ) + if (-not (Test-Path -LiteralPath $TargetCsv -PathType Leaf)) { + throw "ConvertTo-RecordIdExclusion: target CSV not found: '$TargetCsv'" + } + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + $outDir = Split-Path -Parent $OutListPath + if ($outDir -and -not (Test-Path -LiteralPath $outDir -PathType Container)) { + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + } + $reader = $null + $writer = $null + $seen = [System.Collections.Generic.HashSet[string]]::new() + try { + $reader = [System.IO.StreamReader]::new($TargetCsv, [System.Text.Encoding]::UTF8) + $headerLine = $reader.ReadLine() + if (-not $headerLine) { return 0 } + # Tolerate UTF-8 BOM on header + if ($headerLine.Length -gt 0 -and [int][char]$headerLine[0] -eq 0xFEFF) { + $headerLine = $headerLine.Substring(1) + } + $cols = $headerLine.Split(',') + $ridIdx = -1 + for ($i = 0; $i -lt $cols.Length; $i++) { + $name = $cols[$i].Trim().Trim('"') + if ($name -eq 'RecordId') { $ridIdx = $i; break } + } + if ($ridIdx -lt 0) { + throw "ConvertTo-RecordIdExclusion: 'RecordId' column not found in '$TargetCsv'." + } + $writer = [System.IO.StreamWriter]::new($OutListPath, $false, $utf8NoBom) + while (-not $reader.EndOfStream) { + $line = $reader.ReadLine() + if ([string]::IsNullOrEmpty($line)) { continue } + # Lightweight CSV split that respects double-quoted fields containing commas. + $fields = [System.Collections.Generic.List[string]]::new() + $buf = [System.Text.StringBuilder]::new() + $inQ = $false + for ($j = 0; $j -lt $line.Length; $j++) { + $ch = $line[$j] + if ($ch -eq '"') { $inQ = -not $inQ; continue } + if ($ch -eq ',' -and -not $inQ) { + [void]$fields.Add($buf.ToString()); [void]$buf.Clear(); continue + } + [void]$buf.Append($ch) + } + [void]$fields.Add($buf.ToString()) + if ($ridIdx -ge $fields.Count) { continue } + $rid = $fields[$ridIdx].Trim() + if ([string]::IsNullOrEmpty($rid)) { continue } + if ($seen.Add($rid)) { + $writer.WriteLine($rid) + } + } + } + finally { + if ($writer) { $writer.Dispose() } + if ($reader) { $reader.Dispose() } + } + return $seen.Count +} + +function script:Import-CsvDeduped { + <# + .SYNOPSIS + Import-Csv wrapper that tolerates duplicate / BOM-prefixed / whitespace-padded + header columns. Required when reading Users CSVs that may have round-tripped + through the embedded Python processor and acquired duplicate header tokens + (e.g. both 'userPrincipalName' and 'PersonId' present pre-rename). + + .DESCRIPTION + PowerShell's Import-Csv calls Add-Member internally for each header cell, which + throws 'The member "X" is already present' when two header cells normalize to + the same name. This helper rewrites the header line with later duplicates + suffixed (_dup2, _dup3, …) before parsing, so the file always loads. The first + occurrence keeps its original name; callers that care can drop the *_dup* extras. + + Also strips a leading BOM from the header line and trims whitespace from each + header cell. The data rows are passed through unchanged. + + .OUTPUTS + Array of [pscustomobject]. Empty array if file is missing or contains only a + header (or only whitespace). + #> + [CmdletBinding()] + param([Parameter(Mandatory)][string]$LiteralPath) + if (-not (Test-Path -LiteralPath $LiteralPath -PathType Leaf)) { return @() } + $lines = @(Get-Content -LiteralPath $LiteralPath -Encoding UTF8) + if ($lines.Count -eq 0) { return @() } + $hdrLine = $lines[0] -replace '^\uFEFF','' + # Parse header honoring quoted commas using TextFieldParser. + try { Add-Type -AssemblyName Microsoft.VisualBasic -ErrorAction Stop } catch { } + $sr = [System.IO.StringReader]::new($hdrLine) + $tfp = New-Object Microsoft.VisualBasic.FileIO.TextFieldParser($sr) + $tfp.SetDelimiters(',') + $tfp.HasFieldsEnclosedInQuotes = $true + $hdrCells = @() + try { $hdrCells = @($tfp.ReadFields()) } catch { $hdrCells = @($hdrLine -split ',') } + finally { $tfp.Dispose(); $sr.Dispose() } + $seen = [System.Collections.Generic.Dictionary[string,int]]::new([System.StringComparer]::OrdinalIgnoreCase) + $newHdr = New-Object System.Collections.Generic.List[string] + foreach ($c in $hdrCells) { + $n = ($c -as [string]) + if ($null -eq $n) { $n = '' } + $n = $n.Trim() + if ([string]::IsNullOrEmpty($n)) { $n = '_blank' } + if ($seen.ContainsKey($n)) { + $seen[$n] = $seen[$n] + 1 + $newHdr.Add(('{0}_dup{1}' -f $n, $seen[$n])) + } else { + $seen[$n] = 1 + $newHdr.Add($n) + } + } + $newHdrLine = ($newHdr | ForEach-Object { '"' + ($_ -replace '"','""') + '"' }) -join ',' + if ($lines.Count -eq 1) { return @() } + $rest = $lines | Select-Object -Skip 1 + $rewritten = @($newHdrLine) + @($rest) + return @($rewritten | ConvertFrom-Csv) +} + +function Merge-UsersCsv { + <# + .SYNOPSIS + Union-merge a target Users CSV with the current run's freshly-emitted Users CSV. + + .DESCRIPTION + Post-Python helper for -AppendUserInfo. Reads both files and writes a single union + CSV. By default the union is written in place of the current-run CSV (legacy + behavior). Pass -OutputPath to direct the union to a third path so the current-run + CSV remains pristine (the pristine-raw separation pattern used by -AppendUserInfo). + Semantics: + - Retained users (in BOTH): keep target's UserKey + Date_Added; In_Latest_Append=TRUE. + - New users (only in current run): mint Date_Added=today; In_Latest_Append=TRUE. + - Departed users (only in target): carry target row forward; In_Latest_Append=FALSE. + - Latest_Append_Date stamped today on every row. + - TotalEmployees recomputed across the union and rewritten on every row. + + Atomic rewrite: writes to '.merging' then renames over the output path. + + .OUTPUTS + Hashtable with stats (Retained / New / Departed / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUsersCsv, + [Parameter(Mandatory)] [string] $CurrentUsersCsv, + [Parameter()] [string] $OutputPath, + [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd'), + # Fixed in-memory buffer budget for the bounded sorts (bytes). Governs only + # WHEN buffered rows spill; never caps how many rows are processed. + [int64] $MemoryBudgetBytes = 33554432, + # Observability-only progress seam, inert by default. It is forwarded verbatim to + # the bounded sorts, which pass the callback an AGGREGATE snapshot only and swallow + # any exception it raises. Supplying it cannot change the merged output, the row + # order, the classification counts, or the success of the merge. + [scriptblock] $ProgressCallback, + [double] $ProgressIntervalSeconds = 0 + ) + if (-not (Test-Path -LiteralPath $CurrentUsersCsv -PathType Leaf)) { + throw "Merge-UsersCsv: current Users CSV not found: '$CurrentUsersCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentUsersCsv } + + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $u8 = [System.Text.UTF8Encoding]::new($false) + + # Read + dedupe a CSV header record (BOM strip / trim / blank->_blank / _dupN) + # exactly as script:Import-CsvDeduped rewrites it, returning the canonical names. + $readDedupHeader = { + param($reader) + $h = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $h) { return $null } + $seen = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + $out = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $h.Length; $i++) { + $n = [string]$h[$i] + if ($i -eq 0) { $n = $n -replace '^\uFEFF', '' } + $n = $n.Trim() + if ([string]::IsNullOrEmpty($n)) { $n = '_blank' } + if ($seen.ContainsKey($n)) { $seen[$n] = $seen[$n] + 1; $out.Add(('{0}_dup{1}' -f $n, $seen[$n])) } + else { $seen[$n] = 1; $out.Add($n) } + } + return , $out.ToArray() + } + # First index of a name in a header (OrdinalIgnoreCase) or -1. + $findIdx = { + param($header, $name) + for ($i = 0; $i -lt $header.Length; $i++) { if ([string]::Equals([string]$header[$i], $name, [System.StringComparison]::OrdinalIgnoreCase)) { return $i } } + return -1 + } + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_usersmerge_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $tmpPath = "$OutputPath.merging" + $retainedCount = 0; $newCount = 0; $departedCount = 0; $unionCount = 0 + + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + + # --- Resolve headers (bounded: only the header record of each file). --- + $targetExists = Test-Path -LiteralPath $TargetUsersCsv -PathType Leaf + $curHdrReader = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + $currentHeader = $null + try { $currentHeader = & $readDedupHeader $curHdrReader } finally { $curHdrReader.Dispose() } + if ($null -eq $currentHeader) { $currentHeader = @() } + $targetHeader = @() + if ($targetExists) { + $tgtHdrReader = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + try { $th = & $readDedupHeader $tgtHdrReader; if ($null -ne $th) { $targetHeader = $th } } finally { $tgtHdrReader.Dispose() } + } + + # --- Union header: current cols, PersonId_Normalized (derived-when-absent), + # target-only cols, then provenance cols. --- + $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $hdrOrder = New-Object System.Collections.Generic.List[string] + foreach ($c in $currentHeader) { if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } } + if ($hdrSet.Add('PersonId_Normalized')) { $hdrOrder.Add('PersonId_Normalized') } + foreach ($c in $targetHeader) { if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } } + foreach ($c in @('Date_Added', 'Latest_Append_Date', 'In_Latest_Append')) { if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } } + $unionCols = $hdrOrder.ToArray() + $U = $unionCols.Length + $uIndex = @{} + for ($i = 0; $i -lt $U; $i++) { $uIndex[$unionCols[$i]] = $i } + $idxPidNorm = [int]$uIndex['PersonId_Normalized'] + $idxUserKey = if ($uIndex.ContainsKey('UserKey')) { [int]$uIndex['UserKey'] } else { -1 } + $idxDateAdded = [int]$uIndex['Date_Added'] + $idxLatest = [int]$uIndex['Latest_Append_Date'] + $idxInLatest = [int]$uIndex['In_Latest_Append'] + $hasTotalEmp = $uIndex.ContainsKey('TotalEmployees') + $idxTotalEmp = if ($hasTotalEmp) { [int]$uIndex['TotalEmployees'] } else { -1 } + + # Source -> union position maps and PersonId source indices. + $cToU = New-Object 'int[]' $currentHeader.Length + for ($s = 0; $s -lt $currentHeader.Length; $s++) { $cToU[$s] = [int]$uIndex[$currentHeader[$s]] } + $cPN = & $findIdx $currentHeader 'PersonId_Normalized' + $cUPN = & $findIdx $currentHeader 'userPrincipalName' + $cPID = & $findIdx $currentHeader 'PersonId' + $tToU = New-Object 'int[]' $targetHeader.Length + for ($s = 0; $s -lt $targetHeader.Length; $s++) { $tToU[$s] = [int]$uIndex[$targetHeader[$s]] } + $tPN = & $findIdx $targetHeader 'PersonId_Normalized' + $tUPN = & $findIdx $targetHeader 'userPrincipalName' + $tPID = & $findIdx $targetHeader 'PersonId' + + # Build the union-normalized value vector for one source row; also return the + # derived normalized PersonId key and whether it was derived (PN absent/blank). + $normalizeRow = { + param($row, $srcToU, $iPN, $iUPN, $iPID) + $vals = New-Object 'string[]' $U + for ($j = 0; $j -lt $U; $j++) { $vals[$j] = '' } + for ($s = 0; $s -lt $srcToU.Length; $s++) { + $vals[$srcToU[$s]] = if ($s -lt $row.Length) { [string]$row[$s] } else { '' } + } + $pidVal = '' + if ($iPN -ge 0 -and $iPN -lt $row.Length -and -not [string]::IsNullOrWhiteSpace([string]$row[$iPN])) { $pidVal = ([string]$row[$iPN]).Trim().ToLowerInvariant() } + elseif ($iUPN -ge 0 -and $iUPN -lt $row.Length -and -not [string]::IsNullOrWhiteSpace([string]$row[$iUPN])) { $pidVal = ([string]$row[$iUPN]).Trim().ToLowerInvariant() } + elseif ($iPID -ge 0 -and $iPID -lt $row.Length -and -not [string]::IsNullOrWhiteSpace([string]$row[$iPID])) { $pidVal = ([string]$row[$iPID]).Trim().ToLowerInvariant() } + $pnBlank = ($iPN -lt 0) -or ($iPN -ge $row.Length) -or [string]::IsNullOrWhiteSpace([string]$row[$iPN]) + if ($pnBlank) { $vals[$idxPidNorm] = $pidVal } + return [pscustomobject]@{ Vals = $vals; Pid = $pidVal; Derived = ($pnBlank -and -not [string]::IsNullOrWhiteSpace($pidVal)) } + } + + # --- Stream both inputs to a bounded work file (never materialized whole). --- + $workPath = Join-Path $workDir 'work.csv' + $workHdr = @('__PID', '__SEQ', '__SIDE') + $unionCols + $ww = [System.IO.StreamWriter]::new($workPath, $false, $u8) + $seq = [int64]0 + $targetDataRows = [int64]0 + $targetDerived = 0; $currentDerived = 0; $anyTargetPid = $false + try { + script:Write-PaxCsvRecord -Writer $ww -Fields $workHdr + if ($targetExists) { + $tr = [System.IO.StreamReader]::new($TargetUsersCsv, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $tr # skip header + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $tr + if ($null -eq $row) { break } + $targetDataRows++ + $n = & $normalizeRow $row $tToU $tPN $tUPN $tPID + if ($n.Derived) { $targetDerived++ } + if (-not [string]::IsNullOrWhiteSpace($n.Pid)) { + $anyTargetPid = $true + $fields = New-Object System.Collections.Generic.List[string] + $fields.Add($n.Pid); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('0') + foreach ($v in $n.Vals) { $fields.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + } + $seq++ + } + } + finally { $tr.Dispose() } + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetDataRows -eq 0 -and $targetExists -and ((Get-Item -LiteralPath $TargetUsersCsv).Length -gt 0)) { + throw "Merge-UsersCsv: target '$TargetUsersCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + $cr = [System.IO.StreamReader]::new($CurrentUsersCsv, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $cr # skip header + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr + if ($null -eq $row) { break } + $n = & $normalizeRow $row $cToU $cPN $cUPN $cPID + if ($n.Derived) { $currentDerived++ } + $fields = New-Object System.Collections.Generic.List[string] + $pidField = $n.Pid # may be '' (blank-pid current rows are still emitted) + $fields.Add($pidField); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('1') + foreach ($v in $n.Vals) { $fields.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + $seq++ + } + } + finally { $cr.Dispose() } + } + finally { $ww.Flush(); $ww.Dispose() } + + # Auditable derivation log (parity with the prior in-memory path). + if ($targetDataRows -gt 0 -and -not $anyTargetPid) { + Write-LogHost ( + ("WARNING: Merge-UsersCsv: target Users CSV has no derivable dedup key (PersonId_Normalized / userPrincipalName / PersonId are all empty on every row). " + + "Cannot classify Retained / New / Departed; the union will contain only the current-run rows. " + + "Target: {0}") -f $TargetUsersCsv + ) -ForegroundColor Yellow + } + elseif ($targetDerived -gt 0 -or $currentDerived -gt 0) { + Write-LogHost ( + "Merge-UsersCsv: derived PersonId_Normalized on the fly (target rows: {0}, current rows: {1}) — column was absent on the seed/raw shape (expected for M365-bundle EntraUsers CSVs)." -f $targetDerived, $currentDerived + ) -ForegroundColor Cyan + } + + # --- Sort work by (PID, SEQ): equal keys adjacent, target rows before current. --- + $sortedPath = Join-Path $workDir 'sorted.csv' + [void](script:Invoke-PaxExternalSort -InputPath $workPath -OutputPath $sortedPath -KeyColumn @('__PID', '__SEQ') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + # --- Join per PID group; emit ordered result rows [__ORD,__SEQ2]+union cols. + # __ORD '0' = current-section (current order), '1' = departed (target order). --- + $resultPath = Join-Path $workDir 'result.csv' + $sr = [System.IO.StreamReader]::new($sortedPath, $true) + $rw = [System.IO.StreamWriter]::new($resultPath, $false, $u8) + $section1 = 0 + try { + $null = script:Read-PaxCsvRecord -Reader $sr + script:Write-PaxCsvRecord -Writer $rw -Fields (@('__ORD', '__SEQ2') + $unionCols) + $emitRow = { + param($ord, $seq2, $vals) + $out = New-Object System.Collections.Generic.List[string] + $out.Add($ord); $out.Add($seq2) + foreach ($v in $vals) { $out.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $rw -Fields $out.ToArray() + } + $haveGroup = $false; $curPid = $null; $targetFirst = $null; $targetFirstSeq = $null; $sawCurrent = $false + $flush = { + if (-not $haveGroup) { return } + if (-not [string]::IsNullOrEmpty($curPid)) { + if ($sawCurrent) { + if ($null -ne $targetFirst) { $script:__uret++ } else { $script:__unew++ } + } + } + if ((-not $sawCurrent) -and ($null -ne $targetFirst)) { + $dv = $targetFirst.Clone() + if ([string]::IsNullOrWhiteSpace([string]$dv[$idxDateAdded])) { $dv[$idxDateAdded] = $RunDate } + $dv[$idxLatest] = $RunDate; $dv[$idxInLatest] = 'FALSE' + & $emitRow '1' $targetFirstSeq $dv + $script:__udep++ + } + } + $script:__uret = 0; $script:__unew = 0; $script:__udep = 0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $sr + if ($null -eq $row) { break } + $k = [string]$row[0] + $rseq = [string]$row[1] + $side = [string]$row[2] + $vals = New-Object 'string[]' $U + for ($j = 0; $j -lt $U; $j++) { $vals[$j] = if (($j + 3) -lt $row.Length) { [string]$row[$j + 3] } else { '' } } + if ((-not $haveGroup) -or (-not [string]::Equals($k, $curPid, [System.StringComparison]::Ordinal))) { + & $flush + $curPid = $k; $targetFirst = $null; $targetFirstSeq = $null; $sawCurrent = $false; $haveGroup = $true + } + if ($side -eq '0') { + if ($null -eq $targetFirst) { $targetFirst = $vals; $targetFirstSeq = $rseq } + } + else { + $out = $vals.Clone() + if ($null -ne $targetFirst) { + if ($idxUserKey -ge 0 -and -not [string]::IsNullOrWhiteSpace([string]$targetFirst[$idxUserKey])) { $out[$idxUserKey] = $targetFirst[$idxUserKey] } + if (-not [string]::IsNullOrWhiteSpace([string]$targetFirst[$idxDateAdded])) { $out[$idxDateAdded] = $targetFirst[$idxDateAdded] } else { $out[$idxDateAdded] = $RunDate } + } + else { + $out[$idxDateAdded] = $RunDate + } + $out[$idxLatest] = $RunDate; $out[$idxInLatest] = 'TRUE' + & $emitRow '0' $rseq $out + $sawCurrent = $true + $section1++ + } + } + & $flush + $retainedCount = [int]$script:__uret; $newCount = [int]$script:__unew; $departedCount = [int]$script:__udep + Remove-Variable -Name __uret, __unew, __udep -Scope Script -ErrorAction SilentlyContinue + } + finally { $rw.Flush(); $rw.Dispose(); $sr.Dispose() } + + $unionCount = $section1 + $departedCount + + # --- Restore output order (current-section then departed) and publish. --- + $orderedPath = Join-Path $workDir 'ordered.csv' + [void](script:Invoke-PaxExternalSort -InputPath $resultPath -OutputPath $orderedPath -KeyColumn @('__ORD', '__SEQ2') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds) + + $unionColsLocal = $unionCols + $totalEmpStr = "$unionCount" + $objStream = { + $orr = [System.IO.StreamReader]::new($orderedPath, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $orr + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $orr + if ($null -eq $row) { break } + $o = [ordered]@{} + for ($j = 0; $j -lt $unionColsLocal.Length; $j++) { $o[$unionColsLocal[$j]] = if (($j + 2) -lt $row.Length) { [string]$row[$j + 2] } else { '' } } + if ($hasTotalEmp) { $o['TotalEmployees'] = $totalEmpStr } + [pscustomobject]$o + } + } + finally { $orr.Dispose() } + } + & $objStream | Select-Object -Property $unionCols | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $tmpPath) { try { Remove-Item -LiteralPath $tmpPath -Force -ErrorAction SilentlyContinue } catch {} } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + + return @{ + Retained = $retainedCount + New = $newCount + Departed = $departedCount + Union = $unionCount + } +} + +function Get-FactCompositeKeyColumns { + <# + .SYNOPSIS + Return the grain-composite append dedup key column list for a rolled-up + Interactions Fact CSV, derived from its header columns. + .DESCRIPTION + Rolled-up fact rows FAN OUT: many rows share one + Message_Id_Raw, one per distinct grain (e.g. per AccessedResource). Cross-run + append dedup must therefore key on the FULL grain PLUS Message_Id_Raw; keying + on Message_Id_Raw alone collapses every fan-out row for a message to one and + silently discards the rest on merge. + + Grain columns use their stable, cross-run-comparable forms: the per-run INT + surrogates 'UserKey' and 'ThreadId' are replaced by the deid-consistent + normalized user identity (AIO: 'User_Id_Normalized' (FIX 5); AIBV: the + pre-existing 'Audit_UserId_Normalized') and by 'ThreadId_Raw' respectively. + The remaining names mirror the embedded processor grain (GRAIN_KEYS_AIO / + GRAIN_KEYS_AIBV) verbatim. Profile is detected from the header (AIBV carries + 'Is_Agent_Activity'). Returns @() when the header is not a recognizable fact + header (no Message_Id_Raw) so the caller falls back to its own guard. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string[]] $HeaderColumns) + + if (-not ($HeaderColumns -contains 'Message_Id_Raw')) { return @() } + $isAibv = ($HeaderColumns -contains 'Is_Agent_Activity') + $userIdCol = if ($isAibv) { 'Audit_UserId_Normalized' } else { 'User_Id_Normalized' } + $cols = [System.Collections.Generic.List[string]]::new() + $cols.Add($userIdCol) # replaces the per-run 'UserKey' INT surrogate + $cols.Add('InteractionDate') + $cols.Add('AgentId') + $cols.Add('AgentName') + $cols.Add('AppHost') + $cols.Add('Environment') + $cols.Add('License Status') + $cols.Add('Context_Type') + $cols.Add('Behavior_Category') + $cols.Add('Behavior_Enriched') + $cols.Add('AI_Model') + $cols.Add('Is_Sensitive') + $cols.Add('Autonomy_Pattern') + $cols.Add('AppIdentity_AppId') + $cols.Add('AISystemPlugin_Name') + $cols.Add('ThreadId_Raw') # replaces the per-run 'ThreadId' INT surrogate + if ($isAibv) { + $cols.Add('Is_Agent_Activity') + $cols.Add('Web_Grounded_Signal') + $cols.Add('Workflow_Action') + } + $cols.Add('Message_Id_Raw') + # Emit as a flat string[]; callers collect with @(...) (an empty return above + # unrolls to nothing, which @(...) normalizes to an empty array). + return $cols.ToArray() +} +#region PAX-S2 bounded-memory external-sort primitives +# Shared bounded-memory CSV primitives. These helpers stream records through +# fixed-size buffers and spill to disk once a caller-supplied byte budget is +# reached, so peak memory stays bounded by the budget plus a single record +# rather than by the total row count. Nothing in the main flow calls them; they +# are self-contained building blocks. Every function is script-scoped to keep +# the session global namespace clean. The byte budget controls only WHEN data +# spills to disk; it never caps how many rows, fields, files, or dates are +# processed. A single record larger than the budget is still processed intact. + +function script:New-PaxSortMetrics { + # Return a fresh deterministic metrics record. Counters are Int64 aggregate + # numbers only; the object never carries field values, keys, or row contents. + # The run-descriptor and level-occupancy counters prove the sort retains a + # fixed-bounded amount of run bookkeeping regardless of total input size, and + # the managed-heap checkpoints record the live managed allocation at start, + # after every buffer has spilled, and after the final merge. + [CmdletBinding()] + param() + [pscustomobject]@{ + RowsRead = [int64]0 + RowsWritten = [int64]0 + RunCount = [int64]0 + MergePassCount = [int64]0 + MaxBufferedBytes = [int64]0 + MaxBufferedRecords = [int64]0 + MaxAccountedBufferBytes = [int64]0 + LargestRecordBytes = [int64]0 + LargestSerializedRecordBytes = [int64]0 + MaxRetainedRunDescriptors = [int64]0 + MaxLevelOccupancy = [int64]0 + MaxOpenReaders = [int64]0 + TempBytesWritten = [int64]0 + ManagedHeapBefore = [int64]0 + ManagedHeapAfterIngest = [int64]0 + ManagedHeapAfterMerge = [int64]0 + CleanupSucceeded = $true + FailureStage = $null + } +} + +function script:Read-PaxCsvRecord { + # Read the next logical CSV record from an open StreamReader and return its + # fields as an ordered [string[]], or $null at end of input. Behaviour is the + # authoritative RFC-4180 contract defined by Read-PaxCsvRecordFallback below. + # + # By default a compiled state machine (PaxCsvRecordReader) performs the read for + # speed; it implements the SAME four-state machine, the SAME record boundaries, + # the SAME doubled-quote / embedded CR-LF handling, and throws the SAME messages + # as the fallback. If dynamic compilation is unavailable in a restricted host, + # the original PowerShell parser is used instead with identical results. The + # backend is chosen ONCE; a parser error from the compiled backend is surfaced + # verbatim (never used to retry the fallback on an already-advanced stream). + # $script:PaxCsvBackendOverride is a TEST-ONLY seam ('Compiled' | 'Fallback'); + # the production default is automatic. + [CmdletBinding()] + param([Parameter(Mandatory)][System.IO.StreamReader]$Reader) + $backend = $script:PaxCsvBackendOverride + if ($backend -eq 'Fallback') { return , (script:Read-PaxCsvRecordFallback -Reader $Reader) } + if ($backend -ne 'Compiled') { + # Automatic: prefer the compiled backend; fall back ONLY when compilation is + # unavailable (an initialization decision, never a per-record parse retry). + if (-not (script:Initialize-PaxCsvCompiledReader)) { + if (-not $script:PaxCsvFallbackWarned) { + $script:PaxCsvFallbackWarned = $true + $lh = Microsoft.PowerShell.Core\Get-Command -Name 'Write-LogHost' -CommandType Function -ErrorAction SilentlyContinue + if ($lh) { try { & $lh -Message 'PAX CSV read acceleration is unavailable in this host; using the built-in parser (results are unchanged).' -ForegroundColor 'DarkGray' } catch { } } + } + return , (script:Read-PaxCsvRecordFallback -Reader $Reader) + } + } + else { + # Forced compiled (test seam): require the compiled backend. + if (-not (script:Initialize-PaxCsvCompiledReader)) { throw 'PaxCsvParse: compiled backend was forced but is unavailable in this host.' } + } + # Compiled read. Surface the parser's own message verbatim; NEVER fall back after + # the stream has advanced (that would corrupt record position). + try { + $rec = [PaxCsvRecordReader]::ReadRecord($Reader) + } + catch { + $pmsg = $_.Exception.Message + if ($_.Exception.InnerException -and $_.Exception.InnerException.Message) { $pmsg = $_.Exception.InnerException.Message } + throw $pmsg + } + return , $rec +} + +function script:Initialize-PaxCsvCompiledReader { + # Compile the RFC-4180 reader type ONCE per process and cache the outcome so + # routine record reads never re-probe assemblies, recompile, allocate source, + # or re-catch a prior initialization failure. Returns $true when the compiled + # backend is usable, $false when dynamic compilation is unavailable (restricted + # host / language-mode policy). Emits nothing to the pipeline. + if ($null -ne $script:PaxCsvCompiledReady) { return $script:PaxCsvCompiledReady } + try { + if (-not ('PaxCsvRecordReader' -as [type])) { + $paxCsvReaderSource = @' +using System.IO; +using System.Collections.Generic; +using System.Text; +public static class PaxCsvRecordReader { + // Field state: 0 fresh, 1 unquoted, 2 inside quotes, 3 after a closing quote. + public static string[] ReadRecord(StreamReader reader) { + if (reader.Peek() < 0) { return null; } + List fields = new List(); + StringBuilder sb = new StringBuilder(); + int state = 0; + while (true) { + int ci = reader.Read(); + if (ci < 0) { + if (state == 2) { throw new IOException("PaxCsvParse: unterminated quoted field at end of input."); } + fields.Add(sb.ToString()); + return fields.ToArray(); + } + char c = (char)ci; + if (state == 0) { + if (c == '"') { state = 2; } + else if (c == ',') { fields.Add(sb.ToString()); sb.Clear(); } + else if (c == '\r') { if (reader.Peek() == 10) { reader.Read(); } fields.Add(sb.ToString()); return fields.ToArray(); } + else if (c == '\n') { fields.Add(sb.ToString()); return fields.ToArray(); } + else { sb.Append(c); state = 1; } + } + else if (state == 1) { + if (c == '"') { throw new IOException("PaxCsvParse: quote inside an unquoted field."); } + else if (c == ',') { fields.Add(sb.ToString()); sb.Clear(); state = 0; } + else if (c == '\r') { if (reader.Peek() == 10) { reader.Read(); } fields.Add(sb.ToString()); return fields.ToArray(); } + else if (c == '\n') { fields.Add(sb.ToString()); return fields.ToArray(); } + else { sb.Append(c); } + } + else if (state == 2) { + if (c == '"') { + if (reader.Peek() == 34) { reader.Read(); sb.Append('"'); } + else { state = 3; } + } + else { sb.Append(c); } + } + else { + if (c == ',') { fields.Add(sb.ToString()); sb.Clear(); state = 0; } + else if (c == '\r') { if (reader.Peek() == 10) { reader.Read(); } fields.Add(sb.ToString()); return fields.ToArray(); } + else if (c == '\n') { fields.Add(sb.ToString()); return fields.ToArray(); } + else { throw new IOException("PaxCsvParse: unexpected content after a closing quote."); } + } + } + } +} +'@ + Add-Type -TypeDefinition $paxCsvReaderSource -Language CSharp -ErrorAction Stop | Out-Null + } + $script:PaxCsvCompiledReady = $true + } + catch { + $script:PaxCsvCompiledReady = $false + } + return $script:PaxCsvCompiledReady +} + +function script:Read-PaxCsvRecordFallback { + # AUTHORITATIVE RFC-4180 contract (original interpreted parser, preserved verbatim + # as a portable fallback). Read the next logical CSV record from an open + # StreamReader and return its fields as an ordered [string[]], or $null at end of + # input. A single logical record may span several physical lines when a field is + # quoted; quoted fields carry commas, CR, LF, and doubled quotes ("") as literal + # content. A record larger than any read buffer accumulates whole to its + # terminator. Malformed quoting is a terminal error: a quote inside an unquoted + # field, any content after a closing quote, or an unterminated quoted field at end + # of input all throw rather than guess. + [CmdletBinding()] + param([Parameter(Mandatory)][System.IO.StreamReader]$Reader) + if ($Reader.Peek() -lt 0) { return $null } + $fields = [System.Collections.Generic.List[string]]::new() + $sb = [System.Text.StringBuilder]::new() + # Field state: 0 fresh, 1 unquoted, 2 inside quotes, 3 after a closing quote. + $state = 0 + while ($true) { + $ci = $Reader.Read() + if ($ci -lt 0) { + if ($state -eq 2) { throw 'PaxCsvParse: unterminated quoted field at end of input.' } + $fields.Add($sb.ToString()) + return , $fields.ToArray() + } + $c = [char]$ci + switch ($state) { + 0 { + if ($c -eq '"') { $state = 2 } + elseif ($c -eq ',') { $fields.Add($sb.ToString()); [void]$sb.Clear() } + elseif ($c -eq "`r") { if ($Reader.Peek() -eq 10) { [void]$Reader.Read() }; $fields.Add($sb.ToString()); return , $fields.ToArray() } + elseif ($c -eq "`n") { $fields.Add($sb.ToString()); return , $fields.ToArray() } + else { [void]$sb.Append($c); $state = 1 } + } + 1 { + if ($c -eq '"') { throw 'PaxCsvParse: quote inside an unquoted field.' } + elseif ($c -eq ',') { $fields.Add($sb.ToString()); [void]$sb.Clear(); $state = 0 } + elseif ($c -eq "`r") { if ($Reader.Peek() -eq 10) { [void]$Reader.Read() }; $fields.Add($sb.ToString()); return , $fields.ToArray() } + elseif ($c -eq "`n") { $fields.Add($sb.ToString()); return , $fields.ToArray() } + else { [void]$sb.Append($c) } + } + 2 { + if ($c -eq '"') { + if ($Reader.Peek() -eq 34) { [void]$Reader.Read(); [void]$sb.Append('"') } + else { $state = 3 } + } + else { [void]$sb.Append($c) } + } + 3 { + if ($c -eq ',') { $fields.Add($sb.ToString()); [void]$sb.Clear(); $state = 0 } + elseif ($c -eq "`r") { if ($Reader.Peek() -eq 10) { [void]$Reader.Read() }; $fields.Add($sb.ToString()); return , $fields.ToArray() } + elseif ($c -eq "`n") { $fields.Add($sb.ToString()); return , $fields.ToArray() } + else { throw 'PaxCsvParse: unexpected content after a closing quote.' } + } + } + } +} + +function script:Write-PaxCsvRecord { + # Write one RFC-4180-safe record to an open StreamWriter. A field is quoted + # only when it contains a comma, a double-quote, a CR, or an LF; embedded + # quotes are doubled. Output is deterministic, and a parse -> write -> parse + # round trip preserves every field value exactly. Records are terminated with + # CRLF so the reader recognizes them regardless of platform. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamWriter]$Writer, + [Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]]$Fields + ) + $sb = [System.Text.StringBuilder]::new() + $special = [char[]]@(',', '"', "`r", "`n") + for ($i = 0; $i -lt $Fields.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append(',') } + $f = $Fields[$i]; if ($null -eq $f) { $f = '' } + if ($f.IndexOfAny($special) -ge 0) { + [void]$sb.Append('"'); [void]$sb.Append($f.Replace('"', '""')); [void]$sb.Append('"') + } + else { [void]$sb.Append($f) } + } + [void]$sb.Append("`r`n") + $Writer.Write($sb.ToString()) +} + +function script:Get-PaxComparer { + # Map a caller-selected comparer name to the matching ordinal StringComparer. + # The comparer is applied to whole composite keys; keys are never lowercased + # as a shortcut for case-insensitive comparison, so the caller stays in full + # control of case sensitivity. + [CmdletBinding()] + param([Parameter(Mandatory)][ValidateSet('Ordinal', 'OrdinalIgnoreCase')][string]$Name) + if ($Name -eq 'OrdinalIgnoreCase') { return [System.StringComparer]::OrdinalIgnoreCase } + return [System.StringComparer]::Ordinal +} + +function script:Get-PaxCompositeKey { + # Build the composite key for a record by joining the caller-selected columns, + # in caller order, with the U+001F unit separator. That separator is a control + # character absent from normal field text, so distinct column tuples map to + # distinct keys. A key index outside the record contributes an empty segment. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]]$Fields, + [Parameter(Mandatory)][int[]]$KeyIndices + ) + $sep = [char]0x1F + $sb = [System.Text.StringBuilder]::new() + for ($i = 0; $i -lt $KeyIndices.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append($sep) } + $idx = $KeyIndices[$i] + if ($idx -ge 0 -and $idx -lt $Fields.Length) { + $v = $Fields[$idx]; if ($null -ne $v) { [void]$sb.Append($v) } + } + } + return $sb.ToString() +} + +function script:ConvertTo-PaxCsvLine { + # Serialize a field array to a single RFC-4180 line with no terminator. A field + # is quoted only when it contains a comma, a double-quote, a CR, or an LF, and + # embedded quotes are doubled, so the result parses back to the same fields. + # The buffered item keeps this one compact string instead of the field array, + # and the run spill emits the sequence, a comma, and this string unchanged. + [CmdletBinding()] + param([Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]]$Fields) + $sb = [System.Text.StringBuilder]::new() + $special = [char[]]@(',', '"', "`r", "`n") + for ($i = 0; $i -lt $Fields.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append(',') } + $f = $Fields[$i]; if ($null -eq $f) { $f = '' } + if ($f.IndexOfAny($special) -ge 0) { + [void]$sb.Append('"'); [void]$sb.Append($f.Replace('"', '""')); [void]$sb.Append('"') + } + else { [void]$sb.Append($f) } + } + return $sb.ToString() +} + +function script:Get-PaxRetainedRecordBytes { + # Return a conservative upper bound on the RETAINED managed bytes of one + # buffered item: a fixed three-slot object array holding its composite key, its + # boxed Int64 sequence, and its serialized data line. A .NET char occupies two + # managed bytes, so the retained string payloads account for 2*Key.Length plus + # 2*Data.Length. The fixed 512-byte allowance covers both string headers, the + # object-array header and three reference slots, the boxed Int64, the list slot, + # unused list-capacity allowance, alignment, and runtime object bookkeeping. + # This governs + # only WHEN a buffer spills; it never caps how much data flows through, and an + # item larger than the whole budget is still measured and processed intact. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Key, + [Parameter(Mandatory)][AllowEmptyString()][string]$Data + ) + $overhead = [int64]512 + $k = if ($null -ne $Key) { [int64]$Key.Length } else { [int64]0 } + $d = if ($null -ne $Data) { [int64]$Data.Length } else { [int64]0 } + return ([int64]2 * $k) + ([int64]2 * $d) + $overhead +} + +function script:Read-PaxRunFront { + # Read the next row from an internally-sorted run stream and return its front + # view: the Int64 sequence carried in the first field, the original record + # fields, and the recomputed composite key. Returns $null at end of the run. + # A run row missing its sequence field is a terminal error. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamReader]$Reader, + [Parameter(Mandatory)][int[]]$KeyIndices + ) + $row = script:Read-PaxCsvRecord -Reader $Reader + if ($null -eq $row) { return $null } + if ($row.Length -lt 1) { throw 'PaxMerge: malformed run row (missing sequence field).' } + $seq = [int64]::Parse($row[0], [System.Globalization.CultureInfo]::InvariantCulture) + $orig = New-Object System.Collections.Generic.List[string] + for ($i = 1; $i -lt $row.Length; $i++) { $orig.Add($row[$i]) } + $of = $orig.ToArray() + $key = script:Get-PaxCompositeKey -Fields $of -KeyIndices $KeyIndices + [pscustomobject]@{ Seq = $seq; Fields = $of; Key = $key } +} + +function script:Merge-PaxRunGroup { + # Merge up to a fan-in's worth of internally-sorted runs by opening one + # bounded reader per run, holding only the current front record from each open + # reader, and repeatedly emitting the minimum by (composite key under the + # comparer, then Int64 sequence). It never opens more readers than the group + # size and never collects a key group, so a hot key streams through. When + # EmitSequence is set the sequence is carried forward so a later pass keeps + # global order; otherwise the original record fields are written. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$RunPaths, + [Parameter(Mandatory)][int[]]$KeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][System.IO.StreamWriter]$Writer, + [Parameter(Mandatory)][bool]$EmitSequence, + [Parameter(Mandatory)][scriptblock]$ReaderFactory, + [Parameter(Mandatory)][psobject]$Metrics, + # Optional observability-only heartbeat hook (default absent). When supplied it is invoked every + # few thousand emitted rows so a long single-group merge can report active progress. It receives + # no arguments, no data, no keys, and no paths; its exceptions are swallowed and it cannot alter + # ordering, the emitted stream, or the merge result. Absent by default so existing callers and the + # emitted bytes are unchanged. + [scriptblock]$RowObserver + ) + $readers = New-Object System.Collections.Generic.List[System.IO.StreamReader] + $fronts = New-Object System.Collections.Generic.List[object] + $rgN = [int64]0 + try { + foreach ($rp in $RunPaths) { + $r = & $ReaderFactory $rp + $readers.Add($r) + $fronts.Add((script:Read-PaxRunFront -Reader $r -KeyIndices $KeyIndices)) + } + if ([int64]$readers.Count -gt $Metrics.MaxOpenReaders) { $Metrics.MaxOpenReaders = [int64]$readers.Count } + while ($true) { + $minIdx = -1 + for ($i = 0; $i -lt $fronts.Count; $i++) { + $f = $fronts[$i] + if ($null -eq $f) { continue } + if ($minIdx -lt 0) { $minIdx = $i; continue } + $m = $fronts[$minIdx] + $c = $Comparer.Compare($f.Key, $m.Key) + if ($c -lt 0 -or ($c -eq 0 -and $f.Seq -lt $m.Seq)) { $minIdx = $i } + } + if ($minIdx -lt 0) { break } + $sel = $fronts[$minIdx] + if ($EmitSequence) { + $outFields = New-Object System.Collections.Generic.List[string] + $outFields.Add($sel.Seq.ToString([System.Globalization.CultureInfo]::InvariantCulture)) + foreach ($v in $sel.Fields) { $outFields.Add($v) } + script:Write-PaxCsvRecord -Writer $Writer -Fields $outFields.ToArray() + } + else { + script:Write-PaxCsvRecord -Writer $Writer -Fields $sel.Fields + $Metrics.RowsWritten++ + } + if ($RowObserver) { $rgN++; if (($rgN -band 4095) -eq 0) { try { & $RowObserver } catch { } } } + $fronts[$minIdx] = (script:Read-PaxRunFront -Reader $readers[$minIdx] -KeyIndices $KeyIndices) + } + } + finally { + foreach ($r in $readers) { if ($r) { $r.Close(); $r.Dispose() } } + } +} + +function script:Merge-PaxSortedRuns { + # Merge the caller's survivor run set into a single ordered stream while + # honoring a bounded fan-in. The survivor set is the fixed-bounded collection + # gathered from the compaction levels, never the total run count, so the pass + # lists below are bounded by that fixed constant. When the survivor count + # exceeds the fan-in the merge runs in deterministic multiple passes: it + # merges a fan-in's worth of runs at a time into larger runs, deletes each + # consumed run as soon as its group finishes, and repeats until the survivors + # fit one final merge. No pass opens more than the fan-in of readers and no + # pass materializes a whole run or a whole hot-key group. Ordering is stable + # on (composite key under the comparer, then Int64 sequence). The final pass + # writes the original record fields to the output writer; intermediate passes + # carry the sequence forward. + [CmdletBinding()] + param( + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$RunPaths, + [Parameter(Mandatory)][int[]]$KeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][int]$FanIn, + [Parameter(Mandatory)][System.IO.StreamWriter]$OutputWriter, + [Parameter(Mandatory)][string]$WorkDir, + [Parameter(Mandatory)][scriptblock]$ReaderFactory, + [Parameter(Mandatory)][scriptblock]$WriterFactory, + [Parameter(Mandatory)][psobject]$Metrics, + # Optional observability-only observer (default absent). It is invoked as + # & $ProgressObserver at each intermediate-merge pass entry and once + # before the final merge, and (via a derived row hook) periodically during a long pass. It only + # ever receives an aggregate stage name and pass number, never a row, key, field, or path, and it + # cannot alter merge ordering, output, or the result. Absent by default so callers and emitted + # bytes are unchanged. + [scriptblock]$ProgressObserver + ) + if ($FanIn -lt 2) { throw 'PaxMerge: FanIn must be at least 2.' } + $current = New-Object System.Collections.Generic.List[string] + foreach ($p in $RunPaths) { $current.Add($p) } + if ($current.Count -eq 0) { return } + # Derive a row-level heartbeat hook that reports under the stage currently in flight. Absent unless + # an observer was supplied, so the default merge path allocates nothing extra and emits nothing. + $stageBox = [pscustomobject]@{ Stage = 'FinalMerge' } + $rowObs = $null + if ($ProgressObserver) { $rowObs = { & $ProgressObserver $stageBox.Stage -1 } } + $passIndex = 0 + while ($current.Count -gt $FanIn) { + $passIndex++ + $stageBox.Stage = 'IntermediateMerge' + if ($ProgressObserver) { & $ProgressObserver 'IntermediateMerge' $passIndex } + $next = New-Object System.Collections.Generic.List[string] + $groupNo = 0 + for ($i = 0; $i -lt $current.Count; $i += $FanIn) { + $group = New-Object System.Collections.Generic.List[string] + for ($j = $i; $j -lt [Math]::Min($i + $FanIn, $current.Count); $j++) { $group.Add($current[$j]) } + $groupNo++ + $outPath = Join-Path $WorkDir ("pass{0:d3}_grp{1:d4}.paxrun" -f $passIndex, $groupNo) + $w = & $WriterFactory $outPath + try { + script:Merge-PaxRunGroup -RunPaths $group.ToArray() -KeyIndices $KeyIndices -Comparer $Comparer -Writer $w -EmitSequence $true -ReaderFactory $ReaderFactory -Metrics $Metrics -RowObserver $rowObs + } + finally { $w.Flush(); $w.Close(); $w.Dispose() } + $fi = [System.IO.FileInfo]::new($outPath); if ($fi.Exists) { $Metrics.TempBytesWritten += [int64]$fi.Length } + foreach ($cp in $group) { if (Test-Path -LiteralPath $cp) { Remove-Item -LiteralPath $cp -Force -ErrorAction Stop } } + $next.Add($outPath) + } + $Metrics.MergePassCount++ + $current = $next + } + $stageBox.Stage = 'FinalMerge' + if ($ProgressObserver) { & $ProgressObserver 'FinalMerge' 0 } + script:Merge-PaxRunGroup -RunPaths $current.ToArray() -KeyIndices $KeyIndices -Comparer $Comparer -Writer $OutputWriter -EmitSequence $false -ReaderFactory $ReaderFactory -Metrics $Metrics -RowObserver $rowObs + foreach ($cp in $current) { if (Test-Path -LiteralPath $cp) { Remove-Item -LiteralPath $cp -Force -ErrorAction Stop } } + $Metrics.MergePassCount++ +} + +function script:Remove-PaxWorkspace { + # Remove a temp workspace and an optional in-progress publication file after a + # run finishes or fails. A cleanup problem never hides the primary outcome: it + # only clears CleanupSucceeded on the metrics so the caller can report it. + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$WorkDir, + [AllowNull()][AllowEmptyString()][string]$MergingPath, + [Parameter(Mandatory)][psobject]$Metrics + ) + $okAll = $true + if ($MergingPath -and (Test-Path -LiteralPath $MergingPath)) { + try { Remove-Item -LiteralPath $MergingPath -Force -ErrorAction Stop } catch { $okAll = $false } + } + if (Test-Path -LiteralPath $WorkDir) { + try { Remove-Item -LiteralPath $WorkDir -Recurse -Force -ErrorAction Stop } catch { $okAll = $false } + } + if (-not $okAll) { $Metrics.CleanupSucceeded = $false } +} + +function script:Select-PaxFirstRowPerKey { + # Stream the first row of each key from a (key, sequence)-sorted run stream. + # Because the input is ordered by key then ascending sequence, the first row + # seen for a key carries the lowest sequence and is emitted; later rows with + # the same key are skipped. A hot key with many rows streams through without + # ever collecting the group. Returns the number of rows emitted. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamReader]$Reader, + [Parameter(Mandatory)][int[]]$KeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][System.IO.StreamWriter]$Writer, + [bool]$EmitSequence = $false + ) + $haveLast = $false + $lastKey = $null + $emitted = [int64]0 + while ($true) { + $front = script:Read-PaxRunFront -Reader $Reader -KeyIndices $KeyIndices + if ($null -eq $front) { break } + $isNew = (-not $haveLast) -or ($Comparer.Compare($front.Key, $lastKey) -ne 0) + if ($isNew) { + if ($EmitSequence) { + $of = New-Object System.Collections.Generic.List[string] + $of.Add($front.Seq.ToString([System.Globalization.CultureInfo]::InvariantCulture)) + foreach ($v in $front.Fields) { $of.Add($v) } + script:Write-PaxCsvRecord -Writer $Writer -Fields $of.ToArray() + } + else { + script:Write-PaxCsvRecord -Writer $Writer -Fields $front.Fields + } + $emitted++ + $lastKey = $front.Key + $haveLast = $true + } + } + return $emitted +} + +function script:Merge-PaxKeyStreams { + # Co-merge two key-sorted, distinct-key run streams by advancing two bounded + # readers in lockstep. A key present in the current stream is written to the + # present writer; a key found only in the target stream is written to the + # target-only writer. Neither side is collected, so arbitrarily large streams + # classify in bounded memory. Returns present and target-only counts. + [CmdletBinding()] + param( + [Parameter(Mandatory)][System.IO.StreamReader]$CurrentReader, + [Parameter(Mandatory)][System.IO.StreamReader]$TargetReader, + [Parameter(Mandatory)][int[]]$CurrentKeyIndices, + [Parameter(Mandatory)][int[]]$TargetKeyIndices, + [Parameter(Mandatory)][System.Collections.IComparer]$Comparer, + [Parameter(Mandatory)][System.IO.StreamWriter]$PresentWriter, + [Parameter(Mandatory)][System.IO.StreamWriter]$TargetOnlyWriter + ) + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + $present = [int64]0; $targetOnly = [int64]0 + while ($true) { + if ($null -eq $cur -and $null -eq $tgt) { break } + if ($null -eq $tgt) { + script:Write-PaxCsvRecord -Writer $PresentWriter -Fields $cur.Fields; $present++ + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + continue + } + if ($null -eq $cur) { + script:Write-PaxCsvRecord -Writer $TargetOnlyWriter -Fields $tgt.Fields; $targetOnly++ + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + continue + } + $c = $Comparer.Compare($cur.Key, $tgt.Key) + if ($c -lt 0) { + script:Write-PaxCsvRecord -Writer $PresentWriter -Fields $cur.Fields; $present++ + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + } + elseif ($c -gt 0) { + script:Write-PaxCsvRecord -Writer $TargetOnlyWriter -Fields $tgt.Fields; $targetOnly++ + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + } + else { + script:Write-PaxCsvRecord -Writer $PresentWriter -Fields $cur.Fields; $present++ + $cur = script:Read-PaxRunFront -Reader $CurrentReader -KeyIndices $CurrentKeyIndices + $tgt = script:Read-PaxRunFront -Reader $TargetReader -KeyIndices $TargetKeyIndices + } + } + return [pscustomobject]@{ Present = $present; TargetOnly = $targetOnly } +} + +function script:Invoke-PaxExternalSort { + <# + .SYNOPSIS + Sort a CSV by a caller-selected composite key using bounded memory and a + fixed-level online run compactor. + + .DESCRIPTION + Reads the header record of an input CSV, maps the requested key columns to + column indices, keys each data record by the U+001F join of those columns + under the caller-selected comparer, and writes the header plus the sorted + data to the output path. Records buffer in memory only until a caller + supplied byte budget is reached (or a single record alone exceeds it), then + spill to internally-sorted run files. Each buffered item retains only its + composite key, its Int64 sequence, and its serialized data line, so retained + memory is accounted honestly rather than by field payload alone. + + Runs are combined by an online fixed-level compactor rather than by + retaining every run path. A fresh run enters level zero; whenever a level + reaches the fan-in it merges into one promoted run at the next level, the + consumed runs are deleted, and the level clears. A level-L run subsumes at + least FanIn^L runs, so with a fan-in of at least two the sixty-four levels + cover the whole positive Int64 run domain (two to the sixty-third exceeds + Int64.MaxValue). At most fan-in-minus-one runs rest at each level, so the + surviving run set gathered for the final merge is bounded by a fixed + constant, never by the total run count. The final merge combines those + survivors with the same bounded, possibly multi-pass k-way merge, so peak + memory and retained run bookkeeping stay bounded by the budget, one record, + and that fixed constant rather than by total rows or total runs. The byte + budget controls only WHEN data spills; it never caps how many rows, fields, + or files are processed. Every record is preserved, including all duplicate + keys, and a hot key streams through without being collected. + + The output is published only after every read, spill, promotion, merge, + flush, and close succeeds: the sorted data is written to a temporary sibling + file and then moved onto the output path. Any failure surfaces a terminating + error tagged with the stage that failed, leaves an existing output file + byte-for-byte unchanged, and removes partial work where possible. There is + no in-memory fallback after a spill or merge failure. An optional fault hook + is invoked at named internal checkpoints so callers can verify failure + handling; it is inert when not supplied. + + .OUTPUTS + A metrics object with Int64 counters (RowsRead, RowsWritten, RunCount, + MergePassCount, MaxBufferedBytes, MaxBufferedRecords, MaxAccountedBufferBytes, + LargestRecordBytes, LargestSerializedRecordBytes, MaxRetainedRunDescriptors, + MaxLevelOccupancy, MaxOpenReaders, TempBytesWritten, ManagedHeapBefore, + ManagedHeapAfterIngest, ManagedHeapAfterMerge), a CleanupSucceeded flag, and + a FailureStage string. Metrics carry only aggregate numbers, never data. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$InputPath, + [Parameter(Mandatory)][string]$OutputPath, + [Parameter(Mandatory)][string[]]$KeyColumn, + [Parameter(Mandatory)][ValidateSet('Ordinal', 'OrdinalIgnoreCase')][string]$Comparer, + [Parameter(Mandatory)][int64]$MemoryBudgetBytes, + [Parameter(Mandatory)][int]$FanIn, + [string]$WorkRoot, + [scriptblock]$ReaderFactory, + [scriptblock]$WriterFactory, + [scriptblock]$FaultHook, + # Optional observability-only progress seam (default inert). ProgressCallback receives an AGGREGATE + # snapshot object (stage name, elapsed seconds, and Int64 counters copied from the metrics object); + # it is never passed a row, key, field, filename, or path. ProgressIntervalSeconds throttles periodic + # emissions; stage-entry and terminal emissions are always sent. A callback exception is swallowed and + # cannot alter or fail the sort. When ProgressCallback is not supplied the seam is fully inert and the + # output is byte-identical to a run without it. + [scriptblock]$ProgressCallback, + [double]$ProgressIntervalSeconds = 0, + # Optional starting counter values (default 0). Nonzero seeds let the checked + # Int64 overflow guards below be reached deterministically without materializing + # Int64.MaxValue records; at 0 the counters and all output are unchanged. + [int64]$InitialSequence = 0, + [int64]$InitialRunCounter = 0, + [int64]$InitialPromotionCounter = 0 + ) + if ($MemoryBudgetBytes -le 0) { throw 'PaxExternalSort: MemoryBudgetBytes must be positive.' } + if ($FanIn -lt 2) { throw 'PaxExternalSort: FanIn must be at least 2.' } + if (-not $ReaderFactory) { $ReaderFactory = { param($p) [System.IO.StreamReader]::new($p, $true) } } + if (-not $WriterFactory) { $WriterFactory = { param($p) [System.IO.StreamWriter]::new($p, $false, (New-Object System.Text.UTF8Encoding($false))) } } + $cmp = script:Get-PaxComparer -Name $Comparer + $metrics = script:New-PaxSortMetrics + $metrics.ManagedHeapBefore = [int64][System.GC]::GetTotalMemory($true) + # Progress state is allocated ONLY when a callback is supplied, so a run without one starts no + # Stopwatch, allocates no progress state, and never enters the emitter in the hot loops below; the + # managed-heap checkpoints therefore see the same allocations as a run compiled without the seam. + $prog = $null + if ($ProgressCallback) { + $prog = [pscustomobject]@{ + Sw = [System.Diagnostics.Stopwatch]::StartNew() + Last = [double](-1) + Seen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + MergePass = $null + } + } + # Observability-only progress emitter (inert unless a callback is supplied). It reads aggregate + # metrics only and never mutates sort state, buffering, ordering, spill thresholds, or output. The + # first emission for each distinct stage is always sent (stage entry), a forced emission is always + # sent (terminal events), and other emissions are throttled by ProgressIntervalSeconds. Every + # callback stream (success, error, warning, verbose, debug, information) is discarded and a + # terminating callback exception is caught, so callback activity can neither become function output + # nor alter or fail the sort. No row, key, field, filename, or path is ever passed. + $emitProgress = { + param([bool]$Force, [string]$StageName) + if (-not $ProgressCallback) { return } + $el = $prog.Sw.Elapsed.TotalSeconds + $firstForStage = -not $prog.Seen.Contains($StageName) + if ($firstForStage) { [void]$prog.Seen.Add($StageName) } + if (-not ($Force -or $firstForStage)) { + if ($ProgressIntervalSeconds -le 0) { return } + if ($prog.Last -ge 0 -and ($el - $prog.Last) -lt $ProgressIntervalSeconds) { return } + } + $prog.Last = $el + $snapshot = [pscustomobject]@{ + Stage = $StageName + ElapsedSeconds = [Math]::Round($el, 3) + RowsRead = [int64]$metrics.RowsRead + RowsWritten = [int64]$metrics.RowsWritten + RunCount = [int64]$metrics.RunCount + MergePassCount = [int64]$metrics.MergePassCount + MergePass = $prog.MergePass + MaxOpenReaders = [int64]$metrics.MaxOpenReaders + TempBytesWritten = [int64]$metrics.TempBytesWritten + MaxBufferedBytes = [int64]$metrics.MaxBufferedBytes + } + try { & $ProgressCallback $snapshot 2>$null 3>$null 4>$null 5>$null 6>$null | Out-Null } + catch { } + } + # Merge-stage observer: forwards intermediate/final merge-pass entries and long-pass heartbeats + # from the merge primitive to the emitter as aggregate stage events. Inert without a callback. + $mergeObserver = { + param([string]$MergeStage, [int]$PassNo) + if (-not $ProgressCallback) { return } + if ($PassNo -ge 0) { $prog.MergePass = [int64]$PassNo } + & $emitProgress $false $MergeStage + } + # A level-L run subsumes at least FanIn^L runs; two to the sixty-third already + # exceeds Int64.MaxValue, so sixty-four fixed levels span the whole positive + # Int64 run-count domain for any fan-in of at least two. + $MaxLevels = 64 + if (-not $WorkRoot) { $WorkRoot = $env:TEMP; if (-not $WorkRoot) { $WorkRoot = [System.IO.Path]::GetTempPath() } } + $workDir = Join-Path $WorkRoot ('pax_extsort_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $mergingPath = $OutputPath + '.merging' + $st = [pscustomobject]@{ Stage = 'Init' } + $ctr = [pscustomobject]@{ Run = [int64]$InitialRunCounter; Promo = [int64]$InitialPromotionCounter } + $inReader = $null + $outWriter = $null + try { + $st.Stage = 'Workspace'; [void][System.IO.Directory]::CreateDirectory($workDir) + $st.Stage = 'Read' + $inReader = & $ReaderFactory $InputPath + $header = script:Read-PaxCsvRecord -Reader $inReader + if ($null -eq $header) { throw 'PaxExternalSort: input has no header record.' } + $st.Stage = 'KeyMapping' + $keyIndices = New-Object System.Collections.Generic.List[int] + foreach ($kc in $KeyColumn) { + $idx = -1 + for ($i = 0; $i -lt $header.Length; $i++) { if ([System.String]::Equals($header[$i], $kc, [System.StringComparison]::Ordinal)) { $idx = $i; break } } + if ($idx -lt 0) { throw 'PaxExternalSort: a requested key column is not present in the header.' } + $keyIndices.Add($idx) + } + $ki = $keyIndices.ToArray() + # Fixed-level compaction state: one bounded run-descriptor list per level. + $levels = New-Object 'System.Object[]' $MaxLevels + for ($li = 0; $li -lt $MaxLevels; $li++) { $levels[$li] = New-Object System.Collections.Generic.List[string] } + # Record the peak persistent run-descriptor count across the fixed levels and + # the final survivor owner. Fixed fan-in merge argument arrays are transient + # and never scale with total runs. + $updDesc = { + param([int64]$extra) + $sum = [int64]$extra + for ($di = 0; $di -lt $MaxLevels; $di++) { $sum += [int64]$levels[$di].Count } + if ($sum -gt $metrics.MaxRetainedRunDescriptors) { $metrics.MaxRetainedRunDescriptors = $sum } + } + # Insert a fresh run at level zero and cascade: a full level merges its + # fan-in runs into one promoted run, deletes the consumed runs, clears the + # level, and carries the promotion upward. + $insertRun = { + param($newRunPath) + $carry = $newRunPath + $lvl = 0 + while ($true) { + if ($lvl -ge $MaxLevels) { throw 'PaxExternalSort: run-level overflow (Int64 run domain exceeded).' } + $levels[$lvl].Add($carry) + if ([int64]$levels[$lvl].Count -gt $metrics.MaxLevelOccupancy) { $metrics.MaxLevelOccupancy = [int64]$levels[$lvl].Count } + & $updDesc 0 + if ($levels[$lvl].Count -lt $FanIn) { break } + if ($ctr.Promo -eq [System.Int64]::MaxValue) { $st.Stage = 'PromotionCounterOverflow'; throw 'PaxExternalSort: promotion counter reached Int64.MaxValue; refusing to wrap.' } + $ctr.Promo++ + $promoted = Join-Path $workDir ('lvl{0:d2}_{1:d6}.paxrun' -f ($lvl + 1), $ctr.Promo) + $consumed = $levels[$lvl].ToArray() + $st.Stage = 'PromoteWrite' + $pw = & $WriterFactory $promoted + try { + $st.Stage = 'PromoteRead' + script:Merge-PaxRunGroup -RunPaths $consumed -KeyIndices $ki -Comparer $cmp -Writer $pw -EmitSequence $true -ReaderFactory $ReaderFactory -Metrics $metrics + $st.Stage = 'PromoteWrite' + $pw.Flush() + } + finally { $pw.Close(); $pw.Dispose() } + $pfi = [System.IO.FileInfo]::new($promoted); if ($pfi.Exists) { $metrics.TempBytesWritten += [int64]$pfi.Length } + $st.Stage = 'PromoteDelete' + foreach ($cp in $consumed) { if (Test-Path -LiteralPath $cp) { Remove-Item -LiteralPath $cp -Force -ErrorAction Stop } } + $levels[$lvl].Clear() + if ($FaultHook) { $st.Stage = 'PromoteState'; & $FaultHook 'PromoteState' } + $metrics.MergePassCount++ + & $emitProgress $false 'Promote' + $carry = $promoted + $lvl++ + } + } + $spill = { + param($buf) + if ($buf.Count -eq 0) { return } + $buf.Sort([System.Comparison[object[]]] { + param($x, $y) + $k = $cmp.Compare([string]$x[0], [string]$y[0]) + if ($k -ne 0) { return $k } + return ([int64]$x[1]).CompareTo([int64]$y[1]) + }) + if ($ctr.Run -eq [System.Int64]::MaxValue) { $st.Stage = 'RunCounterOverflow'; throw 'PaxExternalSort: run counter reached Int64.MaxValue; refusing to wrap.' } + $ctr.Run++ + $st.Stage = 'RunCreate' + $rp = Join-Path $workDir ('run_{0:d6}.paxrun' -f $ctr.Run) + $w = & $WriterFactory $rp + $st.Stage = 'Spill' + try { + foreach ($it in $buf) { + $w.Write(([int64]$it[1]).ToString([System.Globalization.CultureInfo]::InvariantCulture)) + $w.Write(',') + $w.Write([string]$it[2]) + $w.Write("`r`n") + } + $w.Flush() + } + finally { $w.Close(); $w.Dispose() } + $fi = [System.IO.FileInfo]::new($rp); if ($fi.Exists) { $metrics.TempBytesWritten += [int64]$fi.Length } + $metrics.RunCount++ + & $emitProgress $false 'Spill' + & $insertRun $rp + } + $buffer = New-Object 'System.Collections.Generic.List[object[]]' + $bufferedBytes = [int64]0 + $seq = [int64]$InitialSequence + $st.Stage = 'Read' + & $emitProgress $true 'Ingest' + while ($true) { + $rec = script:Read-PaxCsvRecord -Reader $inReader + if ($null -eq $rec) { break } + $metrics.RowsRead++ + if ($ProgressCallback -and (($metrics.RowsRead -band 255) -eq 0)) { & $emitProgress $false 'Ingest' } + if ($seq -eq [System.Int64]::MaxValue) { $st.Stage = 'SequenceOverflow'; throw 'PaxExternalSort: ingest sequence reached Int64.MaxValue; refusing to wrap.' } + $seq++ + $key = script:Get-PaxCompositeKey -Fields $rec -KeyIndices $ki + $dataLine = script:ConvertTo-PaxCsvLine -Fields $rec + $seqStr = $seq.ToString([System.Globalization.CultureInfo]::InvariantCulture) + $serBytes = [int64][System.Text.Encoding]::UTF8.GetByteCount($seqStr + ',' + $dataLine) + if ($serBytes -gt $metrics.LargestSerializedRecordBytes) { $metrics.LargestSerializedRecordBytes = $serBytes } + $payloadBytes = [int64][System.Text.Encoding]::UTF8.GetByteCount($dataLine) + if ($payloadBytes -gt $metrics.LargestRecordBytes) { $metrics.LargestRecordBytes = $payloadBytes } + $accounted = script:Get-PaxRetainedRecordBytes -Key $key -Data $dataLine + if ($buffer.Count -gt 0 -and ($bufferedBytes + $accounted) -gt $MemoryBudgetBytes) { + & $spill $buffer + $buffer = New-Object 'System.Collections.Generic.List[object[]]' + $bufferedBytes = [int64]0 + $st.Stage = 'Read' + } + $buffer.Add([object[]]@($key, $seq, $dataLine)) + $bufferedBytes += $accounted + if ($bufferedBytes -gt $metrics.MaxBufferedBytes) { $metrics.MaxBufferedBytes = $bufferedBytes } + if ($bufferedBytes -gt $metrics.MaxAccountedBufferBytes) { $metrics.MaxAccountedBufferBytes = $bufferedBytes } + if ([int64]$buffer.Count -gt $metrics.MaxBufferedRecords) { $metrics.MaxBufferedRecords = [int64]$buffer.Count } + if ($accounted -gt $MemoryBudgetBytes) { + & $spill $buffer + $buffer = New-Object 'System.Collections.Generic.List[object[]]' + $bufferedBytes = [int64]0 + $st.Stage = 'Read' + } + } + if ($buffer.Count -gt 0) { & $spill $buffer; $buffer = $null } + $st.Stage = 'Read' + if ($inReader) { $inReader.Close(); $inReader.Dispose(); $inReader = $null } + $metrics.ManagedHeapAfterIngest = [int64][System.GC]::GetTotalMemory($true) + # Move the fixed-bounded surviving run descriptors out of the compaction + # levels. Clearing each source level as its paths transfer prevents retaining + # duplicate references in both the level set and the final survivor owner. + $survivors = New-Object System.Collections.Generic.List[string] + for ($li = 0; $li -lt $MaxLevels; $li++) { + foreach ($rp in $levels[$li]) { $survivors.Add($rp) } + $levels[$li].Clear() + & $updDesc ([int64]$survivors.Count) + } + $st.Stage = 'Publish' + $outWriter = & $WriterFactory $mergingPath + try { + script:Write-PaxCsvRecord -Writer $outWriter -Fields $header + if ($FaultHook) { $st.Stage = 'FinalMerge'; & $FaultHook 'FinalMerge' } + $st.Stage = 'Merge' + script:Merge-PaxSortedRuns -RunPaths $survivors.ToArray() -KeyIndices $ki -Comparer $cmp -FanIn $FanIn -OutputWriter $outWriter -WorkDir $workDir -ReaderFactory $ReaderFactory -WriterFactory $WriterFactory -Metrics $metrics -ProgressObserver $mergeObserver + $st.Stage = 'Publish' + $outWriter.Flush() + } + finally { $outWriter.Close(); $outWriter.Dispose(); $outWriter = $null } + $metrics.ManagedHeapAfterMerge = [int64][System.GC]::GetTotalMemory($true) + $st.Stage = 'Publish' + Move-Item -LiteralPath $mergingPath -Destination $OutputPath -Force -ErrorAction Stop + & $emitProgress $true 'Publish' + $st.Stage = 'Cleanup' + & $emitProgress $true 'Cleanup' + script:Remove-PaxWorkspace -WorkDir $workDir -MergingPath $null -Metrics $metrics + & $emitProgress $true 'Complete' + return $metrics + } + catch { + $primary = $_ + if (-not $metrics.FailureStage) { $metrics.FailureStage = $st.Stage } + & $emitProgress $true 'Failed' + if ($inReader) { try { $inReader.Close(); $inReader.Dispose() } catch { } } + if ($outWriter) { try { $outWriter.Close(); $outWriter.Dispose() } catch { } } + script:Remove-PaxWorkspace -WorkDir $workDir -MergingPath $mergingPath -Metrics $metrics + $ex = [System.Exception]::new(("PaxExternalSort failed at stage '{0}'." -f $metrics.FailureStage), $primary.Exception) + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'PaxExternalSortFailed', [System.Management.Automation.ErrorCategory]::NotSpecified, $metrics) + throw $er + } +} +#endregion PAX-S2 + +function script:Get-PaxDedupedFactHeader { + # Deduplicate a raw CSV header the same way the tolerant importer does: strip a + # leading byte-order mark from the first cell, trim each cell, replace a blank + # cell with _blank, and suffix later case-insensitive duplicates with _dup2, + # _dup3, and so on while the first occurrence keeps its name. The result is the + # stable column identity used for keying, cross-run continuity, and the header + # union. + [CmdletBinding()] + param([Parameter(Mandatory)][AllowEmptyCollection()][string[]]$Cells) + $seen = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + $out = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $Cells.Length; $i++) { + $n = [string]$Cells[$i] + if ($null -eq $n) { $n = '' } + if ($i -eq 0) { $n = $n -replace '^\uFEFF', '' } + $n = $n.Trim() + if ([string]::IsNullOrEmpty($n)) { $n = '_blank' } + if ($seen.ContainsKey($n)) { + $seen[$n] = $seen[$n] + 1 + $out.Add(('{0}_dup{1}' -f $n, $seen[$n])) + } + else { + $seen[$n] = 1 + $out.Add($n) + } + } + return , $out.ToArray() +} + +function script:ConvertTo-PaxFactNormalized { + # Stream a Fact CSV into a normalized work file whose header is the two + # synthetic columns __paxk and __paxseq followed by the deduplicated source + # header, and whose every data row carries its composite key, its zero-padded + # read-order sequence, and the source field values positionally aligned to the + # deduplicated header. The composite key joins the requested key columns with + # U+001F (a single key column contributes its value unseparated); a requested + # column absent from the header contributes an empty segment. Reading uses the + # bounded RFC-4180 record reader and writing uses the RFC-4180 record writer, so + # memory stays bounded regardless of file size. Returns the deduplicated header, + # the data-row count, and whether a header record was present. An absent header + # (empty input) yields a null header and zero rows. + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$InputPath, + [Parameter(Mandatory)][string]$OutputPath, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$KeyColumns + ) + $reader = [System.IO.StreamReader]::new($InputPath, $true) + try { + $header = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $header) { + return [pscustomobject]@{ Header = $null; RowCount = [int64]0; HadHeader = $false } + } + $deduped = script:Get-PaxDedupedFactHeader -Cells $header + $idxMap = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + for ($i = 0; $i -lt $deduped.Length; $i++) { $idxMap[$deduped[$i]] = $i } + $keyIndices = New-Object System.Collections.Generic.List[int] + foreach ($kc in $KeyColumns) { + if ($idxMap.ContainsKey($kc)) { $keyIndices.Add($idxMap[$kc]) } else { $keyIndices.Add(-1) } + } + $ki = $keyIndices.ToArray() + $writer = [System.IO.StreamWriter]::new($OutputPath, $false, (New-Object System.Text.UTF8Encoding($false))) + try { + script:Write-PaxCsvRecord -Writer $writer -Fields (@('__paxk', '__paxseq') + $deduped) + $rowCount = [int64]0 + while ($true) { + $rec = script:Read-PaxCsvRecord -Reader $reader + if ($null -eq $rec) { break } + $paxk = script:Get-PaxCompositeKey -Fields $rec -KeyIndices $ki + $paxseq = $rowCount.ToString('D19') + $data = [string[]]::new($deduped.Length) + for ($j = 0; $j -lt $deduped.Length; $j++) { $data[$j] = if ($j -lt $rec.Length) { $rec[$j] } else { '' } } + script:Write-PaxCsvRecord -Writer $writer -Fields (@($paxk, $paxseq) + $data) + $rowCount++ + } + $writer.Flush() + } + finally { $writer.Dispose() } + return [pscustomobject]@{ Header = $deduped; RowCount = $rowCount; HadHeader = $true } + } + finally { $reader.Dispose() } +} + +function script:Read-PaxFactNextTargetGroup { + # Return the next distinct non-empty target key group from a sorted normalized + # target stream, or $null at end of input. Rows arrive grouped by key (a blank + # key sorts ahead of keyed rows) and, within a key, ordered by read sequence, so + # the first row seen for a group is that key's first original occurrence. No-key + # rows are skipped entirely and duplicate rows of the returned key are consumed + # and discarded; a single peeked row that starts the next thing is stashed on + # the state object so the following call resumes correctly. The state object + # carries the open reader plus a one-record pending slot; the returned value is + # a table with the group Key and the first-occurrence Fields. + [CmdletBinding()] + param([Parameter(Mandatory)][psobject]$State) + while ($true) { + if ($State.HasPending) { $row = $State.Pending; $State.Pending = $null; $State.HasPending = $false } + else { $row = script:Read-PaxCsvRecord -Reader $State.Reader } + if ($null -eq $row) { return $null } + $k = if ($row.Length -gt 0) { [string]$row[0] } else { '' } + if ([string]::IsNullOrWhiteSpace($k)) { continue } + $gk = $k + $first = $row + while ($true) { + $peek = script:Read-PaxCsvRecord -Reader $State.Reader + if ($null -eq $peek) { break } + $pk = if ($peek.Length -gt 0) { [string]$peek[0] } else { '' } + if ([string]::IsNullOrWhiteSpace($pk)) { $State.Pending = $peek; $State.HasPending = $true; break } + if ([System.String]::Equals($pk, $gk, [System.StringComparison]::OrdinalIgnoreCase)) { continue } + $State.Pending = $peek; $State.HasPending = $true; break + } + return @{ Key = $gk; Fields = $first } + } +} + +function Merge-FactCsv { + <# + .SYNOPSIS + Union-merge a target Interactions/Fact CSV with the current run's freshly-emitted Fact CSV. + + .DESCRIPTION + Post-Python helper for -AppendFile. Mirrors Merge-UsersCsv semantics but for the + fact CSV. Keyed on -KeyColumn (default 'Message_Id_Raw' for the rollup Fact CSV; + pass 'RecordId' for the raw audit CSV used by non-rollup -AppendFile). By default + the union is written in place of the current-run CSV (rollup semantics). Pass + -OutputPath to direct the union to a third path so the current-run CSV remains + pristine (the pristine-raw separation pattern used by non-rollup -AppendFile). + + When -KeyColumn is 'Message_Id_Raw' the target's Message_Id INT is carried + forward on retained rows (continuity across runs). For any other key column the + current-run row's values are taken verbatim (target serves only as the source + of departed rows). + + Atomic rewrite: writes to '.merging' then renames over the output path. + + .OUTPUTS + Hashtable with stats (Retained / New / Departed / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetFactCsv, + [Parameter(Mandatory)] [string] $CurrentFactCsv, + [Parameter()] [string] $OutputPath, + [Parameter()] [string] $KeyColumn = 'Message_Id_Raw', + [Parameter()] [string[]] $CompositeKeyColumn = @(), + [Parameter()] [string] $RunDate = (Get-Date -Format 'yyyy-MM-dd'), + [Parameter()] [scriptblock] $ProgressCallback, + [Parameter()] [double] $ProgressIntervalSeconds = 0 + ) + if (-not (Test-Path -LiteralPath $CurrentFactCsv -PathType Leaf)) { + throw "Merge-FactCsv: current Fact CSV not found: '$CurrentFactCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $CurrentFactCsv } + # Grain-composite dedup key. When -CompositeKeyColumn is supplied a row's key is + # the U+001F-joined values of those columns (ASCII Unit Separator 0x1F is a + # control char that never appears in audit field data), so fan-out fact rows + # (many per Message_Id_Raw) are each keyed distinctly instead of collapsing to + # one. Otherwise the single -KeyColumn is used (RecordId raw-audit path). The key + # is computed for every row while it is streamed into a normalized work file. + $useCompositeKey = ($null -ne $CompositeKeyColumn -and @($CompositeKeyColumn).Count -gt 0) + $keyColumns = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } + + # All materialization is disk-backed and bounded: source rows are streamed into + # normalized work files, externally sorted by (key, read-order) under a fixed + # memory budget, joined in a single linear pass that holds at most one record per + # side, and streamed back out. Nothing accumulates a whole file in memory. + $memoryBudgetBytes = [int64]67108864 + $fanIn = 8 + $workDir = Join-Path $env:TEMP ('pax_factmerge_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $mergingPath = "$OutputPath.merging" + $cleanupMetrics = script:New-PaxSortMetrics + + # Coarse stage observability. Merge-FactCsv narrates its bounded pipeline as a + # fixed vocabulary of stage markers (Inspecting inputs, Normalizing existing + # target, Sorting existing target, Normalizing current run, Sorting current run, + # Reconciling rows, Restoring current order, Restoring departed order, Publishing, + # Cleanup, Complete, Failed). Each marker is surfaced through the run's normal + # host/log channel and recorded as a memory observation so a long field-scale + # reconciliation shows forward progress and per-stage memory. The logging helpers + # live in the full processor; when Merge-FactCsv is dot-sourced in isolation (unit + # extraction) they are absent, so emission resolves them once and stays fully + # inert if unavailable. + $logHostCmd = Microsoft.PowerShell.Core\Get-Command -Name 'Write-LogHost' -CommandType Function -ErrorAction SilentlyContinue + $memObsCmd = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxMemoryObservation' -CommandType Function -ErrorAction SilentlyContinue + $emitStage = { + param([string]$StageName, [string]$Detail = '') + $msg = if ($Detail) { "Fact append merge - $StageName ($Detail)" } else { "Fact append merge - $StageName" } + if ($logHostCmd) { try { & $logHostCmd -Message $msg -ForegroundColor 'Gray' } catch { } } + if ($memObsCmd) { try { & $memObsCmd -Stage ("FactMerge:" + $StageName) -TempDir $workDir -Force } catch { } } + } + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + & $emitStage 'Inspecting inputs' + + # 1. Normalize the target: stream every physical data row into tgt_norm with + # its key and read-order sequence. targetRowCount counts ALL data rows + # (including no-key and duplicate-key rows) for the shrink guard. + $hasTarget = Test-Path -LiteralPath $TargetFactCsv -PathType Leaf + $targetLen = if ($hasTarget) { [int64](Get-Item -LiteralPath $TargetFactCsv).Length } else { [int64]0 } + $tgtNorm = Join-Path $workDir 'tgt_norm.csv' + $dedupedTargetHeader = $null + $targetRowCount = [int64]0 + if ($hasTarget) { + & $emitStage 'Normalizing existing target' + $tgtInfo = script:ConvertTo-PaxFactNormalized -InputPath $TargetFactCsv -OutputPath $tgtNorm -KeyColumns $keyColumns + $dedupedTargetHeader = $tgtInfo.Header + $targetRowCount = [int64]$tgtInfo.RowCount + } + + # APPEND SAFETY (data-loss guard): if the target file exists with content but parsed to + # zero rows, the read failed (parse/encoding/memory) or the key cannot be matched. + # Overwriting would replace the existing target file with current-run rows only (a + # row-count shrink). Abort instead so the existing target is left untouched; the caller keeps the + # fresh current-run CSV on disk for a manual merge. + if ($targetRowCount -eq 0 -and $hasTarget -and $targetLen -gt 0) { + throw "Merge-FactCsv: target '$TargetFactCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data). Verify the file's schema/key column ('$KeyColumn')." + } + + # Schema-narrowing warning. The only header we strictly require on the target + # is the dedup key (-KeyColumn) — without it the union cannot correctly + # classify Retained vs New vs Departed. Display / enrichment columns + # (PersonId_Normalized, ThreadId, Message_Id, etc.) are intentionally renamed + # or surrogate-replaced by the rollup processor on the rolled-up Fact CSV, so + # checking for them here would produce false-positive warnings on every + # rollup-shape seed. Header union handles legitimately absent display columns + # gracefully (target rows get blanks for current-only fields; current rows + # get blanks for target-only fields). $targetKeyMissing gates the later + # zero-overlap caution off this data-quality case (retained=0 there means the + # key could not be classified, not that the two nonempty sets were disjoint). + $targetKeyMissing = $false + if ($targetRowCount -gt 0) { + $requiredKeyCols = if ($useCompositeKey) { @($CompositeKeyColumn) } else { @($KeyColumn) } + $missingKeyCols = @($requiredKeyCols | Where-Object { $_ -notin $dedupedTargetHeader }) + if ($missingKeyCols.Count -gt 0) { + $targetKeyMissing = $true + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-FactCsv: target Fact CSV is missing dedup key column(s) '{0}'. " + + "Cannot classify Retained / New / Departed rows reliably; treating ALL current-run rows as New. " + + "Target: {1}") -f ($missingKeyCols -join ', '), $TargetFactCsv + ) -ForegroundColor Yellow + } + } + + # 2. Sort the normalized target by (key, read-order). Duplicate keys stay + # grouped with the first original occurrence first; no-key rows sort ahead + # of keyed rows and are skipped by the join. + $tgtSorted = Join-Path $workDir 'tgt_sorted.csv' + if ($targetRowCount -gt 0) { + & $emitStage 'Sorting existing target' + $null = script:Invoke-PaxExternalSort -InputPath $tgtNorm -OutputPath $tgtSorted -KeyColumn @('__paxk', '__paxseq') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + + # 3. Normalize the current run the same way. + $curNorm = Join-Path $workDir 'cur_norm.csv' + & $emitStage 'Normalizing current run' + $curInfo = script:ConvertTo-PaxFactNormalized -InputPath $CurrentFactCsv -OutputPath $curNorm -KeyColumns $keyColumns + $dedupedCurrentHeader = $curInfo.Header + $currentRowCount = [int64]$curInfo.RowCount + + # 4. Sort the normalized current run by (key, read-order). + $curSorted = Join-Path $workDir 'cur_sorted.csv' + if ($currentRowCount -gt 0) { + & $emitStage 'Sorting current run' + $null = script:Invoke-PaxExternalSort -InputPath $curNorm -OutputPath $curSorted -KeyColumn @('__paxk', '__paxseq') -Comparer 'OrdinalIgnoreCase' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + + # 5. Header union: preserve current-run order, then append target-only columns, + # then provenance — deduplicated case-insensitively. + $hdrSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $hdrOrder = New-Object System.Collections.Generic.List[string] + if ($currentRowCount -gt 0 -and $dedupedCurrentHeader) { + foreach ($n in $dedupedCurrentHeader) { if ($hdrSet.Add($n)) { $hdrOrder.Add($n) } } + } + if ($targetRowCount -gt 0 -and $dedupedTargetHeader) { + foreach ($n in $dedupedTargetHeader) { if ($hdrSet.Add($n)) { $hdrOrder.Add($n) } } + } + foreach ($c in @('Date_Added', 'Latest_Append_Date', 'In_Latest_Append')) { + if ($hdrSet.Add($c)) { $hdrOrder.Add($c) } + } + $hdrArr = [string[]]$hdrOrder.ToArray() + $hdrIndex = [System.Collections.Generic.Dictionary[string, int]]::new([System.StringComparer]::OrdinalIgnoreCase) + for ($i = 0; $i -lt $hdrArr.Length; $i++) { $hdrIndex[$hdrArr[$i]] = $i } + $idxDateAdded = $hdrIndex['Date_Added'] + $idxLatest = $hdrIndex['Latest_Append_Date'] + $idxInLatest = $hdrIndex['In_Latest_Append'] + $hasMessageId = $hdrIndex.ContainsKey('Message_Id') + $idxMessageId = if ($hasMessageId) { $hdrIndex['Message_Id'] } else { -1 } + + # Map each normalized data column to its union position (case-insensitive), and + # locate the target's own Message_Id / Date_Added columns for continuity. + $curColToHdr = [int[]]::new(0) + if ($dedupedCurrentHeader) { + $curColToHdr = [int[]]::new($dedupedCurrentHeader.Length) + for ($j = 0; $j -lt $dedupedCurrentHeader.Length; $j++) { $curColToHdr[$j] = $hdrIndex[$dedupedCurrentHeader[$j]] } + } + $tgtColToHdr = [int[]]::new(0) + $tgtMidIdx = -1 + $tgtDaIdx = -1 + if ($dedupedTargetHeader) { + $tgtColToHdr = [int[]]::new($dedupedTargetHeader.Length) + for ($j = 0; $j -lt $dedupedTargetHeader.Length; $j++) { + $tgtColToHdr[$j] = $hdrIndex[$dedupedTargetHeader[$j]] + if ([System.String]::Equals($dedupedTargetHeader[$j], 'Message_Id', [System.StringComparison]::OrdinalIgnoreCase)) { $tgtMidIdx = $j } + if ([System.String]::Equals($dedupedTargetHeader[$j], 'Date_Added', [System.StringComparison]::OrdinalIgnoreCase)) { $tgtDaIdx = $j } + } + } + + # 6. Message_Id continuity applies when the dedup key is (or contains) + # Message_Id_Raw; then the target's Message_Id INT is carried forward on + # retained rows for cross-run continuity. For other single key columns + # (e.g. RecordId on the raw audit CSV) the current row's values stand. + $messageIdContinuity = ($KeyColumn -eq 'Message_Id_Raw') -or ($useCompositeKey -and (@($CompositeKeyColumn) -contains 'Message_Id_Raw')) + + # Union-row builders. Section-1 rows overlay the current row's own columns + # then apply continuity/provenance; departed rows overlay the target row's + # columns then apply provenance. Both return a full union-width field array. + $buildSection1 = { + param([string[]]$CurData, [bool]$Matched, [object]$HeldFields) + $u = [string[]]::new($hdrArr.Length) + for ($i = 0; $i -lt $u.Length; $i++) { $u[$i] = '' } + for ($j = 0; $j -lt $curColToHdr.Length; $j++) { if ($j -lt $CurData.Length) { $u[$curColToHdr[$j]] = $CurData[$j] } } + if ($Matched) { + if ($messageIdContinuity -and $hasMessageId) { + $tv = if ($tgtMidIdx -ge 0 -and (2 + $tgtMidIdx) -lt $HeldFields.Length) { [string]$HeldFields[2 + $tgtMidIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($tv)) { $u[$idxMessageId] = $tv } + } + $tda = if ($tgtDaIdx -ge 0 -and (2 + $tgtDaIdx) -lt $HeldFields.Length) { [string]$HeldFields[2 + $tgtDaIdx] } else { '' } + if (-not [string]::IsNullOrWhiteSpace($tda)) { $u[$idxDateAdded] = $tda } else { $u[$idxDateAdded] = $RunDate } + } + else { + $u[$idxDateAdded] = $RunDate + } + $u[$idxLatest] = $RunDate + $u[$idxInLatest] = 'TRUE' + return , $u + } + $buildDeparted = { + param([object]$HeldFields) + $u = [string[]]::new($hdrArr.Length) + for ($i = 0; $i -lt $u.Length; $i++) { $u[$i] = '' } + for ($j = 0; $j -lt $tgtColToHdr.Length; $j++) { $s = 2 + $j; if ($s -lt $HeldFields.Length) { $u[$tgtColToHdr[$j]] = [string]$HeldFields[$s] } } + if ([string]::IsNullOrWhiteSpace($u[$idxDateAdded])) { $u[$idxDateAdded] = $RunDate } + $u[$idxLatest] = $RunDate + $u[$idxInLatest] = 'FALSE' + return , $u + } + + # 7. Bounded left-join pass. Walk the sorted current rows, emitting every one + # to the section-1 side file tagged with its read-order sequence. Advance + # the sorted target one distinct key at a time (first occurrence only, + # no-key rows skipped); a target key never matched by a current key becomes + # one departed row. Stats count DISTINCT non-empty current keys. + $sec1Side = Join-Path $workDir 'sec1_side.csv' + $depSide = Join-Path $workDir 'dep_side.csv' + $sideHdr = @('__paxseq') + $hdrArr + $retainedCount = 0 + $newCount = 0 + $departedCount = 0 + & $emitStage 'Reconciling rows' + + $sec1Writer = [System.IO.StreamWriter]::new($sec1Side, $false, (New-Object System.Text.UTF8Encoding($false))) + $depWriter = [System.IO.StreamWriter]::new($depSide, $false, (New-Object System.Text.UTF8Encoding($false))) + $curReader = $null + $tgtState = $null + try { + script:Write-PaxCsvRecord -Writer $sec1Writer -Fields $sideHdr + script:Write-PaxCsvRecord -Writer $depWriter -Fields $sideHdr + + if ($targetRowCount -gt 0) { + $tgtReader = [System.IO.StreamReader]::new($tgtSorted, $true) + $null = script:Read-PaxCsvRecord -Reader $tgtReader + $tgtState = [pscustomobject]@{ Reader = $tgtReader; Pending = $null; HasPending = $false } + } + $held = if ($tgtState) { script:Read-PaxFactNextTargetGroup -State $tgtState } else { $null } + $heldMatched = $false + + $emitDeparted = { + param($Group) + $du = & $buildDeparted $Group.Fields + script:Write-PaxCsvRecord -Writer $depWriter -Fields (@([string]$Group.Fields[1]) + $du) + } + + if ($currentRowCount -gt 0) { + $curReader = [System.IO.StreamReader]::new($curSorted, $true) + $null = script:Read-PaxCsvRecord -Reader $curReader + $havePrev = $false + $prevKey = $null + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $curReader + if ($null -eq $row) { break } + $kc = [string]$row[0] + $curPaxseq = [string]$row[1] + $curData = if ($row.Length -gt 2) { $row[2..($row.Length - 1)] } else { @() } + if ([string]::IsNullOrWhiteSpace($kc)) { + $su = & $buildSection1 $curData $false $null + script:Write-PaxCsvRecord -Writer $sec1Writer -Fields (@($curPaxseq) + $su) + continue + } + while ($null -ne $held -and [System.StringComparer]::OrdinalIgnoreCase.Compare([string]$held.Key, $kc) -lt 0) { + if (-not $heldMatched) { & $emitDeparted $held; $departedCount++ } + $held = script:Read-PaxFactNextTargetGroup -State $tgtState + $heldMatched = $false + } + $matched = ($null -ne $held -and [System.String]::Equals([string]$held.Key, $kc, [System.StringComparison]::OrdinalIgnoreCase)) + if ($matched) { $heldMatched = $true } + $isNewDistinct = (-not $havePrev) -or (-not [System.String]::Equals($kc, $prevKey, [System.StringComparison]::OrdinalIgnoreCase)) + if ($isNewDistinct) { if ($matched) { $retainedCount++ } else { $newCount++ } } + $havePrev = $true + $prevKey = $kc + $heldFieldsForBuild = if ($matched) { $held.Fields } else { $null } + $su = & $buildSection1 $curData $matched $heldFieldsForBuild + script:Write-PaxCsvRecord -Writer $sec1Writer -Fields (@($curPaxseq) + $su) + } + } + # Flush the remaining unmatched target groups as departed rows. + while ($null -ne $held) { + if (-not $heldMatched) { & $emitDeparted $held; $departedCount++ } + $held = if ($tgtState) { script:Read-PaxFactNextTargetGroup -State $tgtState } else { $null } + $heldMatched = $false + } + + $sec1Writer.Flush() + $depWriter.Flush() + } + finally { + if ($curReader) { $curReader.Dispose() } + if ($tgtState -and $tgtState.Reader) { $tgtState.Reader.Dispose() } + $sec1Writer.Dispose() + $depWriter.Dispose() + } + + # 8. Restore original ordering: section-1 rows to current read order, departed + # rows to target first-occurrence order. + $sec1Sorted = Join-Path $workDir 'sec1_sorted.csv' + if ($currentRowCount -gt 0) { + & $emitStage 'Restoring current order' + $null = script:Invoke-PaxExternalSort -InputPath $sec1Side -OutputPath $sec1Sorted -KeyColumn @('__paxseq') -Comparer 'Ordinal' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + $depSorted = Join-Path $workDir 'dep_sorted.csv' + if ($departedCount -gt 0) { + & $emitStage 'Restoring departed order' + $null = script:Invoke-PaxExternalSort -InputPath $depSide -OutputPath $depSorted -KeyColumn @('__paxseq') -Comparer 'Ordinal' -MemoryBudgetBytes $memoryBudgetBytes -FanIn $fanIn -WorkRoot $workDir -ProgressCallback $ProgressCallback -ProgressIntervalSeconds $ProgressIntervalSeconds + } + + # 9. APPEND SAFETY (shrink guard): a union must never be smaller than the existing target. + # If the target keyed but the dedup column did not match (Retained+Departed=0 while the + # target had rows), the union collapses to current-only rows and the file shrinks. Abort + # before any write so the target is preserved. + $unionCount = [int64]$currentRowCount + [int64]$departedCount + if ($targetRowCount -gt 0 -and $unionCount -lt $targetRowCount) { + throw ("Merge-FactCsv: refusing to write a smaller file than the target (union={0} < target={1}); key '{2}' likely did not match. Target left unchanged." -f $unionCount, $targetRowCount, $KeyColumn) + } + + # 10. Publish: write the header, then the ordered section-1 rows, then the + # ordered departed rows, each with the sequence column dropped. Write to a + # sibling .merging file and rename onto the output path so the target is + # byte-unchanged unless every step succeeds. + $pubWriter = [System.IO.StreamWriter]::new($mergingPath, $false, (New-Object System.Text.UTF8Encoding($false))) + & $emitStage 'Publishing' + try { + script:Write-PaxCsvRecord -Writer $pubWriter -Fields $hdrArr + if ($currentRowCount -gt 0) { + $sr = [System.IO.StreamReader]::new($sec1Sorted, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $sr + while ($true) { $rec = script:Read-PaxCsvRecord -Reader $sr; if ($null -eq $rec) { break }; script:Write-PaxCsvRecord -Writer $pubWriter -Fields ($rec[1..($rec.Length - 1)]) } + } + finally { $sr.Dispose() } + } + if ($departedCount -gt 0) { + $dr = [System.IO.StreamReader]::new($depSorted, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $dr + while ($true) { $rec = script:Read-PaxCsvRecord -Reader $dr; if ($null -eq $rec) { break }; script:Write-PaxCsvRecord -Writer $pubWriter -Fields ($rec[1..($rec.Length - 1)]) } + } + finally { $dr.Dispose() } + } + $pubWriter.Flush() + } + finally { $pubWriter.Dispose() } + # Fail closed if the rename cannot complete (e.g. the target is locked by a + # sync client or scanner): a non-terminating move would return success while + # the target still holds its old content and the merged output is discarded. + Move-Item -LiteralPath $mergingPath -Destination $OutputPath -Force -ErrorAction Stop + + # 11. Stats. + & $emitStage 'Cleanup' + & $emitStage 'Complete' + # Zero-overlap caution (count-only). A nonempty target and a nonempty current run + # that share NO keys is a VALID outcome for disjoint windows: every current row is + # New, every unmatched target row is Departed, and the lossless union is preserved. + # Surface it (aggregate counts only — never keys, identities, filenames, or paths) + # so a disjoint-window append is not mistaken for an error. Never thrown, never + # changes counts or provenance. Suppressed when the key column was missing from the + # target ($targetKeyMissing) — that data-quality case owns its own WARNING above and + # is not a genuine disjoint-set overlap. Emitted at most once, via the same resolved + # host helper as the stage markers (inert when Merge-FactCsv is dot-sourced alone). + if ($targetRowCount -gt 0 -and $currentRowCount -gt 0 -and $retainedCount -eq 0 -and -not $targetKeyMissing -and $logHostCmd) { + try { + & $logHostCmd -Message ( + ("Fact append merge found no overlapping keys between the existing target and this run. " + + "All current rows are classified as New and unmatched target rows as Departed. " + + "The lossless union was preserved. " + + "Target rows={0:N0} Current rows={1:N0} Retained={2:N0} New={3:N0} Departed={4:N0} Union={5:N0}") -f ` + [int64]$targetRowCount, [int64]$currentRowCount, [int64]$retainedCount, [int64]$newCount, [int64]$departedCount, [int64]$unionCount + ) -ForegroundColor 'Yellow' + } catch { } + } + # Aggregate-only completion record so the fact merge appears in the unified rollup + # stage stream alongside the earlier preparation stages. Routed through a host + # lookup so the function stays inert when it is extracted and exercised on its own. + $__paxStage = Microsoft.PowerShell.Core\Get-Command -Name 'Write-PaxRollupStage' -CommandType Function -ErrorAction SilentlyContinue + if ($__paxStage) { + try { + & $__paxStage -Stage 'fact-merge' -Status 'complete' -Counters @{ + targetRows = [int64]$targetRowCount + currentRows = [int64]$currentRowCount + retained = [int64]$retainedCount + newRows = [int64]$newCount + departed = [int64]$departedCount + unionRows = [int64]$unionCount + } + } catch { } + } + return @{ + Retained = $retainedCount + New = $newCount + Departed = $departedCount + Union = $unionCount + } + } + catch { + & $emitStage 'Failed' $_.Exception.Message + throw + } + finally { + script:Remove-PaxWorkspace -WorkDir $workDir -MergingPath $mergingPath -Metrics $cleanupMetrics + } +} + +function script:Invoke-PaxKeyedAdditiveMerge { + <# + .SYNOPSIS + Bounded, disk-backed union-merge of a target and current CSV keyed by a + composite key with additive / min / max / OR aggregation. Shared by the two + M365Bundle append reducers. + + .DESCRIPTION + Streams both inputs through the frozen RFC-4180 reader, projects each row onto + a fixed canonical header (missing columns become '' or a supplied default), + spills a bounded work record (composite key, global sequence, side, projected + columns) per row, and folds one key group at a time via the external sort. The + first row of each key group (target rows precede current rows, each in original + order) seeds the group's values; later rows accumulate: summed columns add, + min/max columns take the lexicographic min/max of non-blank values, OR columns + become TRUE if any value is TRUE. The group's UserId-style casing therefore + comes from the first-seen (target-first) row. Output rows are restored to + first-appearance order with a bounded sequence sort and streamed to + '.merging' via Export-Csv, then published with a terminating move. The + complete inputs are never materialized and a hot key streams without being + collected. Any failure raises a terminating error, leaves a pre-existing target + byte-for-byte unchanged, publishes nothing partial, and cleans temporary work. + + .OUTPUTS + PSCustomObject with stats (Retained / New / Updated / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$TargetCsv, + [Parameter(Mandatory)][string]$CurrentCsv, + [Parameter(Mandatory)][string]$OutputPath, + [Parameter(Mandatory)][string[]]$Header, + [Parameter(Mandatory)][string[]]$KeyColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$FoldColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$SumColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$MinColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$MaxColumns, + [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$OrColumns, + [Parameter(Mandatory)][hashtable]$Defaults, + [Parameter(Mandatory)][int64]$MemoryBudgetBytes, + [Parameter(Mandatory)][string]$Label, + [scriptblock]$OnMissingColumns + ) + $inv = [System.Globalization.CultureInfo]::InvariantCulture + $hdrCount = $Header.Length + # Header name -> canonical index. + $hdrIndex = @{} + for ($i = 0; $i -lt $hdrCount; $i++) { $hdrIndex[$Header[$i]] = $i } + $keyIdx = @($KeyColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $foldSet = [System.Collections.Generic.HashSet[int]]::new() + foreach ($c in $FoldColumns) { [void]$foldSet.Add([int]$hdrIndex[$c]) } + $sumIdx = @($SumColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $minIdx = @($MinColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $maxIdx = @($MaxColumns | ForEach-Object { [int]$hdrIndex[$_] }) + $orIdx = @($OrColumns | ForEach-Object { [int]$hdrIndex[$_] }) + + $workRootBase = $env:TEMP; if (-not $workRootBase) { $workRootBase = [System.IO.Path]::GetTempPath() } + $workDir = Join-Path $workRootBase ('pax_kam_{0}' -f ([System.Guid]::NewGuid().ToString('N'))) + $tmpPath = "$OutputPath.merging" + $u8 = [System.Text.UTF8Encoding]::new($false) + + # Resolve a source header record's index map onto the canonical header. + $mapSourceIndices = { + param($srcHeader) + $map = New-Object 'int[]' $hdrCount + for ($j = 0; $j -lt $hdrCount; $j++) { + $map[$j] = -1 + for ($s = 0; $s -lt $srcHeader.Length; $s++) { + if ([string]::Equals([string]$srcHeader[$s], $Header[$j], [System.StringComparison]::OrdinalIgnoreCase)) { $map[$j] = $s; break } + } + } + return , $map + } + # Project a source row (fields + index map) onto canonical values with defaults. + $projectFields = { + param($row, $map) + $vals = New-Object 'string[]' $hdrCount + for ($j = 0; $j -lt $hdrCount; $j++) { + $idx = $map[$j] + $v = if ($idx -ge 0 -and $idx -lt $row.Length) { [string]$row[$idx] } else { '' } + if ([string]::IsNullOrWhiteSpace($v) -and $Defaults.ContainsKey($Header[$j])) { $v = [string]$Defaults[$Header[$j]] } + $vals[$j] = $v + } + return , $vals + } + # Composite key of projected values (folded columns lowercased), '|'-joined. + $makeKey = { + param($vals) + $sb = [System.Text.StringBuilder]::new() + for ($i = 0; $i -lt $keyIdx.Length; $i++) { + if ($i -gt 0) { [void]$sb.Append('|') } + $kv = [string]$vals[$keyIdx[$i]] + if ($foldSet.Contains($keyIdx[$i])) { $kv = $kv.ToLowerInvariant() } + [void]$sb.Append($kv) + } + return $sb.ToString() + } + + $retained = 0; $newCount = 0; $updated = 0; $union = 0 + try { + [void][System.IO.Directory]::CreateDirectory($workDir) + $workPath = Join-Path $workDir 'work.csv' + $workHdr = @('__K', '__SEQ', '__SIDE') + $Header + $ww = [System.IO.StreamWriter]::new($workPath, $false, $u8) + $seq = [int64]0 + $targetDataRows = [int64]0 + $targetExists = Test-Path -LiteralPath $TargetCsv -PathType Leaf + try { + script:Write-PaxCsvRecord -Writer $ww -Fields $workHdr + # Target side (side '0'); may be absent. + if ($targetExists) { + $tr = [System.IO.StreamReader]::new($TargetCsv, $true) + try { + $th = script:Read-PaxCsvRecord -Reader $tr + if ($null -ne $th) { + $tmap = & $mapSourceIndices $th + if ($OnMissingColumns) { + $missing = @($Header | Where-Object { $hdrIndex.ContainsKey($_) -and $tmap[$hdrIndex[$_]] -lt 0 }) + if ($missing.Count -gt 0) { & $OnMissingColumns $missing } + } + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $tr + if ($null -eq $row) { break } + $targetDataRows++ + $vals = & $projectFields $row $tmap + $k = & $makeKey $vals + $fields = New-Object System.Collections.Generic.List[string] + $fields.Add($k); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('0') + foreach ($v in $vals) { $fields.Add($v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + $seq++ + } + } + } + finally { $tr.Dispose() } + } + # APPEND SAFETY (data-loss guard): never overwrite a non-empty target that parsed to 0 rows. + if ($targetDataRows -eq 0 -and $targetExists -and ((Get-Item -LiteralPath $TargetCsv).Length -gt 0)) { + throw "${Label}: target '$TargetCsv' exists with content but parsed to 0 rows; refusing to overwrite (would discard existing data)." + } + # Current side (side '1'). + $cr = [System.IO.StreamReader]::new($CurrentCsv, $true) + try { + $ch = script:Read-PaxCsvRecord -Reader $cr + if ($null -ne $ch) { + $cmap = & $mapSourceIndices $ch + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $cr + if ($null -eq $row) { break } + $vals = & $projectFields $row $cmap + $k = & $makeKey $vals + $fields = New-Object System.Collections.Generic.List[string] + $fields.Add($k); $fields.Add($seq.ToString('D19', $inv)); $fields.Add('1') + foreach ($v in $vals) { $fields.Add($v) } + script:Write-PaxCsvRecord -Writer $ww -Fields $fields.ToArray() + $seq++ + } + } + } + finally { $cr.Dispose() } + } + finally { $ww.Flush(); $ww.Dispose() } + + # Sort work by (key, sequence): equal keys adjacent, target rows before current. + $sortedPath = Join-Path $workDir 'sorted.csv' + [void](script:Invoke-PaxExternalSort -InputPath $workPath -OutputPath $sortedPath -KeyColumn @('__K', '__SEQ') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + + # Reduce one key group at a time; emit [__FSEQ] + accumulated canonical cols. + $resultPath = Join-Path $workDir 'result.csv' + $sr = [System.IO.StreamReader]::new($sortedPath, $true) + $rw = [System.IO.StreamWriter]::new($resultPath, $false, $u8) + try { + $null = script:Read-PaxCsvRecord -Reader $sr + script:Write-PaxCsvRecord -Writer $rw -Fields (@('__FSEQ') + $Header) + $haveGroup = $false + $curKey = $null; $acc = $null; $firstSeq = $null; $tCnt = 0; $cCnt = 0 + $flush = { + if (-not $haveGroup) { return } + $out = New-Object System.Collections.Generic.List[string] + $out.Add($firstSeq) + foreach ($v in $acc) { $out.Add([string]$v) } + script:Write-PaxCsvRecord -Writer $rw -Fields $out.ToArray() + $script:__union++ + if ($tCnt -gt 0) { $script:__ret += $cCnt; $script:__upd += $cCnt } + else { if ($cCnt -ge 1) { $script:__new++ }; if ($cCnt -gt 1) { $script:__upd += ($cCnt - 1) } } + } + $script:__union = 0; $script:__ret = 0; $script:__new = 0; $script:__upd = 0 + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $sr + if ($null -eq $row) { break } + $k = [string]$row[0] + $rowSeq = [string]$row[1] + $side = [string]$row[2] + # Canonical values start at work index 3. + if ((-not $haveGroup) -or ($k -cne $curKey)) { + & $flush + $curKey = $k; $firstSeq = $rowSeq; $tCnt = 0; $cCnt = 0 + $acc = New-Object 'string[]' $hdrCount + for ($j = 0; $j -lt $hdrCount; $j++) { $acc[$j] = [string]$row[3 + $j] } + $haveGroup = $true + } + else { + foreach ($sc in $sumIdx) { + $a = 0; [void][int]::TryParse([string]$acc[$sc], [ref]$a) + $b = 0; [void][int]::TryParse([string]$row[3 + $sc], [ref]$b) + $acc[$sc] = ($a + $b).ToString() + } + foreach ($mc in $minIdx) { + $rv = [string]$row[3 + $mc] + if (-not [string]::IsNullOrWhiteSpace($rv)) { + $av = [string]$acc[$mc] + if ([string]::IsNullOrWhiteSpace($av) -or ($rv -lt $av)) { $acc[$mc] = $rv } + } + } + foreach ($xc in $maxIdx) { + $rv = [string]$row[3 + $xc] + if (-not [string]::IsNullOrWhiteSpace($rv)) { + $av = [string]$acc[$xc] + if ([string]::IsNullOrWhiteSpace($av) -or ($rv -gt $av)) { $acc[$xc] = $rv } + } + } + foreach ($oc in $orIdx) { + if (([string]$row[3 + $oc]).Equals('TRUE', [System.StringComparison]::OrdinalIgnoreCase)) { $acc[$oc] = 'TRUE' } + } + } + if ($side -eq '0') { $tCnt++ } else { $cCnt++ } + } + & $flush + $union = [int]$script:__union; $retained = [int]$script:__ret; $newCount = [int]$script:__new; $updated = [int]$script:__upd + Remove-Variable -Name __union, __ret, __new, __upd -Scope Script -ErrorAction SilentlyContinue + } + finally { $rw.Flush(); $rw.Dispose(); $sr.Dispose() } + + # Restore first-appearance order. + $orderedPath = Join-Path $workDir 'ordered.csv' + [void](script:Invoke-PaxExternalSort -InputPath $resultPath -OutputPath $orderedPath -KeyColumn @('__FSEQ') -Comparer 'Ordinal' -MemoryBudgetBytes $MemoryBudgetBytes -FanIn 8 -WorkRoot $workDir) + + # Stream ordered result rows as objects to Export-Csv (exact prior format). + $hdrLocal = $Header + $objStream = { + $orr = [System.IO.StreamReader]::new($orderedPath, $true) + try { + $null = script:Read-PaxCsvRecord -Reader $orr + while ($true) { + $row = script:Read-PaxCsvRecord -Reader $orr + if ($null -eq $row) { break } + $o = [ordered]@{} + for ($j = 0; $j -lt $hdrLocal.Length; $j++) { $o[$hdrLocal[$j]] = [string]$row[$j + 1] } + [pscustomobject]$o + } + } + finally { $orr.Dispose() } + } + & $objStream | Select-Object -Property $Header | Export-Csv -LiteralPath $tmpPath -NoTypeInformation -Encoding UTF8 + Move-Item -LiteralPath $tmpPath -Destination $OutputPath -Force -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $tmpPath) { try { Remove-Item -LiteralPath $tmpPath -Force -ErrorAction SilentlyContinue } catch {} } + throw + } + finally { + if (Test-Path -LiteralPath $workDir) { try { Remove-Item -LiteralPath $workDir -Recurse -Force -ErrorAction SilentlyContinue } catch {} } + } + + return [pscustomobject]@{ + Retained = $retained + New = $newCount + Updated = $updated + Union = $union + } +} + +function Merge-M365RollupCsv { + <# + .SYNOPSIS + Union-merge a target M365Bundle rollup CSV with the current run's freshly-emitted rollup. + + .DESCRIPTION + Post-Python helper for -AppendFile in M365Bundle mode. Performs a 9-tuple-keyed + union-merge with additive counter semantics: + + Composite key : (lower(UserId), CreationDate, Operation, Workload, + lower(SourceFileExtension), AppHost, AgentId, + AgentName, ContextType) + EventCount : target + current + ItemsAccessedCount : target + current + CreationTime : min(target, current) (lexicographic on ISO-8601) + MaxCreationTime : max(target, current) + IsAgentInteraction : OR (any TRUE -> TRUE) + UserId : target's casing wins on retained groups (first-seen) + + Schema tolerance: target may be a legacy 9- or 10-column rollup. Missing + columns are padded with empty strings; IsAgentInteraction defaults to FALSE. + + Bounded rewrite: the union is computed by streaming both inputs through + bounded temporary storage (script:Invoke-PaxKeyedAdditiveMerge) instead of + loading both files into memory; output rows, ordering, accumulation, casing, + schema, guard, and returned stats are unchanged. + + Atomic rewrite: writes to '.merging' and renames over -OutputPath. + -OutputPath defaults to -TargetRollupCsv (in-place append semantics). + + .OUTPUTS + PSCustomObject with stats (Retained / New / Updated / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetRollupCsv, + [Parameter(Mandatory)] [string] $CurrentRollupCsv, + [Parameter()] [string] $OutputPath, + [int64] $MemoryBudgetBytes = 33554432 + ) + + if (-not (Test-Path -LiteralPath $CurrentRollupCsv -PathType Leaf)) { + throw "Merge-M365RollupCsv: current rollup CSV not found: '$CurrentRollupCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $TargetRollupCsv } + + # Canonical 14-column rollup header (order is contractual for downstream sidecar regen). + $rollupHeader = @( + 'UserId', 'CreationDate', 'Operation', 'Workload', 'SourceFileExtension', + 'AppHost', 'EventCount', 'ItemsAccessedCount', 'CreationTime', 'MaxCreationTime', + 'AgentId', 'AgentName', 'ContextType', 'IsAgentInteraction' + ) + $onMissing = { + param($missing) + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-M365RollupCsv: target rollup is missing {0} of the {1} canonical columns " + + "({2}). Padding with empty values; IsAgentInteraction defaults to FALSE. " + + "Target: {3}") -f $missing.Count, $rollupHeader.Count, ($missing -join ','), $TargetRollupCsv + ) -ForegroundColor Yellow + } + return (script:Invoke-PaxKeyedAdditiveMerge ` + -TargetCsv $TargetRollupCsv -CurrentCsv $CurrentRollupCsv -OutputPath $OutputPath ` + -Header $rollupHeader ` + -KeyColumns @('UserId', 'CreationDate', 'Operation', 'Workload', 'SourceFileExtension', 'AppHost', 'AgentId', 'AgentName', 'ContextType') ` + -FoldColumns @('UserId', 'SourceFileExtension') ` + -SumColumns @('EventCount', 'ItemsAccessedCount') ` + -MinColumns @('CreationTime') -MaxColumns @('MaxCreationTime') -OrColumns @('IsAgentInteraction') ` + -Defaults @{ IsAgentInteraction = 'FALSE' } ` + -MemoryBudgetBytes $MemoryBudgetBytes -Label 'Merge-M365RollupCsv' -OnMissingColumns $onMissing) +} + +function Merge-M365SessionStatsCsv { + <# + .SYNOPSIS + Union-merge a target M365Bundle SessionStats CSV with the current run's freshly-emitted SessionStats. + + .DESCRIPTION + Post-Python helper for -AppendFile in M365Bundle mode. Performs a 3-tuple-keyed + union-merge with additive counter semantics on the SessionStats sidecar: + + Composite key : (lower(UserId), CreationDate, AppHost) + SessionCount : target + current + PromptCount : target + current + AgentPromptCount : target + current + ResponseCount : target + current + AgentSessionCount : target + current + UserId : target's casing wins on retained groups (first-seen) + + Schema tolerance: target may be missing one or more counter columns (e.g. a + placeholder). Missing counters are treated as 0 during accumulation + and emitted as the union value. + + Bounded rewrite: the union is computed by streaming both inputs through + bounded temporary storage (script:Invoke-PaxKeyedAdditiveMerge) instead of + loading both files into memory; output rows, ordering, accumulation, casing, + schema, guard, and returned stats are unchanged. + + Atomic rewrite: writes to '.merging' and renames over -OutputPath. + -OutputPath defaults to -TargetSessionStatsCsv (in-place append semantics). + + .OUTPUTS + PSCustomObject with stats (Retained / New / Updated / Union). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetSessionStatsCsv, + [Parameter(Mandatory)] [string] $CurrentSessionStatsCsv, + [Parameter()] [string] $OutputPath, + [int64] $MemoryBudgetBytes = 33554432 + ) + + if (-not (Test-Path -LiteralPath $CurrentSessionStatsCsv -PathType Leaf)) { + throw "Merge-M365SessionStatsCsv: current SessionStats CSV not found: '$CurrentSessionStatsCsv'" + } + if ([string]::IsNullOrWhiteSpace($OutputPath)) { $OutputPath = $TargetSessionStatsCsv } + + # Canonical 8-column SessionStats header (matches SESSIONSTATS_HEADER in the embedded processor). + $ssHeader = @( + 'UserId', 'CreationDate', 'AppHost', + 'SessionCount', 'PromptCount', 'AgentPromptCount', 'ResponseCount', 'AgentSessionCount' + ) + $onMissing = { + param($missing) + Microsoft.PowerShell.Utility\Write-Host ( + ("WARNING: Merge-M365SessionStatsCsv: target SessionStats is missing {0} of the {1} canonical columns " + + "({2}). Padding with empty values; missing counters treated as 0. Target: {3}") ` + -f $missing.Count, $ssHeader.Count, ($missing -join ','), $TargetSessionStatsCsv + ) -ForegroundColor Yellow + } + return (script:Invoke-PaxKeyedAdditiveMerge ` + -TargetCsv $TargetSessionStatsCsv -CurrentCsv $CurrentSessionStatsCsv -OutputPath $OutputPath ` + -Header $ssHeader ` + -KeyColumns @('UserId', 'CreationDate', 'AppHost') ` + -FoldColumns @('UserId') ` + -SumColumns @('SessionCount', 'PromptCount', 'AgentPromptCount', 'ResponseCount', 'AgentSessionCount') ` + -MinColumns @() -MaxColumns @() -OrColumns @() ` + -Defaults @{} ` + -MemoryBudgetBytes $MemoryBudgetBytes -Label 'Merge-M365SessionStatsCsv' -OnMissingColumns $onMissing) +} + +function Convert-CsvToDelta { + <# + .SYNOPSIS + Convert a local CSV file to a Delta Lake table at a OneLake DFS URI (or local path). + + .DESCRIPTION + Invokes Python with the 'deltalake' package to read the CSV with pyarrow and write + it as a Delta table at the destination. Used by the rollup post-processor to land + Fabric Lakehouse Tables/ outputs after the embedded Python processor has written + the canonical CSV. + + Supports two destination forms: + • Local path (file:// or absolute path) — used for unit testing and for the + pre-publish stage when -OutputPath resolves to a Fabric OneLake URL. + • OneLake DFS URL (https://-onelake.dfs.fabric.microsoft.com/.../Tables/) + — used directly for Fabric Lakehouse Tables/ destinations. The caller supplies + a bearer token (OneLake storage audience). + + Schema evolution: write_deltalake is called with schema_mode='merge' so additive + column changes do not require a full overwrite. The mode itself is the caller's + choice (overwrite for first run, append for subsequent appends). + + .PARAMETER InputCsv + Local path to the source CSV. Must exist and be UTF-8 encoded. + + .PARAMETER TargetUri + Destination Delta table URI. Either a local absolute path or a OneLake DFS https URL. + + .PARAMETER Mode + Delta write mode: 'overwrite' or 'append'. Defaults to 'overwrite'. + + .PARAMETER BearerToken + Optional OneLake storage-audience access token. Required when TargetUri is a + Fabric DFS URL. Ignored for local destinations. + + .PARAMETER PythonExe + Resolved Python interpreter path (from Resolve-PythonExe). + + .PARAMETER LauncherArgs + Launcher arguments (e.g. py -3 prefix) for the resolved interpreter. + + .OUTPUTS + $true on success, $false on failure (errors are logged via Write-LogHost). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $InputCsv, + [Parameter(Mandatory)] [string] $TargetUri, + [Parameter()] [ValidateSet('overwrite','append')] [string] $Mode = 'overwrite', + [Parameter()] [string] $BearerToken = '', + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + if (-not (Test-Path -LiteralPath $InputCsv -PathType Leaf)) { + Write-LogHost "Convert-CsvToDelta: input CSV not found: '$InputCsv'" -ForegroundColor Red + return $false + } + # Inline Python: pyarrow reads CSV, deltalake writes table. Storage options only + # when TargetUri is a remote OneLake URL. Column names are sanitized to satisfy + # Delta's forbidden-character set (' ,;{}()\n\t='); the source CSV on disk is + # left unchanged so PBIP semantic models that read the CSV directly continue + # to bind by the original column names. + $inlinePy = @" +import os, sys, json, re +in_csv = os.environ['PAX_DELTA_IN'] +out_uri = os.environ['PAX_DELTA_OUT'] +mode = os.environ['PAX_DELTA_MODE'] +token = os.environ.get('PAX_DELTA_TOKEN', '') +try: + import pyarrow.csv as pacsv + from deltalake import write_deltalake +except Exception as e: + print(f'Convert-CsvToDelta: import failed: {e}', file=sys.stderr) + sys.exit(2) +# Delta forbids these characters in column names; the Fabric SQL endpoint also +# rejects spaces in column names. Replace each with an underscore. Collisions +# after sanitization (e.g., two source columns differing only in a forbidden +# character) are disambiguated by appending _2, _3, ... . The disambiguated +# name is also recorded so a later raw column whose sanitized form literally +# matches an earlier disambiguated name cannot silently map to the same Delta +# column. +_DELTA_FORBIDDEN = re.compile(r'[ ,;\{\}\(\)\n\t=]') +def _sanitize_col_names(cols): + seen = {} + out = [] + for c in cols: + s = _DELTA_FORBIDDEN.sub('_', c) + if s in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + while candidate in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + s = candidate + seen[s] = 1 + else: + seen[s] = 1 + out.append(s) + return out +try: + import pyarrow as pa + # Stream the CSV as bounded RecordBatches instead of reading the whole file into + # one Arrow table. Each batch is cast to all-string (the Lakehouse model casts at + # query time) and handed to write_deltalake through a self-describing + # RecordBatchReader, so only one bounded batch is materialized at a time. + read_options = pacsv.ReadOptions(block_size=8 * 1024 * 1024) + src_reader = pacsv.open_csv( + in_csv, + read_options=read_options, + convert_options=pacsv.ConvertOptions(strings_can_be_null=False), + ) + cols = list(src_reader.schema.names) + safe_cols = _sanitize_col_names(cols) + target_schema = pa.schema([pa.field(sc, pa.string()) for sc in safe_cols]) + counter = [0] + def _string_batches(): + for batch in src_reader: + counter[0] += batch.num_rows + arrays = [batch.column(i).cast(pa.string()) for i in range(batch.num_columns)] + yield pa.RecordBatch.from_arrays(arrays, names=safe_cols) + batch_reader = pa.RecordBatchReader.from_batches(target_schema, _string_batches()) + storage_options = None + if out_uri.startswith('https://') or out_uri.startswith('abfss://'): + if not token: + print('Convert-CsvToDelta: remote target requires a bearer token', file=sys.stderr) + sys.exit(3) + storage_options = {'bearer_token': token, 'use_fabric_endpoint': 'true'} + write_deltalake( + out_uri, + batch_reader, + mode=mode, + schema_mode='merge', + storage_options=storage_options, + ) + print(f'Convert-CsvToDelta: wrote {counter[0]} rows to {out_uri} (mode={mode}, streamed)') +except Exception as e: + print(f'Convert-CsvToDelta: write failed: {e}', file=sys.stderr) + sys.exit(4) +"@ + $env:PAX_DELTA_IN = $InputCsv + $env:PAX_DELTA_OUT = $TargetUri + $env:PAX_DELTA_MODE = $Mode + $env:PAX_DELTA_TOKEN = $BearerToken + try { + $fullArgs = @($LauncherArgs) + @('-c', $inlinePy) + & $PythonExe @fullArgs 2>&1 | ForEach-Object { Write-Log $_ } + $exit = $LASTEXITCODE + } + finally { + Remove-Item Env:\PAX_DELTA_IN -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_OUT -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_MODE -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_TOKEN -ErrorAction SilentlyContinue + } + if ($exit -eq 0) { return $true } + Write-LogHost "Convert-CsvToDelta: Python exited $exit (input='$InputCsv', target='$TargetUri', mode=$Mode)." -ForegroundColor Red + return $false +} + +function Test-DeltaTableSchemaCompat { + <# + .SYNOPSIS + Pre-flight check: confirm an existing Delta table at TargetUri has a column set + compatible with the CSV the caller is about to convert. + + .DESCRIPTION + Reads the Delta table's schema (column names only — types are all strings in our + write path) and compares against the CSV header. Compatible = every existing-table + column is present in the new CSV (schema_mode='merge' will additively expand it). + Incompatible = at least one existing column is absent from the new CSV (would + require a destructive overwrite-with-different-schema). + + Returns a hashtable: { Compatible=, ExistingCols=[], NewCols=[], Missing=[] }. + When the Delta table does not yet exist, returns Compatible=$true with empty arrays. + + .PARAMETER TargetUri + Delta table URI (local path or OneLake DFS URL). + + .PARAMETER NewCsv + Local CSV whose header is the proposed new schema. + + .PARAMETER BearerToken + Optional OneLake storage-audience token for remote URIs. + + .PARAMETER PythonExe + Resolved Python interpreter. + + .PARAMETER LauncherArgs + Launcher prefix args. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $TargetUri, + [Parameter(Mandatory)] [string] $NewCsv, + [Parameter()] [string] $BearerToken = '', + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + if (-not (Test-Path -LiteralPath $NewCsv -PathType Leaf)) { + return @{ Compatible = $false; ExistingCols = @(); NewCols = @(); Missing = @(); Error = "New CSV not found: '$NewCsv'" } + } + $inlinePy = @" +import os, sys, json, csv, re +target = os.environ['PAX_DELTA_OUT'] +new_csv = os.environ['PAX_DELTA_IN'] +token = os.environ.get('PAX_DELTA_TOKEN', '') +try: + from deltalake import DeltaTable +except Exception as e: + print(json.dumps({'error': f'import: {e}'})) + sys.exit(0) +# Must match the sanitization applied by Convert-CsvToDelta — otherwise the +# compat probe would compare sanitized stored names against unsanitized CSV +# headers and always report drift. +_DELTA_FORBIDDEN = re.compile(r'[ ,;\{\}\(\)\n\t=]') +def _sanitize_col_names(cols): + seen = {} + out = [] + for c in cols: + s = _DELTA_FORBIDDEN.sub('_', c) + if s in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + while candidate in seen: + seen[s] += 1 + candidate = f'{s}_{seen[s]}' + s = candidate + seen[s] = 1 + else: + seen[s] = 1 + out.append(s) + return out +try: + with open(new_csv, 'r', encoding='utf-8-sig', newline='') as f: + rdr = csv.reader(f) + new_cols = _sanitize_col_names(next(rdr, [])) +except Exception as e: + print(json.dumps({'error': f'csv-header: {e}'})) + sys.exit(0) +storage_options = None +if target.startswith('https://') or target.startswith('abfss://'): + if token: + storage_options = {'bearer_token': token, 'use_fabric_endpoint': 'true'} +try: + dt = DeltaTable(target, storage_options=storage_options) + existing = [f.name for f in dt.schema().fields] +except Exception: + print(json.dumps({'existing': [], 'new': new_cols, 'exists': False})) + sys.exit(0) +print(json.dumps({'existing': existing, 'new': new_cols, 'exists': True})) +"@ + $env:PAX_DELTA_OUT = $TargetUri + $env:PAX_DELTA_IN = $NewCsv + $env:PAX_DELTA_TOKEN = $BearerToken + $outJson = '' + try { + $fullArgs = @($LauncherArgs) + @('-c', $inlinePy) + $outJson = (& $PythonExe @fullArgs 2>&1 | Out-String).Trim() + } + finally { + Remove-Item Env:\PAX_DELTA_OUT -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_IN -ErrorAction SilentlyContinue + Remove-Item Env:\PAX_DELTA_TOKEN -ErrorAction SilentlyContinue + } + try { + $parsed = $outJson | ConvertFrom-Json -ErrorAction Stop + } catch { + return @{ Compatible = $false; ExistingCols = @(); NewCols = @(); Missing = @(); Error = "Could not parse schema probe output: $outJson" } + } + if ($parsed.PSObject.Properties['error']) { + return @{ Compatible = $false; ExistingCols = @(); NewCols = @(); Missing = @(); Error = [string]$parsed.error } + } + $existing = @($parsed.existing) + $newCols = @($parsed.new) + if (-not $parsed.exists) { + return @{ Compatible = $true; ExistingCols = $existing; NewCols = $newCols; Missing = @() } + } + $newSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$newCols, [System.StringComparer]::OrdinalIgnoreCase) + $missing = @($existing | Where-Object { -not $newSet.Contains($_) }) + $compatible = ($missing.Count -eq 0) + return @{ Compatible = $compatible; ExistingCols = $existing; NewCols = $newCols; Missing = $missing } +} + +function Write-DeltaAppend { + <# + .SYNOPSIS + Append a local CSV to a Delta table (Fabric Lakehouse Tables/ or local), creating + the table if it does not yet exist. Hard-errors on destructive schema drift. + + .DESCRIPTION + Thin wrapper around Test-DeltaTableSchemaCompat + Convert-CsvToDelta: + 1. Probe the target Delta schema. If unreachable due to nonexistent table, + fall through to Convert-CsvToDelta with mode='append' (deltalake creates + the table on first write). The returned object's IsInit flag is set so + callers can label the banner '[append-init]'. + 2. If the table exists, refuse the write when any existing column is absent + from the new CSV header (column removal / type narrowing). Additive new + columns are absorbed via schema_mode='merge'. + 3. On compatible schema, invoke Convert-CsvToDelta -Mode append. + + .OUTPUTS + Hashtable: { Success=, IsInit=, AddedCols=[], Missing=[], Error= }. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $InputCsv, + [Parameter(Mandatory)] [string] $TargetUri, + [Parameter()] [string] $BearerToken = '', + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs + ) + if (-not (Test-Path -LiteralPath $InputCsv -PathType Leaf)) { + return @{ Success = $false; IsInit = $false; AddedCols = @(); Missing = @(); Error = "Input CSV not found: '$InputCsv'" } + } + $probe = Test-DeltaTableSchemaCompat -TargetUri $TargetUri -NewCsv $InputCsv -BearerToken $BearerToken -PythonExe $PythonExe -LauncherArgs $LauncherArgs + $isInit = (-not ($probe.ExistingCols -and @($probe.ExistingCols).Count -gt 0)) + if (-not $probe.Compatible -and -not $isInit) { + $missingList = ($probe.Missing -join ', ') + $msg = "Write-DeltaAppend: destructive schema drift rejected. Existing column(s) not present in source CSV: $missingList" + Write-LogHost $msg -ForegroundColor Red + return @{ Success = $false; IsInit = $false; AddedCols = @(); Missing = @($probe.Missing); Error = $msg } + } + $existingSet = if ($probe.ExistingCols) { [System.Collections.Generic.HashSet[string]]::new([string[]]@($probe.ExistingCols), [System.StringComparer]::OrdinalIgnoreCase) } else { [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) } + $addedCols = @(@($probe.NewCols) | Where-Object { -not $existingSet.Contains($_) }) + $ok = Convert-CsvToDelta -InputCsv $InputCsv -TargetUri $TargetUri -Mode 'append' -BearerToken $BearerToken -PythonExe $PythonExe -LauncherArgs $LauncherArgs + if (-not $ok) { + return @{ Success = $false; IsInit = $isInit; AddedCols = $addedCols; Missing = @(); Error = 'Convert-CsvToDelta returned failure.' } + } + $tag = if ($isInit) { '[append-init]' } else { '[append]' } + $colNote = if ($addedCols.Count -gt 0) { (" (added columns: {0})" -f ($addedCols -join ', ')) } else { '' } + Write-LogHost "Write-DeltaAppend: $tag '$TargetUri'$colNote" -ForegroundColor Green + return @{ Success = $true; IsInit = $isInit; AddedCols = $addedCols; Missing = @() } +} + +# ============================================================================ +# REMOTE OUTPUT HELPERS — SharePoint (Graph drives API) and Fabric (OneLake DFS) +# ---------------------------------------------------------------------------- +# All helpers are no-ops when $script:RemoteOutputMode -eq 'None'. They are +# invoked from a single dispatch point: Invoke-OutputUpload. +# +# SharePoint path: +# - Reuses the existing Connect-MgGraph token (Graph audience). +# - Resolves the SP URL to siteId/driveId/folder via Graph site lookup. +# - Files <= 250 MB: PUT /drives/{driveId}/root:/path:/content (Graph simple-upload max) +# - Larger files: createUploadSession + 5 MB chunked PUT (must be 320 KB-aligned). +# +# Fabric/OneLake path: +# - Uses a SEPARATE token with audience https://storage.azure.com/ (Az.Accounts). +# - DFS Create -> Append (chunked) -> Flush. Path = //.Lakehouse/Files/ +# ============================================================================ + +function Get-DisplayPath { + <# + .SYNOPSIS + Returns the user-visible path to display for an output artifact, accounting for + a SharePoint or OneLake -OutputPath* value. Local-output runs see the local path unchanged; + remote-output runs see the destination URL the file is uploaded to. + + .DESCRIPTION + Every "Output File: ...", "Output Directory: ...", "Log file: ..." line in the + console / log banner should be filtered through this helper. The local scratch + path is irrelevant to a customer who passed a SharePoint or OneLake -OutputPath*; + showing it implies the artifact lives there, which it does not (it is uploaded + then, on success, the local copy is reaped). + + File mode (default): appends the local file's leaf name to the remote root URL. + -Directory mode: returns the remote root URL with no leaf appended. + + When $script:RemoteOutputMode -eq 'None' (or remote URL not yet set), the input + path is returned verbatim — making this safe to call unconditionally at every + display site. + + .PARAMETER LocalPath + The local scratch path of the artifact. Required. Can be empty/null. + + .PARAMETER Directory + If set, treats LocalPath as a directory and returns the remote root URL alone + (no leaf appended). Use for "Output Directory: " lines. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [AllowNull()] [AllowEmptyString()] [string] $LocalPath, + [switch] $Directory + ) + if ($script:RemoteOutputMode -eq 'None' -or [string]::IsNullOrWhiteSpace($LocalPath) -or [string]::IsNullOrWhiteSpace($script:RemoteOutputUrl)) { + return $LocalPath + } + $base = $script:RemoteOutputUrl.TrimEnd('/') + if ($Directory) { return $base } + $leaf = [System.IO.Path]::GetFileName($LocalPath.TrimEnd('\','/')) + if ([string]::IsNullOrWhiteSpace($leaf)) { return $base } + # Guard against URL doubling: $RemoteOutputUrl can be either a folder URL + # (the normal -OutputPath case) or already a full file URL (when promoted + # from -AppendFile or when -OutputPath was given as a file URL). The + # upstream normalizer in the param-validation block strips the file leaf + # from RemoteOutputUrl when it can recognize the extension, but this + # helper is called from many sites and must remain robust if a caller + # leaves $base as a file URL. + # + # Two doubling cases to defend against: + # 1) Same name ('.../foo.csv' + 'foo.csv' -> '.../foo.csv' ) + # 2) Sibling ('.../foo.csv' + 'foo.log' -> '.../foo.log' ) + # When $baseLeaf carries a data-artifact extension, strip it before + # appending $leaf so siblings land in the correct parent folder. + # .Lakehouse is a folder marker; do NOT strip. + $baseLeaf = ($base -split '/') | Select-Object -Last 1 + if ($baseLeaf -ieq $leaf) { return $base } + if ($baseLeaf -match '\.(csv|log|xlsx|json|parquet|txt)$') { + $base = ($base -replace '/[^/]+$','') + } + return ($base + '/' + $leaf) +} + +function Resolve-SharePointTarget { + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Url) + + # Accepts any of: + # https:///sites//[/] + # https:///teams//[/] + # https:///personal//[/] + # https:///[/] (root site collection) + # where can be any sharepoint.* tenant variant (sharepoint.com, + # sharepoint-df.com, sharepoint.us, sharepoint.de, sharepoint.cn, + # sharepoint-mil.us, ...). + # + # The first one or two URL segments identify the site collection; the remainder + # is library + (optional) folder path. Graph's site-lookup endpoint is the + # authoritative resolver -- we just hand it the server-relative site path. + $u = [Uri]$Url + $hostName = $u.Host + $segments = ($u.AbsolutePath.TrimStart('/') -split '/') | Where-Object { $_ -ne '' } + if (-not $segments -or $segments.Count -lt 1) { + throw "SharePoint URL has no path; expected https://///[/] : '$Url'" + } + + # Determine where the site collection ends and the library begins. + $prefixed = @('sites','teams','personal') -contains $segments[0].ToLower() + if ($prefixed) { + if ($segments.Count -lt 3) { + throw "SharePoint URL is missing a library/folder after the site name: '$Url'" + } + $sitePathSegs = $segments[0..1] # e.g. sites/Analytics or teams/MyTeam + $libAndFolder = $segments[2..($segments.Count-1)] + $siteName = $segments[1] + } else { + # Root site collection: no /sites/ or /teams/ prefix; first segment is the library. + $sitePathSegs = @() + $libAndFolder = $segments + $siteName = '(root)' + } + $libraryName = $libAndFolder[0] + $folderInLibrary = if ($libAndFolder.Count -gt 1) { ($libAndFolder[1..($libAndFolder.Count-1)]) -join '/' } else { '' } + + # Resolve site via Graph. Site path is server-relative; if root, omit the colon-path. + if ($sitePathSegs.Count -gt 0) { + $sitePath = ($sitePathSegs -join '/') + $siteLookupUri = "https://graph.microsoft.com/v1.0/sites/${hostName}:/${sitePath}" + } else { + $siteLookupUri = "https://graph.microsoft.com/v1.0/sites/${hostName}" + } + try { + $site = Invoke-MgGraphRequest -Method GET -Uri $siteLookupUri -ErrorAction Stop + } catch { + # Classify the failure so customers see actionable guidance instead of raw Graph plumbing. + # We inspect the HTTP status, current Graph context state, and granted scopes to decide + # the most likely cause. The structured message is consumed by Test-RemoteDestination's + # error block (which formats the banner with permissions-table back-reference). + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($status -eq 0) { try { if ($_.Exception.Response.StatusCode -is [System.Net.HttpStatusCode]) { $status = [int]$_.Exception.Response.StatusCode } } catch {} } + + $mgCtx = $null + try { $mgCtx = Get-MgContext -ErrorAction SilentlyContinue } catch {} + $hasToken = [bool]$mgCtx + $grantedScopes = @() + if ($mgCtx -and $mgCtx.Scopes) { $grantedScopes = @($mgCtx.Scopes) } + $isAppOnly = $false + if ($mgCtx) { $isAppOnly = ([string]::IsNullOrWhiteSpace($mgCtx.Account)) -or ($mgCtx.AuthType -eq 'AppOnly') } + + # Surface the raw response body for context (truncated; appended to the classified message). + $rawDetail = $_.Exception.Message + try { + if ($_.ErrorDetails -and $_.ErrorDetails.Message) { $rawDetail = $_.ErrorDetails.Message } + elseif ($_.Exception.Response) { + $resp = $_.Exception.Response + $reader = New-Object System.IO.StreamReader($resp.GetResponseStream()) + $body = $reader.ReadToEnd(); $reader.Close() + if ($body) { $rawDetail = "HTTP $status $body" } + } + } catch {} + if ($rawDetail.Length -gt 600) { $rawDetail = $rawDetail.Substring(0, 600) + '...(truncated)' } + + # Build the classified diagnosis. Lines are joined with newline so Test-RemoteDestination + # can split and indent them under its banner. + $diagLines = New-Object System.Collections.Generic.List[string] + $diagLines.Add("SharePoint site lookup failed (host '$hostName', site path '$($sitePathSegs -join '/')').") + + if (-not $hasToken) { + $diagLines.Add('Cause: Microsoft Graph is not connected (no MgContext present).') + $diagLines.Add('Action: This is unexpected at this point in the script — please report it as a bug.') + } + elseif ($status -eq 401) { + if (-not $isAppOnly -and ($grantedScopes -notcontains 'Sites.ReadWrite.All')) { + $diagLines.Add('Cause: The Graph access token is missing the Sites.ReadWrite.All scope.') + $diagLines.Add('Action: Re-run after granting/consenting Sites.ReadWrite.All. See the') + $diagLines.Add(' "Permissions Required for THIS run" table earlier in this output') + $diagLines.Add(' for the full list of required scopes for SharePoint output.') + } else { + $diagLines.Add('Cause: Graph rejected the token (401). For app-only auth this usually means the') + $diagLines.Add(' application permission (Sites.ReadWrite.All)') + $diagLines.Add(' has not been ADMIN-CONSENTED on the app registration / managed identity.') + $diagLines.Add('Action: Have a Global Administrator grant admin consent to the application') + $diagLines.Add(' permissions listed in the "Permissions Required for THIS run" table above.') + } + } + elseif ($status -eq 403) { + $signedIn = if ($mgCtx -and $mgCtx.Account) { $mgCtx.Account } else { '(app identity)' } + $diagLines.Add("Cause: The signed-in identity ('$signedIn') is authenticated but does not have") + $diagLines.Add(' access to the target SharePoint site (HTTP 403 Forbidden).') + $diagLines.Add('Action: In SharePoint, add this identity to the target site as at least Member,') + $diagLines.Add(' OR re-run with an identity that already has access to the site.') + } + elseif ($status -eq 404) { + $diagLines.Add("Cause: The site was not found on host '$hostName' (HTTP 404).") + $diagLines.Add('Action: Verify the URL was copied via SharePoint Details -> Path -> Copy.') + $diagLines.Add(' Check spelling of the /sites/ or /teams/ segment.') + } + elseif ($status -ge 500) { + $diagLines.Add("Cause: Microsoft Graph returned a server error (HTTP $status). This is usually transient.") + $diagLines.Add('Action: Wait a minute and retry. If it persists, check https://status.cloud.microsoft.') + } + else { + # Includes name-resolution failures (no HTTP status) — typically dogfood/PPE hostnames. + if ($hostName -match 'sharepoint-(df|ppe)\.com$') { + $diagLines.Add("Cause: Host '$hostName' is a Microsoft-internal dogfood/PPE SharePoint endpoint") + $diagLines.Add(' and is not reachable from the public Microsoft Graph service. This script') + $diagLines.Add(' does not currently support dogfood/PPE SharePoint targets.') + $diagLines.Add('Action: Use a production SharePoint URL (*.sharepoint.com) instead.') + } else { + $diagLines.Add("Cause: Graph site lookup failed (HTTP status: $status).") + $diagLines.Add('Action: Verify the URL is correct and reachable, and that the signed-in identity') + $diagLines.Add(' has the required Graph scopes and SharePoint site access.') + } + } + $diagLines.Add('') + $diagLines.Add("Graph response: $rawDetail") + + throw ($diagLines -join "`n") + } + $siteId = $site.id + if (-not $siteId) { throw "Unable to resolve SharePoint site '$Url' (Graph returned no id)." } + + # Resolve drive (document library) by name + $drivesUri = "https://graph.microsoft.com/v1.0/sites/${siteId}/drives" + $drives = Invoke-MgGraphRequest -Method GET -Uri $drivesUri -ErrorAction Stop + $drive = $drives.value | Where-Object { $_.name -eq $libraryName -or $_.webUrl -match "/$libraryName(/|$)" } | Select-Object -First 1 + if (-not $drive) { + # Fallback: default drive + $drive = Invoke-MgGraphRequest -Method GET -Uri "https://graph.microsoft.com/v1.0/sites/${siteId}/drive" -ErrorAction Stop + # If the URL named a non-default library, treat the entire path-after-site as folder under default drive + $folderInLibrary = ($libAndFolder -join '/') + } + + return [pscustomobject]@{ + HostName = $hostName + SiteName = $siteName + SiteId = $siteId + DriveId = $drive.id + LibraryName = $libraryName + FolderPath = $folderInLibrary.TrimEnd('/') + WebUrl = $drive.webUrl + } +} + +function Resolve-FabricTarget { + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Url) + + # Accepted URL shapes (all return enough metadata for callers to route correctly): + # 1. https://onelake.dfs.fabric.microsoft.com//.Lakehouse + # (root mode - recommended; operational artifacts auto-routed under Files/, + # Delta tables under Tables/) + # 2. https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Files[/] + # (explicit Files; FilesPath = ) + # 3. https://onelake.dfs.fabric.microsoft.com//.Lakehouse/Tables[/] + # (explicit Tables for table-typed destinations; TablesPath = ) + # All three shapes are accepted here; the call-site helpers + # (Send-FileToOneLake / Get-RemoteFile-OneLake) route operational artifacts + # under /Files/ regardless of TablesPath, so an -OutputPath that points at + # the lakehouse root or at an explicit /Tables/ still lands + # log / checkpoint / mirror artifacts in the correct /Files/ subtree. + $u = [Uri]$Url + $accountUrl = "{0}://{1}" -f $u.Scheme, $u.Host + $segments = ($u.AbsolutePath.TrimStart('/') -split '/' | Where-Object { $_ -ne '' }) + # Force into an array shape so .Count is reliable even for 1- or 0-element results. + $segments = @($segments) + if ($segments.Count -lt 2) { + throw "Fabric URL must include workspace and item: '$Url'" + } + $workspace = $segments[0] + $itemFull = $segments[1] # e.g. MyLakehouse.Lakehouse OR an item GUID + $itemGuidPattern = '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + if ($itemFull -match '\.(Lakehouse|Warehouse)$') { + $itemType = $Matches[1] + $itemName = $itemFull.Substring(0, $itemFull.Length - ($itemType.Length + 1)) + } + elseif ($itemFull -match $itemGuidPattern) { + # GUID-addressed item: OneLake references the item by its GUID with no type + # suffix (https://onelake.dfs.fabric.microsoft.com///...). + # The DFS path is type-agnostic; PAX only emits Lakehouse-shaped output, so + # record the type as Lakehouse for display/diagnostics. ItemFull stays the + # GUID verbatim so every downstream path ("$FilesystemBase/$ItemFull/...") + # builds the correct GUID-form DFS URL. + $itemType = 'Lakehouse' + $itemName = $itemFull + } + else { + throw "Fabric item must end with .Lakehouse/.Warehouse, or be an item GUID: '$itemFull'" + } + + $filesPath = '' + $tablesPath = '' + $rootMode = $false + if ($segments.Count -eq 2) { + $rootMode = $true + } + elseif ($segments[2] -eq 'Files') { + $filesPath = if ($segments.Count -gt 3) { ($segments | Select-Object -Skip 3) -join '/' } else { '' } + } + elseif ($segments[2] -eq 'Tables') { + $tablesPath = if ($segments.Count -gt 3) { ($segments | Select-Object -Skip 3) -join '/' } else { '' } + } + else { + throw "Fabric URL third segment must be 'Files', 'Tables', or absent (lakehouse root): '$Url'" + } + + return [pscustomobject]@{ + AccountUrl = $accountUrl + Workspace = $workspace + ItemName = $itemName + ItemType = $itemType + ItemFull = $itemFull + FilesPath = $filesPath.TrimEnd('/') + TablesPath = $tablesPath.TrimEnd('/') + RootMode = $rootMode + FilesystemBase = "$accountUrl/$workspace" + } +} + +# ============================================== +# AZURE (FABRIC/ONELAKE) TOKEN INFRASTRUCTURE +# ============================================== +# Mirrors the Graph token refresh design ($script:SharedAuthState + +# Refresh-GraphTokenIfNeeded + Invoke-TokenRefresh) but for the OneLake DFS +# audience (https://storage.azure.com/), which is acquired through Az.Accounts. +# +# Layers: +# Get-FabricStorageTokenRaw - private; runs Connect-AzAccount/Get-AzAccessToken; +# normalizes ExpiresOn to UTC DateTime; tags AuthMethod. +# Invoke-AzTokenAcquire - private; calls Raw + stale-token rejection + +# updates $script:AzAuthState + emits log line. +# Refresh-FabricTokenIfNeeded - public; proactive (5-min buffer + 50-min age cap), +# cooldown (auth-mode aware), -Force for 401 reactive. +# Get-FabricStorageToken - public; backward-compatible string-returning wrapper +# (always returns a refresh-checked token). +# Invoke-FabricWebRequest - public; wraps Invoke-WebRequest for OneLake calls. +# Pre-flight refresh + transparent 401 retry once. +# Reactive triggers: 401 detected by Invoke-FabricWebRequest -> -Force refresh + retry. +# Proactive triggers: BufferMinutes (5) or token age >50 min, whichever comes first. +# Cooldown: 45s for app-only/MI (silent client_credentials/IMDS); 5min for interactive +# (avoids re-prompt spam on long upload phases). +function Get-FabricStorageTokenRaw { + [CmdletBinding()] + param() + $resource = 'https://storage.azure.com/' + if (-not (Get-Module -Name Az.Accounts -ListAvailable -ErrorAction SilentlyContinue)) { + throw "Az.Accounts module not installed. Install with: Install-Module Az.Accounts -Scope CurrentUser" + } + Import-Module Az.Accounts -ErrorAction Stop | Out-Null + + $authMethodTag = 'Interactive' + $ctx = Get-AzContext -ErrorAction SilentlyContinue + if (-not $ctx) { + if ($Auth -eq 'ManagedIdentity') { + $authMethodTag = 'ManagedIdentity' + if ($env:AZURE_CLIENT_ID) { + Connect-AzAccount -Identity -AccountId $env:AZURE_CLIENT_ID -ErrorAction Stop | Out-Null + } else { + Connect-AzAccount -Identity -ErrorAction Stop | Out-Null + } + } + elseif ($Auth -eq 'AppRegistration' -and $script:TenantId -and $script:ClientId -and $script:ClientSecret) { + $authMethodTag = 'AppRegistration' + $secureSecret = ConvertTo-SecureString -String $script:ClientSecret -AsPlainText -Force + $cred = New-Object System.Management.Automation.PSCredential($script:ClientId, $secureSecret) + Connect-AzAccount -ServicePrincipal -Tenant $script:TenantId -Credential $cred -ErrorAction Stop | Out-Null + } + else { + Connect-AzAccount -ErrorAction Stop | Out-Null + } + } else { + # Reuse existing context; classify by account type so cooldown/log lines stay accurate. + if ($ctx.Account -and $ctx.Account.Type -eq 'ManagedService') { $authMethodTag = 'ManagedIdentity' } + elseif ($ctx.Account -and $ctx.Account.Type -eq 'ServicePrincipal') { $authMethodTag = 'AppRegistration' } + } + + $tokenObj = Get-AzAccessToken -ResourceUrl $resource -ErrorAction Stop + # Token shape: newer Az returns SecureString; older returns plain string. + $tokenStr = if ($tokenObj.Token -is [System.Security.SecureString]) { + [System.Net.NetworkCredential]::new('', $tokenObj.Token).Password + } else { + [string]$tokenObj.Token + } + # ExpiresOn shape: newer Az returns DateTimeOffset; older returns DateTime; defensive parse for string. + $expiresUtc = if ($tokenObj.ExpiresOn -is [System.DateTimeOffset]) { + $tokenObj.ExpiresOn.UtcDateTime + } elseif ($tokenObj.ExpiresOn -is [datetime]) { + if ($tokenObj.ExpiresOn.Kind -eq [System.DateTimeKind]::Utc) { $tokenObj.ExpiresOn } else { $tokenObj.ExpiresOn.ToUniversalTime() } + } else { + ([datetime]$tokenObj.ExpiresOn).ToUniversalTime() + } + + return [pscustomobject]@{ + Token = $tokenStr + ExpiresOn = $expiresUtc + AuthMethod = $authMethodTag + } +} + +function Invoke-AzTokenAcquire { + [CmdletBinding()] + param([string] $Reason = 'initial') + try { + $tokenObj = Get-FabricStorageTokenRaw + # Stale-token rejection: refuse a token that is already at/near expiry. Protects + # against MSAL cache returning a stale entry after process suspension or system sleep. + $nowUtc = (Get-Date).ToUniversalTime() + $minutesValid = ($tokenObj.ExpiresOn - $nowUtc).TotalMinutes + if ($minutesValid -le 2) { + throw ("Acquired OneLake storage token is already expired or near-expiry (expires in {0:N1} min)." -f $minutesValid) + } + + $script:AzAuthState.Token = $tokenObj.Token + $script:AzAuthState.ExpiresOn = $tokenObj.ExpiresOn + $script:AzAuthState.AcquiredAt = Get-Date + $script:AzAuthState.LastRefresh = Get-Date + $script:AzAuthState.RefreshCount++ + $script:AzAuthState.AuthMethod = $tokenObj.AuthMethod + # Mirror to legacy $script:FabricToken so any cached references stay in sync. + $script:FabricToken = $tokenObj.Token + + $verb = if ($script:AzAuthState.RefreshCount -eq 1) { 'acquired' } else { 'refreshed' } + Write-LogHost (" [AZ-TOKEN] OneLake storage token {0} ({1}); auth: {2}; expires {3} UTC; refresh #{4}" -f $verb, $Reason, $tokenObj.AuthMethod, $tokenObj.ExpiresOn.ToString('yyyy-MM-dd HH:mm:ss'), $script:AzAuthState.RefreshCount) -ForegroundColor Gray + return $true + } + catch { + Write-LogHost (" [AZ-TOKEN] [!] Failed to acquire OneLake storage token ({0}): {1}" -f $Reason, $_.Exception.Message) -ForegroundColor Red + return $false + } +} + +function Refresh-FabricTokenIfNeeded { + <# + .SYNOPSIS + Proactively refreshes the Fabric/OneLake storage token if nearing expiry. + .PARAMETER BufferMinutes + Refresh if token expires within this many minutes. Default: 5. + .PARAMETER Force + Bypass cooldown and force re-acquisition (used by 401 reactive retry). + .OUTPUTS + $true - token is valid (either still-valid or freshly refreshed) + $false - acquisition/refresh failed; caller must surface the error + #> + [CmdletBinding()] + param( + [int] $BufferMinutes = 5, + [switch] $Force + ) + $now = (Get-Date).ToUniversalTime() + + # First-time acquisition: no state yet. + if (-not $script:AzAuthState.Token -or -not $script:AzAuthState.ExpiresOn) { + return (Invoke-AzTokenAcquire -Reason 'initial') + } + + $minutesRemaining = ($script:AzAuthState.ExpiresOn - $now).TotalMinutes + $tokenAge = if ($script:AzAuthState.AcquiredAt) { + ((Get-Date) - $script:AzAuthState.AcquiredAt).TotalMinutes + } else { 999 } + + # Trigger conditions: + # - Forced (typically 401 reactive) + # - Within BufferMinutes of expiry (proactive) + # - Token age > 50 min (belt-and-suspenders for any auth mode where ExpiresOn might be optimistic) + $needRefresh = $Force.IsPresent -or ($minutesRemaining -le $BufferMinutes) -or ($tokenAge -gt 50) + if (-not $needRefresh) { return $true } + + # Cooldown: app-only / MI refresh silently via client_credentials or IMDS (cheap), + # so 45s is enough. Interactive may prompt the user, so 5 min keeps that bearable. + $isAppOnly = ($script:AzAuthState.AuthMethod -in @('ManagedIdentity', 'AppRegistration')) + $cooldownMinutes = if ($isAppOnly) { 0.75 } else { 5.0 } + if (-not $Force -and $script:AzAuthState.LastRefreshAttempt) { + $sinceLast = ((Get-Date) - $script:AzAuthState.LastRefreshAttempt).TotalMinutes + if ($sinceLast -lt $cooldownMinutes) { + # Still in cooldown; report current token as valid only if it actually is. + return ($minutesRemaining -gt 0) + } + } + $script:AzAuthState.LastRefreshAttempt = Get-Date + + $reason = if ($Force.IsPresent) { + 'forced (401 retry or explicit)' + } elseif ($minutesRemaining -le $BufferMinutes) { + ("near-expiry ({0:N1} min remaining)" -f $minutesRemaining) + } else { + ("age cap ({0:N1} min)" -f $tokenAge) + } + return (Invoke-AzTokenAcquire -Reason $reason) +} + +function Get-FabricStorageToken { + <# + .SYNOPSIS + Returns a valid OneLake storage-audience bearer token (string) for OneLake DFS calls. + .DESCRIPTION + Backward-compatible wrapper around the Az token state machine. Always invokes + Refresh-FabricTokenIfNeeded first so callers receive a token guaranteed to have at + least BufferMinutes of validity (or a fresh one). Throws if acquisition fails. + #> + [CmdletBinding()] param() + $ok = Refresh-FabricTokenIfNeeded + if (-not $ok -or -not $script:AzAuthState.Token) { + throw "Failed to acquire Fabric/OneLake storage token (see prior [AZ-TOKEN] messages)." + } + return $script:AzAuthState.Token +} + +function Invoke-FabricWebRequest { + <# + .SYNOPSIS + Invoke-WebRequest wrapper for OneLake DFS calls with proactive token refresh + and transparent 401 reactive retry. + .DESCRIPTION + Before each call: ensures token is fresh via Refresh-FabricTokenIfNeeded. + During the call: if a 401 Unauthorized is returned, forces a refresh and + retries the request once with the new token. Any other status (429, 5xx, + 403, 404, etc.) is re-thrown unchanged so the caller's existing retry/error + handling (e.g., chunk-upload 429/5xx backoff loops) continues to work. + .PARAMETER Headers + Caller-supplied headers. The Authorization header is ALWAYS overwritten with + the current $script:AzAuthState.Token. x-ms-version defaults to '2021-06-08' + if the caller does not supply it. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $Uri, + [Parameter(Mandatory)] [string] $Method, + [hashtable] $Headers, + $Body, + [string] $ContentType, + [string] $OutFile + ) + # Pre-flight refresh. + $null = Refresh-FabricTokenIfNeeded + if (-not $script:AzAuthState.Token) { + throw "Invoke-FabricWebRequest: no OneLake storage token available." + } + + # Build effective headers; always set Authorization from current state. + $h = @{} + if ($Headers) { + foreach ($k in $Headers.Keys) { + if ($k -ne 'Authorization') { $h[$k] = $Headers[$k] } + } + } + $h['Authorization'] = "Bearer $($script:AzAuthState.Token)" + if (-not $h.ContainsKey('x-ms-version')) { $h['x-ms-version'] = '2021-06-08' } + + $invokeParams = @{ + Uri = $Uri + Method = $Method + Headers = $h + UseBasicParsing = $true + ErrorAction = 'Stop' + } + if ($PSBoundParameters.ContainsKey('Body')) { $invokeParams.Body = $Body } + if ($ContentType) { $invokeParams.ContentType = $ContentType } + if ($OutFile) { $invokeParams.OutFile = $OutFile } + + try { + return Invoke-WebRequest @invokeParams + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 401) { + Write-LogHost " [AZ-TOKEN] OneLake returned 401 Unauthorized - forcing token refresh and retrying once..." -ForegroundColor Yellow + $refreshed = Refresh-FabricTokenIfNeeded -Force + if (-not $refreshed) { + throw ("OneLake 401 Unauthorized and token refresh failed: {0}" -f $_.Exception.Message) + } + $h['Authorization'] = "Bearer $($script:AzAuthState.Token)" + $invokeParams.Headers = $h + return Invoke-WebRequest @invokeParams + } + throw + } +} + +function Test-RemoteDestination { + [CmdletBinding()] param() + if ($script:RemoteOutputMode -eq 'None') { return } + if ($script:RemoteOutputMode -eq 'SharePoint') { + $resolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl + $script:SPResolved = $resolved + # Ensure folder exists (create if missing) + if ($resolved.FolderPath) { + $folderUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/$($resolved.FolderPath)" + try { + Invoke-MgGraphRequest -Method GET -Uri $folderUri -ErrorAction Stop | Out-Null + } catch { + # Create folder hierarchy + $parts = $resolved.FolderPath -split '/' + $accumulated = '' + foreach ($p in $parts) { + $parentUri = if ($accumulated) { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${accumulated}:/children" } else { "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root/children" } + # FolderPath segments come from [Uri].AbsolutePath, which percent-encodes spaces + # and other reserved characters (e.g. 'PAX Exports' -> 'PAX%20Exports'). The + # parent addressing above (root:/...:/children) is colon-path syntax that Graph + # DECODES, but the 'name' field below is a literal display name that Graph does + # NOT decode — so passing the raw segment created a folder literally named + # 'PAX%20Exports' alongside the decoded 'PAX Exports' that the upload path + # (Send-FileToSharePoint, also colon-path) resolves to. Decode the segment for + # the display name so both mechanisms agree and only one folder is created. + $folderDisplayName = [System.Uri]::UnescapeDataString($p) + $body = @{ name = $folderDisplayName; folder = @{}; '@microsoft.graph.conflictBehavior' = 'replace' } | ConvertTo-Json + try { Invoke-MgGraphRequest -Method POST -Uri $parentUri -Body $body -ContentType 'application/json' -ErrorAction Stop | Out-Null } catch {} + $accumulated = if ($accumulated) { "$accumulated/$p" } else { $p } + } + } + } + } + elseif ($script:RemoteOutputMode -eq 'Fabric') { + $resolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl + $script:FabricResolved = $resolved + # Initial token acquisition (populates $script:AzAuthState). Classify Az/auth + # failures here so customers see Cause/Action lines instead of raw exceptions. + try { + $null = Get-FabricStorageToken + } catch { + $tokenErr = $_.Exception.Message + $tokenDiag = New-Object System.Collections.Generic.List[string] + $tokenDiag.Add("OneLake storage token acquisition failed for workspace '$($resolved.Workspace)'.") + if ($tokenErr -match 'Az\.Accounts module not installed') { + $tokenDiag.Add('Cause: The Az.Accounts PowerShell module is not installed.') + $tokenDiag.Add('Action: Install-Module Az.Accounts -Scope CurrentUser') + } elseif ($tokenErr -match 'AADSTS' -or $tokenErr -match 'consent' -or $tokenErr -match 'admin') { + $tokenDiag.Add('Cause: Azure AD rejected the sign-in for the storage.azure.com audience.') + $tokenDiag.Add('Action: Verify the identity has Azure AD sign-in to your tenant, and that') + $tokenDiag.Add(' any conditional access / MFA requirements are satisfied.') + } elseif ($tokenErr -match 'IDENTITY|managed identity|MSI|IMDS') { + $tokenDiag.Add('Cause: Managed identity token acquisition failed (no IMDS endpoint reachable,') + $tokenDiag.Add(' or AZURE_CLIENT_ID points to an identity not assigned to this host).') + $tokenDiag.Add('Action: Re-run from a host with an assigned managed identity, or use') + $tokenDiag.Add(' -Auth Interactive / -Auth AppRegistration instead.') + } else { + $tokenDiag.Add('Cause: Az.Accounts could not acquire a token for https://storage.azure.com/.') + $tokenDiag.Add('Action: Verify your authentication mode (-Auth) and that you can sign in via') + $tokenDiag.Add(' Connect-AzAccount manually.') + } + $tokenDiag.Add('') + $tokenDiag.Add("Az response: $tokenErr") + throw ($tokenDiag -join "`n") + } + # HEAD the filesystem (workspace) to verify access. Invoke-FabricWebRequest sets + # Authorization + x-ms-version automatically and handles 401 reactive refresh. + $probeUri = "$($resolved.FilesystemBase)?resource=filesystem" + try { + $null = Invoke-FabricWebRequest -Uri $probeUri -Method HEAD + } catch { + # Classify OneLake DFS responses. Unlike Graph, OneLake rarely returns a JSON + # body on auth failures — the HTTP status is the primary signal. + $status = 0 + try { $status = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($status -eq 0) { try { if ($_.Exception.Response.StatusCode -is [System.Net.HttpStatusCode]) { $status = [int]$_.Exception.Response.StatusCode } } catch {} } + $rawDetail = $_.Exception.Message + if ($rawDetail.Length -gt 600) { $rawDetail = $rawDetail.Substring(0, 600) + '...(truncated)' } + + $diagLines = New-Object System.Collections.Generic.List[string] + $diagLines.Add("OneLake filesystem probe failed (workspace '$($resolved.Workspace)', item '$($resolved.ItemFull)').") + + if ($status -eq 401) { + $diagLines.Add('Cause: OneLake rejected the storage token (401). The token audience is correct') + $diagLines.Add(' but the identity is not recognized by this Fabric workspace.') + $diagLines.Add('Action: Verify the identity is signed in to the same tenant that owns the Fabric') + $diagLines.Add(' workspace, and that the storage.azure.com token is not expired/blocked.') + } + elseif ($status -eq 403) { + $diagLines.Add('Cause: The identity is authenticated but lacks permissions on the Fabric workspace') + $diagLines.Add(' (HTTP 403 Forbidden from OneLake DFS).') + $diagLines.Add('Action: In the Fabric portal -> Workspace settings -> Manage access, grant the') + $diagLines.Add(' identity at least Contributor. See the "Permissions Required for THIS run"') + $diagLines.Add(' table earlier in this output for the exact Azure RBAC roles required.') + } + elseif ($status -eq 404) { + $diagLines.Add("Cause: Workspace or item not found (HTTP 404). One of these is wrong:") + $diagLines.Add(" workspace = '$($resolved.Workspace)'") + $diagLines.Add(" item = '$($resolved.ItemFull)' (must exist as a Lakehouse/Warehouse)") + $diagLines.Add('Action: Verify the URL by opening the lakehouse in the Fabric portal and using') + $diagLines.Add(' the OneLake "Copy ABFS path" option, then converting to https:// form.') + } + elseif ($status -ge 500) { + $diagLines.Add("Cause: OneLake returned a server error (HTTP $status). This is usually transient.") + $diagLines.Add('Action: Wait a minute and retry. If it persists, check Fabric service health.') + } + else { + $diagLines.Add("Cause: OneLake DFS probe failed (HTTP status: $status).") + $diagLines.Add('Action: Verify the Fabric workspace URL is correct and that the identity has') + $diagLines.Add(' the Azure RBAC roles listed in the "Permissions Required for THIS run" table.') + } + $diagLines.Add('') + $diagLines.Add("OneLake response: $rawDetail") + + throw ($diagLines -join "`n") + } + } +} + +function Get-GraphErrorDetail { + # Best-effort extraction of the Microsoft Graph error body from a caught error. + # PowerShell attaches the raw HTTP response body to $_.ErrorDetails.Message for both + # Invoke-MgGraphRequest and Invoke-WebRequest; fall back to the response stream, then + # to the plain exception message. Returns a short single-line string for logging. + [CmdletBinding()] + param([Parameter(Mandatory)] $ErrorRecord) + $raw = $null + if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { + $raw = $ErrorRecord.ErrorDetails.Message + } + elseif ($ErrorRecord.Exception -and $ErrorRecord.Exception.Response) { + try { + $stream = $ErrorRecord.Exception.Response.GetResponseStream() + if ($stream) { + $reader = New-Object System.IO.StreamReader($stream) + $raw = $reader.ReadToEnd() + $reader.Dispose() + } + } catch { } + } + if (-not $raw) { return $ErrorRecord.Exception.Message } + # Try to pull error.code / error.message out of the JSON body. + try { + $j = $raw | ConvertFrom-Json -ErrorAction Stop + if ($j.error) { + $code = $j.error.code + $msg = $j.error.message + return ("{0}: {1}" -f $code, $msg).Trim(': ') + } + } catch { } + # Non-JSON body - return a trimmed single line. + return ($raw -replace '\s+', ' ').Trim() +} + +function script:New-PaxGraphUploadSession { + # Direct-REST createUploadSession for large SharePoint uploads (files above the Graph simple-upload + # ceiling). The request body is built as a plain ordered object and serialized with ConvertTo-Json, + # then sent via Invoke-RestMethod with an explicit bearer header - deliberately BYPASSING the Graph SDK + # object serializer used by Invoke-MgGraphRequest. Handing a hashtable body that carries a PowerShell + # string to that serializer fails, because the string's extended-type '.Chars' member is a + # PSParameterizedProperty that triggers a self-referencing-loop serialization error at 'item.name.Chars' + # and no HTTP request is ever sent. Returns the parsed session object (with .uploadUrl); throws with + # Graph error detail on failure. Never logs the token or the returned uploadUrl. + param( + [Parameter(Mandatory)][string]$SessionUri, + [Parameter(Mandatory)][string]$RemoteFileName + ) + try { $null = Refresh-GraphTokenIfNeeded } catch { } + $token = Get-GraphAccessToken + if ([string]::IsNullOrWhiteSpace([string]$token)) { throw 'createUploadSession: no Graph access token available.' } + # Plain values only -> plain JSON. No PowerShell object reaches an SDK serializer. + $bodyObj = [ordered]@{ item = [ordered]@{ '@microsoft.graph.conflictBehavior' = 'replace'; name = [string]$RemoteFileName } } + $json = $bodyObj | ConvertTo-Json -Depth 4 -Compress + $jsonBytes = [System.Text.Encoding]::UTF8.GetBytes($json) + $headers = @{ Authorization = ('Bearer ' + [string]$token) } + return Invoke-RestMethod -Uri $SessionUri -Method POST -Headers $headers -ContentType 'application/json' -Body $jsonBytes -ErrorAction Stop +} + +function Send-FileToSharePoint { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $LocalPath, + [string] $RemoteFileName, + # Optional per-data-type SharePoint parent URL. When supplied, the destination + # is resolved against $ParentOverride instead of the script-wide Purview + # RemoteOutputUrl. Used by per-data-type append surfaces (UserInfo / Agent365 / + # Log) when their -OutputPath* points at a different SharePoint folder than + # the primary Purview output. + [string] $ParentOverride + ) + if (-not (Test-Path -LiteralPath $LocalPath)) { throw "Send-FileToSharePoint: source not found: $LocalPath" } + if ($ParentOverride) { + $resolved = Resolve-SharePointTarget -Url $ParentOverride + } + else { + if (-not $script:SPResolved) { $script:SPResolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl } + $resolved = $script:SPResolved + } + $fileName = if ($RemoteFileName) { $RemoteFileName } else { Split-Path -Leaf $LocalPath } + # Per-segment URI escaping for the filename. The auto-defaulted basenames used by + # this script are safe ASCII (timestamps + underscores), but routing custom basenames + # through the same encoding pipeline avoids any per-call-site special casing. + $fileNameEnc = [System.Uri]::EscapeDataString($fileName) + $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$fileNameEnc" } else { $fileNameEnc } + $fileInfo = Get-Item -LiteralPath $LocalPath + $sizeMb = $fileInfo.Length / 1MB + + # Graph simple upload (PUT .../content) supports up to 250 MB; use it for everything + # up to that ceiling so as many files as possible avoid the resumable-upload-session + # API (which some locked-down libraries / egress proxies reject even when a simple PUT + # is allowed). Files above 250 MB still use createUploadSession + chunked PUT. + $simpleUploadMax = 250MB + if ($fileInfo.Length -le $simpleUploadMax) { + $putUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/content" + $bytes = [System.IO.File]::ReadAllBytes($LocalPath) + try { + Invoke-MgGraphRequest -Method PUT -Uri $putUri -Body $bytes -ContentType 'application/octet-stream' -ErrorAction Stop | Out-Null + } + catch { + $detail = Get-GraphErrorDetail -ErrorRecord $_ + throw "SharePoint simple upload (PUT) failed for '$relPath': $detail" + } + } + else { + # Create upload session + $sessUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/createUploadSession" + try { + # Direct REST (New-PaxGraphUploadSession): the body is never handed to the Graph SDK object + # serializer, which fails on a PowerShell string's ETS '.Chars' property before any HTTP send. + $session = script:New-PaxGraphUploadSession -SessionUri $sessUri -RemoteFileName $fileName + } + catch { + $detail = Get-GraphErrorDetail -ErrorRecord $_ + $httpStatus = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + $sizeGb = $fileInfo.Length / 1GB + throw ("createUploadSession failed for '{0}' (HTTP {1}; file size {2:N0} bytes / {3:N2} GiB): {4}" -f $relPath, $httpStatus, $fileInfo.Length, $sizeGb, $detail) + } + $uploadUrl = $session.uploadUrl + if (-not $uploadUrl) { throw "SharePoint createUploadSession returned no uploadUrl for '$relPath'." } + + # Aggregate upload progress heartbeat (large files only). Reports transfer status only - never + # the uploadUrl, bearer token, or any response secret. + $hbInterval = if ($StatusIntervalSeconds -gt 0) { [int]$StatusIntervalSeconds } else { 60 } + $uploadStart = Get-Date + $lastHb = $uploadStart + $chunkCount = 0 + $retryTotal = 0 + Write-LogHost (" SharePoint resumable upload session created: {0} ({1:N0} bytes / {2:N2} MiB); uploading in chunks..." -f $relPath, $fileInfo.Length, ($fileInfo.Length / 1MB)) -ForegroundColor Cyan + + $chunkSize = 5MB # multiple of 320 KB + $fs = [System.IO.File]::OpenRead($LocalPath) + try { + $buffer = New-Object byte[] $chunkSize + $offset = 0L + while ($offset -lt $fileInfo.Length) { + $toRead = [int][Math]::Min([int64]$chunkSize, $fileInfo.Length - $offset) + # FileStream.Read may return fewer bytes than requested; loop until the + # fragment is fully populated (or EOF) so every Content-Range fragment + # carries exactly the byte count its header declares. + $read = 0 + while ($read -lt $toRead) { + $n = $fs.Read($buffer, $read, $toRead - $read) + if ($n -le 0) { break } + $read += $n + } + if ($read -le 0) { break } + $end = $offset + $read - 1 + $range = "bytes $offset-$end/$($fileInfo.Length)" + # Send a true byte[] body. PowerShell range-indexing ($buffer[0..N]) + # returns an Object[] of boxed bytes, which Invoke-WebRequest stringifies + # instead of sending as raw bytes — producing a body whose length does not + # match the declared Content-Range and a 400 (Bad Request) from Graph on + # the final partial fragment. Copy into a right-sized byte[] instead. + if ($read -eq $chunkSize) { + $slice = $buffer + } + else { + $slice = New-Object byte[] $read + [System.Array]::Copy($buffer, 0, $slice, 0, $read) + } + $attempt = 0 + while ($true) { + try { + $null = Invoke-WebRequest -Uri $uploadUrl -Method PUT -Body $slice -Headers @{ 'Content-Range' = $range } -ContentType 'application/octet-stream' -UseBasicParsing -ErrorAction Stop + break + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $retryTotal++ + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } + $offset += $read + $chunkCount++ + $nowHb = Get-Date + if ((($nowHb - $lastHb).TotalSeconds -ge $hbInterval) -or ($offset -ge $fileInfo.Length)) { + $elapsedSec = ($nowHb - $uploadStart).TotalSeconds + $pct = if ($fileInfo.Length -gt 0) { [Math]::Round(100.0 * $offset / $fileInfo.Length, 1) } else { 100 } + $rateMb = if ($elapsedSec -gt 0) { [Math]::Round(($offset / 1MB) / $elapsedSec, 2) } else { 0 } + Write-LogHost (" upload progress: {0:N0}/{1:N0} bytes ({2}%); chunk {3:N0}; elapsed {4:N0}s; {5} MiB/s; retries {6:N0}" -f $offset, $fileInfo.Length, $pct, $chunkCount, $elapsedSec, $rateMb, $retryTotal) -ForegroundColor DarkGray + $lastHb = $nowHb + } + } + } + finally { $fs.Dispose() } + } + Write-LogHost (" -> SharePoint upload OK: {0} ({1:N2} MB)" -f $relPath, $sizeMb) -ForegroundColor DarkGray +} + +function Send-FileToOneLake { + <# + .SYNOPSIS + Uploads a local file to OneLake Files/ via the DFS create/append/flush sequence. + .PARAMETER FailIfExists + Create-if-absent publication mode for IMMUTABLE resume-protocol objects (generation + artifacts and authority records). The DFS Create File request carries an + If-None-Match: * precondition, so an already-existing target answers 409/412 and is + raised as a HARD COLLISION instead of being overwritten - an existence conflict is + never success. Callers retry by allocating a completely fresh unique target, never by + replacing the conflicted object. + + This is OPT-IN and scoped to resume-generation publication. Customer-facing output + uploads omit the switch and keep the existing overwrite semantics unchanged. + + ASSUMPTION: the OneLake/ADLS Gen2 surface honours the If-None-Match: * conditional + on Create File. Where it does not, the caller's nonce-bearing target names remain + mandatory and any collision that is detected still fails closed. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $LocalPath, + [string] $RemoteFileName, + # Optional per-data-type Fabric/OneLake parent URL. See Send-FileToSharePoint + # for the rationale. When omitted, falls back to the script-wide Purview URL. + [string] $ParentOverride, + [switch] $FailIfExists + ) + if (-not (Test-Path -LiteralPath $LocalPath)) { throw "Send-FileToOneLake: source not found: $LocalPath" } + if ($ParentOverride) { + $resolved = Resolve-FabricTarget -Url $ParentOverride + } + else { + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + } + $fileName = if ($RemoteFileName) { $RemoteFileName } else { Split-Path -Leaf $LocalPath } + # Per-SEGMENT URI escaping. $RemoteFileName may contain '/' separators + # (e.g. ".pax_resume//" from Sync-FabricResumeMirror); escaping the + # whole string as one data segment would percent-encode the slashes ('%2F'), + # malforming the DFS path so PUT (Create File) and PATCH (Append) target + # different resolved files, surfacing as "400 (position not equal to length)". + # Split-then-escape-per-segment preserves the hierarchical path. + $fileNameEnc = ($fileName -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/$fileNameEnc" } else { $fileNameEnc } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem" + $fileInfo = Get-Item -LiteralPath $LocalPath + $sizeMb = $fileInfo.Length / 1MB + + # Authorization + x-ms-version are set per-request by Invoke-FabricWebRequest using the + # current $script:AzAuthState. Do not cache headers locally - the token may rotate. + + # Step 0: Pre-create any parent directories. OneLake Lakehouse Files/ uses a + # hierarchical namespace; PUT ?resource=file does not auto-create intermediate + # directories. Without this, mirroring artifacts to e.g. ".pax_resume//" + # can land the leaf file in an inconsistent state that surfaces downstream as + # a 400 (position not equal to length) on the first append. Idempotent: a 409 + # response means the directory already exists and is treated as success. + $parentRel = $null + if ($relInItem -match '/') { $parentRel = $relInItem.Substring(0, $relInItem.LastIndexOf('/')) } + if ($parentRel) { + $dirUri = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$parentRel`?resource=directory" + try { + $null = Invoke-FabricWebRequest -Uri $dirUri -Method PUT + } + catch { + $dirStatus = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($dirStatus -ne 409) { throw } + } + } + + # Step 1: Create (PUT ?resource=file) - overwrites any existing file, unless + # -FailIfExists asks for create-if-absent semantics. + $createUri = "$dfsPath`?resource=file" + if ($FailIfExists) { + # If-None-Match: * makes the create succeed ONLY when the target does not already + # exist. A 409/412 means something is already published at this exact name; for an + # immutable resume-protocol object that is a hard collision, not a success, so it is + # raised and the caller allocates a fresh unique target instead of replacing bytes. + try { + $null = Invoke-FabricWebRequest -Uri $createUri -Method PUT -Headers @{ 'If-None-Match' = '*' } + } + catch { + $createStatus = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($createStatus -eq 409 -or $createStatus -eq 412) { + throw ("Send-FileToOneLake: create-if-absent conflict for {0} - the target object already exists and an immutable resume object is never overwritten." -f $relInItem) + } + throw + } + } + else { + $null = Invoke-FabricWebRequest -Uri $createUri -Method PUT + } + + # Step 2: Append in chunks (PATCH ?action=append&position=N). + # Inner retry loop handles 429/5xx with exponential backoff. 401 is handled transparently + # inside Invoke-FabricWebRequest (force-refresh + retry once); any 401 that escapes here + # means refresh itself failed, which is fatal for this upload and propagates up. + $chunkSize = 4MB + # Production ALWAYS streams from disk via File.OpenRead. $script:PaxOneLakeSourceStreamOverride + # is an internal test seam (null in every non-test run) that lets a focused test inject a + # stream whose Read() returns deliberately short fragments, exercising the fill loop below + # without changing the production read path. + $fs = if ($script:PaxOneLakeSourceStreamOverride) { $script:PaxOneLakeSourceStreamOverride } else { [System.IO.File]::OpenRead($LocalPath) } + try { + $buffer = New-Object byte[] $chunkSize + $position = 0L + while ($position -lt $fileInfo.Length) { + $toRead = [int][Math]::Min([int64]$chunkSize, $fileInfo.Length - $position) + # FileStream.Read may return fewer bytes than requested; fill the fragment fully (or to EOF) + # so the PATCH Content-Length and the declared append position stay exactly consistent. + $read = 0 + while ($read -lt $toRead) { + $n = $fs.Read($buffer, $read, $toRead - $read) + if ($n -le 0) { break } + $read += $n + } + if ($read -le 0) { break } + # Send a TRUE byte[] body. PowerShell range-indexing ($buffer[0..N]) returns an Object[] of + # boxed bytes, which the web cmdlet stringifies instead of transmitting as raw bytes - the wire + # length then disagrees with the declared append position and OneLake rejects the PATCH with + # "uploaded data is not contiguous" / "append position does not equal current file length". + # Partial chunks (checkpoint JSON and small shards below the 4 MiB chunk size) are the NORMAL + # path, so a right-sized byte[] copy is required for correctness here. + if ($read -eq $chunkSize) { + $slice = $buffer + } + else { + $slice = [byte[]]::new($read) + [System.Array]::Copy($buffer, 0, $slice, 0, $read) + } + $appendUri = "$dfsPath`?action=append&position=$position" + $attempt = 0 + while ($true) { + try { + $null = Invoke-FabricWebRequest -Uri $appendUri -Method PATCH -Body $slice -ContentType 'application/octet-stream' + break + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } + $position += $read + } + } + finally { $fs.Dispose() } + + # Local integrity guard: the append loop must have transmitted every source byte before + # we ask OneLake to flush the declared length. A short/interrupted read (unexpected EOF, or + # the local file mutated mid-upload) leaves $position < length; flushing that length would + # commit a remote file whose tail was never sent. Fail locally instead of asking OneLake to + # flush a length that was not uploaded. Normal successful uploads satisfy $position == length. + if ($position -ne $fileInfo.Length) { + throw ("Send-FileToOneLake: incomplete source read for {0} - sent {1} of {2} bytes before flush; aborting upload." -f $relInItem, $position, $fileInfo.Length) + } + + # Step 3: Flush (PATCH ?action=flush&position=) + $flushUri = "$dfsPath`?action=flush&position=$($fileInfo.Length)" + $null = Invoke-FabricWebRequest -Uri $flushUri -Method PATCH + + Write-LogHost (" -> OneLake upload OK: {0} ({1:N2} MB)" -f $relInItem, $sizeMb) -ForegroundColor DarkGray +} + +function Get-RemoteFile-SharePoint { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RelativeName, + [Parameter(Mandatory)] [string] $DestinationPath, + # Optional per-data-type SharePoint parent URL (see Send-FileToSharePoint). + [string] $ParentOverride + ) + if ($ParentOverride) { + $resolved = Resolve-SharePointTarget -Url $ParentOverride + } + else { + if (-not $script:SPResolved) { $script:SPResolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl } + $resolved = $script:SPResolved + } + $relName = [System.Uri]::EscapeDataString($RelativeName) + $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$relName" } else { $relName } + $dlUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}:/content" + # Stream the response body straight to disk. The Graph SDK's + # Invoke-MgGraphRequest -OutputFilePath buffers the ENTIRE body in memory before + # writing it out, so a large append target (roughly 1 GB or more) exceeds the + # single-object memory ceiling and fails. Invoke-WebRequest -OutFile on PowerShell 7 + # streams to disk instead, keeping peak memory flat regardless of file size (this + # mirrors Get-RemoteFile-OneLake). Graph accepts a raw bearer token directly; a + # missing file still surfaces as an HTTP 404 whose + # $_.Exception.Response.StatusCode.value__ is 404, so every caller's + # first-run / not-found classification is unchanged. + $graphToken = Get-GraphAccessToken + if (-not $graphToken) { throw "Get-RemoteFile-SharePoint: no Microsoft Graph access token available." } + Invoke-WebRequest -Uri $dlUri -Method GET -Headers @{ Authorization = "Bearer $graphToken" } -OutFile $DestinationPath -UseBasicParsing -ErrorAction Stop | Out-Null +} + +function Get-RemoteFile-OneLake { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RelativeName, + [Parameter(Mandatory)] [string] $DestinationPath, + # Optional per-data-type Fabric/OneLake parent URL (see Send-FileToSharePoint). + [string] $ParentOverride + ) + if ($ParentOverride) { + $resolved = Resolve-FabricTarget -Url $ParentOverride + } + else { + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + } + # Per-SEGMENT URI escaping. $RelativeName may contain '/' separators (e.g. + # ".pax_resume//" from Restore-FabricResumeMirror); escaping the + # whole string as one data segment would percent-encode the slashes to '%2F' + # and resolve to a different (or non-existent) DFS path than the uploader + # wrote to. Mirrors the Send-FileToOneLake fix. + $relName = ($RelativeName -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/$relName" } else { $relName } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem" + # Invoke-FabricWebRequest handles Authorization, x-ms-version, proactive refresh, 401 retry. + $null = Invoke-FabricWebRequest -Uri $dfsPath -Method GET -OutFile $DestinationPath +} + +function Get-RemoteAppendTargetSize { + <# + .SYNOPSIS + Return the byte size of an append/merge target WITHOUT downloading its body, + or $null when the size cannot be determined. + .DESCRIPTION + A cheap, read-only pre-flight probe used to warn when an append target is large + enough that the in-memory merge could strain RAM. Dispatches by tier: + * SharePoint : a driveItem metadata GET (the download URL WITHOUT the + ':/content' suffix) whose JSON carries a 'size' field. + * Fabric : a DFS HEAD to the same path Get-RemoteFile-OneLake reads, + returning Content-Length with no body. + * Local : (Get-Item).Length. + FAIL-OPEN: any error, unexpected response shape, or a not-found (404) target + returns $null so the caller falls through to normal first-run / merge handling + with no warning and no abort. A probe failure is never read as "huge" or + "missing" - it simply yields "size unknown". + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [ValidateSet('SharePoint', 'Fabric', 'Local')] [string] $Tier, + [string] $RelativeName, + [string] $Path, + [string] $ParentOverride + ) + try { + switch ($Tier) { + 'Local' { + if (-not $Path) { return $null } + return [int64](Get-Item -LiteralPath $Path -ErrorAction Stop).Length + } + 'SharePoint' { + if ($ParentOverride) { $resolved = Resolve-SharePointTarget -Url $ParentOverride } + else { + if (-not $script:SPResolved) { $script:SPResolved = Resolve-SharePointTarget -Url $script:RemoteOutputUrl } + $resolved = $script:SPResolved + } + $relName = [System.Uri]::EscapeDataString($RelativeName) + $relPath = if ($resolved.FolderPath) { "$($resolved.FolderPath)/$relName" } else { $relName } + # Metadata GET: the download URL WITHOUT ':/content' returns the driveItem JSON. + $metaUri = "https://graph.microsoft.com/v1.0/drives/$($resolved.DriveId)/root:/${relPath}" + $item = Invoke-MgGraphRequest -Method GET -Uri $metaUri -ErrorAction Stop + if ($null -ne $item -and $null -ne $item.size) { return [int64]$item.size } + return $null + } + 'Fabric' { + if ($ParentOverride) { $resolved = Resolve-FabricTarget -Url $ParentOverride } + else { + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + } + $relName = ($RelativeName -split '/' | ForEach-Object { [System.Uri]::EscapeDataString($_) }) -join '/' + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/$relName" } else { $relName } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem" + $resp = Invoke-FabricWebRequest -Uri $dfsPath -Method HEAD + $cl = $null + if ($resp -and $resp.Headers) { $cl = $resp.Headers['Content-Length'] } + if ($cl -is [array]) { $cl = $cl[0] } + if ($null -ne $cl -and "$cl" -ne '') { return [int64]$cl } + return $null + } + } + } + catch { + # Fail-open: size unknown -> caller proceeds normally, no warning, no abort. + return $null + } + return $null +} + +function Get-AppendTargetSizeWarning { + <# + .SYNOPSIS + Given an append-target size in bytes (or $null) and a threshold, return a + human-readable advisory warning string when the size exceeds the threshold, + otherwise $null. Pure (no side effects) so the decision is independently testable. + #> + [CmdletBinding()] + param( + [AllowNull()] [System.Nullable[int64]] $SizeBytes, + [Parameter(Mandatory)] [int64] $ThresholdBytes, + [string] $Label + ) + if ($null -eq $SizeBytes) { return $null } + if ($SizeBytes -le $ThresholdBytes) { return $null } + $gb = [math]::Round($SizeBytes / 1GB, 2) + $name = if ($Label) { " '$Label'" } else { '' } + return ( + "WARNING: The append target$name is $gb GB. The merge step reads the entire " + + "target into memory to deduplicate against it, which can require several times " + + "that size in RAM. Large downloads themselves are streamed safely, but a very " + + "large target can still exhaust memory during the merge. Consider archiving or " + + "rolling the target to a fresh file, or ensure the machine has enough memory. " + + "This is an advisory only - the run will continue." + ) +} + +function script:Test-AuditPartitionConsistency { + <# + .SYNOPSIS + Bounded, deterministic consistency check for a REPEATED fetch of a single 'successful' audit + partition. Compares two RecordId samples (or counts) of the SAME partition and classifies the + repeat as 'Stable' (identical set) or 'Unstable' (any difference). It NEVER picks the larger + sample and NEVER unions the two - it only DETECTS/CLASSIFIES so the caller can refuse to + silently certify an unstable partition (data-safe reconciliation). Observed live: SharePoint + partition 26 returned 13,350 vs 15,465 (2,115 RecordIds silently lost, a strict subset). + .OUTPUTS + [pscustomobject] Consistent(bool), Verdict('Stable'|'Unstable'), FirstCount, RepeatCount, + MissingFromRepeat, ExtraInRepeat, Basis('Set'|'Count'). + #> + [CmdletBinding()] + param([AllowNull()] $FirstRecordIds, [AllowNull()] $RepeatRecordIds, [AllowNull()] $FirstCount = $null, [AllowNull()] $RepeatCount = $null) + $haveSets = ($null -ne $FirstRecordIds) -and ($null -ne $RepeatRecordIds) + if ($haveSets) { + $first = New-Object 'System.Collections.Generic.HashSet[string]' ([StringComparer]::Ordinal) + foreach ($id in @($FirstRecordIds)) { if ($null -ne $id) { [void]$first.Add([string]$id) } } + $repeat = New-Object 'System.Collections.Generic.HashSet[string]' ([StringComparer]::Ordinal) + foreach ($id in @($RepeatRecordIds)) { if ($null -ne $id) { [void]$repeat.Add([string]$id) } } + $missing = 0; foreach ($id in $first) { if (-not $repeat.Contains($id)) { $missing++ } } + $extra = 0; foreach ($id in $repeat) { if (-not $first.Contains($id)) { $extra++ } } + $consistent = ($missing -eq 0 -and $extra -eq 0) + return [pscustomobject]@{ Consistent = $consistent; Verdict = $(if ($consistent) { 'Stable' } else { 'Unstable' }); FirstCount = $first.Count; RepeatCount = $repeat.Count; MissingFromRepeat = $missing; ExtraInRepeat = $extra; Basis = 'Set' } + } + $fc = if ($null -ne $FirstCount) { [int]$FirstCount } else { -1 } + $rc = if ($null -ne $RepeatCount) { [int]$RepeatCount } else { -1 } + $consistent = ($fc -ge 0 -and $rc -ge 0 -and $fc -eq $rc) + return [pscustomobject]@{ Consistent = $consistent; Verdict = $(if ($consistent) { 'Stable' } else { 'Unstable' }); FirstCount = $fc; RepeatCount = $rc; MissingFromRepeat = $(if ($fc -gt $rc) { $fc - $rc } else { 0 }); ExtraInRepeat = $(if ($rc -gt $fc) { $rc - $fc } else { 0 }); Basis = 'Count' } +} + +function script:Get-PartitionStabilityRecordIds { + <# + .SYNOPSIS + FIRST-pass RecordId source for -VerifyPartitionStability's bounded repeat check. Materializes + the partition's original-fetch RecordIds from the run's persisted incremental JSONL (a disk + read of what the first fetch already wrote - NOT a new service query). The REPEAT pass is NO + LONGER a re-page of the partition's (now-DELETED) Graph audit QueryId; it is an INDEPENDENT + create->poll->page->delete query for the same window handled by + Invoke-PartitionStabilityRepeatQuery. Returns a [string[]] of RecordIds (identity resolved via + the codebase's RecordId/Identity/Id fallback), never the raw records, so no full set is kept. + .OUTPUTS + [string[]] RecordIds (possibly empty). + #> + [CmdletBinding()] + param([Parameter(Mandatory)] $PartitionStatus, [bool] $IsRepeat) + $resolveId = { param($r) if ($null -eq $r) { return $null }; if ($r.RecordId) { return [string]$r.RecordId }; if ($r.Identity) { return [string]$r.Identity }; if ($r.Id) { return [string]$r.Id }; return $null } + $ids = [System.Collections.Generic.List[string]]::new() + # This helper is FIRST-pass only. The independent repeat query lives in + # Invoke-PartitionStabilityRepeatQuery; $IsRepeat is retained only for the injectable -Fetcher + # shape and, if ever passed, still returns the persisted first-fetch set (never a QueryId re-page). + # FIRST pass: read the partition's persisted first-fetch JSONL (the data the first fetch wrote). + try { + if ($script:PartialOutputPath) { + $incDir = Join-Path (Split-Path $script:PartialOutputPath -Parent) '.pax_incremental' + if (Test-Path -LiteralPath $incDir) { + $pat = "Part$($PartitionStatus.Partition.Index)_$($global:ScriptRunTimestamp)_*.jsonl" + foreach ($f in @(Get-ChildItem -LiteralPath $incDir -Filter $pat -File -ErrorAction SilentlyContinue)) { + foreach ($line in [System.IO.File]::ReadLines($f.FullName)) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + try { $rec = $line | ConvertFrom-Json -ErrorAction Stop; $v = & $resolveId $rec; if ($v) { [void]$ids.Add($v) } } catch {} + } + } + } + } + } catch {} + return ,([string[]]$ids.ToArray()) +} + +function script:Invoke-PartitionStabilityRepeatQuery { + <# + .SYNOPSIS + REPEAT-pass executor for -VerifyPartitionStability. Runs ONE bounded, INDEPENDENT audit query + for a completed partition's EXACT window + operation/recordType/service filters (a NEW QueryId, + never a re-page of the partition's already-DELETED first-fetch QueryId) so it can detect the + CROSS-QUERY under-delivery a re-page can never see (observed live: a fresh query for the same + window returned a very different count). Flow: create -> bounded poll to 'succeeded' -> page the + complete record set -> best-effort DELETE the repeat query. Any create / poll (failed|cancelled| + timeout) / paging failure yields Outcome='Failed' (the caller treats that as Unstable/Unknown -> + exit 40); cleanup is housekeeping and cannot invalidate a completed comparison. Returns only compact + RecordId strings (identity resolved via the RecordId/Identity/Id fallback), never raw records. + .OUTPUTS + [pscustomobject] RecordIds([string[]]), Outcome('Complete'|'Failed'), FailureStage, + RepeatQueryId, CleanupSucceeded. + .PARAMETER CreateOp + Injectable create op ($DisplayName,$StartDate,$EndDate,$Operations,$RecordTypes,$ServiceTypes) + -> new queryId or $null. Defaults to the real Invoke-GraphAuditQuery (main-thread Graph wrapper). + .PARAMETER StatusOp + Injectable status op ($QueryId) -> @{Status;RecordCount}. Defaults to Get-GraphAuditQueryStatus. + .PARAMETER RecordsOp + Injectable records op ($QueryId) -> record objects. Defaults to Get-GraphAuditRecords. + .PARAMETER DeleteOp + Injectable delete op ($QueryId) -> void. Defaults to the real Graph queries/$id DELETE. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] $PartitionStatus, + [AllowNull()][string[]] $TargetUsers = $null, + [scriptblock] $CreateOp = $null, + [scriptblock] $StatusOp = $null, + [scriptblock] $RecordsOp = $null, + [scriptblock] $DeleteOp = $null, + [int] $MaxPollAttempts = 960, + [int] $PollIntervalSeconds = 15 + ) + if ($null -eq $CreateOp) { $CreateOp = { param($dn, $sd, $ed, $ops, $rt, $svc) Invoke-GraphAuditQuery -DisplayName $dn -StartDate $sd -EndDate $ed -Operations $ops -RecordTypes $rt -ServiceTypes $svc } } + if ($null -eq $StatusOp) { $StatusOp = { param($qid) Get-GraphAuditQueryStatus -QueryId $qid } } + if ($null -eq $RecordsOp) { $RecordsOp = { param($qid) Get-GraphAuditRecords -QueryId $qid } } + if ($null -eq $DeleteOp) { $DeleteOp = { param($qid) Invoke-MgGraphRequest -Method DELETE -Uri (Get-GraphAuditApiUri -Path "queries/$qid") -ErrorAction Stop | Out-Null } } + $resolveId = { param($r) if ($null -eq $r) { return $null }; if ($r.RecordId) { return [string]$r.RecordId }; if ($r.Identity) { return [string]$r.Identity }; if ($r.Id) { return [string]$r.Id }; return $null } + $fail = { param($stage, $qid) [pscustomobject]@{ RecordIds = [string[]]@(); Outcome = 'Failed'; FailureStage = $stage; RepeatQueryId = $qid; CleanupSucceeded = $false } } + $pt = $PartitionStatus.Partition + if ($null -eq $pt) { return (& $fail 'Create' $null) } + # Build the repeat create payload from the partition's OWN window + filters (exact parity, so a + # test can capture and assert the create args equal the original partition's window/filters). + $ops = if ($pt.Activities) { @($pt.Activities) } elseif ($pt.Activity) { @($pt.Activity) } else { @() } + $svc = $null; if ($pt.ServiceFilter) { $svc = @($pt.ServiceFilter) } + $rt = $null; if ($pt.RecordTypes) { $rt = @($pt.RecordTypes) } + $dn = "PAX_StabilityRepeat_P$($pt.Index)of$($pt.Total)_$([DateTime]::UtcNow.ToString('yyyyMMddHHmmss'))" + # CREATE an independent NEW query (never reuse / re-page $PartitionStatus.QueryId). + $repeatId = $null + try { $repeatId = & $CreateOp $dn $pt.PStart $pt.PEnd $ops $rt $svc } catch { return (& $fail 'Create' $null) } + if (-not $repeatId) { return (& $fail 'Create' $null) } + # POLL to 'succeeded' - bounded (finite attempts, NOT unbounded). + $succeeded = $false + for ($a = 0; $a -lt $MaxPollAttempts; $a++) { + $st = $null + try { $st = & $StatusOp $repeatId } catch { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Poll' $repeatId) } + $state = if ($st) { [string]$st.Status } else { $null } + if ($state -eq 'succeeded') { $succeeded = $true; break } + if ($state -in @('failed', 'cancelled')) { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Poll' $repeatId) } + if ($PollIntervalSeconds -gt 0) { Start-Sleep -Seconds $PollIntervalSeconds } + } + if (-not $succeeded) { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Timeout' $repeatId) } + # PAGE the complete record set, resolving compact RecordId strings (never retain raw records). + $ids = [System.Collections.Generic.List[string]]::new() + try { foreach ($r in @(& $RecordsOp $repeatId)) { if ($TargetUsers -and $TargetUsers.Count -gt 0) { $__upn = if ($null -ne $r) { [string]$r.userPrincipalName } else { $null }; if (-not ($TargetUsers -contains $__upn)) { continue } } $v = & $resolveId $r; if ($v) { [void]$ids.Add($v) } } } + catch { try { & $DeleteOp $repeatId } catch {}; return (& $fail 'Page' $repeatId) } + # DELETE is best-effort housekeeping after retrieval has completed. A delete failure does not + # change the retrieved RecordId set or invalidate the comparison. + $deleteOk = $true + try { & $DeleteOp $repeatId } catch { $deleteOk = $false } + return [pscustomobject]@{ RecordIds = [string[]]$ids.ToArray(); Outcome = 'Complete'; FailureStage = $null; RepeatQueryId = $repeatId; CleanupSucceeded = $deleteOk } +} + +function script:Invoke-PartitionStabilityVerification { + <# + .SYNOPSIS + Bounded, opt-in repeat-fetch stability verification for completed audit partitions + (-VerifyPartitionStability). For each 'Complete' partition it captures the first-fetch + RecordId set (.RecordIds) and runs EXACTLY ONE independent repeat fetch of the same partition + window (.StabilityRepeatIds), then classifies the repeat via Test-AuditPartitionConsistency. + Any 'Unstable' verdict (any RecordId-set difference - missing OR extra) is treated as a gap: + $script:HadTerminalFailures is set (exit 40) - never silently certified, never blind-accept + -larger, never unioned, never count-only (Set basis). Memory is bounded: RecordId sets are + materialized per-partition inside the loop and ONLY when enabled - default OFF performs no + repeat fetch and retains nothing (byte-identical to a run without the switch). + .PARAMETER Fetcher + Injectable FIRST-pass RecordId source scriptblock ($PartitionStatus, [bool]$IsRepeat) -> + [string[]] of RecordIds. Defaults to Get-PartitionStabilityRecordIds (persisted-JSONL first + pass only). Injected by tests to model the first-fetch persisted set. + .PARAMETER RepeatExecutor + Injectable REPEAT-pass executor scriptblock ($PartitionStatus) -> { RecordIds=[string[]]; + Outcome='Complete'|'Failed'; FailureStage; RepeatQueryId }. Defaults to + Invoke-PartitionStabilityRepeatQuery (one independent create->poll->page->delete query for the + SAME window/filters, a NEW QueryId). Injected by tests to model the repeat query without a + live tenant. + #> + [CmdletBinding()] + param( + [AllowNull()] $CompletedPartitions, + [AllowNull()][string[]] $TargetUsers = $null, + [scriptblock] $Fetcher = $null, + [scriptblock] $RepeatExecutor = $null, + [switch] $Enabled + ) + # Default OFF: no repeat query, no RecordId retention -> byte-identical to a run without the switch. + if (-not $Enabled) { return } + if ($null -eq $CompletedPartitions) { return } + if ($null -eq $Fetcher) { $Fetcher = ${function:script:Get-PartitionStabilityRecordIds} } + if ($null -eq $RepeatExecutor) { $RepeatExecutor = ${function:script:Invoke-PartitionStabilityRepeatQuery} } + foreach ($__ps in @($CompletedPartitions)) { + if ($null -eq $__ps) { continue } + $__pt = $__ps.Partition + # FIRST set: original-fetch RecordIds (a disk read of the first fetch's persisted output, not a + # new service query). Retained only now (switch ON) and only one partition at a time (bounded). + $__ps.RecordIds = @(& $Fetcher $__ps $false) + # REPEAT set: exactly ONE bounded INDEPENDENT query (a NEW QueryId) for the SAME window/filters + # via create->poll->page->delete. This detects the CROSS-QUERY under-delivery that a re-page of + # the (already-deleted) first-fetch QueryId never could. + $__rep = & $RepeatExecutor $__ps $TargetUsers + if ($null -eq $__rep -or $__rep.Outcome -ne 'Complete') { + # Any create/poll/page/cleanup failure => Unknown: never silently certified (gap -> exit 40). + $script:HadTerminalFailures = $true + $__ps.StabilityVerdict = 'Unknown' + $__ps.StabilityFirstCount = @($__ps.RecordIds).Count + $__ps.StabilityRepeatCount = -1 + $__ps.StabilityMissing = -1 + $__ps.StabilityExtra = -1 + $__stage = if ($__rep) { [string]$__rep.FailureStage } else { 'Executor' } + Write-LogHost (" [STABILITY] Partition {0}/{1} NOT CERTIFIED - independent repeat query failed at stage '{2}'; treated as a gap (exit 40)." -f $__pt.Index, $__pt.Total, $__stage) -ForegroundColor Red + # Bounded memory: drop RecordId arrays immediately; keep only compact verdict metadata. + $__ps.RecordIds = $null + $__ps.StabilityRepeatIds = $null + continue + } + $__ps.StabilityRepeatIds = @($__rep.RecordIds) + # Classify via the pure detector (Set basis: never blind-larger, never union, never count-only). + $__cons = script:Test-AuditPartitionConsistency -FirstRecordIds $__ps.RecordIds -RepeatRecordIds $__ps.StabilityRepeatIds + # Retain ONLY compact verdict/count metadata (no RecordId arrays held across partitions). + $__ps.StabilityVerdict = $__cons.Verdict + $__ps.StabilityFirstCount = $__cons.FirstCount + $__ps.StabilityRepeatCount = $__cons.RepeatCount + $__ps.StabilityMissing = $__cons.MissingFromRepeat + $__ps.StabilityExtra = $__cons.ExtraInRepeat + if (-not $__cons.Consistent) { + $script:HadTerminalFailures = $true + Write-LogHost (" [STABILITY] Partition {0}/{1} UNSTABLE on independent repeat verification: {2} RecordId(s) missing, {3} extra (first={4}, repeat={5}); NOT silently certified (gap -> exit 40)." -f $__pt.Index, $__pt.Total, $__cons.MissingFromRepeat, $__cons.ExtraInRepeat, $__cons.FirstCount, $__cons.RepeatCount) -ForegroundColor Red + } + # Bounded memory: drop RecordId arrays immediately after the verdict; keep only compact metadata. + $__ps.RecordIds = $null + $__ps.StabilityRepeatIds = $null + } +} + +function Get-DataTypeForOutputFile { + # Map a customer-visible artifact basename to its data-type key. Used by the + # upload sweep to route each file to its per-data-type destination (-OutputPath*). + # Returns one of: 'Purview' | 'UserInfo' | 'Agent365Info' | 'DefenderUsage' | 'Log'. + # The twelve fixed-name AISID dashboard leaves (ai_activity_sessions.csv, + # ai_offhours_geo.csv, ai_file_proximity.csv, ai_oauth_consents.csv, + # ai_sso_signins.csv, ai_client_channel.csv, ai_copilot_usage_graph.csv, + # ai_solutions_catalog.csv, EntraUsers.csv) route to 'DefenderUsage' — matched + # against the authoritative $script:AISIDTableSchemas fixed-name set BEFORE the + # UserInfo pattern so the AISID EntraUsers.csv projection can never be misrouted + # to UserInfo (the timestamped '_Users.csv' stream) or fall through to + # Purview. The internal '.aisid_cache' delta-cache directory (and any file under + # it) is a working area, never a dashboard artifact, so it is excluded from + # DefenderUsage and treated as a non-dashboard Purview fallback. + # Unknown / unmatched names fall back to 'Purview' (primary audit destination). + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $FileName) + $n = [System.IO.Path]::GetFileName($FileName) + # The '.aisid_cache' delta-cache is internal scratch, never a dashboard artifact — + # exclude any path segment named '.aisid_cache' from every dashboard classification. + if (($FileName -replace '\\', '/') -match '(^|/)\.aisid_cache(/|$)') { return 'Purview' } + # AISID dashboard leaves -> DefenderUsage (authoritative fixed-name set kept in + # sync with $script:AISIDTableSchemas), checked BEFORE UserInfo so EntraUsers.csv + # routes to DefenderUsage, never UserInfo/Purview. + $aisidLeaves = @() + if ($script:AISIDTableSchemas) { $aisidLeaves = @($script:AISIDTableSchemas.Values | ForEach-Object { $_.FileName }) } + if ($aisidLeaves -notcontains 'EntraUsers.csv') { $aisidLeaves += 'EntraUsers.csv' } + foreach ($aisidLeaf in $aisidLeaves) { if ($aisidLeaf -and $n -ieq $aisidLeaf) { return 'DefenderUsage' } } + if ($n -like 'EntraUsers_*' -or $n -like '*_Users.csv' -or $n -like '*_Users_*.csv') { return 'UserInfo' } + if ($n -like 'Agent365_*' -or $n -like 'Agent365.csv') { return 'Agent365Info' } + if ($n -like '*.log' -or $n -like '*.partial.log') { return 'Log' } + return 'Purview' +} + +function Invoke-OutputUpload { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $LocalPath, + # Optional per-data-type parent URL. When supplied, the upload targets this + # URL instead of the script-wide Purview destination. Used to honor + # -OutputPathUserInfo / -OutputPathAgent365Info / -OutputPathLog routing. + [string] $ParentOverride + ) + if ($script:RemoteOutputMode -eq 'None') { return } + if (-not (Test-Path -LiteralPath $LocalPath)) { + Write-Verbose "Invoke-OutputUpload: skipping (file not found): $LocalPath" + return + } + # NOTE: No catch wrapper here — every caller either (a) has its own try/catch with a + # context-specific WARNING (upload sweep / metrics / log file) or (b) wants silent + # best-effort behavior with Write-Verbose only (checkpoint mirror). An inner WARNING + # would always be either redundant or unwanted. Let exceptions propagate untouched. + switch ($script:RemoteOutputMode) { + 'SharePoint' { Send-FileToSharePoint -LocalPath $LocalPath -ParentOverride $ParentOverride } + 'Fabric' { Send-FileToOneLake -LocalPath $LocalPath -ParentOverride $ParentOverride } + } +} + +function Invoke-EmbeddedProcessor { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [ValidateSet('CopilotInteraction', 'M365Bundle')] + [string] $ProcessorMode, + + [Parameter(Mandatory)] [string] $PythonExe, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $LauncherArgs, + [Parameter(Mandatory)] [string[]] $ProcessorArgs, + [Parameter(Mandatory)] [string] $IncrementalDir + ) + + # Materialize the appropriate embedded source as a temp .py inside .pax_incremental, + # named with the global run timestamp so it is auto-reaped by the end-of-run cleanup + # AND the finally-block safety-net even if we crash. Returns the processor's exit code. + + if (-not (Test-Path -LiteralPath $IncrementalDir -PathType Container)) { + New-Item -ItemType Directory -Path $IncrementalDir -Force | Out-Null + } + + switch ($ProcessorMode) { + 'CopilotInteraction' { + $label = 'CopilotInteractionProcessor' + $source = $Script:EMBEDDED_PROCESSOR_COPILOT + $ver = $Script:EMBEDDED_PROCESSOR_COPILOT_VERSION + } + 'M365Bundle' { + $label = 'M365BundleProcessor' + $source = $Script:EMBEDDED_PROCESSOR_M365 + $ver = $Script:EMBEDDED_PROCESSOR_M365_VERSION + } + } + + $tempPyName = "PAX_${label}_$($global:ScriptRunTimestamp).py" + $tempPyPath = Join-Path $IncrementalDir $tempPyName + + $utf8NoBom = [Text.UTF8Encoding]::new($false) + [IO.File]::WriteAllText($tempPyPath, $source, $utf8NoBom) + + Write-LogHost "Rollup: invoking embedded $label v$ver ($PythonExe)" -ForegroundColor Cyan + Write-LogFile "Rollup: temp script -> $tempPyPath" + Write-LogFile "Rollup: args -> $($ProcessorArgs -join ' ')" + # Additive memory/stage observation at the processor boundary (never fatal). + try { Write-PaxMemoryObservation -Stage ("EmbeddedProcessor:" + $label) -TempDir $IncrementalDir -Force } catch {} + + $exitCode = 1 + # Force UTF-8 across the Python subprocess boundary so Unicode glyphs the + # processor emits to stdout / stderr (right-arrow, section separators, + # bullets) are not mojibaked when captured through PowerShell on hosts + # whose console output encoding defaults to a legacy OEM code page. + # Both sides matter: + # * [Console]::OutputEncoding controls how PowerShell decodes the bytes + # it reads back from the subprocess. + # * $env:PYTHONIOENCODING tells the Python interpreter what to encode + # its stdout / stderr text streams as. + # Outer values are captured here and restored in the finally block so the + # host's session-wide encoding is not perturbed. + $prevConsoleOutEnc = [Console]::OutputEncoding + $prevPyIoEnc = $env:PYTHONIOENCODING + try { + [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false) + $env:PYTHONIOENCODING = 'utf-8' + $fullArgs = @($LauncherArgs) + @($tempPyPath) + @($ProcessorArgs) + # In remote-output mode, suppress the Python processors' echoed input/output PATH + # lines (e.g. "Purview input:", "Output file:"). Those values are LOCAL scratch + # paths; surfacing them to the customer when a remote -OutputPath* is + # in effect is misleading. The remote-aware path summary is emitted by the + # PowerShell wrapper after the processor returns. All other Python output + # (header, counts, timing, errors) is preserved verbatim. + $pathLineRegex = if ($script:RemoteOutputMode -ne 'None') { + '^\s*(Purview input:|Entra input:|Purview output:|Entra output:|Input:|Output:|Output file:)\s' + } else { $null } + & $PythonExe @fullArgs 2>&1 | ForEach-Object { + $line = [string]$_ + if ($pathLineRegex -and $line -match $pathLineRegex) { return } + # Stream stdout+stderr through Write-Log (which mirrors to host AND log file). + Write-Log $line + } + $exitCode = $LASTEXITCODE + } + catch { + Write-LogHost "Rollup: $label threw an exception: $($_.Exception.Message)" -ForegroundColor Red + $exitCode = 1 + } + finally { + # Restore the outer console output encoding and PYTHONIOENCODING regardless + # of outcome so subsequent host writes (and any later subprocess) see the + # same environment they would have without this call. + try { [Console]::OutputEncoding = $prevConsoleOutEnc } catch { } + if ($null -eq $prevPyIoEnc) { + Remove-Item Env:PYTHONIOENCODING -ErrorAction SilentlyContinue + } else { + $env:PYTHONIOENCODING = $prevPyIoEnc + } + try { + if (Test-Path -LiteralPath $tempPyPath) { Remove-Item -LiteralPath $tempPyPath -Force -ErrorAction SilentlyContinue } + } + catch { } + } + + return $exitCode +} + +# Validate MaxConcurrency range (Microsoft Purview enforces 10 concurrent search job limit per user account) +# ============================================================================ +# APPENDFILE COLUMN VALIDATION FUNCTIONS +# ============================================================================ +# Early validation to prevent wasting time on Graph API queries when explosion +# parameters don't match between existing file and new data parameters. +# Note: For explosion modes, actual column schemas are dynamic and vary by data, +# so we validate explosion parameter compatibility rather than exact columns. +# ============================================================================ + +function Get-LikelyExplosionParams { + param([string[]]$Columns) + + # Check for deep explosion indicators (CopilotEventData.* columns) + $hasDeepColumns = $Columns | Where-Object { $_ -match '^CopilotEventData\.' } + if ($hasDeepColumns) { + return @{ Mode = "ExplodeDeep"; DisplayName = "-ExplodeDeep" } + } + + # Check for array explosion indicators (exploded field names like Message_, Context_, AgentId, etc.) + $hasArrayColumns = $Columns | Where-Object { $_ -match '^(Message_|Context_|Interaction_|AgentId|AgentName|AgentVersion|AccessedResource_|AISystemPlugin_)' } + if ($hasArrayColumns) { + return @{ Mode = "ExplodeArrays"; DisplayName = "-ExplodeArrays" } + } + + # Check for standard mode indicators (AuditData JSON column present) + $hasAuditData = $Columns -contains 'AuditData' + if ($hasAuditData) { + return @{ Mode = "Standard"; DisplayName = "Standard (no explosion)" } + } + + # Unable to determine + return @{ Mode = "Unknown"; DisplayName = "Unknown mode" } +} + +function Get-AppendFileRecordIds { + <# + .SYNOPSIS + Stream the RecordId column from an existing AppendFile (CSV or XLSX) into a HashSet. + + .DESCRIPTION + Cross-run dedup helper for -AppendFile. Reads only the RecordId column from the + existing target file and returns a HashSet[string] of identifiers. Used to seed + Merge-IncrementalSaves-Streaming's -ExcludeRecordIds parameter so this run skips + records already present in the target. + + CSV path: streams via StreamReader + manual header parse — never loads the entire + file into memory (the target may be hundreds of MB). + Excel path: uses Import-Excel (RecordId column only). The Excel branch is bounded + by ImportExcel's own behavior; very large workbooks should use CSV. + + .PARAMETER Path + Path to the AppendFile. Missing file returns an empty set (first-run semantic). + + .OUTPUTS + HashSet[string] of RecordIds (empty when the file or column is absent). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $Path + ) + $set = New-Object System.Collections.Generic.HashSet[string] + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return ,$set } + + $ext = [System.IO.Path]::GetExtension($Path).ToLowerInvariant() + if ($ext -eq '.xlsx') { + try { + if (-not (Get-Module -Name ImportExcel -ListAvailable)) { return ,$set } + Import-Module ImportExcel -ErrorAction Stop + $rows = Import-Excel -Path $Path -ErrorAction Stop + foreach ($r in $rows) { + $rid = $r.RecordId + if ($rid -and -not [string]::IsNullOrWhiteSpace([string]$rid)) { + [void]$set.Add([string]$rid) + } + } + } catch { + Write-LogHost "Get-AppendFileRecordIds: failed to read '$Path': $($_.Exception.Message). Cross-run dedup disabled for this file." -ForegroundColor Yellow + } + return ,$set + } + + # CSV path — manual streaming for memory safety on large files. + try { + $reader = [System.IO.StreamReader]::new($Path, [System.Text.Encoding]::UTF8) + try { + $header = $reader.ReadLine() + if (-not $header) { return ,$set } + $cols = $header.Split(',') | ForEach-Object { $_.Trim('"') } + $ridIdx = -1 + for ($i = 0; $i -lt $cols.Count; $i++) { + if ($cols[$i] -ieq 'RecordId') { $ridIdx = $i; break } + } + if ($ridIdx -lt 0) { return ,$set } + $line = $null + while ($null -ne ($line = $reader.ReadLine())) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + # Simple field split; AppendFiles use Export-Csv quoting so RecordId (a GUID) + # never contains commas or quotes. Fall back to ConvertFrom-Csv for the rare + # row that has embedded commas before the RecordId column. + $fields = $line.Split(',') + if ($fields.Count -gt $ridIdx) { + $raw = $fields[$ridIdx].Trim('"').Trim() + if ($raw) { [void]$set.Add($raw) } + } + } + } finally { + $reader.Dispose() + } + } catch { + Write-LogHost "Get-AppendFileRecordIds: failed to read '$Path': $($_.Exception.Message). Cross-run dedup disabled for this file." -ForegroundColor Yellow + } + return ,$set +} + +function Test-AppendFileCompatibility { + param( + [string]$FilePath, + [bool]$IsExcel, + [bool]$ExplodeArrays, + [bool]$ExplodeDeep, + [string]$TargetSheet = $null + ) + + $result = @{ + Compatible = $true + ExistingMode = $null + CurrentMode = $null + ExistingColumns = @() + ExistingCount = 0 + ErrorMessage = $null + } + + try { + # Determine current explosion mode + if ($ExplodeDeep) { + $result.CurrentMode = @{ Mode = "ExplodeDeep"; DisplayName = "-ExplodeDeep" } + } + elseif ($ExplodeArrays) { + $result.CurrentMode = @{ Mode = "ExplodeArrays"; DisplayName = "-ExplodeArrays" } + } + else { + $result.CurrentMode = @{ Mode = "Standard"; DisplayName = "Standard (no explosion)" } + } + + # Read existing file columns + if ($IsExcel) { + # Validate Excel file and read columns + if (-not (Get-Module -Name ImportExcel -ListAvailable)) { + $result.ErrorMessage = "ImportExcel module not available for validation" + $result.Compatible = $false + return $result + } + + Import-Module ImportExcel -ErrorAction Stop + + # Get sheet info + $sheets = Get-ExcelSheetInfo -Path $FilePath -ErrorAction Stop + + if ($TargetSheet) { + # Validate specific sheet + $sheet = $sheets | Where-Object { $_.Name -eq $TargetSheet } + if (-not $sheet) { + $result.ErrorMessage = "Target sheet '$TargetSheet' not found in workbook" + $result.Compatible = $false + return $result + } + } + else { + # Use first sheet + $sheet = $sheets | Select-Object -First 1 + } + + # Read header row from Excel + $headerData = Import-Excel -Path $FilePath -WorksheetName $sheet.Name -StartRow 1 -EndRow 1 -NoHeader -ErrorAction Stop + $existingCols = $headerData[0].PSObject.Properties.Value | Where-Object { $_ } + } + else { + # CSV: Read first line (header) + $firstLine = Get-Content -Path $FilePath -First 1 -Encoding UTF8 -ErrorAction Stop + $existingCols = ($firstLine -split ',') | ForEach-Object { $_.Trim('"') } + } + + $result.ExistingColumns = $existingCols + $result.ExistingCount = $existingCols.Count + + # Detect explosion mode of existing file + $result.ExistingMode = Get-LikelyExplosionParams -Columns $existingCols + + # Check if explosion modes match + if ($result.ExistingMode.Mode -ne $result.CurrentMode.Mode) { + $result.Compatible = $false + $result.ErrorMessage = "Explosion parameter mismatch: existing file is '$($result.ExistingMode.DisplayName)' but current command uses '$($result.CurrentMode.DisplayName)'" + } + else { + # Modes match - compatible + # Note: We don't validate exact columns because explosion schemas are dynamic + # and vary based on actual data content. As long as explosion params match, + # the append will work correctly. + $result.Compatible = $true + } + } + catch { + $result.ErrorMessage = $_.Exception.Message + $result.Compatible = $false + } + + return $result +} + +# ============================================================================ +# END APPENDFILE VALIDATION FUNCTIONS +# ============================================================================ + +if ($MaxConcurrency -lt 1 -or $MaxConcurrency -gt 10) { + Write-Host "ERROR: -MaxConcurrency must be between 1 and 10." -ForegroundColor Red + Write-Host "Microsoft Purview enforces a maximum of 10 concurrent search jobs per user account." -ForegroundColor Yellow + Write-Host "Current value: $MaxConcurrency" -ForegroundColor Yellow + Write-Host "Please specify a value between 1 and 10 and re-run." -ForegroundColor Yellow + exit 1 +} + +# Establish date defaults / validation depending on mode. +if ($RAWInputCSV) { + $parsedStart = $null; $parsedEnd = $null + if ($PSBoundParameters.ContainsKey('StartDate')) { + try { $parsedStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } catch { Write-Host "ERROR: StartDate must be yyyy-MM-dd if provided." -ForegroundColor Red; exit 1 } + } + if ($PSBoundParameters.ContainsKey('EndDate')) { + try { $parsedEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } catch { Write-Host "ERROR: EndDate must be yyyy-MM-dd if provided." -ForegroundColor Red; exit 1 } + } + if ($parsedStart -and $parsedEnd -and $parsedEnd -lt $parsedStart) { Write-Host "ERROR: EndDate ($EndDate) is earlier than StartDate ($StartDate)." -ForegroundColor Red; exit 1 } + if (-not $PSBoundParameters.ContainsKey('StartDate')) { $StartDate = '*' } + if (-not $PSBoundParameters.ContainsKey('EndDate')) { $EndDate = '*' } +} +else { + if (-not $PSBoundParameters.ContainsKey('StartDate') -and -not $PSBoundParameters.ContainsKey('EndDate')) { + $yesterdayUtc = (Get-Date).ToUniversalTime().Date.AddDays(-1) + $StartDate = $yesterdayUtc.ToString('yyyy-MM-dd') + $EndDate = $yesterdayUtc.AddDays(1).ToString('yyyy-MM-dd') + } + elseif (-not $PSBoundParameters.ContainsKey('StartDate')) { + $StartDate = '*' + try { + $parsedEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + } catch { Write-Host "ERROR: EndDate must be yyyy-MM-dd format." -ForegroundColor Red; exit 1 } + } + elseif (-not $PSBoundParameters.ContainsKey('EndDate')) { + $EndDate = '*' + try { + $parsedStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + } catch { Write-Host "ERROR: StartDate must be yyyy-MM-dd format." -ForegroundColor Red; exit 1 } + } + else { + try { + $parsedStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + $parsedEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) + } + catch { Write-Host "ERROR: StartDate/EndDate must be in yyyy-MM-dd format." -ForegroundColor Red; exit 1 } + if ($parsedEnd -lt $parsedStart) { Write-Host "ERROR: EndDate ($EndDate) is earlier than StartDate ($StartDate)." -ForegroundColor Red; exit 1 } + } +} + +# Client-side date-range trim boundaries — Purview's partition-based indexing can +# return records outside the requested date range (observed up to ~10 h past EndDate). +# These UTC boundaries are used after dedup to trim any out-of-range records. +# SpecifyKind(Utc) is critical: ParseExact returns Kind=Unspecified, and .ToUniversalTime() +# on Unspecified assumes LOCAL time, shifting the boundary by the machine's UTC offset. +$script:TrimStartDateUTC = if ($StartDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } +$script:TrimEndDateUTC = if ($EndDate -ne '*') { [datetime]::SpecifyKind([datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture), [System.DateTimeKind]::Utc) } else { $null } +$script:DateTrimCount = 0 + +# ============================================================ +# AISID date-window contract - fresh runs only +# ============================================================ +# -Dashboard AISID requires TWO concrete date boundaries after normal defaulting: both dates omitted +# -> the [yesterday..today) UTC default (both concrete) is accepted; both supplied -> the existing +# format/order validation applies; exactly one supplied -> the other is the '*' sentinel (unbounded) +# here, which is INVALID for AISID. Reject the one-sided shape now - before authentication, checkpoint +# creation, or any customer output - with a clear both-or-neither message and no prompt. Resume runs +# restore the original run's concrete dates from the checkpoint, so this fresh-run gate is skipped +# under -Resume. Non-AISID one-sided / unbounded runs are unaffected. +if ($Dashboard -eq 'AISID' -and $null -eq $Resume) { + if ($StartDate -eq '*' -or $EndDate -eq '*' -or $null -eq $script:TrimStartDateUTC -or $null -eq $script:TrimEndDateUTC) { + $missingBoundary = if ($StartDate -eq '*' -and $EndDate -eq '*') { 'both StartDate and EndDate' } elseif ($StartDate -eq '*') { 'StartDate' } else { 'EndDate' } + Write-LogHost "ERROR: -Dashboard AISID requires BOTH -StartDate and -EndDate to be concrete dates (missing: $missingBoundary)." -ForegroundColor Red + Write-LogHost " AISID intersects a bounded requested window with the 30-day retention window, so a one-sided (unbounded) range is not valid." -ForegroundColor Yellow + Write-LogHost " Supply BOTH -StartDate and -EndDate (yyyy-MM-dd), or omit BOTH to use the default [yesterday..today) UTC window." -ForegroundColor Yellow + Write-LogHost " No customer data files were written and no service connection or query was started." -ForegroundColor Yellow + if ($script:LogFile) { Write-LogHost (" Diagnostic log preserved at: {0}" -f $script:LogFile) -ForegroundColor Yellow } + exit 1 + } + # Freeze ONE reference instant for the whole run and resolve the requested-vs-available window once. + $script:AISIDReferenceNowUtc = (Get-Date).ToUniversalTime() + $script:AISIDWindow = Resolve-AISIDDateWindow -RequestedStartUtc $script:TrimStartDateUTC -RequestedEndUtc $script:TrimEndDateUTC -ReferenceNowUtc $script:AISIDReferenceNowUtc -RetentionDays (Get-AISIDEffectiveLookbackDays) + $fmtW = 'yyyy-MM-ddTHH:mm:ssZ' + $reqW = ('[{0} .. {1})' -f $script:AISIDWindow.RequestedStartUtc.ToString($fmtW), $script:AISIDWindow.RequestedEndUtc.ToString($fmtW)) + $availW = ('[{0} .. {1})' -f $script:AISIDWindow.AvailableStartUtc.ToString($fmtW), $script:AISIDWindow.AvailableEndUtc.ToString($fmtW)) + $effW = ('[{0} .. {1})' -f $script:AISIDWindow.EffectiveStartUtc.ToString($fmtW), $script:AISIDWindow.EffectiveEndUtc.ToString($fmtW)) + switch ($script:AISIDWindow.Status) { + 'NONE' { + # No overlap between the requested window and the 30-day retention window. Stop BEFORE any + # authentication, Purview/Entra/Defender query, licensing probe, checkpoint, rollup, or customer + # CSV. Controlled exit; this is a date-selection decision, not a Ctrl+C / interrupt. + Write-LogHost "AISID collection window: NONE - the requested date range does not overlap the available 30-day retention window; no collection was started." -ForegroundColor Red + Write-LogHost (" Requested window : {0}" -f $reqW) -ForegroundColor Yellow + Write-LogHost (" Available window : {0}" -f $availW) -ForegroundColor Yellow + Write-LogHost " Choose a -StartDate/-EndDate range that overlaps the last 30 days, then re-run." -ForegroundColor Yellow + Write-LogHost " No customer data files were written and no service connection or query was started." -ForegroundColor Yellow + if ($script:LogFile) { Write-LogHost (" Diagnostic log preserved at: {0}" -f $script:LogFile) -ForegroundColor Yellow } + exit 1 + } + 'PARTIAL' { + # The requested window extends beyond the 30-day retention window. Purview retains the FULL requested + # range; the AISID data sets use ONLY the requested-intersect-available intersection. One warning, no + # input, continue immediately. + Write-LogHost ("WARNING: AISID collection window PARTIAL - the requested range extends beyond the 30-day retention window; AISID uses only the intersection.") -ForegroundColor Yellow + Write-LogHost (" Requested window : {0}" -f $reqW) -ForegroundColor Yellow + Write-LogHost (" Available window : {0}" -f $availW) -ForegroundColor Yellow + Write-LogHost (" Effective window : {0}" -f $effW) -ForegroundColor Yellow + Write-LogHost (" Reason : {0}" -f $script:AISIDWindow.ClipReason) -ForegroundColor Yellow + Write-LogHost (" Purview retains the FULL requested range; only the AISID data sets use the intersection. No input is required; processing continues automatically.") -ForegroundColor Yellow + } + default { + # FULL - the requested window is entirely within retention. No clipping, no warning. + } + } +} + +if ($BlockHours -le 0) { Write-Host "ERROR: BlockHours must be positive." -ForegroundColor Red; exit 1 } + +try { if ($PSVersionTable.PSEdition -eq 'Core' -and ($global:InformationPreference -in @('SilentlyContinue', 'Ignore'))) { $global:InformationPreference = 'Continue' } } catch {} + +if ($RAWInputCSV) { + $rawConflictParams = @('BlockHours', 'ResultSize', 'PacingMs', 'Auth', 'ParallelMode', 'MaxParallelGroups', 'MaxConcurrency', 'EnableParallel', 'GroupNames') + $specifiedConflicts = @() + foreach ($cp in $rawConflictParams) { if ($PSBoundParameters.ContainsKey($cp)) { $specifiedConflicts += $cp } } + if ($specifiedConflicts.Count -gt 0) { + Write-Host "ERROR: -RAWInputCSV cannot be combined with live query parameter(s): $($specifiedConflicts -join ', ')" -ForegroundColor Red + Write-Host "Remove those conflicting parameters and re-run. Allowed with RAWInputCSV: StartDate, EndDate, ActivityTypes, AgentId, AgentsOnly, UserIds, OutputFile, AppendFile, explosion switches." -ForegroundColor Yellow + Write-Host "Note: -GroupNames requires authentication and cannot be used in replay mode. Use -UserIds with explicit email addresses instead." -ForegroundColor Yellow + exit 1 + } +} + +# Validate -UseEOM compatibility with parallel processing +if ($UseEOM) { + $parallelConflicts = @() + + # Check for explicit parallel mode settings + if ($PSBoundParameters.ContainsKey('EnableParallel') -and $EnableParallel) { + $parallelConflicts += '-EnableParallel' + } + + if ($PSBoundParameters.ContainsKey('ParallelMode') -and $ParallelMode -ne 'Off') { + $parallelConflicts += "-ParallelMode $ParallelMode" + } + + if ($parallelConflicts.Count -gt 0) { + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: -UseEOM Incompatible with Parallel Processing" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host "Exchange Online Management mode (-UseEOM) only supports SERIAL processing." -ForegroundColor Yellow + Write-Host "The Search-UnifiedAuditLog cmdlet cannot be used in parallel ThreadJobs due to" -ForegroundColor Gray + Write-Host "implicit remoting architecture limitations in the EOM PowerShell module." -ForegroundColor Gray + Write-Host "" + Write-Host "CONFLICTING PARAMETERS DETECTED:" -ForegroundColor Yellow + foreach ($conflict in $parallelConflicts) { + Write-Host " • $conflict" -ForegroundColor Red + } + Write-Host "" + Write-Host "RESOLUTION OPTIONS:" -ForegroundColor Cyan + Write-Host " 1. Remove -UseEOM switch to enable Graph API mode (supports parallel processing)" -ForegroundColor White + Write-Host " 2. Remove parallel parameters and use serial-only processing with -UseEOM" -ForegroundColor White + Write-Host " 3. Set -ParallelMode Off explicitly: -UseEOM -ParallelMode Off" -ForegroundColor White + Write-Host "" + Write-Host "NOTE: Graph API mode (default, no -UseEOM) supports parallel processing in PowerShell 7+." -ForegroundColor Gray + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + + # Log to file if log initialized + if ($script:logFile -and (Test-Path $script:logFile)) { + $timestamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss' + Add-Content -Path $script:logFile -Value "[$timestamp] ERROR: -UseEOM incompatible with parallel processing parameters: $($parallelConflicts -join ', ')" + Add-Content -Path $script:logFile -Value "[$timestamp] Script terminated. Resolution: Remove -UseEOM or disable parallel mode." + } + + exit 1 + } + + # Force ParallelMode Off in EOM mode even if Auto is set + if ($ParallelMode -ne 'Off') { + Write-Host "" + Write-Host "NOTE: -UseEOM mode requires serial processing. Forcing -ParallelMode Off." -ForegroundColor Yellow + Write-Host "" + $ParallelMode = 'Off' + } +} + +$script:learnedActivityBlockSize = @{} +$script:globalLearnedBlockSize = $BlockHours +$script:subdivisionSequence = @(0.5, 0.25, 0.133333, 0.066667, 0.033333, 0.016667, 0.010417, 0.005556, 0.002778, 0.001389) # 12h, 6h, 3.2h, 1.6h, 48m, 24m, 15m, 8m, 4m, 2m +$script:Hit10KLimit = $false +$script:Hit1MLimit = $false # Graph API 1,000,000 record limit per query +$script:LimitTimeWindow = "" +$script:SubdividedPartitions = @{} # Track partitions that needed subdivision (key=original range, value=count) +$script:Connected = $false + +# ============================================================================ +# GRAPH API SECURITY AUDIT ENDPOINT VERSION CONFIGURATION +# ============================================================================ +# Manually configure these variables if Microsoft updates the API version +# PAX will try CURRENT version first, then fallback to PREVIOUS version +# ============================================================================ +$script:GraphAuditApiVersion_Current = 'v1.0' # Try this version first +$script:GraphAuditApiVersion_Previous = 'beta' # Fallback to this version if current unavailable +$script:GraphAuditApiVersion = $null # Runtime-detected version (do not edit) +# ============================================================================ + +# Suppress PowerShell's web request progress bar (prevents "Reading web response stream" noise) +$ProgressPreference = 'SilentlyContinue' + +# Telemetry tracking for Graph API parallel queries (per-slice lifecycle data) +$script:telemetryData = @() + +# ============================================================================ +# SHAPE CONTRACTS +# ---------------------------------------------------------------------------- +# These two helpers freeze the canonical shapes of $script:metrics and of each +# entry in $script:partitionStatus. They are intentionally cheap, fail-fast, +# and called only at well-known initialization points. They do NOT police every +# mutation; their purpose is to catch a future change that introduces an +# incompatible re-init (e.g. drops a required key) at script-start time rather +# than at consumer-time hours later. +# +# If you add a new REQUIRED field to either structure, add it here so the +# contract drift is caught early. +# ============================================================================ +$script:MetricsRequiredFields = @( + 'StartTime', 'QueryMs', 'ExplosionMs', 'ExportMs', + 'PagesFetched', 'TotalRecordsFetched', 'TotalStructuredRows', + 'ExplosionEvents', 'ExplosionRowsFromEvents', 'ExplosionMaxPerRecord', 'ExplosionTruncated', + 'Activities', + 'FilteringSkippedRecords', 'FilteringMissingAuditData', + 'FilteringPromptFiltered', 'FilteringParseFailures' +) +$script:PartitionStatusRequiredFields = @( + 'Partition', 'AttemptNumber', 'QueryId', 'QueryName', + 'Status', 'LastError', 'RecordCount' +) + +function Assert-MetricsShape { + <# + .SYNOPSIS + Fail-fast contract check for $script:metrics. + .DESCRIPTION + Verifies the script-scope metrics hashtable exists and contains every + required field. Throws a single descriptive error on first mismatch so + the failure surfaces at startup rather than via a NullReferenceException + deep in the summary writer. + #> + param([hashtable]$Metrics = $script:metrics) + if (-not $Metrics) { + throw "Assert-MetricsShape: \$script:metrics is null. A required initialization step was skipped." + } + foreach ($k in $script:MetricsRequiredFields) { + if (-not $Metrics.ContainsKey($k)) { + throw "Assert-MetricsShape: \$script:metrics is missing required field '$k'. This indicates a structural mismatch." + } + } +} + +function Assert-PartitionStatusEntry { + <# + .SYNOPSIS + Fail-fast contract check for a single $script:partitionStatus entry. + .DESCRIPTION + Verifies the supplied entry contains every field required by the + downstream STATUS display, retry logic, and summary writer. Optional + fields (ParentPartition, SubdivisionReason) are intentionally not + asserted. + #> + param( + [Parameter(Mandatory = $true)] $Entry, + [string]$Context = '' + ) + if ($null -eq $Entry) { + throw "Assert-PartitionStatusEntry [$Context]: entry is null." + } + foreach ($k in $script:PartitionStatusRequiredFields) { + if (-not $Entry.ContainsKey($k)) { + throw "Assert-PartitionStatusEntry [$Context]: missing required field '$k'. This indicates a structural mismatch." + } + } +} + +# Number of consecutive same-category failures (signature = partition index + +# failure category) a partition may accumulate before it is flagged as a terminal +# failure. Count only; there is no elapsed-time component. Maintained by +# Set-PartitionFailure. +$script:MaxConsecutiveIdenticalFailures = 5 + +# Defensive runaway guard: the maximum number of times a single collection window +# may be subdivided by record-limit auto-subdivision. A legitimate high-volume +# window reaches well under this depth; hitting it indicates a pathological or +# spurious limit signal. A window still over the record limit at this depth is +# stopped and reported as a DISTINCT subdivision-limit state (Status +# 'SubdivisionLimit'), separate from the terminal create/fetch failure class. This +# is a depth count only — not a time limit. +$script:MaxSubdivisionLevel = 20 + +function Get-PartitionFailureCategory { + <# + .SYNOPSIS + Maps a specific failure reason to the coarse category used for the + terminal-detection signature. + .DESCRIPTION + All "the attempt failed at runtime" reasons collapse to FAILED so the SAME + underlying failure accumulates toward terminal regardless of which + observation site (live monitoring, reconcile, retry) reports it (a job error + can surface as JOB_ERROR live and ERROR_STREAM at reconcile for one problem). + AUTH (token) and SLOW (query still running) stay distinct so genuine + transients reset the consecutive-failure streak instead of driving a false + terminal. + #> + param([string]$Reason) + switch ($Reason) { + '401' { 'AUTH'; break } + 'STILL_RUNNING' { 'SLOW'; break } + default { 'FAILED' } + } +} + +function Set-PartitionFailure { + <# + .SYNOPSIS + Central sink for marking a partition Failed while tracking consecutive + same-category failure runs. + .DESCRIPTION + Sets Status='Failed' (identical to the inline assignment it replaces) and, + in addition, maintains a per-partition consecutive-failure counter keyed on a + coarse signature of (partition index + failure category). The category + collapses all runtime-failure reasons to FAILED (see + Get-PartitionFailureCategory) so the same underlying failure accumulates + regardless of which observation site reports it; AUTH and SLOW stay distinct. + When the same category repeats $script:MaxConsecutiveIdenticalFailures times + in a row, IsTerminalFailure is set; a differing category resets the counter + to 1. The most-recent Stage and Reason are recorded (LastFailureStage / + LastFailureReason) for the diagnostic terminal-failure report — they are NOT + part of the counting signature. LastError is left to the caller (unchanged). + The tracking fields are additive and are not consumed by any re-queue or + retry predicate here. + .PARAMETER Index + Partition index key into $script:partitionStatus. + .PARAMETER Stage + Coarse phase where the failure was observed (e.g. CREATE, FETCH, RECONCILE). + Recorded for the report only. + .PARAMETER Reason + Specific failure token (e.g. 401, EMPTY_QUERYID, JOB_FAILED). Recorded for the + report and mapped to a category for the counting signature. + #> + param( + [Parameter(Mandatory)] $Index, + [Parameter(Mandatory)] [string]$Stage, + [Parameter(Mandatory)] [string]$Reason + ) + if (-not ($script:partitionStatus -and $script:partitionStatus.ContainsKey($Index))) { return } + $entry = $script:partitionStatus[$Index] + $category = Get-PartitionFailureCategory -Reason $Reason + $signature = "$Index|$category" + if ($entry.LastFailureSignature -eq $signature) { + $entry.ConsecutiveIdenticalFailures = [int]$entry.ConsecutiveIdenticalFailures + 1 + } else { + $entry.ConsecutiveIdenticalFailures = 1 + $entry.LastFailureSignature = $signature + } + $entry.LastFailureStage = $Stage + $entry.LastFailureReason = $Reason + # A persistent create denial permits one token-renewal retry before becoming terminal. + $terminalThreshold = if ($Reason -eq 'PersistentForbidden') { 2 } else { $script:MaxConsecutiveIdenticalFailures } + $entry.IsTerminalFailure = ($entry.ConsecutiveIdenticalFailures -ge $terminalThreshold) + # Structured lifecycle category preservation: map the coarse Reason + # to the bounded vocabulary and NEVER let a generic parent reconciliation result + # (JOB_FAILED / ERROR_STREAM / NO_DATA / INVALID_COMPLETE) overwrite a more + # specific worker-established category. LastError stays diagnostic-only and is + # not consulted here. Additive: does not affect the terminal-failure counting. + $incomingCategory = Get-GraphAuditReasonCategory -Reason $Reason + $existingCategory = if ($entry.ContainsKey('FailureCategory') -and $null -ne $entry.FailureCategory) { [string]$entry.FailureCategory } else { 'None' } + $entry.FailureCategory = Merge-GraphAuditLifecycleCategory -Existing $existingCategory -Incoming $incomingCategory + $entry.Status = 'Failed' +} + +function Test-TransientAuthSignal { + <# + .SYNOPSIS + Returns $true only when error text carries a genuine transient-authentication + signal (expired/invalid token), never a coincidental '401' substring. + .DESCRIPTION + Matches explicit auth tokens (Unauthorized, InvalidAuthenticationToken, + token expired, authentication failed, lifetime validation failed) and the + HTTP status 401 ONLY in a status context — "(401)", "HTTP 401", or + "401 Unauthorized" — so a bare number such as a record count ("4013 records") + is not mistaken for an auth failure. Fail-closed: ambiguous or empty text + returns $false, so the failure then counts normally against the retry limit. + #> + param([string]$Text) + if ([string]::IsNullOrWhiteSpace($Text)) { return $false } + return [bool]($Text -match '(?i)(unauthorized|invalidauthenticationtoken|token(?:\s+has)?\s+expired|authentication\s+failed|lifetime\s+validation\s+failed|\(401\)|http[ /]?401\b|401\s+unauthorized)') +} + +$script:metrics = @{ + StartTime = (Get-Date).ToUniversalTime() + QueryMs = 0 + ExplosionMs = 0 + ExportMs = 0 + PagesFetched = 0 + TotalRecordsFetched = 0 + TotalStructuredRows = 0 + ExplosionEvents = 0 + ExplosionRowsFromEvents = 0 + ExplosionMaxPerRecord = 0 + ExplosionTruncated = $false + ShrinkEvents = 0 + Activities = @{} + EffectiveChunkSize = 0 + ParallelBatchSizeFinal = 0 + ParallelThrottleFinal = 0 + AgentFilterApplied = $false + AgentFilterPreCount = 0 + AgentFilterPostCount = 0 + AgentFilterRemovedCount = 0 + AgentFilterElapsedSec = 0 + ExcludeAgentsApplied = $false + ExcludeAgentsPreCount = 0 + ExcludeAgentsPostCount = 0 + ExcludeAgentsRemoved = 0 + ExcludeAgentsElapsedSec = 0 + PromptFilterApplied = $false + PromptFilterType = '' + PromptFilterPreCount = 0 + PromptFilterPostCount = 0 + PromptFilterRemovedCount = 0 + PromptFilterElapsedSec = 0 + PromptFilterMsgBefore = 0 + PromptFilterMsgAfter = 0 + PromptFilterMsgRemoved = 0 + PromptFilterRecordsMixed = 0 + PromptFilterRecordsPromptOnly = 0 + PromptFilterRecordsResponseOnly = 0 + PromptFilterRecordsNoMessages = 0 + FilteringSkippedRecords = 0 + FilteringMissingAuditData = 0 + FilteringParseFailures = 0 + FilteringPromptFiltered = 0 + FilteringAgentFiltered = 0 + FilteringExcludeAgents = 0 + FilteringUserIds = 0 + FilteringGroupNames = 0 + ScopeResolvedGroups = 0 + ScopeFailedGroups = 0 + ScopeExpandedMembers = 0 + ScopeFinalTargetUsers = 0 + DirectoryRowsBeforeScope = 0 + DirectoryRowsAfterScope = 0 + DirectoryRowsExcludedByScope = 0 + FilteringOther = 0 + AdaptiveEvents = @() + AdaptiveMemoryReductions = 0 + AdaptiveLatencyReductions = 0 + AdaptiveLatencyIncreases = 0 + ThroughputBaselineRps = 0 + CircuitBreakerTrips = 0 + BackoffTotalDelaySeconds = 0 + PartitionCapsApplied = 0 + PartitionCapHighestRequested = 0 +} + +# v6 shape-contract guard: catches accidental drops of required metrics fields +# at startup rather than at summary-writer time. See Assert-MetricsShape above. +Assert-MetricsShape + +$script:summaryWritten = $false + +# Streaming dataset profiler (live & replay) +$script:profiler = @{ + Rows = 0 + Operations = @{} + RecordTypes = @{} + HasCopilot = 0 + MaxDepth = 0 + DepthCounts = @{} + MaxArrayLen = 0 +} + +$script:shapeCache = @{} + +function Get-RecordShapeKey { + param([object]$AuditData) + try { + $rt = $AuditData.RecordType + } catch { $rt = '' } + try { + $op = $AuditData.Operation + } catch { $op = '' } + try { + $hasCopilot = $AuditData.PSObject.Properties['CopilotEventData'] -ne $null + } catch { $hasCopilot = $false } + return "$rt|$op|$hasCopilot" +} + +function Get-RecordShape { + param([object]$AuditData) + if ($null -eq $AuditData) { return $null } + $key = Get-RecordShapeKey $AuditData + if ($script:shapeCache.ContainsKey($key)) { return $script:shapeCache[$key] } + $shape = @{} + try { + $shape.RecordType = $AuditData.RecordType + $shape.Operation = $AuditData.Operation + } catch {} + try { $shape.HasCopilot = $AuditData.PSObject.Properties['CopilotEventData'] -ne $null } catch { $shape.HasCopilot = $false } + try { $shape.Depth = Get-JsonDepth $AuditData 0 } catch { $shape.Depth = 0 } + $shape.Mode = if ($shape.HasCopilot) { 'Copilot' } else { 'AuditData' } + $script:shapeCache[$key] = $shape + return $shape +} + +function Reset-Profiler { + $script:profiler = @{ + Rows = 0 + Operations = @{} + RecordTypes = @{} + HasCopilot = 0 + MaxDepth = 0 + DepthCounts = @{} + MaxArrayLen = 0 + } +} + +function Get-JsonDepth([object]$node, [int]$d = 0) { + if ($null -eq $node -or (Test-ScalarValue $node)) { return $d } + if ($node -is [System.Collections.IDictionary]) { + $maxd = $d + foreach ($v in $node.Values) { $maxd = [math]::Max($maxd, (Get-JsonDepth $v ($d + 1))) } + return $maxd + } + if ($node -is [System.Collections.IEnumerable] -and -not ($node -is [string])) { + $maxd = $d + $i = 0 + foreach ($el in $node) { $maxd = [math]::Max($maxd, (Get-JsonDepth $el ($d + 1))); $i++ } + if ($i -gt $script:profiler.MaxArrayLen) { $script:profiler.MaxArrayLen = $i } + return $maxd + } + return $d +} + +function Profile-AuditData { + param([object]$AuditData) + if ($null -eq $AuditData) { return } + try { + $script:profiler.Rows++ + # Operation + try { + $op = $AuditData.Operation + if (-not [string]::IsNullOrWhiteSpace($op)) { + if (-not $script:profiler.Operations.ContainsKey($op)) { $script:profiler.Operations[$op] = 0 } + $script:profiler.Operations[$op] += 1 + } + } catch {} + # RecordType + try { + $rt = $AuditData.RecordType + if (-not [string]::IsNullOrWhiteSpace([string]$rt)) { + if (-not $script:profiler.RecordTypes.ContainsKey([string]$rt)) { $script:profiler.RecordTypes[[string]$rt] = 0 } + $script:profiler.RecordTypes[[string]$rt] += 1 + } + } catch {} + # CopilotEventData presence + try { if ($AuditData.PSObject.Properties['CopilotEventData']) { $script:profiler.HasCopilot++ } } catch {} + # Depth & arrays + $depth = Get-JsonDepth $AuditData 0 + if ($depth -gt $script:profiler.MaxDepth) { $script:profiler.MaxDepth = $depth } + if (-not $script:profiler.DepthCounts.ContainsKey($depth)) { $script:profiler.DepthCounts[$depth] = 0 } + $script:profiler.DepthCounts[$depth] += 1 + } catch {} +} + +function Write-ProfilerSummary { + param([int]$TopOps = 20, [int]$TopDepths = 10) + try { + Write-LogHost "Profiler: Rows=$($script:profiler.Rows), MaxDepth=$($script:profiler.MaxDepth), MaxArrayLen=$($script:profiler.MaxArrayLen), HasCopilot=$($script:profiler.HasCopilot)" -ForegroundColor Gray + if ($script:profiler.Operations.Count -gt 0) { + Write-LogHost "Profiler: Operations (top $TopOps):" -ForegroundColor Gray + $script:profiler.Operations.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First $TopOps | ForEach-Object { Write-LogHost " $($_.Key): $($_.Value)" -ForegroundColor Gray } + } + if ($script:profiler.DepthCounts.Count -gt 0) { + Write-LogHost "Profiler: Depth distribution (top $TopDepths):" -ForegroundColor Gray + $script:profiler.DepthCounts.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First $TopDepths | ForEach-Object { Write-LogHost " Depth $($_.Key): $($_.Value)" -ForegroundColor Gray } + } + } catch {} +} +$script:adaptiveThroughputBaseline = $null +$script:adaptiveLowLatencyStreak = 0 +$script:consecutiveBlockFailures = 0 +$script:circuitBreakerOpen = $false +$script:circuitBreakerOpenUntil = $null + +# Finalize-time "completed with gaps" signals for the process exit code. Set true in +# the query-submission summary when the run finalizes with one or more partitions +# terminally failed (Issue 3A) or windows that hit the subdivision-depth guard +# (Issue 3B). Drive exit code 40 (lowest precedence — existing 10/20/30 dominate). +$script:HadTerminalFailures = $false +$script:HadSubdivisionLimit = $false +$script:AISIDHadGaps = $false +# Set true when requested Agent 365 catalog output is incomplete: an incomplete catalog +# listing, a package whose detail could not be retrieved, a row that could not be built, or +# a listed entry without a usable identifier. Drives exit code 40 at the same lowest +# precedence as the other completed-with-gaps signals. +$script:Agent365HadGaps = $false +# Leaves of Agent 365 recovery material written when the catalog listing did not complete. +# Recovery material is never published as a completed catalog, so both the end-of-run local +# listing and the remote upload sweep exclude these leaves (same convention as the Fact-append +# recovery leaves). +$script:Agent365RecoveryLeafs = $null +$script:AISIDCompletionStatus = $null +# D3: set true when the CopilotInteraction rollup post-processor fails (throw / nonzero exit) or does not +# produce a required deliverable (Fact / Users). Consumed ONLY under -Dashboard AISID (a requested-output +# failure -> completed-with-gaps, exit 40); non-AISID best-effort rollup exit semantics are unchanged. +$script:RollupProcessorFailed = $false + +# 1C: a REQUESTED Fact-append (-AppendFile) reconciliation FAILED this run. This is a +# requested-output failure -> completed-with-gaps (exit 40, driven via HadTerminalFailures), +# and it must ALSO preserve the recovery artifacts (the unmerged current-run Fact CSV, the +# remote scratch dir, and the Fabric resume mirror) that the normal clean-run cleanup would +# otherwise delete. It is DELIBERATELY DISTINCT from AnyUploadFailed (no upload actually +# failed) so diagnostics never mislabel a local merge failure as an upload failure. The +# exact leaves registered in FactAppendRecoveryLeafs are excluded from the end-of-run upload +# sweep and the local output listing so the unmerged current-run Fact is never published as +# though it were the completed -AppendFile target (nor as a second normal Fact output). +$script:FactAppendFailed = $false +$script:FactAppendRecoveryLeafs = $null + +# Advisory append-target size guardrail. When an append/merge target exceeds this +# many bytes, a non-blocking warning is emitted before the merge (which reads the +# whole target into memory to deduplicate against it). Default 1 GB; override in +# bytes via the PAX_APPEND_WARN_BYTES environment variable. The size probe itself is +# fail-open (see Get-RemoteAppendTargetSize) so it can never block or mis-handle a run. +$script:AppendTargetSizeWarnBytes = if ($env:PAX_APPEND_WARN_BYTES -and (($env:PAX_APPEND_WARN_BYTES -as [int64]) -gt 0)) { [int64]$env:PAX_APPEND_WARN_BYTES } else { 1GB } + +# ============================================== +# GRAPH API VERSION DETECTION HELPER +# ============================================== +# Automatically detects and uses configured current version or falls back to previous version +# Version configuration is at top of script for easy manual updates + +function Get-GraphAuditApiUri { + <# + .SYNOPSIS + Builds Graph API audit endpoint URI with automatic version detection. + + .DESCRIPTION + Attempts to use the configured current version first. If not available, + falls back to the previous version. Version detection is cached per session. + + Configure versions at top of script: + $script:GraphAuditApiVersion_Current = 'v1.0' (try first) + $script:GraphAuditApiVersion_Previous = 'beta' (fallback) + + .PARAMETER Path + The audit API path (e.g., "queries", "queries/{id}", "queries/{id}/records") + + .OUTPUTS + String - Full Graph API URI with appropriate version + + .EXAMPLE + $uri = Get-GraphAuditApiUri -Path "queries" + # Returns: https://graph.microsoft.com/v1.0/security/auditLog/queries + # or: https://graph.microsoft.com/beta/security/auditLog/queries (if v1.0 unavailable) + #> + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + # Select the audit-query lifecycle version on first use (cached for session). + # + # A successful collection GET does NOT prove that per-item status, records, + # pagination and cleanup all work on a given version, so it must never by + # itself select the version used for the full query lifecycle. Field evidence + # showed a version whose collection GET and create succeeded but whose + # item/status and records operations returned 404 for queries that had + # actually completed server-side, while the alternate version served the same + # lifecycle reliably. Cross-version query-id visibility is unproven, so the + # lifecycle is never mixed across versions within a single query. + # + # This release therefore pins the version with a proven end-to-end lifecycle + # in this service ($script:GraphAuditApiVersion_Previous). The other version + # ($script:GraphAuditApiVersion_Current) remains configured for a future + # capability-gated path that is enabled only after a read-only cross-version + # discriminator proves its per-operation support and cross-version visibility. + if ($null -eq $script:GraphAuditApiVersion) { + $script:GraphAuditApiVersion = $script:GraphAuditApiVersion_Previous + Write-LogHost "Graph API: security/auditLog lifecycle pinned to version $($script:GraphAuditApiVersion) (proven end-to-end; collection availability alone does not select a version)" -ForegroundColor Green + } + + return "https://graph.microsoft.com/$($script:GraphAuditApiVersion)/security/auditLog/$Path" +} + +# ============================================================================ +# GRAPH AUDIT-QUERY LIFECYCLE DECISION LAYER +# ---------------------------------------------------------------------------- +# One shared, injectable decision layer for the Graph security/auditLog query +# lifecycle (create / status / records / pagination / delete). These are pure +# functions driven by STRUCTURED inputs (HTTP status codes + exception TYPE), +# never by error-message text. They are exercised directly by +# tests/Purview.GraphAuditLifecycleStateMachine.Tests.ps1 with injected HTTP +# operations, so the tested logic is the shipped logic. The same rules govern +# both the parent path and the isolated ThreadJob worker runspace. +# +# Stage-A-grounded invariants (beta is the default; v1.0 CREATION stays disabled): +# * beta is the production default; new query CREATION always uses beta. +# * A single endpoint/version 404 is QueryNotVisible, never QueryConfirmedAbsent. +# * Version fallback is a same-ID GET on the alternate version only; it NEVER +# issues a POST. Status and records are decided INDEPENDENTLY (Stage A proved +# v1.0 status 404 while v1.0 records 200 for the same id). +# * 401/403, 429, network, and 5xx keep existing handling with NO version +# fallback. +# * @odata.nextLink is followed verbatim; no $top / cap / truncation is added. +# * A successful zero-record retrieval is Complete, never NO_DATA / JOB_FAILED. +# * Delete uses the effective version and is best-effort; cleanup failure is +# observable but never erases saved records or fabricates a data gap. +# ============================================================================ + +# Bounded failure-category vocabulary. Extending this list is a deliberate +# contract change and must be mirrored in the classifier + tests. +$script:GraphAuditFailureCategories = @( + 'None', 'EndpointUnavailable', 'QueryNotVisible', 'QueryConfirmedAbsent', + 'Authentication', 'Throttled', 'Network', 'Service', 'Cleanup', 'Other' +) +# Categories that carry a specific, worker-established meaning. A generic parent +# reconciliation result (JOB_FAILED / ERROR_STREAM / NO_DATA / INVALID_COMPLETE) +# must never overwrite one of these. +$script:GraphAuditSpecificFailureCategories = @( + 'EndpointUnavailable', 'QueryNotVisible', 'QueryConfirmedAbsent', + 'Authentication', 'Throttled', 'Network', 'Service', 'Cleanup' +) + +function Get-GraphAuditFailureCategory { + <# + .SYNOPSIS + Maps a structured HTTP status + exception type to the bounded lifecycle + failure vocabulary. NEVER inspects message text. + #> + param( + [Nullable[int]]$HttpStatus = $null, + [string]$ExceptionType = $null, + [ValidateSet('Create', 'Status', 'Records', 'Delete', 'List', '')] + [string]$Operation = '', + [switch]$ConfirmedAbsent + ) + # Delete/cleanup failures are always Cleanup (best-effort; never data-erasing). + # A 404 on delete means the query is already gone => successful cleanup (None). + if ($Operation -eq 'Delete') { + if ($HttpStatus -eq 404) { return 'None' } + if (($null -ne $HttpStatus -and $HttpStatus -ge 400) -or $ExceptionType) { return 'Cleanup' } + return 'None' + } + if ($null -ne $HttpStatus) { + switch ([int]$HttpStatus) { + 401 { return 'Authentication' } + 403 { return 'Authentication' } + 429 { return 'Throttled' } + 404 { if ($ConfirmedAbsent) { return 'QueryConfirmedAbsent' } else { return 'QueryNotVisible' } } + default { + if ($HttpStatus -ge 500 -and $HttpStatus -le 599) { return 'Service' } + if ($HttpStatus -ge 200 -and $HttpStatus -le 299) { return 'None' } + return 'Other' + } + } + } + # No HTTP status: network/transport-class exceptions (by TYPE, not text). + if ($ExceptionType) { + $net = @('HttpRequestException', 'WebException', 'SocketException', 'TaskCanceledException', 'TimeoutException', 'IOException') + foreach ($n in $net) { if ($ExceptionType -like "*$n*") { return 'Network' } } + return 'Other' + } + return 'None' +} + +function New-GraphAuditLifecycleState { + <# + .SYNOPSIS + Factory for the additive per-query structured lifecycle fields. Ordered so + checkpoint serialization is stable. + #> + param( + [string]$QueryApiVersion = $null, + [string]$LifecycleOperation = $null + ) + return [ordered]@{ + QueryApiVersion = $QueryApiVersion + LifecycleOperation = $LifecycleOperation + FailureCategory = 'None' + AlternateVersionChecked = $false + AlternateVersionResult = $null # 'Visible' | 'NotVisible' | $null + QueryConfirmedAbsent = $false + ReplacementAttempted = $false + ReplacementQueryId = $null + LastLifecycleHttpStatus = $null + LifecycleRetryCount = 0 + ZeroRecordComplete = $false + # Durable uncertain-create state. UncertainCreate is a boolean flag; when true the + # partition has an unproven (possibly-accepted) first create POST and must only ever + # be resolved by READ-ONLY exact-name reconciliation, never another POST. + # UncertainCreateInfo is a structured sub-object (Fingerprint, DisplayName, PStart, + # PEnd, Activities, RecordTypes, ServiceFilter, FirstSeenUtc, AttemptMarker) used to + # rebuild the exact fingerprinted name for reconciliation across attempts/resume. + UncertainCreate = $false + UncertainCreateInfo = $null + } +} + +function Set-GraphAuditLifecycleFields { + <# + .SYNOPSIS + Ensures the lifecycle fields exist on a partitionStatus/checkpoint entry + WITHOUT discarding existing values. + .DESCRIPTION + Legacy-safe: a missing field is added with its default; an existing field + (including a stored QueryId elsewhere on the entry) is NEVER overwritten or + cleared. Loading a legacy checkpoint that lacks these fields therefore can + never discard or recreate a stored QueryId. + #> + param([Parameter(Mandatory)] $Entry) + if ($null -eq $Entry) { return $Entry } + $defaults = New-GraphAuditLifecycleState + foreach ($k in $defaults.Keys) { + $has = $false + if ($Entry -is [hashtable]) { $has = $Entry.ContainsKey($k) } + else { $has = ($null -ne $Entry.PSObject.Properties[$k]) } + if (-not $has) { + if ($Entry -is [hashtable]) { $Entry[$k] = $defaults[$k] } + else { Add-Member -InputObject $Entry -NotePropertyName $k -NotePropertyValue $defaults[$k] -Force } + } + } + return $Entry +} + +function Merge-GraphAuditLifecycleCategory { + <# + .SYNOPSIS + Reconciliation preservation: a generic parent result never overwrites a + more specific worker-established category. Returns the category to keep. + #> + param([string]$Existing, [string]$Incoming) + if ([string]::IsNullOrWhiteSpace($Existing)) { return $Incoming } + if (($script:GraphAuditSpecificFailureCategories -contains $Existing) -and + (-not ($script:GraphAuditSpecificFailureCategories -contains $Incoming))) { + return $Existing + } + return $Incoming +} + +function Get-GraphAuditReasonCategory { + <# + .SYNOPSIS + Maps a coarse parent Reason token to the bounded category vocabulary. + .DESCRIPTION + Generic runtime reasons (JOB_FAILED / ERROR_STREAM / NO_DATA / + INVALID_COMPLETE) collapse to 'Other' (non-specific) so they cannot mask a + worker-established specific category. Driven by the structured token only. + #> + param([string]$Reason) + switch -Regex ($Reason) { + '^401$|^403$' { 'Authentication'; break } + '^429$' { 'Throttled'; break } + '^QUERY-?GONE$|^QueryNotVisible$' { 'QueryNotVisible'; break } + '^QueryConfirmedAbsent$' { 'QueryConfirmedAbsent'; break } + '^Cleanup$' { 'Cleanup'; break } + '^Network$' { 'Network'; break } + '^Service$|^5\d\d$' { 'Service'; break } + default { 'Other' } + } +} + +function Invoke-GraphAuditLifecycleOperation { + <# + .SYNOPSIS + Shared decision layer for ONE lifecycle operation (Status/Records/Delete). + .DESCRIPTION + Routes a single operation through: (1) call on the effective version; + (2) on 404, bounded eventual-consistency retry on the SAME version; + (3) then a same-ID GET on the alternate version (never a POST); (4) if the + alternate GET succeeds, retain the id, record the result, switch the + effective QueryApiVersion, and continue. 401/403/429/network/5xx keep their + existing meaning with NO version fallback. Status and records are decided + INDEPENDENTLY. The -Invoke callback isolates the HTTP transport so the same + function is exercised by injected tests and by production callers. + .PARAMETER Invoke + scriptblock param($Version,$QueryId,$Operation) -> + @{ HttpStatus=; Data=; ExceptionType= } + #> + param( + [Parameter(Mandatory)][ValidateSet('Status', 'Records', 'Delete')] [string]$Operation, + [Parameter(Mandatory)][string]$QueryId, + [Parameter(Mandatory)][scriptblock]$Invoke, + [string]$EffectiveVersion = $null, + [string]$AlternateVersion = $null, + [Parameter(Mandatory)] $State, + [int]$MaxConsistencyRetries = 3, + [bool]$EnableAlternateVersion = $true + ) + if ([string]::IsNullOrWhiteSpace($EffectiveVersion)) { $EffectiveVersion = $script:GraphAuditApiVersion_Previous } # legacy/unknown => beta + if ([string]::IsNullOrWhiteSpace($AlternateVersion)) { + $AlternateVersion = if ($EffectiveVersion -eq $script:GraphAuditApiVersion_Previous) { $script:GraphAuditApiVersion_Current } else { $script:GraphAuditApiVersion_Previous } + } + $State.LifecycleOperation = $Operation + $ver = $EffectiveVersion + $attempt = 0 + while ($true) { + $r = & $Invoke $ver $QueryId $Operation + $status = if ($r) { $r.HttpStatus } else { $null } + $exType = if ($r) { $r.ExceptionType } else { $null } + $State.LastLifecycleHttpStatus = $status + # Success. + if ($null -ne $status -and $status -ge 200 -and $status -le 299) { + $State.QueryApiVersion = $ver + $State.FailureCategory = 'None' + $data = if ($r) { $r.Data } else { $null } + if ($Operation -eq 'Records') { + $count = 0 + if ($data -and $data.value) { $count = @($data.value).Count } + if ($count -eq 0) { $State.ZeroRecordComplete = $true } + } + return @{ Success = $true; HttpStatus = $status; Version = $ver; Category = 'None'; Data = $data; State = $State } + } + # 404 => bounded eventual-consistency retry on the SAME version first. + if ($status -eq 404) { + if ($attempt -lt $MaxConsistencyRetries) { + $attempt++ + $State.LifecycleRetryCount = [int]$State.LifecycleRetryCount + 1 + continue + } + # Delete: a 404 after retries means already-gone => successful cleanup. + if ($Operation -eq 'Delete') { + $State.QueryApiVersion = $ver + $State.FailureCategory = 'None' + return @{ Success = $true; HttpStatus = 404; Version = $ver; Category = 'None'; Data = $null; State = $State } + } + # Same-ID GET on the alternate version (never a POST). + if ($EnableAlternateVersion -and $AlternateVersion -and $AlternateVersion -ne $ver) { + $State.AlternateVersionChecked = $true + $ra = & $Invoke $AlternateVersion $QueryId $Operation + $as = if ($ra) { $ra.HttpStatus } else { $null } + if ($null -ne $as -and $as -ge 200 -and $as -le 299) { + $State.AlternateVersionResult = 'Visible' + $State.QueryApiVersion = $AlternateVersion # switch effective version for subsequent ops + $State.FailureCategory = 'None' + $State.LastLifecycleHttpStatus = $as + $adata = if ($ra) { $ra.Data } else { $null } + if ($Operation -eq 'Records') { + $count = 0; if ($adata -and $adata.value) { $count = @($adata.value).Count } + if ($count -eq 0) { $State.ZeroRecordComplete = $true } + } + return @{ Success = $true; HttpStatus = $as; Version = $AlternateVersion; Category = 'None'; Data = $adata; State = $State } + } + $State.AlternateVersionResult = 'NotVisible' + } + # Both versions 404 (or alternate disabled): NOT visible. Absence is NOT + # confirmed here — confirmation additionally requires a fully-paginated + # exact-name miss (see Resolve-GraphAuditDuplicateSafeQuery). + $State.FailureCategory = 'QueryNotVisible' + return @{ Success = $false; HttpStatus = 404; Version = $ver; Category = 'QueryNotVisible'; Data = $null; State = $State } + } + # Non-404 failures: classify and NEVER version-fallback. + $cat = Get-GraphAuditFailureCategory -HttpStatus $status -ExceptionType $exType -Operation $Operation + $State.FailureCategory = $cat + return @{ Success = $false; HttpStatus = $status; Version = $ver; Category = $cat; Data = $null; State = $State } + } +} + +function Resolve-GraphAuditDuplicateSafeQuery { + <# + .SYNOPSIS + Duplicate-safe creation/replacement around a deterministic exact partition + displayName. + .DESCRIPTION + Rules: fully-paginated exact-name lookup (ordinal exact match) BEFORE any + recovery POST; exactly one match => adopt (no POST); multiple exact matches + => fail closed (no POST); confirmed absence requires both item versions 404 + for the same id AND an exact-name miss; at most ONE replacement POST, only + after absence is confirmed and persisted ReplacementAttempted is false + (persisted BEFORE the POST so a resume cannot double-create); an ambiguous + POST error is reconciled by exact-name lookup, never a blind re-POST. A + partition with no prior id may create its initial beta query but still uses + the exact-name + ambiguous-POST guard. + #> + param( + [Parameter(Mandatory)][string]$DisplayName, + [string]$ExistingQueryId = $null, + [Parameter(Mandatory)] $State, + [Parameter(Mandatory)][scriptblock]$ListQueries, # -> array of @{ id; displayName } (fully paginated) + [scriptblock]$CheckItem = $null, # param($Version,$QueryId) -> @{ HttpStatus } + [scriptblock]$CreateQuery = $null, # param($Version,$DisplayName) -> @{ HttpStatus; QueryId; Ambiguous } + [scriptblock]$PersistReplacementIntent = $null, # param($State) -> durably persist ReplacementAttempted; fired ONCE, immediately before the sole replacement POST + [string]$EffectiveVersion = $null, + [string]$AlternateVersion = $null, + [int]$MaxConsistencyRetries = 3 + ) + if ([string]::IsNullOrWhiteSpace($EffectiveVersion)) { $EffectiveVersion = $script:GraphAuditApiVersion_Previous } + if ([string]::IsNullOrWhiteSpace($AlternateVersion)) { + $AlternateVersion = if ($EffectiveVersion -eq $script:GraphAuditApiVersion_Previous) { $script:GraphAuditApiVersion_Current } else { $script:GraphAuditApiVersion_Previous } + } + $exactMatch = { + param($items) + @($items | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $DisplayName, [System.StringComparison]::Ordinal) }) + } + # 1) Fully-paginated exact-name lookup (established ordinal case semantics). + $all = @(& $ListQueries) + $matches = @(& $exactMatch $all) + if ($matches.Count -gt 1) { + # Multiple exact matches => fail closed. Never choose arbitrarily; never POST. + $State.FailureCategory = 'Other' + return @{ Action = 'FailClosed'; Reason = 'MultipleExactMatches'; MatchCount = $matches.Count; QueryId = $null } + } + if ($matches.Count -eq 1) { + # Exactly one existing exact match => adopt; no POST. + return @{ Action = 'Adopt'; Reason = 'ExactNameMatch'; QueryId = [string]$matches[0].id } + } + # Zero exact-name matches. + if (-not [string]::IsNullOrWhiteSpace($ExistingQueryId)) { + # Replacement path: only after positively-confirmed absence. + $absent = $true + if ($CheckItem) { + foreach ($v in @($EffectiveVersion, $AlternateVersion | Select-Object -Unique)) { + if ([string]::IsNullOrWhiteSpace($v)) { continue } + for ($i = 0; $i -le $MaxConsistencyRetries; $i++) { + $ci = & $CheckItem $v $ExistingQueryId + $cs = if ($ci) { $ci.HttpStatus } else { $null } + if ($null -ne $cs -and $cs -ge 200 -and $cs -le 299) { $absent = $false; break } + if ($cs -eq 404) { if ($i -ge $MaxConsistencyRetries) { break } else { continue } } + # Non-404, non-2xx (auth/throttle/5xx/network): cannot confirm absence. + $absent = $false; break + } + if (-not $absent) { break } + } + } + if (-not $absent) { + # The id is still visible somewhere => retain it, never replace. + $State.AlternateVersionChecked = $true + $State.AlternateVersionResult = 'Visible' + return @{ Action = 'Adopt'; Reason = 'StillVisible'; QueryId = $ExistingQueryId } + } + # Positively confirmed absent: both-version 404 + exact-name miss. + $State.QueryConfirmedAbsent = $true + if ($State.ReplacementAttempted) { + # At-most-once: a replacement was already persisted; never create again. + return @{ Action = 'NoReplacement'; Reason = 'ReplacementAlreadyAttempted'; QueryId = $null } + } + # Persist ReplacementAttempted BEFORE creating, so a crash/resume cannot double-create. + # Durable persistence is delegated to the caller-supplied intent callback and runs + # ONLY here: after confirmed absence + exact-name miss, immediately before the sole + # replacement POST (never for an Adopt/FailClosed/StillVisible outcome that POSTs nothing). + $State.ReplacementAttempted = $true + if ($PersistReplacementIntent) { & $PersistReplacementIntent $State } + if ($CreateQuery) { + $cr = & $CreateQuery $EffectiveVersion $DisplayName + if ($cr -and $cr.Ambiguous) { + $again = @(& $exactMatch (@(& $ListQueries))) + if ($again.Count -eq 1) { return @{ Action = 'Adopt'; Reason = 'AmbiguousResolvedByName'; QueryId = [string]$again[0].id } } + if ($again.Count -gt 1) { $State.FailureCategory = 'Other'; return @{ Action = 'FailClosed'; Reason = 'MultipleExactMatches'; MatchCount = $again.Count; QueryId = $null } } + return @{ Action = 'NoReplacement'; Reason = 'AmbiguousUnresolved'; QueryId = $null } + } + $newId = if ($cr) { [string]$cr.QueryId } else { $null } + $State.ReplacementQueryId = $newId + return @{ Action = 'Replaced'; Reason = 'ConfirmedAbsent'; QueryId = $newId } + } + return @{ Action = 'NoReplacement'; Reason = 'NoCreateCallback'; QueryId = $null } + } + # No existing id and no exact-name match => initial creation (beta). Still guard + # against duplicates via exact-name lookup + ambiguous-POST reconciliation. + if ($CreateQuery) { + $cr = & $CreateQuery $EffectiveVersion $DisplayName + if ($cr -and $cr.Ambiguous) { + $again = @(& $exactMatch (@(& $ListQueries))) + if ($again.Count -eq 1) { return @{ Action = 'Adopt'; Reason = 'AmbiguousResolvedByName'; QueryId = [string]$again[0].id } } + if ($again.Count -gt 1) { $State.FailureCategory = 'Other'; return @{ Action = 'FailClosed'; Reason = 'MultipleExactMatches'; MatchCount = $again.Count; QueryId = $null } } + return @{ Action = 'NoReplacement'; Reason = 'AmbiguousUnresolved'; QueryId = $null } + } + $newId = if ($cr) { [string]$cr.QueryId } else { $null } + return @{ Action = 'Created'; Reason = 'InitialCreate'; QueryId = $newId } + } + return @{ Action = 'NoReplacement'; Reason = 'NoCreateCallback'; QueryId = $null } +} + +function New-GraphAuditQueryFingerprint { + <# + .SYNOPSIS + Deterministic short contract fingerprint for a partition audit query. + .DESCRIPTION + Derives a stable hex fingerprint from the CANONICAL creation contract - the + UTC start/end window plus the operation, record-type, and service filters - + so two partitions that share a date window and partition index but differ in + ANY filter yield DIFFERENT fingerprints (and therefore different display + names). Each string array is normalized identically before hashing (null and + blank dropped, trimmed, invariant upper-cased, ordinal-sorted, joined with a + fixed unit separator); the two dates are rendered in invariant UTC; the parts + are concatenated in a fixed order with a fixed record separator; the whole is + hashed with SHA256 and the first twelve lower-case hex characters are returned. + The same contract always yields the same fingerprint regardless of input + order, element casing, or surrounding whitespace, so initial create, + replacement, checkpoint naming, and exact-name reconciliation all derive one + identity from the same producer. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + $Activities, + $RecordTypes, + $ServiceFilter + ) + $normArray = { + param($arr) + if ($null -eq $arr) { return '' } + $vals = @(@($arr) | ForEach-Object { if ($null -eq $_) { '' } else { ([string]$_).Trim().ToUpperInvariant() } } | Where-Object { $_ -ne '' }) + if ($vals.Count -eq 0) { return '' } + $arrSorted = [string[]]$vals + [System.Array]::Sort($arrSorted, [System.StringComparer]::Ordinal) + return ($arrSorted -join ([char]0x1F)) + } + $svc = if ($null -eq $ServiceFilter) { '' } else { ([string]$ServiceFilter).Trim().ToUpperInvariant() } + $parts = @( + $PStart.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture), + $PEnd.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture), + (& $normArray $Activities), + (& $normArray $RecordTypes), + $svc + ) + $canonical = $parts -join ([char]0x1E) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $hash = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($canonical)) + } + finally { $sha.Dispose() } + $hex = -join ($hash | ForEach-Object { $_.ToString('x2') }) + return $hex.Substring(0, 12) +} + +function Get-PaxCheckpointEntryFingerprint { + <# + .SYNOPSIS + Returns the durable contract fingerprint carried by a checkpoint bucket entry, or + $null when the entry predates fingerprint identity (a legacy checkpoint). + .DESCRIPTION + A fingerprinted entry stores the stable New-GraphAuditQueryFingerprint value in a + top-level `fingerprint` field; durable uncertain-create entries additionally carry it + inside their structured UncertainCreateInfo. This reader is tolerant of both the + in-memory hashtable shape and the ConvertFrom-Json object shape and NEVER throws on a + missing field (a legacy entry yields $null so callers fall back to the historical + index-only identity). + #> + param($Entry) + if ($null -eq $Entry) { return $null } + $fp = $Entry.fingerprint + if (-not [string]::IsNullOrWhiteSpace([string]$fp)) { return [string]$fp } + $uci = $Entry.UncertainCreateInfo + if ($uci) { + $ufp = $uci.fingerprint + if (-not [string]::IsNullOrWhiteSpace([string]$ufp)) { return [string]$ufp } + } + return $null +} + +function Get-PaxEntryIdentityKey { + <# + .SYNOPSIS + Bucket-exclusivity identity key for a checkpoint entry: its fingerprint when present + (the durable per-contract identity), otherwise its integer index (legacy fallback). + #> + param($Entry) + $fp = Get-PaxCheckpointEntryFingerprint $Entry + if (-not [string]::IsNullOrWhiteSpace($fp)) { return "fp:$fp" } + if ($null -ne $Entry -and $null -ne $Entry.index) { return "idx:$([int]$Entry.index)" } + return $null +} + +function Test-PaxCheckpointIdentityMatch { + <# + .SYNOPSIS + $true when a checkpoint bucket entry denotes the SAME durable partition identity as + the supplied (Index, Fingerprint). + .DESCRIPTION + Fingerprint is the authoritative identity: when BOTH the entry and the query carry a + non-empty fingerprint, ONLY equal fingerprints match, so two workload-pass contracts + that reuse the same partition index remain distinct identities and never collide. + Defect 2 (fail-closed identity): fingerprint is the SOLE durable identity in the + resume-classification and persistence paths. A legacy (fingerprint-less) checkpoint + entry is migrated to a proven-unique fingerprint by the pre-auth legacy-mapping + resolver (Invoke-PaxResumeLegacyContractMapping) BEFORE any partition is classified + or persisted, so an un-migrated legacy entry - or an identity-less caller - reaching + this matcher is NEVER silently index-matched onto a fingerprinted contract. Index-only + "matching" is exactly the cross-workload-pass collision this defect closes, so when a + fingerprint is not available on both sides this matcher fails closed (returns $false). + #> + param($Entry, [int]$Index, [string]$Fingerprint) + if ($null -eq $Entry) { return $false } + $efp = Get-PaxCheckpointEntryFingerprint $Entry + if (-not [string]::IsNullOrWhiteSpace($Fingerprint) -and -not [string]::IsNullOrWhiteSpace($efp)) { + return ($efp -eq [string]$Fingerprint) + } + # Tightened (defect 2, Option 1): no permissive index fallback. Legacy entries are + # migrated to a durable fingerprint pre-auth; an un-migrated legacy entry (or a caller + # with no contract fingerprint in hand) must not be silently index-matched onto a + # fingerprinted contract. Fail closed. + return $false +} + +function Test-PaxLegacyContractCandidate { + <# + .SYNOPSIS + $true when a current partition contract is a candidate match for a legacy + (fingerprint-less) checkpoint bucket entry, under the read-only pre-auth + legacy-mapping resolver. + .DESCRIPTION + A legacy entry identifies its partition only by integer index plus its saved + start/end window (and, when present, its stored contract fields). This tester matches + a reconstructed current contract to that entry by: + * integer partition index (when the entry carries one), AND + * exact UTC-instant equality of the start and end window, AND + * every stored contract field the entry actually carries (serviceFilter/service, + recordTypes) - a field absent from the legacy entry is not used to reject. + It NEVER computes or trusts a fingerprint; disambiguation among several contracts that + share a window/index is deliberately impossible here, so such a legacy entry yields + more than one candidate and the resolver fails closed rather than guessing. + #> + param($Entry, $Index, $Start, $End, $Contract) + if ($null -eq $Contract) { return $false } + if ($null -ne $Index -and $null -ne $Contract.Index -and ([int]$Contract.Index -ne [int]$Index)) { return $false } + if ($Start -and $Contract.PStart) { + if (([datetime]$Contract.PStart).ToUniversalTime() -ne ([datetime]$Start).ToUniversalTime()) { return $false } + } + if ($End -and $Contract.PEnd) { + if (([datetime]$Contract.PEnd).ToUniversalTime() -ne ([datetime]$End).ToUniversalTime()) { return $false } + } + $eSvc = $Entry.serviceFilter + if ($null -eq $eSvc) { $eSvc = $Entry.service } + if ($null -ne $eSvc -and -not [string]::IsNullOrWhiteSpace([string]$eSvc)) { + if ([string]$eSvc -ne [string]$Contract.ServiceFilter) { return $false } + } + $eRt = $Entry.recordTypes + if ($null -ne $eRt) { + $eRtJoin = (@($eRt) | Sort-Object) -join '|' + $cRtJoin = (@($Contract.RecordTypes) | Sort-Object) -join '|' + if ($eRtJoin -ne $cRtJoin) { return $false } + } + return $true +} + +function Test-PaxCheckpointEntryCompatible { + <# + .SYNOPSIS + $true when two checkpoint bucket entries that claim the SAME durable contract identity + carry PROVABLY equivalent stored metadata, so one may supersede the other inside a + single atomic migration. $false whenever equivalence cannot be proven. + .DESCRIPTION + Bucket exclusivity (completed > queryCreated > uncertainCreate) may only be enforced by + REMOVING a lower-precedence duplicate when that duplicate is provably the SAME work. A + shared fingerprint alone is NOT that evidence: two entries can share a contract + fingerprint yet carry different server query ids, indexes, windows, or filters, which + means the checkpoint records genuinely different state and silently discarding either + would lose it. + + Every field is compared ONLY when BOTH entries carry a usable value - an absent or + blank field on a legacy entry is not evidence of a conflict. Any field present on both + sides that DIFFERS (server query id, partition index, start / end instant, record + types, or service filter) is a CONFLICT and yields $false so the caller fails closed + instead of guessing. A start / end value present on both sides that cannot be parsed on + either side is likewise unproven, and therefore a conflict - never a match. + #> + param($A, $B) + if ($null -eq $A -or $null -eq $B) { return $false } + $looseEq = { + param($x, $y) + if ($null -eq $x -or $null -eq $y) { return $true } + $xs = ([string]$x).Trim(); $ys = ([string]$y).Trim() + if ($xs -eq '' -or $ys -eq '') { return $true } + return ([string]::Equals($xs, $ys, [System.StringComparison]::OrdinalIgnoreCase)) + } + if (-not (& $looseEq $A.queryId $B.queryId)) { return $false } + if ($null -ne $A.index -and $null -ne $B.index) { + if ([int]$A.index -ne [int]$B.index) { return $false } + } + foreach ($fld in @('start', 'end')) { + $av = $A.$fld + $bv = $B.$fld + if ([string]::IsNullOrWhiteSpace([string]$av) -or [string]::IsNullOrWhiteSpace([string]$bv)) { continue } + $ad = script:Parse-DateSafe $av + $bd = script:Parse-DateSafe $bv + if ($null -eq $ad -or $null -eq $bd) { return $false } + if (([datetime]$ad).ToUniversalTime() -ne ([datetime]$bd).ToUniversalTime()) { return $false } + } + if ($null -ne $A.recordTypes -and $null -ne $B.recordTypes) { + $aRt = (@($A.recordTypes) | Sort-Object) -join '|' + $bRt = (@($B.recordTypes) | Sort-Object) -join '|' + if ($aRt -ne $bRt) { return $false } + } + $aSvc = if ($null -ne $A.serviceFilter) { $A.serviceFilter } else { $A.service } + $bSvc = if ($null -ne $B.serviceFilter) { $B.serviceFilter } else { $B.service } + if (-not (& $looseEq $aSvc $bSvc)) { return $false } + return $true +} + +function Get-PaxResumeCurrentContractSet { + <# + .SYNOPSIS + READ-ONLY reconstruction of the COMPLETE set of current partition contracts across + ALL serviceRuns x query groups, from restored checkpoint parameters. No Graph call, + no authentication, no mutation. + .DESCRIPTION + Deterministically replays the main partition build (serviceRuns split, per-workload + operation/recordType alignment, partition-hours defaulting + MaxPartitions capping, + Get-QueryPlan grouping, and per-group time slicing) so a legacy (fingerprint-less) + checkpoint entry can be matched to exactly one durable contract BEFORE authentication. + It reproduces the single-partition variable-carry quirk of the main build byte-for-byte: + $partitionRecordTypes / $partitionServiceFilter are assigned ONLY inside the + multi-partition branch and otherwise carry across groups and service passes, so a + single-partition group inherits the prior multi-partition group's record/service + values exactly as the live build does. The normal per-group build remains the sole + producer of the contracts actually processed; this helper only ENUMERATES them. + #> + param( + [datetime]$StartDateObj, + [datetime]$EndDateObj, + [string[]]$ActivityTypes, + [string[]]$RecordTypes, + [string[]]$ServiceTypes, + [int]$PartitionHours = 0, + [int]$MaxPartitions = 50, + [int]$MaxConcurrency = 10, + [switch]$IncludeM365Usage, + [switch]$UseEOM, + [hashtable]$ServiceOperationMap, + [hashtable]$RecordTypeWorkloadMap + ) + $contracts = New-Object System.Collections.ArrayList + if (-not $UseEOM -and $ServiceTypes -and $ServiceTypes.Count -gt 0) { $serviceRuns = $ServiceTypes } else { $serviceRuns = @($null) } + # Variable-carry quirk (mirrors main build): declared outside both loops and assigned + # ONLY in the multi-partition branch, so a single-partition group carries the prior value. + $partitionRecordTypes = $null + $partitionServiceFilter = $null + foreach ($currentServiceFilter in $serviceRuns) { + $serviceActivities = $ActivityTypes + if ($currentServiceFilter -and $ServiceOperationMap -and $ServiceOperationMap.ContainsKey($currentServiceFilter)) { + $mappedOps = @($ServiceOperationMap[$currentServiceFilter] | Where-Object { $ActivityTypes -contains $_ }) + if ($mappedOps.Count -gt 0) { $serviceActivities = $mappedOps } + } + $serviceRecordTypes = $RecordTypes + if ($currentServiceFilter -and $RecordTypes) { + $matched = @() + foreach ($rt in $RecordTypes) { + $rtServices = $RecordTypeWorkloadMap[$rt] + if ($null -eq $rtServices -or $rtServices.Count -eq 0) { $matched += $rt; continue } + if ($rtServices -contains $currentServiceFilter) { $matched += $rt } + } + if ($matched.Count -gt 0) { $serviceRecordTypes = $matched | Select-Object -Unique } else { $serviceRecordTypes = $null } + } + $queryPlan = @(Get-QueryPlan -RequestedActivities $serviceActivities) + foreach ($grp in $queryPlan) { + $effectivePartitionHours = $PartitionHours + $userSpecifiedPartitionHours = ($PartitionHours -gt 0) + if (-not $UseEOM -and $effectivePartitionHours -eq 0) { $effectivePartitionHours = 12 } + if (-not $UseEOM -and $effectivePartitionHours -gt 0) { + $totalHours = ($EndDateObj - $StartDateObj).TotalHours + $calculatedPartitions = [Math]::Ceiling($totalHours / $effectivePartitionHours) + if ($calculatedPartitions -gt $MaxPartitions) { + if (-not $userSpecifiedPartitionHours) { + $effectivePartitionHours = 12 + $calculatedPartitions = [Math]::Ceiling($totalHours / $effectivePartitionHours) + } + if ($calculatedPartitions -gt $MaxPartitions) { + $calculatedPartitions = $MaxPartitions + $effectivePartitionHours = $totalHours / $calculatedPartitions + } + } + $degree = $calculatedPartitions + } else { + $degree = [Math]::Min($grp.Concurrency, $MaxConcurrency) + } + $totalPartitions = $degree + $activities = $grp.Activities + $activity = $grp.Activities[0] + if ($totalPartitions -gt 1) { + $totalHours = ($EndDateObj - $StartDateObj).TotalHours + if (-not $UseEOM -and $effectivePartitionHours -gt 0) { $sliceHours = $effectivePartitionHours } else { $sliceHours = $totalHours / $totalPartitions } + $partitionRecordTypes = $serviceRecordTypes + $partitionServiceFilter = $currentServiceFilter + if ($IncludeM365Usage) { $partitionRecordTypes = $null; $partitionServiceFilter = $null } + for ($pi = 0; $pi -lt $totalPartitions; $pi++) { + $pStart = $StartDateObj.AddHours($sliceHours * $pi) + $pEnd = if ($pi -eq ($totalPartitions - 1)) { $EndDateObj } else { $StartDateObj.AddHours($sliceHours * ($pi + 1)) } + [void]$contracts.Add([pscustomobject]@{ + Activities = $activities + Activity = $activity + PStart = $pStart + PEnd = $pEnd + Index = ($pi + 1) + Total = $totalPartitions + RecordTypes = $partitionRecordTypes + ServiceFilter = $partitionServiceFilter + }) + } + } else { + [void]$contracts.Add([pscustomobject]@{ + Activities = $activities + Activity = $activity + PStart = $StartDateObj + PEnd = $EndDateObj + Index = 1 + Total = 1 + RecordTypes = $partitionRecordTypes + ServiceFilter = $partitionServiceFilter + }) + } + } + } + return @($contracts.ToArray()) +} + +function Invoke-PaxResumeLegacyContractMapping { + <# + .SYNOPSIS + Pre-auth, read-only cross-run legacy-mapping resolver (defect 2). Migrates every + legacy (fingerprint-less) checkpoint bucket entry to a proven-unique durable + fingerprint, or fails closed WITHOUT authentication or mutation. + .DESCRIPTION + Runs on resume BEFORE authentication and the per-workload partition build. For each + legacy entry (completed / queryCreated / uncertainCreate WITHOUT a fingerprint) it + finds candidate current contracts (Test-PaxLegacyContractCandidate) from the complete + reconstructed set. EXACTLY ONE candidate stages a unique mapping (the contract's + New-GraphAuditQueryFingerprint); ZERO fails closed (never discard/recreate); MORE THAN + ONE fails closed as LegacyContractAmbiguous. + + BUCKET EXCLUSIVITY. A contract identity may occupy EXACTLY ONE lifecycle bucket, under + the precedence completed > queryCreated > uncertainCreate. Every candidate fingerprint + is therefore checked against ALL THREE buckets, not only its source bucket. A collision + is resolved only with EXPLICIT evidence (Test-PaxCheckpointEntryCompatible proves the + stored query id / index / window / record types / service filter are equivalent): a + legacy row already owned by a HIGHER-precedence entry is classified superseded and + removed as part of this same atomic migration, a LOWER-precedence duplicate of a + migrated row is normalized in the same transaction, a same-bucket collision stays + LegacyContractAmbiguous, and any CONFLICTING metadata fails closed as + LegacyContractConflict rather than guessing. Residual cross-bucket duplicates among + untouched identities are normalized under the same rule, so the resume classifier never + depends on a normalization that happens later. + + The ENTIRE migration + normalization plan is validated first; only on full success are + fingerprints stamped in-memory and published ATOMICALLY and EXACTLY ONCE (via the + supplied -Publish block, default Save-CheckpointStrict) BEFORE any fetch/reconcile/ + create. Publication must be PROVEN durable - an explicit $true, or a throw on failure; + any other outcome restores the exact pre-migration in-memory state (bucket membership + AND every fingerprint stamp), leaves the durable checkpoint byte-for-byte unchanged, + latches $script:GenericFatal, and throws PAX-FATAL. On any zero/ambiguous/collision/ + conflict outcome it likewise latches $script:GenericFatal and + throws a PAX-FATAL error so the caller can emit actionable output and exit nonzero with + no auth and no mutation. Returns a status object on success or when there is nothing to + migrate. + #> + param( + [Parameter(Mandatory = $true)]$CheckpointData, + [object[]]$CurrentContracts = @(), + [scriptblock]$Publish + ) + if ($null -eq $CheckpointData -or $null -eq $CheckpointData.partitions) { + return [pscustomobject]@{ Status = 'NoLegacy'; Migrated = 0; Reason = 'no checkpoint partitions present' } + } + $buckets = @('completed', 'queryCreated', 'uncertainCreate') + # Lifecycle precedence: a single contract identity may occupy EXACTLY ONE bucket, and a + # higher-precedence record always supersedes a lower-precedence one. + $rankOf = @{ completed = 3; queryCreated = 2; uncertainCreate = 1 } + $legacy = New-Object System.Collections.ArrayList + foreach ($b in $buckets) { + foreach ($e in @($CheckpointData.partitions.$b)) { + if ($null -eq $e) { continue } + $efp = Get-PaxCheckpointEntryFingerprint $e + if ([string]::IsNullOrWhiteSpace($efp)) { [void]$legacy.Add([pscustomobject]@{ Bucket = $b; Entry = $e }) } + } + } + if ($legacy.Count -eq 0) { + return [pscustomobject]@{ Status = 'NoLegacy'; Migrated = 0; Reason = 'all checkpoint entries already carry a durable fingerprint' } + } + # Validate the ENTIRE migration first - no mutation until every legacy entry maps uniquely. + $claimed = @{} + $stage = New-Object System.Collections.ArrayList + foreach ($item in $legacy) { + $e = $item.Entry + $idx = if ($null -ne $e.index) { [int]$e.index } else { $null } + $eStart = if ($e.start) { script:Parse-DateSafe $e.start } else { $null } + $eEnd = if ($e.end) { script:Parse-DateSafe $e.end } else { $null } + $cands = @($CurrentContracts | Where-Object { Test-PaxLegacyContractCandidate -Entry $e -Index $idx -Start $eStart -End $eEnd -Contract $_ }) + $where = "index $idx, window $($e.start)..$($e.end), bucket $($item.Bucket)" + if ($cands.Count -eq 0) { + $script:GenericFatal = $true + throw "PAX-FATAL-LEGACY-NOMATCH: a legacy (fingerprint-less) checkpoint entry ($where) matches NONE of the current partition contracts. Refusing to discard or recreate it. Confirm the resume parameters (date range, activity/record/service filters, partition hours) match the original run, then retry." + } + if ($cands.Count -gt 1) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractAmbiguous: a legacy (fingerprint-less) checkpoint entry ($where) matches $($cands.Count) current partition contracts and cannot be assigned a durable identity safely. This checkpoint predates per-contract fingerprints and spans multiple workload contracts at one index. Re-run the original collection to completion, or narrow -ServiceTypes/-RecordTypes so the entry maps to a single contract." + } + $c = $cands[0] + $fp = New-GraphAuditQueryFingerprint -PStart $c.PStart -PEnd $c.PEnd -Activities $c.Activities -RecordTypes $c.RecordTypes -ServiceFilter $c.ServiceFilter + if ([string]::IsNullOrWhiteSpace($fp)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LEGACY-FP: could not derive a durable fingerprint for the current contract matching a legacy checkpoint entry ($where)." + } + if ($claimed.ContainsKey($fp)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractAmbiguous: two legacy checkpoint entries resolve to the same current contract (fingerprint $fp); the mapping is not one-to-one. Re-run the original collection to completion." + } + # Bucket exclusivity is validated across ALL THREE lifecycle buckets, never only the + # SOURCE bucket: a fingerprint may occupy EXACTLY ONE bucket. Every existing owner of + # this candidate identity is located, its stored metadata is compared for PROVABLE + # equivalence, and the outcome is resolved by precedence - all BEFORE any mutation. + $owners = New-Object System.Collections.ArrayList + foreach ($ob in $buckets) { + foreach ($oe in @($CheckpointData.partitions.$ob)) { + if ($null -eq $oe) { continue } + if ([object]::ReferenceEquals($oe, $e)) { continue } + if ((Get-PaxCheckpointEntryFingerprint $oe) -eq $fp) { + [void]$owners.Add([pscustomobject]@{ Bucket = $ob; Entry = $oe }) + } + } + } + $legacyRank = [int]$rankOf[$item.Bucket] + $supersededLegacy = $false + $demoted = New-Object System.Collections.ArrayList + foreach ($o in $owners) { + if (-not (Test-PaxCheckpointEntryCompatible $e $o.Entry)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractConflict: migrating a legacy checkpoint entry ($where) to fingerprint $fp would collide with an existing '$($o.Bucket)' entry whose stored query id, partition index, window, or contract metadata DIFFERS. The two records are not provably the same work, so neither may be discarded. Re-run the original collection to completion, or narrow -ServiceTypes/-RecordTypes so the entry maps to a single contract." + } + $ownerRank = [int]$rankOf[$o.Bucket] + if ($ownerRank -eq $legacyRank) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractAmbiguous: migrating a legacy checkpoint entry ($where) to fingerprint $fp would collide with an existing durable entry in the same bucket. Re-run the original collection to completion." + } + if ($ownerRank -gt $legacyRank) { $supersededLegacy = $true } + else { [void]$demoted.Add($o) } + } + $claimed[$fp] = $idx + if ($supersededLegacy) { + # A HIGHER-precedence entry already owns this contract and is PROVABLY the same work, + # so the legacy lower-precedence row is superseded. It is removed ONLY as part of the + # atomic migration and is never stamped into a second bucket. + [void]$stage.Add([pscustomobject]@{ Entry = $e; Fingerprint = $fp; Bucket = $item.Bucket; Action = 'Supersede'; Demoted = @() }) + } + else { + # No owner, or only LOWER-precedence owners: stamp the legacy entry and normalize + # every lower-precedence duplicate inside the SAME atomic transaction. + [void]$stage.Add([pscustomobject]@{ Entry = $e; Fingerprint = $fp; Bucket = $item.Bucket; Action = 'Migrate'; Demoted = @($demoted.ToArray()) }) + } + } + # Residual cross-bucket duplicates among identities this migration does not touch are + # resolved under the SAME precedence and the SAME evidence rule, planned BEFORE any + # mutation so a fail-closed outcome still leaves the checkpoint untouched. This is what + # lets Get-PartitionsToProcess classify from an already-exclusive checkpoint instead of + # depending on a normalization that happens later. + $existingByFp = @{} + foreach ($b in $buckets) { + foreach ($xe in @($CheckpointData.partitions.$b)) { + if ($null -eq $xe) { continue } + $xfp = Get-PaxCheckpointEntryFingerprint $xe + if ([string]::IsNullOrWhiteSpace($xfp)) { continue } + if (-not $existingByFp.ContainsKey($xfp)) { $existingByFp[$xfp] = New-Object System.Collections.ArrayList } + [void]$existingByFp[$xfp].Add([pscustomobject]@{ Bucket = $b; Entry = $xe }) + } + } + $residual = New-Object System.Collections.ArrayList + foreach ($k in @($existingByFp.Keys)) { + $grp = @($existingByFp[$k].ToArray()) + if ($grp.Count -le 1) { continue } + if (@($grp | ForEach-Object { $_.Bucket } | Select-Object -Unique).Count -le 1) { continue } + $winner = @($grp | Sort-Object @{ Expression = { [int]$rankOf[$_.Bucket] }; Descending = $true })[0] + foreach ($g in $grp) { + if ([object]::ReferenceEquals($g.Entry, $winner.Entry)) { continue } + if ([int]$rankOf[$g.Bucket] -eq [int]$rankOf[$winner.Bucket]) { continue } + if (-not (Test-PaxCheckpointEntryCompatible $winner.Entry $g.Entry)) { + $script:GenericFatal = $true + throw "PAX-FATAL-LegacyContractConflict: contract $k is recorded in both the '$($winner.Bucket)' and '$($g.Bucket)' lifecycle buckets with DIFFERING stored query id, partition index, window, or contract metadata, so neither record may be discarded. Re-run the original collection to completion." + } + [void]$residual.Add([pscustomobject]@{ Bucket = $g.Bucket; Entry = $g.Entry }) + } + } + # Full migration + normalization validated. Snapshot every lifecycle bucket and the exact + # set of entries about to be stamped so an unproven publication rolls back to the EXACT + # pre-migration in-memory state, then mutate and publish EXACTLY ONCE. + $snapCompleted = @($CheckpointData.partitions.completed) + $snapQueryCreated = @($CheckpointData.partitions.queryCreated) + $snapUncertain = @($CheckpointData.partitions.uncertainCreate) + $stamped = New-Object System.Collections.ArrayList + $removals = New-Object System.Collections.ArrayList + foreach ($s in $stage) { + if ($s.Action -eq 'Supersede') { + [void]$removals.Add([pscustomobject]@{ Bucket = $s.Bucket; Entry = $s.Entry }) + continue + } + if ($s.Entry -is [System.Collections.IDictionary]) { $s.Entry['fingerprint'] = $s.Fingerprint } + else { $s.Entry | Add-Member -NotePropertyName fingerprint -NotePropertyValue $s.Fingerprint -Force } + [void]$stamped.Add($s.Entry) + foreach ($d in @($s.Demoted)) { if ($null -ne $d) { [void]$removals.Add([pscustomobject]@{ Bucket = $d.Bucket; Entry = $d.Entry }) } } + } + foreach ($r in @($residual.ToArray())) { [void]$removals.Add($r) } + foreach ($b in $buckets) { + $drop = @(@($removals.ToArray()) | Where-Object { $_.Bucket -eq $b } | ForEach-Object { $_.Entry }) + if ($drop.Count -eq 0) { continue } + $CheckpointData.partitions.$b = @(@($CheckpointData.partitions.$b) | Where-Object { + $candidate = $_ + $keep = $true + foreach ($d in $drop) { if ([object]::ReferenceEquals($candidate, $d)) { $keep = $false; break } } + $keep + }) + } + # Publish EXACTLY ONCE. Publication must be PROVEN durable (an explicit $true, or a throw + # on failure); anything else is a failed publication and rolls the checkpoint back. + $published = $false + $publishErr = $null + try { + if ($Publish) { + $publishResult = @(& $Publish) | Select-Object -Last 1 + $published = ($publishResult -is [bool]) -and ([bool]$publishResult) + } + elseif (Get-Command Save-CheckpointStrict -ErrorAction SilentlyContinue) { + $published = [bool](Save-CheckpointStrict) + } + } + catch { $publishErr = $_ } + if (-not $published) { + # Fail closed: restore the EXACT pre-migration in-memory state (bucket membership AND + # every fingerprint stamp). The strict publisher leaves the durable copy byte-for-byte + # unchanged, so memory and disk agree again and no fetch, reconcile, or create is + # authorized on a migration that was never proven durable. + $CheckpointData.partitions.completed = $snapCompleted + $CheckpointData.partitions.queryCreated = $snapQueryCreated + $CheckpointData.partitions.uncertainCreate = $snapUncertain + foreach ($se in @($stamped.ToArray())) { + if ($se -is [System.Collections.IDictionary]) { + if ($se.Contains('fingerprint')) { [void]$se.Remove('fingerprint') } + } + elseif ($null -ne $se.PSObject.Properties['fingerprint']) { + $se.PSObject.Properties.Remove('fingerprint') + } + } + $script:GenericFatal = $true + $publishDetail = if ($publishErr) { [string]$publishErr.Exception.Message } else { 'the publication callback did not return a proven durable publication.' } + if ($publishDetail -match 'PAX-FATAL-ROLLBACK') { + throw "PAX-FATAL-ROLLBACK: durable publication of the legacy contract migration failed AND the prior checkpoint could not be restored; disk state is indeterminate - manual checkpoint inspection required. $publishDetail" + } + throw "PAX-FATAL: durable publication of the legacy contract migration failed; fail-closed, the checkpoint is unchanged and no fetch, reconcile, or create is authorized. $publishDetail" + } + return [pscustomobject]@{ Status = 'Migrated'; Migrated = $stamped.Count; Superseded = @($stage | Where-Object { $_.Action -eq 'Supersede' }).Count; Normalized = @($removals.ToArray()).Count; Reason = 'legacy entries migrated to proven-unique fingerprints, normalized to one lifecycle bucket per contract, and published atomically' } +} + +function New-GraphAuditPartitionDisplayName { + <# + .SYNOPSIS + Single deterministic producer of a partition query displayName. + .DESCRIPTION + Every path that names a partition query (parent create/replacement, + exact-name lookup, checkpoint, and completion/retry display) routes through + this one function so the same partition always yields a byte-identical name. + The end boundary is exclusive, so the name shows the last INCLUDED minute + (PEnd minus one minute), matching the Purview UI and the created query. + + Identity is contract-inclusive: when the creation contract (Activities, + RecordTypes, ServiceFilter) or an explicit -Fingerprint is supplied, a + deterministic fingerprint of that contract is appended so two partitions with + the same date window and partition index but a DIFFERENT operation, record + type, service, or date filter produce different names and cannot cross-adopt. + An explicit -Fingerprint wins; otherwise the fingerprint is derived here from + the supplied contract through the single fingerprint producer. With no contract + and no fingerprint the legacy (fingerprint-less) name is returned unchanged, so + a pre-fingerprint checkpoint name can still be reproduced for legacy recovery. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + [Parameter(Mandatory)]$Index, + [Parameter(Mandatory)]$Total, + $Activities, + $RecordTypes, + $ServiceFilter, + [string]$Fingerprint + ) + $pEndDisplay = $PEnd.AddMinutes(-1) + $base = "PAX_Query_$($PStart.ToString('yyyyMMdd_HHmm'))-$($pEndDisplay.ToString('yyyyMMdd_HHmm'))_Part$Index/$Total" + $fp = $Fingerprint + if ([string]::IsNullOrWhiteSpace($fp) -and ($PSBoundParameters.ContainsKey('Activities') -or $PSBoundParameters.ContainsKey('RecordTypes') -or $PSBoundParameters.ContainsKey('ServiceFilter'))) { + $fp = New-GraphAuditQueryFingerprint -PStart $PStart -PEnd $PEnd -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + } + if (-not [string]::IsNullOrWhiteSpace($fp)) { return "${base}_$fp" } + return $base +} + +function New-GraphAuditQueryRequest { + <# + .SYNOPSIS + Single producer of a partition audit-query request body (filters only). + .DESCRIPTION + Every recovery/replacement create path routes its filter payload through + this one function so an initial create and a replacement create for the same + partition produce EQUIVALENT bodies. Emits filterStartDateTime and + filterEndDateTime in invariant-culture .fffZ, an unconditional + operationFilters array, and the recordTypeFilters / serviceFilter keys gated + exactly as the production create. The displayName is added by the caller (see + New-GraphAuditPartitionDisplayName), so the name and the filters each have a + single source. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + $Activities, + $RecordTypes, + $ServiceFilter + ) + $body = @{ + filterStartDateTime = $PStart.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + filterEndDateTime = $PEnd.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + operationFilters = @($Activities) + } + if ($RecordTypes -and $RecordTypes.Count -gt 0) { $body.recordTypeFilters = @($RecordTypes) } + if ($ServiceFilter) { $body.serviceFilter = $ServiceFilter } + return $body +} + +function Invoke-GraphAuditHardenedCreate { + <# + .SYNOPSIS + Parent-callable hardened creation of a single partition audit query. + .DESCRIPTION + Reproduces the per-partition create robustness in a parent-scope function so + query creation can run in the main runspace (where the token can be + refreshed) before workers are launched. The HTTP transport is injected via + -Invoke so the same logic serves both production (real Invoke-MgGraphRequest / + Invoke-RestMethod) and injected-transport validation. + + The name comes from New-GraphAuditPartitionDisplayName and the filters from + New-GraphAuditQueryRequest, so a parent create and a worker/replacement create + for the same partition are byte-equivalent. Creation always targets the beta + surface; v1.0 creation is disabled. + + Failure handling preserves the established caps exactly: + * transient 403: retry up to 3 times with 15s / 30s / 60s backoff; + * permanent 403 / CAE (InsufficientPrivileges | Authorization_RequestDenied | + AccessDenied | InvalidAuthenticationToken in the body, or a WWW-Authenticate + claims challenge): fast-fail, no retry; + * 429: honour Retry-After when present else exponential 60/120/240 capped at + 300s, abort once the shared attempt counter reaches 20; + * transient network / 5xx: retry inside a time-based outage window + (MaxNetworkOutageMinutes) with a 30-60s randomized delay, then abort; + * an ambiguous create response (name conflict) is reconciled by a + fully-paginated exact-name lookup: exactly one match is adopted, multiple + fail closed, none aborts - never a blind re-POST. + + The injected transport callback receives ($Version, $BodyJson, $ClientRequestId) + and returns @{ HttpStatus; Id; ResponseBody; Headers }. A connectivity failure + is surfaced as HttpStatus 0 (or null) or >= 500. Returns a structured result: + @{ QueryId; FailureCategory; HttpStatus; DisplayName; Version; Adopted }. + #> + param( + [Parameter(Mandatory)][datetime]$PStart, + [Parameter(Mandatory)][datetime]$PEnd, + [Parameter(Mandatory)]$Index, + [Parameter(Mandatory)]$Total, + $Activities, + $RecordTypes, + $ServiceFilter, + [Parameter(Mandatory)][scriptblock]$Invoke, # param($Version,$BodyJson,$ClientRequestId) -> @{ HttpStatus; Id; ResponseBody; Headers } + [string]$Version = $null, # creation always beta; v1.0 disabled + [int]$MaxNetworkOutageMinutes = 30, + [scriptblock]$ListQueries = $null, # -> array of @{ id; displayName } (fully paginated), for ambiguous-POST reconcile + [switch]$PreCreateReconcile, # with $ListQueries, run a fully-paginated exact-name lookup BEFORE the first POST (adopt one, fail closed on many, POST only on a miss) + [switch]$ReconcileOnly, # with $ListQueries, resolve a durable uncertain-create partition READ-ONLY: adopt one exact match, fail closed on many, stay unresolved on none - NEVER a POST + [scriptblock]$FetchItem = $null, # param($QueryId) -> the query item with its filter contract (operationFilters/recordTypeFilters/serviceFilter/filterStartDateTime/filterEndDateTime); required to safely adopt a legacy fingerprint-less same-name match + [scriptblock]$Sleep = $null, # param($Seconds) - injectable delay (defaults to Start-Sleep) + [scriptblock]$Log = $null # param($Message) - optional diagnostic sink + ) + if ([string]::IsNullOrWhiteSpace($Version)) { $Version = $script:GraphAuditApiVersion_Previous } + if (-not $Sleep) { $Sleep = { param($s) Start-Sleep -Seconds $s } } + $emit = { param($m) if ($Log) { & $Log $m } } + + # Contract-inclusive identity: the created name, every exact-name reconcile lookup, + # and the POST body all derive from the SAME contract, so a differing operation / + # record-type / service / date contract yields a different name and cannot cross-adopt. + $displayName = New-GraphAuditPartitionDisplayName -PStart $PStart -PEnd $PEnd -Index $Index -Total $Total -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + $bodyObj = New-GraphAuditQueryRequest -PStart $PStart -PEnd $PEnd -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + $bodyObj.displayName = $displayName + $bodyJson = $bodyObj | ConvertTo-Json -Depth 5 + $clientRequestId = [guid]::NewGuid().ToString() + + $new = { + param($queryId, $category, $status, $adopted, $requestId = $null, $responseBody = $null) + return @{ + QueryId = $queryId + FailureCategory = $category + HttpStatus = $status + DisplayName = $displayName + Version = $Version + Adopted = [bool]$adopted + ClientRequestId = $clientRequestId + RequestId = $requestId + ResponseBody = $responseBody + } + } + + # Reconcile an ambiguous create by exact-name lookup: adopt one, fail closed on + # many, abort on none. Never a blind re-POST. + $reconcile = { + param($status) + if (-not $ListQueries) { return & $new $null 'AmbiguousUnresolved' $status $false } + $all = @(& $ListQueries) + $exact = @($all | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $displayName, [System.StringComparison]::Ordinal) }) + if ($exact.Count -eq 1) { return & $new ([string]$exact[0].id) 'None' $status $true } + if ($exact.Count -gt 1) { return & $new $null 'MultipleExactMatches' $status $false } + return & $new $null 'AmbiguousUnresolved' $status $false + } + + # Exact-name lookup shared by the pre-create and uncertain-outcome reconcile + # paths: an ordinal displayName match against the fully-paginated collection. + $findExactByName = { + $all = @(& $ListQueries) + @($all | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $displayName, [System.StringComparison]::Ordinal) }) + } + + # Legacy fingerprint-less recovery. A pre-fingerprint checkpoint created queries + # whose names lack the contract fingerprint, so a query that matches the LEGACY + # (fingerprint-less) name may actually carry a DIFFERENT operation / record-type / + # service / date contract. Such a match is adopted ONLY after its COMPLETE server + # contract is retrieved and compared to the requested contract; a fingerprinted-name + # match already encodes the contract and never reaches this path. The compare reuses + # the deterministic fingerprint producer over the server fields, so it is stable and + # order/case-insensitive in exactly the same way the created name is. + $legacyDisplayName = New-GraphAuditPartitionDisplayName -PStart $PStart -PEnd $PEnd -Index $Index -Total $Total + $requestedFingerprint = New-GraphAuditQueryFingerprint -PStart $PStart -PEnd $PEnd -Activities $Activities -RecordTypes $RecordTypes -ServiceFilter $ServiceFilter + $serverContractFingerprint = { + param($item) + if ($null -eq $item) { return $null } + $fsRaw = if ($item -is [hashtable]) { $item['filterStartDateTime'] } else { $item.filterStartDateTime } + $feRaw = if ($item -is [hashtable]) { $item['filterEndDateTime'] } else { $item.filterEndDateTime } + if ([string]::IsNullOrWhiteSpace([string]$fsRaw) -or [string]::IsNullOrWhiteSpace([string]$feRaw)) { return $null } + $opsRaw = if ($item -is [hashtable]) { $item['operationFilters'] } else { $item.operationFilters } + $rtRaw = if ($item -is [hashtable]) { $item['recordTypeFilters'] } else { $item.recordTypeFilters } + $svcRaw = if ($item -is [hashtable]) { $item['serviceFilter'] } else { $item.serviceFilter } + $styles = [System.Globalization.DateTimeStyles]::AdjustToUniversal -bor [System.Globalization.DateTimeStyles]::AssumeUniversal + $fs = $null; $fe = $null + try { $fs = [datetime]::Parse([string]$fsRaw, [System.Globalization.CultureInfo]::InvariantCulture, $styles) } catch { return $null } + try { $fe = [datetime]::Parse([string]$feRaw, [System.Globalization.CultureInfo]::InvariantCulture, $styles) } catch { return $null } + return New-GraphAuditQueryFingerprint -PStart $fs -PEnd $fe -Activities $opsRaw -RecordTypes $rtRaw -ServiceFilter $svcRaw + } + # Classify a legacy fingerprint-less same-name candidate set against the requested + # contract. A candidate is adopted ONLY after its COMPLETE server contract is + # retrieved and its deterministic fingerprint matches. A candidate whose contract + # cannot be verified (missing filter fields with no fetch, a failed/throwing fetch, + # or a malformed/incomplete fetched item) is never adopted and forces a fail-closed + # outcome so an unverifiable candidate can never be duplicated by a fresh POST. + # Returns @{ Outcome; Id; Category } where Outcome is None | Match | Mismatch | + # Unresolved. + $findLegacyContractMatch = { + if (-not $ListQueries) { return @{ Outcome = 'None'; Id = $null; Category = $null } } + $all = @(& $ListQueries) + $legacy = @($all | Where-Object { $null -ne $_ -and [string]::Equals([string]$_.displayName, $legacyDisplayName, [System.StringComparison]::Ordinal) }) + if ($legacy.Count -eq 0) { return @{ Outcome = 'None'; Id = $null; Category = $null } } + $matched = New-Object System.Collections.Generic.List[string] + $anyUnverifiable = $false + foreach ($q in $legacy) { + $qid = if ($q -is [hashtable]) { $q['id'] } else { $q.id } + $sfp = & $serverContractFingerprint $q + if (-not $sfp -and $FetchItem) { + # Read-only item fetch with a bounded retry consistent with the create + # retry policy; a fetch that throws or never yields a complete contract + # leaves the candidate unverifiable. + $fetched = $null + $fetchOk = $false + for ($fa = 1; $fa -le 3; $fa++) { + try { $fetched = & $FetchItem ([string]$qid); $fetchOk = $true; break } + catch { $fetched = $null; $fetchOk = $false; if ($fa -lt 3) { & $Sleep 2 } } + } + if ($fetchOk) { $sfp = & $serverContractFingerprint $fetched } + } + if (-not $sfp) { $anyUnverifiable = $true; continue } + if ([string]::Equals([string]$sfp, [string]$requestedFingerprint, [System.StringComparison]::Ordinal)) { $matched.Add([string]$qid) } + } + if ($anyUnverifiable) { return @{ Outcome = 'Unresolved'; Id = $null; Category = 'LegacyContractUnverified' } } + if ($matched.Count -gt 1) { return @{ Outcome = 'Unresolved'; Id = $null; Category = 'MultipleExactMatches' } } + if ($matched.Count -eq 1) { return @{ Outcome = 'Match'; Id = $matched[0]; Category = $null } } + return @{ Outcome = 'Mismatch'; Id = $null; Category = $null } + } + + # Pre-create reconciliation: before the first POST, adopt an already-present + # query by exact name (zero POST), fail closed on multiple exact matches, and + # fall through to the create POST only on an exact-name miss. + if ($PreCreateReconcile -and $ListQueries) { + $preExact = @(& $findExactByName) + if ($preExact.Count -eq 1) { + & $emit "[PRE-RECONCILE] Partition $Index/$Total - adopting existing query by exact name (no POST)" + return & $new ([string]$preExact[0].id) 'None' $null $true + } + if ($preExact.Count -gt 1) { + & $emit "[PRE-RECONCILE] Partition $Index/$Total - multiple exact-name matches - failing closed (no POST)" + return & $new $null 'MultipleExactMatches' $null $false + } + # No fingerprinted-name match: consider a legacy fingerprint-less same-name query, + # adopting ONLY after its full server contract is fetched and matches (no POST on a + # contract mismatch, which falls through to a fresh create). + $preLegacy = & $findLegacyContractMatch + if ($preLegacy.Outcome -eq 'Match') { + & $emit "[PRE-RECONCILE] Partition $Index/$Total - adopting legacy same-name query after contract match (no POST)" + return & $new ([string]$preLegacy.Id) 'None' $null $true + } + if ($preLegacy.Outcome -eq 'Unresolved') { + # A legacy same-name candidate exists but its full contract could not be + # safely verified, or several candidates verify at once; fail closed with zero + # POST so an unverified candidate is never duplicated by a fresh create. + & $emit "[PRE-RECONCILE] Partition $Index/$Total - legacy same-name query could not be safely verified ($($preLegacy.Category)) - failing closed (no POST)" + return & $new $null $preLegacy.Category $null $false + } + # Outcome None (no legacy candidate) or Mismatch (every candidate verified to a + # different contract) falls through to exactly one fingerprinted create POST. + } + + # Read-only reconcile-only mode: NEVER POST. Resolves a partition that carries durable + # uncertain-create state (a possibly-accepted first POST) across a later attempt, + # subdivision, or resume, so the create is never repeated. One exact fingerprinted-name + # match is adopted (real id); multiple fail closed; none stays UncertainCreateUnresolved. + # Never QueryConfirmedAbsent - a collection miss cannot prove the first POST was rejected. + if ($ReconcileOnly) { + if (-not $ListQueries) { return & $new $null 'UncertainCreateUnresolved' $null $false } + $roExact = @(& $findExactByName) + if ($roExact.Count -eq 1) { + & $emit "[RECONCILE-ONLY] Partition $Index/$Total - uncertain create resolved by exact name (no POST)" + return & $new ([string]$roExact[0].id) 'None' $null $true + } + if ($roExact.Count -gt 1) { + & $emit "[RECONCILE-ONLY] Partition $Index/$Total - multiple exact-name matches - failing closed (no POST)" + return & $new $null 'MultipleExactMatches' $null $false + } + & $emit "[RECONCILE-ONLY] Partition $Index/$Total - uncertain create still not visible - remains unresolved (no POST)" + return & $new $null 'UncertainCreateUnresolved' $null $false + } + + $createRetries = 0 + $max403Retries = 3 + $maxNetworkOutageSeconds = $MaxNetworkOutageMinutes * 60 + $networkErrorStart = $null + + while ($true) { + $resp = & $Invoke $Version $bodyJson $clientRequestId + $status = if ($resp) { $resp.HttpStatus } else { $null } + $body = if ($resp) { [string]$resp.ResponseBody } else { $null } + $headers = if ($resp) { $resp.Headers } else { $null } + + # Success. + if ($null -ne $status -and $status -ge 200 -and $status -le 299) { + $newId = if ($resp) { [string]$resp.Id } else { $null } + return & $new $newId 'None' $status $false + } + + # Classify. Network first, then 429, then 403, then ambiguous, then other. + $isNetwork = $false + if ($null -eq $status -or $status -eq 0 -or $status -ge 500) { $isNetwork = $true } + if (-not $isNetwork -and $body -match 'timed out|connection|unable to connect|could not be resolved|Bad Gateway|Service Unavailable|Gateway Timeout|Internal Server Error') { $isNetwork = $true } + + $is429 = (-not $isNetwork) -and ($status -eq 429) + $is403 = (-not $isNetwork) -and (-not $is429) -and ($status -eq 403) + $isAmbiguous = (-not $isNetwork) -and (-not $is429) -and (-not $is403) -and ($status -eq 409) + + if ($is403) { + $wwwAuth = $null + $requestId = $null + if ($headers) { try { $wwwAuth = [string]$headers['WWW-Authenticate'] } catch {} } + if ($headers) { try { $requestId = [string]$headers['request-id'] } catch {} } + $isPermanent = $false + if ($body -match 'InsufficientPrivileges|Authorization_RequestDenied|AccessDenied|InvalidAuthenticationToken') { $isPermanent = $true } + if ($wwwAuth -match 'claims') { $isPermanent = $true } + & $emit "[403-DIAG] Partition $Index/$Total client-request-id: $clientRequestId permanent: $isPermanent" + if ($isPermanent) { + & $emit "[403-PERM] Partition $Index/$Total - permanent 403 (permissions/CAE) - failing" + return & $new $null 'PermanentForbidden' $status $false $requestId $body + } + $createRetries++ + if ($createRetries -le $max403Retries) { + $retryAfter = [Math]::Min(15 * [Math]::Pow(2, $createRetries - 1), 60) + & $emit "[403-CREATE] Partition $Index/$Total - transient 403 (Attempt $createRetries/$max403Retries) - retrying in ${retryAfter}s" + & $Sleep $retryAfter + continue + } + & $emit "[403-PERSISTENT] Partition $Index/$Total - generic 403 persisted after all create retries" + return & $new $null 'PersistentForbidden' $status $false $requestId $body + } + elseif ($is429) { + $createRetries++ + $retryAfter = $null + if ($headers) { try { if ($headers['Retry-After']) { $retryAfter = [int]$headers['Retry-After'] } } catch {} } + if (-not $retryAfter) { $retryAfter = [Math]::Min(60 * [Math]::Pow(2, $createRetries - 1), 300) } + & $emit "[THROTTLE] Partition $Index/$Total - throttled (Attempt $createRetries) - waiting ${retryAfter}s" + & $Sleep $retryAfter + if ($createRetries -ge 20) { + & $emit "[THROTTLE-ABORT] Partition $Index/$Total - throttled after $createRetries attempts - aborting" + return & $new $null 'ThrottleAbort' $status $false + } + continue + } + elseif ($isNetwork) { + if (-not $networkErrorStart) { + $networkErrorStart = Get-Date + & $emit "[NETWORK] Partition $Index/$Total - uncertain create outcome (HTTP $status) - starting read-only reconcile window (max ${MaxNetworkOutageMinutes}m), no further POST" + } + # An uncertain create outcome (status 0/null, timeout, connection reset, or a + # 5xx where acceptance cannot be disproven) may have been accepted by the + # server, so the create is NEVER re-POSTed for this partition on this attempt. + # Recovery is read-only: reconcile the fingerprinted exact name against the + # fully-paginated collection until a single match appears (adopt), several + # appear (fail closed), or the window expires with no match (fail closed as + # unresolved). A collection miss cannot prove the POST was rejected, so it can + # only fail closed - never issue another POST. + if (-not $ListQueries) { + & $emit "[NETWORK-UNRESOLVED] Partition $Index/$Total - no collection transport to reconcile an uncertain create - failing closed (no re-POST)" + return & $new $null 'UncertainCreateUnresolved' $status $false + } + while ($true) { + $netExact = @(& $findExactByName) + if ($netExact.Count -eq 1) { + & $emit "[NETWORK-RECONCILE] Partition $Index/$Total - uncertain create adopted by exact name (no re-POST)" + return & $new ([string]$netExact[0].id) 'None' $status $true + } + if ($netExact.Count -gt 1) { + & $emit "[NETWORK-RECONCILE] Partition $Index/$Total - multiple exact-name matches after uncertain create - failing closed" + return & $new $null 'MultipleExactMatches' $status $false + } + $elapsedOutageSeconds = ((Get-Date) - $networkErrorStart).TotalSeconds + if ($elapsedOutageSeconds -ge $maxNetworkOutageSeconds) { + & $emit "[NETWORK-UNRESOLVED] Partition $Index/$Total - uncertain create not visible within $MaxNetworkOutageMinutes minute window - failing closed (no re-POST)" + return & $new $null 'UncertainCreateUnresolved' $status $false + } + $retryDelay = 30 + (Get-Random -Minimum 10 -Maximum 30) + & $emit "[NETWORK] Partition $Index/$Total - uncertain create not yet visible (${elapsedOutageSeconds}s elapsed) - re-listing after delay (no re-POST)" + & $Sleep $retryDelay + } + } + elseif ($isAmbiguous) { + & $emit "[AMBIGUOUS] Partition $Index/$Total - ambiguous create response - reconciling by exact name" + return & $reconcile $status + } + else { + & $emit "[CREATE-FAILED] Partition $Index/$Total - non-retriable create failure (HTTP $status)" + return & $new $null 'Other' $status $false + } + } +} + +function Write-PaxCreatePoolHeartbeat { + <# + .SYNOPSIS + Sanitized aggregate progress heartbeat for the bounded parent-create pool. Emits at + most one line per interval and ONLY safe aggregate counts - never a QueryId, token, + identifier, display name, operation name, URL, request body, or header. + .DESCRIPTION + Clock-driven and side-effect-minimal: given the pass start, the last-emit time, and + the current time (all supplied by the caller's injectable now-provider) plus the + interval, it emits one heartbeat line through the injected sink ONLY when at least a + full interval has elapsed since the last emit, and returns the updated last-emit time. + Before the first full interval nothing is emitted and the last-emit time is returned + unchanged, so a pass that finishes quickly produces no heartbeat. It never touches the + jobs, scheduling, throttle, timeout, token refresh, result order, checkpoint state, or + POST count. + .PARAMETER Start + Wall-clock time the wait pass began (from the injectable now-provider). + .PARAMETER LastEmit + Time the previous heartbeat was emitted (Start when none yet). + .PARAMETER Now + Current time sample (from the injectable now-provider). + .PARAMETER IntervalSeconds + Minimum seconds between heartbeats; <= 0 disables emission. + .PARAMETER Total / Completed / Active / Queued / Uncertain + Safe aggregate partition counts for this pass. + .PARAMETER Emit + Injectable single-argument sink for the heartbeat line (defaults to Write-Host). + .OUTPUTS + The (possibly-updated) last-emit [datetime]; equal to -LastEmit when nothing was emitted. + #> + param( + [Parameter(Mandatory)][datetime]$Start, + [Parameter(Mandatory)][datetime]$LastEmit, + [Parameter(Mandatory)][datetime]$Now, + [Parameter(Mandatory)][int]$IntervalSeconds, + [Parameter()][int]$Total = 0, + [Parameter()][int]$Completed = 0, + [Parameter()][int]$Active = 0, + [Parameter()][int]$Queued = 0, + [Parameter()][int]$Uncertain = 0, + [Parameter()][scriptblock]$Emit = $null + ) + if ($IntervalSeconds -le 0) { return $LastEmit } + if (($Now - $LastEmit).TotalSeconds -lt $IntervalSeconds) { return $LastEmit } + $elapsedSeconds = [int][Math]::Floor(($Now - $Start).TotalSeconds) + if ($elapsedSeconds -lt 0) { $elapsedSeconds = 0 } + $elapsedText = if ($elapsedSeconds -ge 60) { "$([int][Math]::Floor($elapsedSeconds / 60))m" } else { "$($elapsedSeconds)s" } + # 'localQueued' is the count still waiting in THIS pool, and 'pending' is everything + # not yet finished. Wording only: the counts, the pass, and the work are unchanged. + $pendingCount = $Active + $Queued + $message = " [PARENT-CREATE] $elapsedText elapsed | total $Total | completed $Completed | active $Active | localQueued $Queued | pending $pendingCount" + if ($Uncertain -gt 0) { $message += " | uncertain $Uncertain" } + if ($Emit) { & $Emit $message } else { Write-Host $message -ForegroundColor DarkCyan } + return $Now +} + +function Invoke-PaxBoundedCreatePool { + <# + .SYNOPSIS + Bounded-parallel executor for parent-scope partition query creation. + .DESCRIPTION + Runs Invoke-GraphAuditHardenedCreate for each supplied partition inside + isolated ThreadJob runspaces under a fixed concurrency cap. Because a + ThreadJob runspace does not inherit parent-scope functions or variables, the + create functions, version constants, and HTTP transport are injected as + text/arguments into every runspace. The transport is injected via + -TransportText so the same executor serves production (a real HTTP POST) and + validation (a runspace-local mock). The concurrency cap is passed straight to + Start-ThreadJob -ThrottleLimit, so the supplied bound is never exceeded. + Returns one result object per partition: + @{ Index; QueryId; Version; FailureCategory; HttpStatus }. + .PARAMETER Items + Partition-shaped objects carrying Index, Total, PStart, PEnd, Activities, + RecordTypes, ServiceFilter. + .PARAMETER ThrottleLimit + Maximum number of concurrent create runspaces. + .PARAMETER TransportText + Text of the injected transport scriptblock: param($Version,$BodyJson, + $ClientRequestId) returning @{ HttpStatus; Id; ResponseBody; Headers }. + .PARAMETER FunctionTexts + Verbatim function definitions dot-sourced into each runspace. + .PARAMETER InjectVariables + Name/value pairs set at script scope inside each runspace (version constants, + token material). + .PARAMETER ListTransportText + Optional text of an injected collection-list scriptblock: returns the + fully-paginated array of @{ id; displayName } for exact-name reconciliation. + When supplied, every create routes through exact-name lookup before and around + its POST; when omitted, the executor behaves as a plain create pool. + .PARAMETER FetchTransportText + Optional text of an injected item-fetch scriptblock: param($QueryId) returning + the query item with its filter contract, so a legacy fingerprint-less same-name + match can be adopted ONLY after its full server contract is fetched and matched. + .PARAMETER TokenRefresh + Optional parent-scope scriptblock invoked periodically WHILE the create jobs run + so a token nearing expiry is refreshed on the shared auth reference mid-pool. + Because the shared auth state is a synchronized hashtable, a refresh landing here + is observed by an already-started retrying create job before its next request. + Defaults to Refresh-GraphTokenIfNeeded; pass an injected refresher for validation. + #> + param( + [Parameter(Mandatory)]$Items, + [Parameter(Mandatory)][int]$ThrottleLimit, + [Parameter(Mandatory)][string]$TransportText, + [Parameter()][string[]]$FunctionTexts = @(), + [Parameter()][hashtable]$InjectVariables = @{}, + [Parameter()][string]$ListTransportText = $null, + [Parameter()][string]$FetchTransportText = $null, + [Parameter()][scriptblock]$TokenRefresh = $null, + [Parameter()][int]$MaxNetworkOutageMinutes = 30, + [Parameter()][int]$HeartbeatIntervalSeconds = 60, + [Parameter()][scriptblock]$NowProvider = $null, + [Parameter()][scriptblock]$HeartbeatEmit = $null + ) + if ($ThrottleLimit -lt 1) { $ThrottleLimit = 1 } + # Injectable clock + emit sink for the create-pool progress heartbeat. Production uses the + # wall clock and the run log/host; validation injects a synthetic clock + collector so the + # heartbeat is exercised deterministically without waiting a real interval. Neither affects + # scheduling, throttle, timeout, token refresh, result order, checkpoint state, or POST count. + if (-not $NowProvider) { $NowProvider = { Get-Date } } + if (-not $HeartbeatEmit) { + $HeartbeatEmit = { + param($HeartbeatMessage) + if (Get-Command -Name Write-LogHost -ErrorAction SilentlyContinue) { Write-LogHost $HeartbeatMessage -ForegroundColor DarkCyan } + else { Write-Host $HeartbeatMessage -ForegroundColor DarkCyan } + } + } + # Synchronized per-partition PHASE MARKER shared into every worker runspace by + # reference (like the shared auth state). It lets an interrupted pool tell a + # ThreadJob that never began (provably no create POST) apart from one whose create + # POST was in flight, so a never-started partition is never conflated with a + # started-but-unknown one. Keyed by partition Index; a missing entry means the + # worker never ran. + $phaseMap = [hashtable]::Synchronized(@{}) + $poolWorker = { + param($it, $fnTexts, $vars, $transportText, $listTransportText, $fetchTransportText, $outageMin, $phaseMap) + # First worker action: record that this partition's worker actually began. A + # missing phase entry at the interrupt therefore proves the ThreadJob never + # started and no create POST could have been sent. + if ($phaseMap) { $phaseMap[$it.Index] = 'Started' } + foreach ($ft in $fnTexts) { . ([scriptblock]::Create($ft)) } + if ($vars) { foreach ($vk in $vars.Keys) { Set-Variable -Name $vk -Value $vars[$vk] -Scope Script } } + $rawTransport = [scriptblock]::Create($transportText) + # Wrap the injected create transport so a 'PostInFlight' marker is set IMMEDIATELY + # BEFORE the real create POST and 'PostReturned' AFTER the response. ThreadJob + # 'Running' alone does not prove a POST started - only this marker does. The marker + # is set conservatively (before the call), so a worker stopped around its POST + # classifies as outcome-unknown, never as never-started. + $transport = { + param($Version, $BodyJson, $ClientRequestId) + if ($phaseMap) { $phaseMap[$it.Index] = 'PostInFlight' } + try { return (& $rawTransport $Version $BodyJson $ClientRequestId) } + finally { if ($phaseMap) { $phaseMap[$it.Index] = 'PostReturned' } } + }.GetNewClosure() + # Injected alongside the create transport so every create routes through + # exact-name reconciliation before and around its POST (adopt one, fail closed + # on many, POST only on a miss). + $listQueries = if ($listTransportText) { [scriptblock]::Create($listTransportText) } else { $null } + # Item-fetch transport: retrieves a query's full contract so a legacy + # fingerprint-less same-name match is adopted only after a contract compare. + $fetchItem = if ($fetchTransportText) { [scriptblock]::Create($fetchTransportText) } else { $null } + $res = Invoke-GraphAuditHardenedCreate -PStart $it.PStart -PEnd $it.PEnd -Index $it.Index -Total $it.Total -Activities $it.Activities -RecordTypes $it.RecordTypes -ServiceFilter $it.ServiceFilter -Invoke $transport -ListQueries $listQueries -PreCreateReconcile:([bool]$listQueries) -FetchItem $fetchItem -MaxNetworkOutageMinutes $outageMin + # Per-partition EXECUTION STATE for a COMPLETED worker, mirroring the parent's + # own create classification exactly: a proven QueryId is a create; an + # unproven-acceptance outcome stays outcome-unknown (durable uncertain); every + # other completion is a determinable failure that provably produced no query. + $execState = if ($res.QueryId) { 'CompletedWithQueryId' } elseif ([string]$res.FailureCategory -eq 'UncertainCreateUnresolved') { 'StartedOutcomeUnknown' } else { 'CompletedFailureBeforePOST' } + [pscustomobject]@{ + Index = $it.Index + QueryId = $res.QueryId + Version = $res.Version + FailureCategory = $res.FailureCategory + HttpStatus = $res.HttpStatus + ExecutionState = $execState + ClientRequestId = $res.ClientRequestId + RequestId = $res.RequestId + ResponseBody = $res.ResponseBody + } + } + $poolJobs = @() + $poolJobIndex = @() + foreach ($item in $Items) { + $poolJobs += Start-ThreadJob -ThrottleLimit $ThrottleLimit -ScriptBlock $poolWorker -ArgumentList $item, $FunctionTexts, $InjectVariables, $TransportText, $ListTransportText, $FetchTransportText, $MaxNetworkOutageMinutes, $phaseMap + $poolJobIndex += $item.Index + } + # Token monitoring during a prolonged create pass: while any create job is still + # running, invoke the token refresher periodically instead of blocking on a single + # Wait-Job. Because the shared auth state is a synchronized hashtable, a refresh + # landing here is visible to an already-started retrying create job BEFORE its next + # request - so a job that started under a token that later neared expiry can still + # observe a fresh token mid-retry, not only jobs launched after a pre-pass refresh. + # A token refresher can signal an interactive quit ('Quit') or throw. Either outcome + # fails the pool closed: no further create work is awaited as success, still-active + # jobs are stopped, only already-completed job results are collected, and a sentinel + # row carries the quit/error decision back to the caller so the parent orchestration + # can enter its graceful authentication-exit path instead of silently reporting + # success. + $authQuit = $false + $refresherError = $null + if ($poolJobs.Count -gt 0) { + if ($TokenRefresh) { + # Sanitized aggregate progress heartbeat: at most one line per interval while the + # create pool is still active, emitting ONLY safe aggregate counts (elapsed, total, + # completed, active, queued) - never a QueryId, token, identifier, name, URL, or + # body. The clock and interval are injectable so the emit is exercised without a + # real wait; the heartbeat never changes scheduling, throttle, timeout, token + # refresh, result order, checkpoint behaviour, or POST count. + $hbStart = & $NowProvider + $hbLastEmit = $hbStart + while (@($poolJobs | Where-Object { $_.State -in 'Running', 'NotStarted' }).Count -gt 0) { + $refreshOutcome = $null + try { $refreshOutcome = & $TokenRefresh } catch { $refresherError = $_.Exception.Message; break } + if ($refreshOutcome -is [string] -and $refreshOutcome -eq 'Quit') { $authQuit = $true; break } + $null = $poolJobs | Wait-Job -Timeout 5 + $hbCompleted = @($poolJobs | Where-Object { $_.State -in 'Completed', 'Failed', 'Stopped' }).Count + $hbActive = @($poolJobs | Where-Object { $_.State -eq 'Running' }).Count + $hbQueued = @($poolJobs | Where-Object { $_.State -eq 'NotStarted' }).Count + if (Get-Command Write-PaxCreatePoolHeartbeat -ErrorAction SilentlyContinue) { + $hbLastEmit = Write-PaxCreatePoolHeartbeat -Start $hbStart -LastEmit $hbLastEmit -Now (& $NowProvider) -IntervalSeconds $HeartbeatIntervalSeconds -Total $poolJobs.Count -Completed $hbCompleted -Active $hbActive -Queued $hbQueued -Emit $HeartbeatEmit + } + } + if (-not $authQuit -and -not $refresherError) { + try { $finalOutcome = & $TokenRefresh; if ($finalOutcome -is [string] -and $finalOutcome -eq 'Quit') { $authQuit = $true } } catch { $refresherError = $_.Exception.Message } + } + } + else { + $null = $poolJobs | Wait-Job + } + } + if ($authQuit -or $refresherError) { + $stillActive = @($poolJobs | Where-Object { $_.State -in 'Running', 'NotStarted' }) + if ($stillActive.Count -gt 0) { $stillActive | Stop-Job -ErrorAction SilentlyContinue } + $poolResults = @() + foreach ($pj in $poolJobs) { if ($pj.State -eq 'Completed') { try { $poolResults += Receive-Job -Job $pj -ErrorAction SilentlyContinue } catch {} } } + if ($poolJobs.Count -gt 0) { $poolJobs | Remove-Job -Force -ErrorAction SilentlyContinue } + # Emit a per-partition row for every requested job that did NOT complete, using the + # shared phase marker to distinguish a never-started worker (no phase entry, or only + # 'Started' - provably no create POST) from one whose create POST was in flight + # ('PostInFlight'/'PostReturned' - outcome unknown). This prevents the interrupt path + # from conflating never-started partitions with started-but-unknown ones. + $completedIdx = New-Object System.Collections.Generic.HashSet[string] + foreach ($pr in $poolResults) { if ($null -ne $pr -and $null -ne $pr.Index) { $null = $completedIdx.Add([string]$pr.Index) } } + foreach ($ji in $poolJobIndex) { + if ($completedIdx.Contains([string]$ji)) { continue } + $ph = $null + if ($phaseMap -and $phaseMap.ContainsKey($ji)) { $ph = [string]$phaseMap[$ji] } + $execState = if ($ph -eq 'PostInFlight' -or $ph -eq 'PostReturned') { 'StartedOutcomeUnknown' } else { 'NeverStarted' } + $poolResults += [pscustomobject]@{ Index = $ji; QueryId = $null; Version = $null; FailureCategory = 'Interrupted'; HttpStatus = $null; ExecutionState = $execState } + } + $poolResults += [pscustomobject]@{ Index = $null; QueryId = $null; Version = $null; FailureCategory = $(if ($authQuit) { 'AuthQuit' } else { 'TokenRefreshError' }); HttpStatus = $null; AuthQuit = $authQuit; RefresherError = $refresherError } + return $poolResults + } + $poolResults = @() + foreach ($pj in $poolJobs) { + try { $poolResults += Receive-Job -Job $pj -ErrorAction SilentlyContinue } catch {} + } + if ($poolJobs.Count -gt 0) { $poolJobs | Remove-Job -Force -ErrorAction SilentlyContinue } + return $poolResults +} + +function Invoke-PaxParentQueryCreation { + <# + .SYNOPSIS + Parent-side bounded-parallel pre-pass that creates the audit query for each + pending partition before its worker is launched. + .DESCRIPTION + For every supplied partition this creates the query in the main runspace, + where the token can be refreshed, under the same concurrency bound used for + worker launch. Creation runs through an injectable runner: production uses a + real bounded ThreadJob pool (Invoke-PaxBoundedCreatePool) with the parent + create functions and a real HTTP transport injected into each runspace, while + validation supplies its own runner with an injected transport. Returns an + index-keyed result map: + @{ = @{ Index; QueryId; Version; FailureCategory; HttpStatus } }. + Persistence and worker wiring stay with the caller: a successful entry carries + a non-null QueryId to store and checkpoint before launch; a failed entry + carries a null QueryId and must not launch a worker. + .PARAMETER Partitions + Partition objects needing creation (each with Index, Total, PStart, PEnd, + RecordTypes, ServiceFilter). + .PARAMETER ThrottleLimit + Concurrency bound reused from the launch phase. + .PARAMETER Activities + Run-level activity list applied to every create body. + .PARAMETER Runner + Optional injected runner: param($Items,$Throttle,$OutageMinutes) returning the + per-partition result objects. When omitted a production bounded ThreadJob pool + is used. + #> + param( + [Parameter(Mandatory)]$Partitions, + [Parameter(Mandatory)][int]$ThrottleLimit, + [Parameter()]$Activities = @(), + [Parameter()][int]$MaxNetworkOutageMinutes = 30, + [Parameter()][scriptblock]$Runner = $null + ) + $items = @() + foreach ($p in $Partitions) { + $items += [pscustomobject]@{ + Index = $p.Index + Total = $p.Total + PStart = $p.PStart + PEnd = $p.PEnd + Activities = $Activities + RecordTypes = $p.RecordTypes + ServiceFilter = $p.ServiceFilter + } + } + if (-not $Runner) { + $Runner = { + param($RunnerItems, $RunnerThrottle, $RunnerOutage) + $fnTexts = @( + "function New-GraphAuditQueryFingerprint {`n$(${function:New-GraphAuditQueryFingerprint})`n}", + "function New-GraphAuditPartitionDisplayName {`n$(${function:New-GraphAuditPartitionDisplayName})`n}", + "function New-GraphAuditQueryRequest {`n$(${function:New-GraphAuditQueryRequest})`n}", + "function Resolve-GraphAuditDuplicateSafeQuery {`n$(${function:Resolve-GraphAuditDuplicateSafeQuery})`n}", + "function Invoke-GraphAuditHardenedCreate {`n$(${function:Invoke-GraphAuditHardenedCreate})`n}" + ) + $vars = @{ + GraphAuditApiVersion_Previous = $script:GraphAuditApiVersion_Previous + GraphAuditApiVersion_Current = $script:GraphAuditApiVersion_Current + PaxSharedAuthState = $script:SharedAuthState + } + $transportText = @' +param($Version, $BodyJson, $ClientRequestId) +$uri = "https://graph.microsoft.com/$Version/security/auditLog/queries" +$headers = @{ Authorization = "Bearer $($script:PaxSharedAuthState.Token)"; 'Content-Type' = 'application/json' } +if ($ClientRequestId) { $headers['client-request-id'] = $ClientRequestId } +try { + $r = Invoke-WebRequest -Method POST -Uri $uri -Headers $headers -Body $BodyJson -ErrorAction Stop + $obj = $null; try { $obj = $r.Content | ConvertFrom-Json } catch {} + return @{ HttpStatus = [int]$r.StatusCode; Id = $obj.id; ResponseBody = $r.Content; Headers = $r.Headers } +} catch { + $resp = $_.Exception.Response + $code = 0; $body = $_.Exception.Message; $rh = @{} + if ($resp) { + try { $code = [int]$resp.StatusCode } catch {} + try { $sr = New-Object System.IO.StreamReader($resp.GetResponseStream()); $body = $sr.ReadToEnd() } catch {} + try { foreach ($hn in $resp.Headers) { $rh[$hn] = $resp.Headers[$hn] } } catch {} + } + return @{ HttpStatus = $code; Id = $null; ResponseBody = $body; Headers = $rh } +} +'@ + # Fully-paginated exact-name collection transport. Reads the live token from + # the shared auth state so a token refreshed by the parent is picked up by + # queued and retrying create jobs, and walks every page of the audit-query + # collection so exact-name reconciliation sees the full server state. + $listTransportText = @' +$__lver = $script:GraphAuditApiVersion_Previous +$__luri = "https://graph.microsoft.com/$__lver/security/auditLog/queries" +$__acc = @() +while ($__luri) { + $__lheaders = @{ Authorization = "Bearer $($script:PaxSharedAuthState.Token)" } + $__lr = Invoke-WebRequest -Method GET -Uri $__luri -Headers $__lheaders -ErrorAction Stop + $__lobj = $null; try { $__lobj = $__lr.Content | ConvertFrom-Json } catch {} + if ($__lobj -and $__lobj.value) { foreach ($__q in $__lobj.value) { $__acc += @{ id = $__q.id; displayName = $__q.displayName } } } + $__luri = if ($__lobj) { $__lobj.'@odata.nextLink' } else { $null } +} +, @($__acc) +'@ + # Single-item fetch transport. Retrieves a query's full filter contract so a + # legacy fingerprint-less same-name match is adopted only after its server + # contract matches the requested one. Reads the live shared-auth token. + $fetchTransportText = @' +param($QueryId) +$__fver = $script:GraphAuditApiVersion_Previous +$__furi = "https://graph.microsoft.com/$__fver/security/auditLog/queries/$QueryId" +$__fheaders = @{ Authorization = "Bearer $($script:PaxSharedAuthState.Token)" } +try { + $__fr = Invoke-WebRequest -Method GET -Uri $__furi -Headers $__fheaders -ErrorAction Stop + $__fobj = $null; try { $__fobj = $__fr.Content | ConvertFrom-Json } catch {} + return $__fobj +} catch { return $null } +'@ + # Refresh the shared-auth token WHILE the create jobs run so an already-started + # retrying job observes a fresh token before its next request. The refresher + # outcome (including an interactive 'Quit' or a thrown error) is propagated so + # the pool can fail closed instead of discarding a quit decision. + $tokenRefresh = { Refresh-GraphTokenIfNeeded -BufferMinutes 5 } + Invoke-PaxBoundedCreatePool -Items $RunnerItems -ThrottleLimit $RunnerThrottle -TransportText $transportText -FunctionTexts $fnTexts -InjectVariables $vars -ListTransportText $listTransportText -FetchTransportText $fetchTransportText -TokenRefresh $tokenRefresh -MaxNetworkOutageMinutes $RunnerOutage + } + } + $results = @(& $Runner $items $ThrottleLimit $MaxNetworkOutageMinutes) + $map = @{} + foreach ($r in $results) { + if ($null -ne $r) { + # A pool that failed closed on an interactive quit or a token-refresh error + # returns a sentinel row instead of a partition result; surface it under a + # reserved key so the caller enters its graceful authentication-exit path and + # never persists a partition as created off a quit/error decision. + $isQuit = if ($r -is [hashtable]) { [bool]$r['AuthQuit'] } else { [bool]$r.AuthQuit } + $rerr = if ($r -is [hashtable]) { $r['RefresherError'] } else { $r.RefresherError } + if ($isQuit) { $map['__AuthQuit'] = $true; continue } + if ($rerr) { $map['__RefresherError'] = [string]$rerr; continue } + $idx = if ($r -is [hashtable]) { $r['Index'] } else { $r.Index } + if ($null -ne $idx) { $map[[int]$idx] = $r } + } + } + return $map +} + +function Save-PaxInterruptedCreatePool { + <# + .SYNOPSIS + On a create-pool auth-quit / refresher-error interrupt, durably persist the + partitions the pool actually completed AND the partitions still in flight, + BEFORE the graceful exit. + .DESCRIPTION + Invoke-PaxBoundedCreatePool returns the results of jobs that completed BEFORE + the interrupt as integer-keyed entries in the map, alongside the reserved + __AuthQuit / __RefresherError sentinel. Discarding that map on the interrupt + would (a) throw away a query that was actually created (forcing a duplicate + create on resume) and (b) leave a possibly-accepted in-flight POST unrecorded. + + This helper reconciles the interrupt map against the list of partitions the + pass requested ($Requested): + + * A requested partition whose map entry carries a real QueryId is persisted + via the SAME success path a normal completion uses (status QueryId + + effective QueryApiVersion + Create operation + a QueryCreated checkpoint). + * A partition whose worker never began (ExecutionState 'NeverStarted' - + provably no create POST was ever issued) is left untouched so a resume + creates it fresh; it is NEVER persisted as UncertainCreate. This is the + correction for conflating never-started jobs with started-but-unknown ones. + * A partition whose worker completed with a determinable non-accepting failure + (ExecutionState 'CompletedFailureBeforePOST') is recorded via + Set-PartitionFailure, matching the normal-completion classification. + * Every OTHER requested partition was still in flight at the interrupt + (ExecutionState 'StartedOutcomeUnknown', or a legacy map entry with no + ExecutionState) and is persisted as durable UncertainCreate, so a resume + routes it to a READ-ONLY reconcile instead of a duplicate create. + + A stopped / incomplete job is NEVER persisted as successful: only a map entry + with a non-null QueryId is treated as completed. + #> + param( + [Parameter(Mandatory)]$CreateMap, + [Parameter(Mandatory)]$Requested, + [Parameter()]$Activities = @(), + [Parameter(Mandatory)][ValidateSet('initial', 'retry')][string]$Phase + ) + $phaseTag = if ($Phase -eq 'retry') { 'retry ' } else { '' } + $completedIndices = [System.Collections.Generic.HashSet[int]]::new() + # 1) Persist every partition the pool actually completed (has a real QueryId). + foreach ($cp in $Requested) { + $cr = $CreateMap[[int]$cp.Index] + if ($cr -and $cr.QueryId) { + $cp | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue ([string]$cr.QueryId) -Force + $script:partitionStatus[$cp.Index].QueryId = [string]$cr.QueryId + $script:partitionStatus[$cp.Index].QueryApiVersion = $cr.Version + $script:partitionStatus[$cp.Index].LifecycleOperation = 'Create' + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -QueryId ([string]$cr.QueryId) -State 'QueryCreated' -PartitionStart $cp.PStart -PartitionEnd $cp.PEnd -LifecycleState $script:partitionStatus[$cp.Index] + } + [void]$completedIndices.Add([int]$cp.Index) + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) query created before ${phaseTag}interrupt (QueryId: $($cr.QueryId)) - persisted, not discarded" -ForegroundColor DarkCyan + } + } + # 2) Persist the remaining requested partitions by execution state: never-started + # stays fresh (no create POST happened), a determinable failure is recorded as a + # failure, and only a genuinely in-flight partition becomes durable UncertainCreate. + foreach ($cp in $Requested) { + if ($completedIndices.Contains([int]$cp.Index)) { continue } + $cr = $CreateMap[[int]$cp.Index] + $execState = if ($cr) { [string]$cr.ExecutionState } else { '' } + if ($execState -eq 'NeverStarted') { + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) never started before ${phaseTag}interrupt - no create POST issued, left fresh for create on resume" -ForegroundColor DarkGray + continue + } + if ($execState -eq 'CompletedFailureBeforePOST') { + Set-PartitionFailure -Index $cp.Index -Stage 'CREATE' -Reason ([string]$cr.FailureCategory) + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) failed before ${phaseTag}interrupt (no query created) - recorded as failure" -ForegroundColor DarkYellow + continue + } + $script:partitionStatus[$cp.Index].UncertainCreate = $true + $script:partitionStatus[$cp.Index].UncertainCreateInfo = @{ + displayName = New-GraphAuditPartitionDisplayName -PStart $cp.PStart -PEnd $cp.PEnd -Index $cp.Index -Total $cp.Total -Activities $Activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + fingerprint = New-GraphAuditQueryFingerprint -PStart $cp.PStart -PEnd $cp.PEnd -Activities $Activities -RecordTypes $cp.RecordTypes -ServiceFilter $cp.ServiceFilter + pStart = $cp.PStart.ToString('o') + pEnd = $cp.PEnd.ToString('o') + attempt = [int]$script:partitionStatus[$cp.Index].AttemptNumber + phase = $Phase + } + $script:partitionStatus[$cp.Index].LastError = "Parent-side query creation in-flight at ${phaseTag}interrupt - durable reconcile pending" + if ($script:CheckpointEnabled) { + Save-Checkpoint -PartitionIndex $cp.Index -State 'UncertainCreate' -PartitionStart $cp.PStart -PartitionEnd $cp.PEnd -LifecycleState $script:partitionStatus[$cp.Index] + } + Write-LogHost " [PARENT-CREATE] Partition $($cp.Index)/$($cp.Total) in flight at ${phaseTag}interrupt - persisted for read-only reconcile, no re-POST" -ForegroundColor Yellow + } +} + +function Invoke-PaxResumeUncertainReconcile { + <# + .SYNOPSIS + Resume-side READ-ONLY reconcile of durable uncertain-create partitions (never a POST). + .DESCRIPTION + Consumes the ToReconcileOnly bucket produced by Get-PartitionsToProcess on a + resume. Each such partition carries durable uncertain-create state: a first + create POST that MAY have been accepted but was never proven. In the MAIN + runspace (where the token can refresh) this runs a READ-ONLY exact-name + reconcile via Invoke-GraphAuditHardenedCreate -ReconcileOnly against the + fully-paginated audit-query collection - it NEVER issues a POST. + + Per partition: + * exactly one exact fingerprinted-name match -> ADOPT: store the real QueryId, + checkpoint it as QueryCreated, vacate the durable uncertain entry, and route + the partition to fetch-only; + * multiple exact matches -> FAIL CLOSED: no adopt, no POST, + the durable uncertain entry is preserved for an operator decision; + * none after the bounded collection window -> REMAIN UNCERTAIN: no adopt, no + POST, never QueryConfirmedAbsent, the durable uncertain entry is preserved. + Only adopted partitions are returned for launch; unresolved and fail-closed + partitions are deliberately withheld, so no worker is launched for an unresolved + uncertain create (worker-launch gating). + + The identity is recomputed from the partition's contract (operation / record-type + / service / date), so a differing contract yields a different name and can never + cross-adopt a different query. Transports are injectable: production builds a real + fully-paginated GET collection and a single-item fetch that read the live + shared-auth token; validation injects list / fetch / invoke scriptblocks. + .PARAMETER Partitions + The ToReconcileOnly partition objects (each with Index, Total, PStart, PEnd, + RecordTypes, ServiceFilter, UncertainCreateState). + .PARAMETER Activities + Run-level activity list applied to the reconciled contract identity. + .PARAMETER ListQueries + Optional injected fully-paginated collection transport (-> array of + @{ id; displayName }). Omitted in production to build the real transport. + .PARAMETER FetchItem + Optional injected single-item fetch transport for legacy contract verification. + .PARAMETER Invoke + Optional injected transport required by the shared Invoke-GraphAuditHardenedCreate + signature; it is NEVER called in reconcile-only mode (a fail-closed no-op default). + .PARAMETER Persist + Optional callback param($PartitionIndex,$QueryId,$PStart,$PEnd,$Fingerprint) used to + persist an adopted partition with its EXACT reconciled contract fingerprint. + + CONTRACT (fail-closed): the callback MUST stage the complete bucket transition and + publish it durably, then return the literal boolean $true as its LAST output to signal a + PROVEN durable publication - or throw. Any other outcome (no output, $null, $false, a + non-boolean value) is treated as a FAILED publication: the pre-adoption in-memory state + is restored exactly, the partition is NOT adopted and NOT routed to fetch-only, no POST + is authorized, $script:GenericFatal is latched, and a PAX-FATAL error is thrown. The + default callback stages the transition in memory only (Save-Checkpoint -DeferDiskWrite) + and publishes it EXACTLY ONCE through the strict local+Fabric transaction + (Save-CheckpointStrict), whose verify-after-write result is returned; the warning-only + Save-CheckpointToDisk path is deliberately never used here because a swallowed local + write failure would let the run proceed as though the adoption were durable. + .PARAMETER Log + Optional diagnostic sink param($Message). + .OUTPUTS + @{ AdoptedFetchOnly = @(); Unresolved = @(); FailedClosed = @() } + #> + param( + [Parameter(Mandatory)]$Partitions, + [Parameter()]$Activities = @(), + [Parameter()][scriptblock]$ListQueries = $null, + [Parameter()][scriptblock]$FetchItem = $null, + [Parameter()][scriptblock]$Invoke = $null, + [Parameter()][scriptblock]$Persist = $null, + [Parameter()][scriptblock]$Log = $null + ) + $emit = { param($m) if ($Log) { & $Log $m } } + $result = @{ AdoptedFetchOnly = @(); Unresolved = @(); FailedClosed = @() } + if (-not $Partitions -or @($Partitions).Count -eq 0) { return $result } + + # Reconcile-only NEVER posts; this default transport fails closed if ever reached. + if (-not $Invoke) { $Invoke = { param($v, $b, $c) @{ HttpStatus = 0; Id = $null; ResponseBody = 'reconcile-only-no-post'; Headers = @{} } } } + + # Production list / fetch transports: a fully-paginated exact-name collection and a + # single-item contract fetch, both reading the live shared-auth token so a parent + # token refresh is observed. Injected transports (validation) take precedence. + if (-not $ListQueries) { + $ListQueries = { + $__lver = $script:GraphAuditApiVersion_Previous + $__luri = "https://graph.microsoft.com/$__lver/security/auditLog/queries" + $__acc = @() + while ($__luri) { + $__lheaders = @{ Authorization = "Bearer $($script:SharedAuthState.Token)" } + $__lr = Invoke-WebRequest -Method GET -Uri $__luri -Headers $__lheaders -ErrorAction Stop + $__lobj = $null; try { $__lobj = $__lr.Content | ConvertFrom-Json } catch {} + if ($__lobj -and $__lobj.value) { foreach ($__q in $__lobj.value) { $__acc += @{ id = $__q.id; displayName = $__q.displayName } } } + $__luri = if ($__lobj) { $__lobj.'@odata.nextLink' } else { $null } + } + , @($__acc) + } + } + if (-not $FetchItem) { + $FetchItem = { + param($QueryId) + $__fver = $script:GraphAuditApiVersion_Previous + $__furi = "https://graph.microsoft.com/$__fver/security/auditLog/queries/$QueryId" + $__fheaders = @{ Authorization = "Bearer $($script:SharedAuthState.Token)" } + try { + $__fr = Invoke-WebRequest -Method GET -Uri $__furi -Headers $__fheaders -ErrorAction Stop + $__fobj = $null; try { $__fobj = $__fr.Content | ConvertFrom-Json } catch {} + return $__fobj + } catch { return $null } + } + } + if (-not $Persist) { + $Persist = { + param($PartitionIndex, $QueryId, $PStart, $PEnd, $Fingerprint) + # Adoption persists the proven QueryId with the EXACT reconciled contract fingerprint so + # the queryCreated entry preserves per-contract identity even when this runs BEFORE + # $script:partitionStatus is initialized on resume (Save-Checkpoint cannot re-derive the + # fingerprint from partitionStatus at that point). A legacy fingerprint-less contract + # passes no fingerprint and retains the historical index identity. + # + # The bucket transition is staged IN MEMORY ONLY (-DeferDiskWrite) and then published + # EXACTLY ONCE through the STRICT verify-after-write local+Fabric transaction, so the + # adoption can only affect processing after its durability has been PROVEN. Returning + # the strict publisher's boolean result is what makes an unproven publication fail + # closed in the caller. + if ([string]::IsNullOrWhiteSpace([string]$Fingerprint)) { + Save-Checkpoint -PartitionIndex $PartitionIndex -State 'QueryCreated' -QueryId $QueryId -PartitionStart $PStart -PartitionEnd $PEnd -DeferDiskWrite + } + else { + Save-Checkpoint -PartitionIndex $PartitionIndex -State 'QueryCreated' -QueryId $QueryId -PartitionStart $PStart -PartitionEnd $PEnd -Fingerprint $Fingerprint -DeferDiskWrite + } + return [bool](Save-CheckpointStrict) + } + } + + foreach ($p in @($Partitions)) { + $res = Invoke-GraphAuditHardenedCreate -PStart $p.PStart -PEnd $p.PEnd -Index $p.Index -Total $p.Total -Activities $Activities -RecordTypes $p.RecordTypes -ServiceFilter $p.ServiceFilter -Invoke $Invoke -ListQueries $ListQueries -FetchItem $FetchItem -ReconcileOnly + if ($res.Adopted -and $res.QueryId) { + # Adopt: store the proven real id, checkpoint as created, and vacate the durable + # uncertain entry so a later resume sees a normal fetch-only (never a re-POST). + # + # ONE identity drives BOTH the persist and the vacate so the adopted queryCreated entry + # and the vacated uncertain entry are the SAME contract. Prefer the exact fingerprint + # carried on the partition's durable uncertain state (so the adopted entry PRESERVES that + # precise identity and a sibling same-index contract is never vacated); else reconstruct + # it from the SAME contract the reconcile matched on - the RUN-LEVEL $Activities plus the + # partition record-type / service / date window (NOT $p.Activities, which is absent before + # partitionStatus is initialized on resume). A fingerprinted durable entry whose + # fingerprint cannot be resolved FAILS CLOSED for this partition - it is NEVER adopted + # with an index-only identity, which would persist an identity-less entry and vacate a + # sibling same-index contract. + $__ucState = if ($p.PSObject.Properties['UncertainCreateState']) { $p.UncertainCreateState } else { $null } + $__storedFp = if ($null -ne $__ucState) { Get-PaxCheckpointEntryFingerprint $__ucState } else { $null } + $__entryFingerprinted = (-not [string]::IsNullOrWhiteSpace([string]$__storedFp)) + $__fp = $null + if ($__entryFingerprinted) { + $__fp = [string]$__storedFp + } + else { + try { $__fp = New-GraphAuditQueryFingerprint -PStart $p.PStart -PEnd $p.PEnd -Activities $Activities -RecordTypes $p.RecordTypes -ServiceFilter $p.ServiceFilter } catch { $__fp = $null } + } + if ($__entryFingerprinted -and [string]::IsNullOrWhiteSpace([string]$__fp)) { + # Fingerprinted entry, unresolved fingerprint: never index-fallback (a sibling same-index + # contract would be destroyed). Hold this partition, do not adopt, never POST. + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - fingerprinted uncertain entry could not resolve its contract identity - failing closed (no adopt, no POST)" + $result.FailedClosed += $p + continue + } + # ONE transaction owns the COMPLETE bucket transition. The post-adoption state is built + # in memory FIRST (vacate the matching uncertain contract here, add the queryCreated + # entry inside the persist callback), then published EXACTLY ONCE through the STRICT + # local+Fabric transaction. An EXACT snapshot of every lifecycle bucket is captured + # BEFORE the mutation so a failed publication is rolled back to the precise pre-adoption + # state rather than a best-effort re-derivation. + $__cpParts = if ($script:CheckpointData) { $script:CheckpointData.partitions } else { $null } + $__snapCompleted = @() + $__snapQueryCreated = @() + $__snapUncertain = @() + if ($__cpParts) { + # Assigned as statements (never as an `if` expression) so an EMPTY bucket snapshots + # as a real empty array rather than collapsing to $null, which would restore a + # one-element @($null) bucket instead of an empty one. + $__snapCompleted = @($__cpParts.completed) + $__snapQueryCreated = @($__cpParts.queryCreated) + $__snapUncertain = @($__cpParts.uncertainCreate) + } + if ($__cpParts -and $__cpParts.uncertainCreate) { + # Vacate ONLY the adopted contract, using the SAME single fingerprint used to persist, + # so a sibling same-index contract is preserved byte-for-byte. + $__cpParts.uncertainCreate = @( + $__cpParts.uncertainCreate | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $p.Index $__fp) } + ) + } + $__persistProven = $false + $__persistErr = $null + try { + $__persistResult = @(& $Persist $p.Index ([string]$res.QueryId) $p.PStart $p.PEnd $__fp) | Select-Object -Last 1 + $__persistProven = ($__persistResult -is [bool]) -and ([bool]$__persistResult) + } + catch { $__persistErr = $_ } + if (-not $__persistProven) { + # Fail closed: the adoption's durable publication was NOT proven. Restore the EXACT + # pre-adoption in-memory state (the strict publisher leaves the durable copy + # byte-for-byte unchanged), so the uncertain contract survives in memory AND on disk; + # withhold the partition (never adopted, never routed to fetch-only, ZERO POSTs); + # latch the classified fatal and throw so the caller terminates nonzero. + if ($__cpParts) { + $__cpParts.completed = $__snapCompleted + $__cpParts.queryCreated = $__snapQueryCreated + $__cpParts.uncertainCreate = $__snapUncertain + } + $script:GenericFatal = $true + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - durable publication of the adopted create was NOT proven - failing closed (no adopt, no fetch, no POST)" + $__persistDetail = if ($__persistErr) { [string]$__persistErr.Exception.Message } else { 'the persistence callback did not return a proven durable publication.' } + if ($__persistDetail -match 'PAX-FATAL-ROLLBACK') { + throw "PAX-FATAL-ROLLBACK: durable publication of the adopted uncertain create failed for partition $([int]$p.Index) AND the prior checkpoint could not be restored; disk state is indeterminate - manual checkpoint inspection required. $__persistDetail" + } + throw "PAX-FATAL: durable publication of the adopted uncertain create failed for partition $([int]$p.Index); fail-closed, the durable uncertain contract is preserved and no fetch or fresh create is authorized. $__persistDetail" + } + # PROVEN durable publication: only NOW may the adoption affect processing - the real id + # is stamped on the partition and the partition is classified fetch-only. + $p | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue ([string]$res.QueryId) -Force + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - durable uncertain create adopted by exact name (no POST)" + $result.AdoptedFetchOnly += $p + } + elseif ($res.FailureCategory -eq 'MultipleExactMatches') { + # Fail closed: several exact matches; never adopt, never POST, keep the durable + # uncertain state for an operator decision (-ClearUncertainCreate). + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - multiple exact-name matches - failing closed (no POST, uncertain preserved)" + $result.FailedClosed += $p + } + else { + # Remain uncertain: not visible after the bounded window. Never POST, never + # QueryConfirmedAbsent (a collection miss cannot prove the first POST was rejected); + # the durable uncertain entry is preserved. + & $emit "[RESUME-RECONCILE] Partition $($p.Index)/$($p.Total) - uncertain create still not visible - remains unresolved (no POST, uncertain preserved)" + $result.Unresolved += $p + } + } + return $result +} + +function Clear-PaxUncertainCreateState { + <# + .SYNOPSIS + Operator-directed SELECTIVE clear of durable uncertain-create checkpoint state. + .DESCRIPTION + Durable uncertain-create entries are NEVER auto-cleared - not by elapsed time, a + collection miss, a process restart, or failure-count exhaustion. The ONLY way to + clear them is an explicit operator request that names specific partition indexes (the + top-level -ClearUncertainCreate parameter). The clear is always + SELECTIVE and bucket-exclusive: only the durable entries whose index is in + -SelectedIndexes are removed; every other uncertain entry is preserved exactly. A + selection that is empty, contains a duplicate, or names an index that is not currently + in the durable uncertain-create set is REJECTED and nothing is cleared. With + -ValidateOnly the selection is validated and returned WITHOUT mutating the checkpoint + (used at resume time to fail an invalid selection before any auth or create). + .PARAMETER CheckpointData + The in-memory checkpoint hashtable ($script:CheckpointData is used when omitted). + .PARAMETER Requested + $true only when the operator explicitly passed -ClearUncertainCreate. + .PARAMETER SelectedIndexes + The explicit operator-supplied list of partition indexes to clear. + .PARAMETER SelectedContracts + The explicit operator-supplied list of ":" tokens naming durable + uncertain-create contracts to clear (the -ClearUncertainContract companion). Used to + disambiguate an index that maps to more than one durable uncertain contract: each token + must resolve to EXACTLY one durable entry by (index, fingerprint); a token that is + malformed, unknown, matches more than one, or is duplicated is REJECTED and nothing is + cleared. When supplied it takes precedence over -SelectedIndexes and clears strictly by + contract identity (bucket-exclusive by fingerprint); every unselected contract is kept. + .PARAMETER ValidateOnly + Validate and return the selection without removing anything from the checkpoint. + .OUTPUTS + A pscustomobject: Rejected (bool), Reason (string), Selected (int[] validated indexes), + Cleared (int[] indexes actually removed), DurableIndexes (int[] indexes present at call), + SelectedContracts (string[] validated ":" tokens), ClearedContracts + (string[] tokens actually removed), DurableContracts (string[] tokens present at call). + #> + param( + [Parameter()]$CheckpointData = $null, + [Parameter()][bool]$Requested, + [Parameter()][int[]]$SelectedIndexes = @(), + [Parameter()][string[]]$SelectedContracts = @(), + [Parameter()][switch]$ValidateOnly + ) + $outcome = [pscustomobject]@{ Rejected = $false; Reason = ''; Selected = @(); Cleared = @(); DurableIndexes = @(); SelectedContracts = @(); ClearedContracts = @(); DurableContracts = @() } + if (-not $Requested) { return $outcome } + $cp = if ($null -ne $CheckpointData) { $CheckpointData } else { $script:CheckpointData } + $durable = @() + if ($cp -and $cp.partitions -and $cp.partitions.uncertainCreate) { + $durable = @(@($cp.partitions.uncertainCreate) | Where-Object { $null -ne $_ } | ForEach-Object { [int]$_.index }) + } + $outcome.DurableIndexes = $durable + # EXPLICIT-CONTRACT selection (-ClearUncertainContract companion): each token + # ":" names EXACTLY one durable uncertain contract by (index, + # fingerprint) identity, so an index that maps to more than one contract can be + # disambiguated. This path takes precedence when supplied: zero/multiple/unknown/malformed/ + # duplicate tokens fail closed WITHOUT mutation, and a valid clear removes strictly the + # named contract(s) (bucket-exclusive by fingerprint) leaving every unselected entry exact. + if (@($SelectedContracts).Count -gt 0) { + $durableEntries = @() + if ($cp -and $cp.partitions -and $cp.partitions.uncertainCreate) { + $durableEntries = @(@($cp.partitions.uncertainCreate) | Where-Object { $null -ne $_ }) + } + $outcome.DurableContracts = @($durableEntries | ForEach-Object { "$([int]$_.index):$(Get-PaxCheckpointEntryFingerprint $_)" }) + $normTokens = @() + $matchedEntries = @() + foreach ($tok in @($SelectedContracts)) { + $raw = [string]$tok + $ci = $raw.IndexOf(':') + if ($ci -lt 1 -or $ci -ge ($raw.Length - 1)) { + $outcome.Rejected = $true; $outcome.Reason = "malformed explicit contract selector '$raw' (expected ':')"; return $outcome + } + $idxRaw = $raw.Substring(0, $ci).Trim() + $fpRaw = $raw.Substring($ci + 1).Trim() + $idxParsed = 0 + if (-not [int]::TryParse($idxRaw, [ref]$idxParsed)) { + $outcome.Rejected = $true; $outcome.Reason = "malformed explicit contract selector '$raw' (index is not an integer)"; return $outcome + } + if ([string]::IsNullOrWhiteSpace($fpRaw)) { + $outcome.Rejected = $true; $outcome.Reason = "malformed explicit contract selector '$raw' (missing fingerprint)"; return $outcome + } + $normTok = "$($idxParsed):$fpRaw" + $hits = @($durableEntries | Where-Object { ([int]$_.index -eq $idxParsed) -and ([string](Get-PaxCheckpointEntryFingerprint $_) -eq $fpRaw) }) + if (@($hits).Count -eq 0) { + $outcome.Rejected = $true; $outcome.Reason = "explicit contract selector '$normTok' matches no durable uncertain-create contract"; return $outcome + } + if (@($hits).Count -gt 1) { + $outcome.Rejected = $true; $outcome.Reason = "explicit contract selector '$normTok' matches more than one durable uncertain-create contract"; return $outcome + } + $normTokens += $normTok + $matchedEntries += $hits[0] + } + $dupeTok = @($normTokens | Group-Object | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name }) + if (@($dupeTok).Count -gt 0) { + $outcome.Rejected = $true; $outcome.Reason = "duplicate explicit contract selector token ($(@($dupeTok).Count) duplicate value(s))"; return $outcome + } + $outcome.SelectedContracts = $normTokens + if ($ValidateOnly) { return $outcome } + $cp.partitions.uncertainCreate = @( + @($cp.partitions.uncertainCreate) | Where-Object { + $entry = $_ + $drop = $false + foreach ($m in $matchedEntries) { + if (([int]$entry.index -eq [int]$m.index) -and ([string](Get-PaxCheckpointEntryFingerprint $entry) -eq [string](Get-PaxCheckpointEntryFingerprint $m))) { $drop = $true; break } + } + -not $drop + } + ) + $outcome.ClearedContracts = $normTokens + return $outcome + } + $selected = @($SelectedIndexes | ForEach-Object { [int]$_ }) + if ($selected.Count -eq 0) { + $outcome.Rejected = $true; $outcome.Reason = 'no partition index was supplied (selection is empty)'; return $outcome + } + $dupes = @($selected | Group-Object | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name }) + if ($dupes.Count -gt 0) { + $outcome.Rejected = $true; $outcome.Reason = "duplicate partition index selected ($($dupes.Count) duplicate value(s))"; return $outcome + } + $unknown = @($selected | Where-Object { $_ -notin $durable }) + if ($unknown.Count -gt 0) { + $outcome.Rejected = $true; $outcome.Reason = "$($unknown.Count) selected index(es) are not in the durable uncertain-create set of $($durable.Count) partition(s)"; return $outcome + } + # Ambiguity guard: because Graph workload passes RESTART partition indexes at 1, a single + # partition index can map to MORE THAN ONE durable uncertain contract (distinct + # fingerprints). An index-only selector cannot deterministically name which contract to + # clear, so REJECT before any mutation - clearing the wrong contract's durable uncertain + # state is unrecoverable. An explicit fingerprint selector is required to disambiguate. + $ambiguous = @() + foreach ($si in $selected) { + $cnt = @($durable | Where-Object { $_ -eq $si }).Count + if ($cnt -gt 1) { $ambiguous += $si } + } + if ($ambiguous.Count -gt 0) { + $cand = @() + foreach ($ai in @($ambiguous | Sort-Object -Unique)) { + foreach ($e in @($cp.partitions.uncertainCreate | Where-Object { $null -ne $_ -and ([int]$_.index -eq [int]$ai) })) { + $efp = Get-PaxCheckpointEntryFingerprint $e + if (-not [string]::IsNullOrWhiteSpace([string]$efp)) { $cand += "$([int]$ai):$efp" } + } + } + $hint = if (@($cand).Count -gt 0) { " re-run with -ClearUncertainContract naming the exact contract token(s): $(@($cand) -join ', ')" } else { ' the durable entries carry no fingerprint (legacy) and cannot be safely disambiguated - no clear is possible' } + $outcome.Rejected = $true; $outcome.Reason = "$($ambiguous.Count) selected index(es) map to more than one durable uncertain contract (ambiguous index-only selection);$hint"; return $outcome + } + $outcome.Selected = $selected + if ($ValidateOnly) { return $outcome } + $cp.partitions.uncertainCreate = @( + @($cp.partitions.uncertainCreate) | Where-Object { $null -ne $_ -and ([int]$_.index) -notin $selected } + ) + $outcome.Cleared = $selected + return $outcome +} + +function Resolve-PaxOperatorClearedUncertain { + <# + .SYNOPSIS + Selective operator recovery for a single NOT-VISIBLE durable-uncertain partition on + resume. Returns $true only when the operator explicitly selected this partition index + via -ClearUncertainCreate, in which case its durable uncertain marker is cleared + SELECTIVELY and atomically (single durable write, bucket-exclusive) BEFORE any fresh + create and the partition is authorized for a fresh create this run. + .DESCRIPTION + This is only ever called for a partition the READ-ONLY reconcile could NOT resolve by + exact name (Unresolved / not visible). A still-visible partition is adopted or fails + closed by the reconcile before this point, so a visible query is never cleared into a + duplicate POST. When the partition is not in the operator selection this is a no-op + that returns $false and preserves the durable uncertain state exactly. + .PARAMETER Index + The partition index being considered for operator-selected clear. + .PARAMETER Fingerprint + The contract fingerprint of the partition being considered, used to match an explicit + -ClearUncertainContract ":" selection so a clear removes EXACTLY that + contract even when another contract shares the index. + .PARAMETER Selection + The staged operator selection set (defaults to $script:ClearUncertainSelection). + .PARAMETER ContractSelection + The staged explicit-contract selection set of ":" tokens (defaults to + $script:ClearUncertainContractSelection). + .PARAMETER CheckpointData + The in-memory checkpoint (defaults to $script:CheckpointData). + .PARAMETER Persist + Optional durable-write callback (defaults to Save-CheckpointStrict when checkpointing + is enabled) invoked immediately after the selective clear, BEFORE any fresh create. The + write must be PROVEN durable: a failure rolls back the clear and fails closed. + .OUTPUTS + $true when the partition was operator-cleared for a fresh create; otherwise $false. + #> + param( + [Parameter(Mandatory)][int]$Index, + [Parameter()][string]$Fingerprint = $null, + [Parameter()]$Selection = $null, + [Parameter()]$ContractSelection = $null, + [Parameter()]$CheckpointData = $null, + [Parameter()][scriptblock]$Persist = $null + ) + $sel = $Selection + if ($null -eq $sel) { $sel = $script:ClearUncertainSelection } + $csel = $ContractSelection + if ($null -eq $csel) { $csel = $script:ClearUncertainContractSelection } + # This partition is operator-selected either by an explicit contract token + # (":", the precise identity that disambiguates two contracts sharing an + # index) or by an unambiguous index-only selection. The explicit contract selector is checked + # first so a fingerprinted clear removes EXACTLY that contract; index selection is only ever + # staged for an UNAMBIGUOUS index (ambiguous index selectors are rejected at resume time). + $contractToken = if (-not [string]::IsNullOrWhiteSpace($Fingerprint)) { "$([int]$Index):$Fingerprint" } else { $null } + $byContract = ($null -ne $contractToken -and $null -ne $csel -and $csel.Contains($contractToken)) + $byIndex = ($null -ne $sel -and $sel.Contains([int]$Index)) + if (-not $byContract -and -not $byIndex) { return $false } + $cp = if ($null -ne $CheckpointData) { $CheckpointData } else { $script:CheckpointData } + # Copy-on-write snapshot of the durable uncertain bucket so a failed durable write + # can be rolled back to EXACTLY the pre-clear state (fail-closed, no data loss). + $origUncertain = @($cp.partitions.uncertainCreate) + # Selective + atomic durable clear of ONLY this partition's uncertain marker - by the exact + # contract identity when a fingerprint selection matched, else by unambiguous index. + if ($byContract) { + $null = Clear-PaxUncertainCreateState -CheckpointData $cp -Requested:$true -SelectedContracts @($contractToken) + } + else { + $null = Clear-PaxUncertainCreateState -CheckpointData $cp -Requested:$true -SelectedIndexes @([int]$Index) + } + # Single durable write BEFORE any fresh create POST for this partition. The write + # must be PROVEN durable: the default path uses the strict verify-after-write + # publisher, and any failure (default OR injected callback) fails closed below. + $persistOk = $false + $persistError = $null + try { + if ($Persist) { & $Persist } + elseif ($script:CheckpointEnabled) { $null = Save-CheckpointStrict } + $persistOk = $true + } + catch { + $persistOk = $false + $persistError = $_ + } + if (-not $persistOk) { + # Fail-closed: durable persistence of the operator-cleared uncertain marker was + # NOT proven. Roll back the in-memory clear, leave the operator selection + # UNCONSUMED (no fresh create authorized), latch the classified fatal so the run + # exits nonzero, and re-throw so the caller loop stops without adding a + # create-capable partition. + $cp.partitions.uncertainCreate = $origUncertain + $script:GenericFatal = $true + # If the strict publisher could NOT restore the prior checkpoint bytes, the disk state + # is INDETERMINATE - surface that distinct rollback-failure category rather than the + # normal fail-closed message (which implies the prior state was restored intact). + if ($persistError -and ([string]$persistError.Exception.Message) -match 'PAX-FATAL-ROLLBACK') { + throw "PAX-FATAL-ROLLBACK: durable persistence of operator-cleared uncertain create failed for partition $([int]$Index) AND the prior checkpoint could not be restored; disk state is indeterminate - manual checkpoint inspection required. $([string]$persistError.Exception.Message)" + } + throw "PAX-FATAL: durable persistence of operator-cleared uncertain create failed for partition $([int]$Index); fail-closed, no fresh create authorized." + } + # Only a PROVEN durable write consumes the selection and authorizes the fresh create. + if ($byContract) { [void]$csel.Remove($contractToken) } + else { [void]$sel.Remove([int]$Index) } + return $true +} + +# ============================================== +# CTRL+C GRACEFUL EXIT HANDLER +# ============================================== +# Track Ctrl+C state for graceful exit messaging in finally block + +$script:CtrlCPressed = $false +$script:ScriptCompleted = $false +$script:EarlyExit = $false +# Track whether ANY file upload (SharePoint/Fabric output, metrics, or run log) failed +# this run. When true, local run files are preserved at end of run - treated the same as +# Ctrl+C / early-exit / crash - so nothing is lost when a destination upload did not complete. +$script:AnyUploadFailed = $false +$script:ResumeValidationFailed = $false +# 1D-a: a GENERIC, unexpected terminating error reached the top-level catch (i.e. NOT a +# Ctrl+C / graceful early-exit, and NOT one of the classified failure conditions that already +# drive a specific exit code). Without this flag such a crash falls through the finally exit +# selector with every failure flag false and reports exit 0 (false success) to ACA/automation, +# contradicting the exit contract (1 = Fatal / aborted). It maps to exit 1 at top precedence, +# suppresses the finally's Ctrl+C mis-inference, and (like any non-clean end) preserves the +# durable recovery artifacts the clean-run cleanup would otherwise delete. +$script:GenericFatal = $false +$script:AISIDCleanTail = $false + +function script:Invoke-PaxGraphDisconnect { + param([System.Management.Automation.ActionPreference]$ErrorAction = [System.Management.Automation.ActionPreference]::Stop) + $writer = $null + $originalWarning = $null + try { + try { + $writer = [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.OutputWriter + if ($writer -and $writer.WriteWarning) { + $originalWarning = $writer.WriteWarning + $warningPrefix = 'Failed to clear the persisted MSAL token cache during Disconnect-MgGraph:' + $writer.WriteWarning = [System.Action[string]]{ + param($message) + if (-not ([string]$message).StartsWith($warningPrefix, [System.StringComparison]::Ordinal)) { + $originalWarning.Invoke([string]$message) + } + }.GetNewClosure() + } + } catch {} + Disconnect-MgGraph -ErrorAction $ErrorAction | Out-Null + } + finally { + if ($writer -and $originalWarning) { + $writer.WriteWarning = $originalWarning + } + } +} + +# Register exit handler that ALWAYS runs when PowerShell exits +# This works even when Ctrl+C is pressed before the try block (e.g., during module loading) +# Uses environment variable for cross-runspace communication since Register-EngineEvent runs in isolated scope +$env:PAX_GRACEFUL_EXIT_DONE = $null +$env:PAX_REPLAY_MODE = $null # Will be set to "1" when RAWInputCSV is used +Register-EngineEvent -SourceIdentifier PowerShell.Exiting -Action { + if (-not $env:PAX_GRACEFUL_EXIT_DONE) { + # Skip interrupt messaging in replay mode - no Graph connection to disconnect + if (-not $env:PAX_REPLAY_MODE) { + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host " Script Interrupted - Performing Graceful Cleanup" -ForegroundColor Yellow + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "" + Write-Host " Cleanup complete. Exiting..." -ForegroundColor Green + Write-Host "" + } + } +} | Out-Null + +# Define cleanup function (used by catch block for PipelineStoppedException) +function Invoke-GracefulExit { + param([string]$Reason = "Script interrupted") + + if ($script:CtrlCPressed) { return } # Prevent multiple invocations + $script:CtrlCPressed = $true + + # Signal to engine event handler that graceful exit is handling this + $env:PAX_GRACEFUL_EXIT_DONE = "1" + + # Skip interrupt messaging and Graph disconnect in replay mode - no connections to clean up + if ($env:PAX_REPLAY_MODE) { + exit 0 + } + + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host " Script Interrupted - Performing Graceful Cleanup" -ForegroundColor Yellow + Write-Host "============================================================================================================" -ForegroundColor Yellow + Write-Host "" + + # Disconnect from Microsoft Graph - ALWAYS attempt disconnect + Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Cyan + try { + script:Invoke-PaxGraphDisconnect -ErrorAction Stop + Write-Host " Microsoft Graph disconnected" -ForegroundColor Green + } + catch { + if ($_.Exception.Message -match 'No application to sign out from') { + Write-Host " (Not connected to Microsoft Graph)" -ForegroundColor DarkGray + } else { + Write-Host " Microsoft Graph session cleared" -ForegroundColor Green + } + } + + # Disconnect from Exchange Online (if connected via EOM mode) + try { + $eomSession = Get-PSSession | Where-Object { $_.ConfigurationName -eq 'Microsoft.Exchange' -and $_.State -eq 'Opened' } + if ($eomSession) { + Write-Host " Disconnecting from Exchange Online Management..." -ForegroundColor Cyan + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue | Out-Null + Write-Host " Exchange Online disconnected" -ForegroundColor Green + } + } + catch { + Write-Host " (Exchange Online cleanup completed)" -ForegroundColor Gray + } + + # Log the graceful exit + if ($LogFile -and (Test-Path $LogFile)) { + Write-Output "" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] Script interrupted by user (Ctrl+C)" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')] Graceful cleanup completed" | Out-File -FilePath $LogFile -Append -Encoding utf8 + } + + # Show checkpoint resume message if checkpoint is enabled + if ($script:CheckpointEnabled -and $script:CheckpointPath -and (Test-Path $script:CheckpointPath)) { + Show-CheckpointExitMessage + } + + # Release the exclusive checkpoint lock so a future -Resume of THIS checkpoint + # (from any host) is not blocked by our stale lock. The lock file itself is removed + # by Release-CheckpointLock; the checkpoint JSON is preserved for resume. + try { script:Release-CheckpointLock } catch {} + + Write-Host "" + Write-Host " Cleanup complete. Exiting..." -ForegroundColor Green + Write-Host "" + + # Exit cleanly (env var PAX_GRACEFUL_EXIT_DONE already set at function start) + exit 0 +} + +# Trap for catching terminating errors (including Ctrl+C) +trap { + if ($_.Exception -is [System.Management.Automation.PipelineStoppedException]) { + Invoke-GracefulExit + break + } + # Re-throw other exceptions + throw $_ +} + +# ============================================== +# MODULE PREREQUISITES +# ============================================== +# Load required modules based on mode selection (-UseEOM vs Graph API default) + +# Emit PAX banner + sensitive-data warning to terminal/log BEFORE any other startup output +Write-LogHost "=== Portable Audit eXporter (PAX) - Purview Audit Log Exporter ===" -ForegroundColor Cyan +Write-LogHost ("Script Version: v$ScriptVersion") -ForegroundColor White +if (-not $SkipVersionCheck) { Invoke-PaxVersionCheck -CurrentVersion $ScriptVersion } +Write-LogHost "" +Write-LogHost "=========================================================================" -ForegroundColor Yellow +Write-LogHost " !! SENSITIVE DATA WARNING - CUSTOMER RESPONSIBILITY !!" -ForegroundColor Yellow +Write-LogHost "=========================================================================" -ForegroundColor Yellow +Write-LogHost "The audit data exported by this script is HIGHLY SENSITIVE. Output may" -ForegroundColor Yellow +Write-LogHost "contain user identifiers (UPN, email, GUID), file/site/resource paths," -ForegroundColor Yellow +Write-LogHost "conversation and message IDs, agent identifiers, prompt/response metadata" -ForegroundColor Yellow +Write-LogHost "(timestamps, lengths, classifications), and other personally identifiable" -ForegroundColor Yellow +Write-LogHost "information drawn directly from your tenant's Unified Audit Log." -ForegroundColor Yellow +Write-LogHost "" +Write-LogHost " * Data is NOT hashed, masked, redacted, anonymized, or de-identified." -ForegroundColor Yellow +Write-LogHost " Records are exported in raw, attributable form as Purview returns them." -ForegroundColor Yellow +Write-LogHost " * Outputs (CSV/Excel/JSON metrics, checkpoint files, logs) may contain" -ForegroundColor Yellow +Write-LogHost " confidential business content, regulated data (PII, PHI, financial," -ForegroundColor Yellow +Write-LogHost " IP), and end-user communications." -ForegroundColor Yellow +Write-LogHost " * The customer (you / your organization) is SOLELY RESPONSIBLE for the" -ForegroundColor Yellow +Write-LogHost " secure handling, storage, transmission, retention, disclosure, access" -ForegroundColor Yellow +Write-LogHost " control, and deletion of all data produced by this script, and for" -ForegroundColor Yellow +Write-LogHost " ensuring its use complies with all applicable laws, regulations," -ForegroundColor Yellow +Write-LogHost " contractual obligations, and internal policies - including but not" -ForegroundColor Yellow +Write-LogHost " limited to GDPR, HIPAA, CCPA, employee monitoring laws, works-council" -ForegroundColor Yellow +Write-LogHost " agreements, and data-residency requirements." -ForegroundColor Yellow +Write-LogHost " * Microsoft has no visibility into, control over, or responsibility" -ForegroundColor Yellow +Write-LogHost " for the data customers extract using this tool or how that data is" -ForegroundColor Yellow +Write-LogHost " subsequently used, shared, or stored. Microsoft disclaims any and all" -ForegroundColor Yellow +Write-LogHost " liability arising from or related to customer use of this script and" -ForegroundColor Yellow +Write-LogHost " its output." -ForegroundColor Yellow +Write-LogHost " * Treat all output files as HIGHLY CONFIDENTIAL. Restrict access to" -ForegroundColor Yellow +Write-LogHost " authorized personnel with a documented business need. Encrypt at rest" -ForegroundColor Yellow +Write-LogHost " and in transit. Apply tenant DLP / sensitivity labels as appropriate." -ForegroundColor Yellow +Write-LogHost "=========================================================================" -ForegroundColor Yellow +Write-LogHost "" + +if ($RAWInputCSV) { + # Set replay mode flag for graceful exit handling (skip Graph disconnect messaging) + $env:PAX_REPLAY_MODE = "1" + Write-LogHost "`nReplay mode: Skipping module loading`n" -ForegroundColor Cyan +} +elseif (-not $UseEOM) { + # DEFAULT MODE: Microsoft Graph Security API + # Requires Microsoft.Graph.Authentication and Microsoft.Graph.Security modules + Write-LogHost "" + Write-LogHost "Loading Microsoft Graph modules..." -ForegroundColor Cyan + + try { + # ============================================ + # AUTO-UPDATE CHECK: Ensure latest SDK version + # ============================================ + # The Graph Security auditLog API has known issues with older SDK versions. + # Always check for and install the latest version to ensure compatibility. + + Write-LogHost " Checking for Microsoft Graph SDK updates..." -ForegroundColor Gray + + # Get currently installed version + $installedAuth = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + $installedAuthVersion = if ($installedAuth) { $installedAuth.Version } else { [Version]"0.0.0" } + + # Check PSGallery for latest version (with 15-second timeout to avoid hangs) + $latestAuthVersion = $null + try { + $updateCheckJob = Start-Job -ScriptBlock { Find-Module -Name Microsoft.Graph.Authentication -Repository PSGallery -ErrorAction Stop } + $jobCompleted = Wait-Job -Job $updateCheckJob -Timeout 15 + if ($jobCompleted) { + $galleryAuth = Receive-Job -Job $updateCheckJob -ErrorAction Stop + $latestAuthVersion = [Version]$galleryAuth.Version + } + else { + Write-LogHost " PSGallery check timed out (15s) - skipping update check" -ForegroundColor Yellow + } + Remove-Job -Job $updateCheckJob -Force -ErrorAction SilentlyContinue + } + catch { + Write-LogHost " Warning: Could not check PSGallery for updates: $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " Continuing with installed version..." -ForegroundColor Yellow + } + + # Update if newer version available + $updatePerformed = $false + if ($latestAuthVersion -and ($latestAuthVersion -gt $installedAuthVersion)) { + Write-LogHost " Update available: v$installedAuthVersion → v$latestAuthVersion" -ForegroundColor Yellow + Write-LogHost " Installing Microsoft.Graph.Authentication v$latestAuthVersion..." -ForegroundColor Yellow + + try { + # Install latest Authentication module + Install-Module -Name Microsoft.Graph.Authentication -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Authentication updated to v$latestAuthVersion" -ForegroundColor Green + + # Install matching Security module + Write-LogHost " Installing Microsoft.Graph.Security v$latestAuthVersion..." -ForegroundColor Yellow + Install-Module -Name Microsoft.Graph.Security -RequiredVersion $latestAuthVersion -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Security updated to v$latestAuthVersion" -ForegroundColor Green + + $updatePerformed = $true + + # Refresh module info after update + $installedAuth = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + $installedAuthVersion = $installedAuth.Version + } + catch { + Write-LogHost " Warning: Update failed: $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " Continuing with existing version v$installedAuthVersion..." -ForegroundColor Yellow + } + } + elseif ($latestAuthVersion) { + Write-LogHost " Microsoft Graph SDK is up to date (v$installedAuthVersion)" -ForegroundColor Green + } + else { + Write-LogHost " Using installed version v$installedAuthVersion" -ForegroundColor Gray + } + + # ============================================ + # LOAD MODULES + # ============================================ + + $authModule = Get-Module -Name Microsoft.Graph.Authentication | Select-Object -First 1 + if (-not $authModule) { + $authModule = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + } + if (-not $authModule) { + Write-LogHost " Installing Microsoft.Graph.Authentication module (CurrentUser scope)..." -ForegroundColor Yellow + Install-Module -Name Microsoft.Graph.Authentication -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + $authModule = Get-Module -ListAvailable -Name Microsoft.Graph.Authentication | Sort-Object Version -Descending | Select-Object -First 1 + } + $authVersion = $authModule.Version + Write-LogHost " Importing Microsoft.Graph.Authentication v$authVersion..." -ForegroundColor Gray + Import-Module Microsoft.Graph.Authentication -RequiredVersion $authVersion -Force -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Authentication v$authVersion loaded" -ForegroundColor Green + + # Load Microsoft.Graph.Security matching auth version (exact if possible, otherwise same major/minor) + $securityModule = Get-Module -Name Microsoft.Graph.Security | Where-Object { $_.Version -eq $authVersion } | Select-Object -First 1 + if (-not $securityModule) { + $securityModule = Get-Module -ListAvailable -Name Microsoft.Graph.Security | + Where-Object { $_.Version.Major -eq $authVersion.Major -and $_.Version.Minor -eq $authVersion.Minor } | + Sort-Object Version -Descending | + Select-Object -First 1 + } + if (-not $securityModule) { + Write-LogHost " Installing Microsoft.Graph.Security v$authVersion (CurrentUser scope)..." -ForegroundColor Yellow + Install-Module -Name Microsoft.Graph.Security -RequiredVersion $authVersion -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + $securityModule = Get-Module -ListAvailable -Name Microsoft.Graph.Security | Where-Object { $_.Version -eq $authVersion } | Select-Object -First 1 + } + $secVersion = $securityModule.Version + Write-LogHost " Importing Microsoft.Graph.Security v$secVersion..." -ForegroundColor Gray + Import-Module Microsoft.Graph.Security -RequiredVersion $secVersion -Force -ErrorAction Stop + Write-LogHost " Microsoft.Graph.Security v$secVersion loaded" -ForegroundColor Green + } + catch { + Write-LogHost " ERROR: Failed to load Microsoft Graph module: $($_.Exception.Message)" -ForegroundColor Red + Write-LogHost "`nTroubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure PowerShell Gallery access is available" -ForegroundColor White + Write-LogHost " 2. Try manual installation: Install-Module -Name Microsoft.Graph -Force" -ForegroundColor White + Write-LogHost " 3. Use -UseEOM switch to fall back to Exchange Online Management mode" -ForegroundColor White + throw + } + + Write-LogHost "Microsoft Graph modules loaded successfully`n" -ForegroundColor Green +} +else { + # EOM MODE: Exchange Online Management + # Graph modules not required in EOM mode + Write-LogHost "`nEOM Mode: Skipping Microsoft Graph module loading`n" -ForegroundColor Cyan +} + +# ============================================== +# DUAL-MODE AUTHENTICATION FUNCTION +# ============================================== +# Unified authentication supporting both EOM and Graph API modes + +function Connect-PurviewAudit { + <# + .SYNOPSIS + Unified authentication for Purview audit log access via EOM or Graph API. + + .DESCRIPTION + Authenticates to Microsoft 365 using either Exchange Online Management (EOM) + or Microsoft Graph Security API based on the -UseEOM switch. + + EOM Mode (-UseEOM): + - Uses Connect-ExchangeOnline cmdlet + - Requires Exchange Online RBAC roles + - Serial processing only + + Graph API Mode (Default): + - Uses Connect-MgGraph with AuditLogsQuery.Read.All scope + - Requires Azure AD roles + Graph API permissions + - Supports parallel processing + + .PARAMETER AuthMethod + Authentication method: WebLogin, DeviceCode, Credential, Silent, AppRegistration + + .PARAMETER UseEOMMode + If true, use EOM mode. If false, use Graph API mode. + #> + + param( + [Parameter(Mandatory = $true)] + [ValidateSet('WebLogin', 'DeviceCode', 'Credential', 'Silent', 'AppRegistration', 'ManagedIdentity')] + [string]$AuthMethod, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Exchange Online Management + # ======================================== + + if ($script:Connected) { + Write-LogHost "Already connected to Exchange Online." -ForegroundColor Gray + return + } + + Write-LogHost "Connecting to Microsoft 365 Security & Compliance Center (EOM)..." -ForegroundColor Cyan + + # Ensure ExchangeOnlineManagement module is available + try { + $existingEOM = Get-Module -ListAvailable -Name ExchangeOnlineManagement | Sort-Object Version -Descending | Select-Object -First 1 + if (-not $existingEOM) { + Write-LogHost "Installing ExchangeOnlineManagement module (CurrentUser scope)..." -ForegroundColor Yellow + Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop + } + Import-Module ExchangeOnlineManagement -Force -ErrorAction Stop + + $eomVersion = (Get-Module ExchangeOnlineManagement).Version + Write-LogHost " ExchangeOnlineManagement v$eomVersion loaded" -ForegroundColor Green + } + catch { + Write-LogHost "ERROR: Module load/install failure: $($_.Exception.Message)" -ForegroundColor Red + throw + } + + # Authenticate based on method + try { + switch ($AuthMethod.ToLower()) { + 'appregistration' { + Write-LogHost "AppRegistration authentication is not supported with -UseEOM. Remove -UseEOM to use Graph mode." -ForegroundColor Yellow + throw "AppRegistration authentication is only available in Graph API mode" + } + 'weblogin' { + $exoCmd = Get-Command Connect-ExchangeOnline -ErrorAction Stop + $hasUseWeb = $exoCmd.Parameters.ContainsKey('UseWebLogin') + + if ($hasUseWeb) { + Write-LogHost "Using Connect-ExchangeOnline -UseWebLogin..." -ForegroundColor Gray + Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null + } + else { + Write-LogHost "UseWebLogin parameter not available; using standard interactive auth..." -ForegroundColor Yellow + Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null + } + } + + 'devicecode' { + Write-LogHost "Using device code flow..." -ForegroundColor Gray + Connect-ExchangeOnline -ShowBanner:$false -Device -ErrorAction Stop | Out-Null + } + + 'credential' { + Write-LogHost "Using credential-based authentication..." -ForegroundColor Gray + $cred = Get-Credential -Message 'Enter admin credentials for Exchange Online' + Connect-ExchangeOnline -ShowBanner:$false -Credential $cred -ErrorAction Stop | Out-Null + } + + 'silent' { + Write-LogHost "Attempting silent authentication..." -ForegroundColor Gray + $silentOk = $true + try { + Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null + } + catch { + $silentOk = $false + } + + if (-not $silentOk) { + Write-LogHost "Silent auth failed, falling back to WebLogin..." -ForegroundColor Yellow + try { + Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null + } + catch { + Write-LogHost "ERROR: Silent + fallback auth failed: $($_.Exception.Message)" -ForegroundColor Red + throw + } + } + } + 'managedidentity' { + Write-LogHost "ERROR: -Auth ManagedIdentity is not supported with -UseEOM." -ForegroundColor Red + Write-LogHost " Exchange Online Management does not accept managed-identity auth." -ForegroundColor Yellow + Write-LogHost " Use -Auth ManagedIdentity without -UseEOM (Graph API path)." -ForegroundColor Yellow + throw "ManagedIdentity auth is not supported with -UseEOM" + } + } + + $script:Connected = $true + Write-LogHost "Successfully connected to Exchange Online" -ForegroundColor Green + + # Verify connection + try { + $connInfo = Get-ConnectionInformation -ErrorAction SilentlyContinue | Where-Object { $_.TokenStatus -ne 'Expired' } | Select-Object -First 1 + if ($connInfo) { + Write-LogHost " Tenant ID: $($connInfo.TenantId)" -ForegroundColor Gray + Write-LogHost " User: $($connInfo.UserPrincipalName)" -ForegroundColor Gray + } + } + catch { + # Connection info not critical, continue + } + } + catch { + Write-LogHost "ERROR: EOM authentication failed: $($_.Exception.Message)" -ForegroundColor Red + Write-LogHost "`nTroubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Verify you have required Exchange Online roles" -ForegroundColor White + Write-LogHost " 2. Check Multi-Factor Authentication requirements" -ForegroundColor White + Write-LogHost " 3. Try a different auth method (-Auth parameter)" -ForegroundColor White + throw + } + } + else { + # ======================================== + # GRAPH API MODE: Microsoft Graph Security + # ======================================== + + # Clear any stale Graph session from a previous script run or Ctrl+C in this terminal. + # This forces a fresh Connect-MgGraph with a new token, preventing issues where MSAL + # silently returns a cached expired token or a token from a different user account. + Write-LogHost "Clearing any previous Graph session..." -ForegroundColor Gray + try { script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue } catch { } + + Write-LogHost "Connecting to Microsoft Graph Security API..." -ForegroundColor Cyan + + # Define required scopes for Purview audit log access via the Microsoft Graph + # Security API. The exact scope set is built dynamically based on which features + # the caller has activated, so users only consent to what they actually use. + # + # Conditional (added only when the relevant switch is set): + # Audit query (skipped under -OnlyUserInfo since no audit calls are made): + # AuditLogsQuery.Read.All - umbrella scope for /security/auditLog/queries + # (covers CopilotInteraction and other workload- + # agnostic record types) + # -IncludeM365Usage: + # AuditLogsQuery-Exchange.Read.All + # AuditLogsQuery-OneDrive.Read.All + # AuditLogsQuery-SharePoint.Read.All + # -IncludeUserInfo / -OnlyUserInfo (Graph API mode only): + # User.Read.All (read /users for Entra directory + license map) + # Organization.Read.All (read /subscribedSkus for license SKU lookup) + # -GroupNames (Graph API mode only): + # GroupMember.Read.All (least-privilege scope per Microsoft Graph docs + # for GET /groups and GET /groups/{id}/transitiveMembers; + # used by Expand-PurviewGroupMembership) + $RequiredScopes = [System.Collections.Generic.List[string]]::new() + if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { + [void]$RequiredScopes.Add('AuditLogsQuery.Read.All') + } + if ($IncludeM365Usage) { + [void]$RequiredScopes.Add('AuditLogsQuery-Exchange.Read.All') + [void]$RequiredScopes.Add('AuditLogsQuery-OneDrive.Read.All') + [void]$RequiredScopes.Add('AuditLogsQuery-SharePoint.Read.All') + } + if ($IncludeUserInfo -or $OnlyUserInfo) { + if ($RequiredScopes -notcontains 'User.Read.All') { [void]$RequiredScopes.Add('User.Read.All') } + if ($RequiredScopes -notcontains 'Organization.Read.All') { [void]$RequiredScopes.Add('Organization.Read.All') } + } + if ($GroupNames -and $GroupNames.Count -gt 0) { + if ($RequiredScopes -notcontains 'GroupMember.Read.All') { [void]$RequiredScopes.Add('GroupMember.Read.All') } + } + # Microsoft Agent 365 enrichment scopes - DELEGATED auth modes only. + # Delegated modes (WebLogin / DeviceCode / Credential / Silent) request + # CopilotPackages.Read.All (+ Application.Read.All) as DELEGATED scopes here at sign-in. + # App-only modes (AppRegistration certificate/secret, ManagedIdentity) do NOT request + # delegated scopes: the Agent 365 catalog is read with the matching APPLICATION app-roles + # granted + admin-consented on the app / managed-identity service principal out-of-band + # (see Connect-Agent365InteractiveContext). The $AuthMethod -notin guard below therefore + # correctly excludes both app-only modes from delegated scope requests. + if (($IncludeAgent365Info -or $OnlyAgent365Info) -and $AuthMethod -notin @('AppRegistration','ManagedIdentity')) { + if ($RequiredScopes -notcontains 'CopilotPackages.Read.All') { [void]$RequiredScopes.Add('CopilotPackages.Read.All') } + if ($RequiredScopes -notcontains 'Application.Read.All') { [void]$RequiredScopes.Add('Application.Read.All') } + } + # AISID (Defender / AI Solutions) hunting scope - DELEGATED auth modes only. + # -Dashboard AISID pulls Defender data via the Graph advanced-hunting endpoint + # (POST /v1.0/security/runHuntingQuery) using THIS SAME Graph token — no separate login. + # Delegated modes (WebLogin / DeviceCode) request ThreatHunting.Read.All here at sign-in. + # App-only modes (Credential client-secret, AppRegistration, Silent/ManagedIdentity) do NOT + # request delegated scopes: ThreatHunting.Read.All must be granted + admin-consented as an + # APPLICATION permission on the app registration / managed-identity service principal + # out-of-band (documented customer prerequisite). The -notin guard mirrors the Agent 365 + # block above so app-only modes are correctly excluded from the delegated request. + if (($Dashboard -eq 'AISID') -and $AuthMethod -notin @('AppRegistration','ManagedIdentity')) { + if ($RequiredScopes -notcontains 'ThreatHunting.Read.All') { [void]$RequiredScopes.Add('ThreatHunting.Read.All') } + } + # SharePoint remote-output destination needs delegated/app drive write scopes. + # Fabric/OneLake uses a separate storage-audience token (Az.Accounts), not a Graph scope. + # Sites.ReadWrite.All alone authorizes every SharePoint drive/file endpoint used here + # (site/drive resolve, folder create via POST children, PUT / createUploadSession, download) + # for both delegated and application auth, so Files.ReadWrite.All is not requested. + if ($script:RemoteOutputMode -eq 'SharePoint') { + if ($RequiredScopes -notcontains 'Sites.ReadWrite.All') { [void]$RequiredScopes.Add('Sites.ReadWrite.All') } + } + $RequiredScopes = $RequiredScopes.ToArray() + + try { + switch ($AuthMethod.ToLower()) { + 'weblogin' { + Write-LogHost "Using interactive browser authentication..." -ForegroundColor Gray + Connect-MgGraph -Scopes $RequiredScopes -NoWelcome -ErrorAction Stop + } + + 'devicecode' { + Write-LogHost "Using device code flow..." -ForegroundColor Gray + Write-LogHost "A browser window will open. Follow the instructions to authenticate." -ForegroundColor Yellow + Connect-MgGraph -Scopes $RequiredScopes -UseDeviceCode -NoWelcome -ErrorAction Stop + } + + 'credential' { + Write-LogHost "Using client secret credential..." -ForegroundColor Gray + + # Check for required environment variables + $tenantId = $env:GRAPH_TENANT_ID + $clientId = $env:GRAPH_CLIENT_ID + $clientSecret = $env:GRAPH_CLIENT_SECRET + + if (-not $tenantId -or -not $clientId -or -not $clientSecret) { + Write-LogHost "ERROR: Credential authentication requires environment variables:" -ForegroundColor Red + Write-LogHost " GRAPH_TENANT_ID : Your Azure AD Tenant ID" -ForegroundColor Yellow + Write-LogHost " GRAPH_CLIENT_ID : Your App Registration Client ID" -ForegroundColor Yellow + Write-LogHost " GRAPH_CLIENT_SECRET : Your App Registration Client Secret" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "Set these variables before running the script:" -ForegroundColor Yellow + Write-LogHost " `$env:GRAPH_TENANT_ID = 'your-tenant-id'" -ForegroundColor White + Write-LogHost " `$env:GRAPH_CLIENT_ID = 'your-client-id'" -ForegroundColor White + Write-LogHost " `$env:GRAPH_CLIENT_SECRET = 'your-client-secret'" -ForegroundColor White + throw "Missing required environment variables for credential authentication" + } + + $secureSecret = ConvertTo-SecureString -String $clientSecret -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential($clientId, $secureSecret) + + # Clear plain-text secret from memory + Clear-Variable -Name clientSecret -Force -ErrorAction SilentlyContinue + + Connect-MgGraph -TenantId $tenantId -ClientSecretCredential $credential -NoWelcome -ErrorAction Stop + } + + 'silent' { + Write-LogHost "Using managed identity or existing token..." -ForegroundColor Gray + Connect-MgGraph -Identity -NoWelcome -ErrorAction Stop + } + 'managedidentity' { + Write-LogHost "Using managed identity (system-assigned or user-assigned)..." -ForegroundColor Gray + if ($env:AZURE_CLIENT_ID) { + Write-LogHost (" -> User-assigned MI client id: {0}" -f $env:AZURE_CLIENT_ID) -ForegroundColor DarkGray + Connect-MgGraph -Identity -ClientId $env:AZURE_CLIENT_ID -NoWelcome -ErrorAction Stop + } else { + Connect-MgGraph -Identity -NoWelcome -ErrorAction Stop + } + } + 'appregistration' { + Write-LogHost "Using app registration authentication..." -ForegroundColor Gray + + $appTenantId = $script:TenantId + if ([string]::IsNullOrWhiteSpace($appTenantId)) { $appTenantId = $env:GRAPH_TENANT_ID } + if ([string]::IsNullOrWhiteSpace($appTenantId)) { + Write-LogHost "ERROR: -TenantId or GRAPH_TENANT_ID is required for AppRegistration auth." -ForegroundColor Red + throw "Missing TenantId for AppRegistration authentication" + } + + $appClientId = $script:ClientId + if ([string]::IsNullOrWhiteSpace($appClientId)) { $appClientId = $env:GRAPH_CLIENT_ID } + if ([string]::IsNullOrWhiteSpace($appClientId)) { + Write-LogHost "ERROR: -ClientId or GRAPH_CLIENT_ID is required for AppRegistration auth." -ForegroundColor Red + throw "Missing ClientId for AppRegistration authentication" + } + + # Store auth config for potential re-authentication during long-running operations + $script:AuthConfig.Method = 'AppRegistration' + $script:AuthConfig.TenantId = $appTenantId + $script:AuthConfig.ClientId = $appClientId + $script:AuthConfig.CertStoreLocation = $script:ClientCertificateStoreLocation + + $secretValue = $script:ClientSecret + if ([string]::IsNullOrWhiteSpace($secretValue)) { $secretValue = $env:GRAPH_CLIENT_SECRET } + + $certThumbprint = $script:ClientCertificateThumbprint + if ([string]::IsNullOrWhiteSpace($certThumbprint)) { $certThumbprint = $env:GRAPH_CLIENT_CERT_THUMBPRINT } + + $certPath = $script:ClientCertificatePath + if ([string]::IsNullOrWhiteSpace($certPath)) { $certPath = $env:GRAPH_CLIENT_CERT_PATH } + + $certPasswordSecure = $script:ClientCertificatePassword + if (-not $certPasswordSecure -and $env:GRAPH_CLIENT_CERT_PASSWORD) { + $certPasswordSecure = ConvertTo-SecureString $env:GRAPH_CLIENT_CERT_PASSWORD -AsPlainText -Force + } + + $certPasswordPlain = $null + if ($certPasswordSecure) { + $certPasswordPlain = [System.Net.NetworkCredential]::new('', $certPasswordSecure).Password + } + + if (-not [string]::IsNullOrWhiteSpace($secretValue)) { + Write-LogHost " -> Authenticating with client secret" -ForegroundColor Gray + $secureSecret = ConvertTo-SecureString -String $secretValue -AsPlainText -Force + $credential = New-Object System.Management.Automation.PSCredential($appClientId, $secureSecret) + # Store secret securely for re-authentication (keep a copy before clearing) + $script:AuthConfig.ClientSecret = $secureSecret.Copy() + $script:AuthConfig.CanReauthenticate = $true + Clear-Variable -Name secretValue -Force -ErrorAction SilentlyContinue + Connect-MgGraph -TenantId $appTenantId -ClientSecretCredential $credential -NoWelcome -ErrorAction Stop + Clear-Variable -Name secureSecret -Force -ErrorAction SilentlyContinue + Clear-Variable -Name credential -Force -ErrorAction SilentlyContinue + } + elseif (-not [string]::IsNullOrWhiteSpace($certThumbprint)) { + # Normalize: strip spaces and any non-hex paste artifacts (the Windows cert + # dialog can prepend a hidden left-to-right mark), then upper-case. + $certThumbprint = ($certThumbprint -replace '[^0-9A-Fa-f]', '').ToUpperInvariant() + Write-LogHost " -> Authenticating with certificate thumbprint $certThumbprint" -ForegroundColor Gray + # Search the requested store first, then the OTHER store. App-registration + # certs on servers are commonly in LocalMachine\My even when the run defaults + # to CurrentUser; this finds the cert in either location. + $requestedLoc = $script:ClientCertificateStoreLocation + $otherLoc = if ($requestedLoc -eq 'LocalMachine') { 'CurrentUser' } else { 'LocalMachine' } + $certificate = $null + $resolvedLoc = $null + foreach ($loc in @($requestedLoc, $otherLoc)) { + $storeLocation = [System.Security.Cryptography.X509Certificates.StoreLocation]::$loc + $store = New-Object System.Security.Cryptography.X509Certificates.X509Store("My", $storeLocation) + try { + $store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadOnly) + $match = @($store.Certificates | Where-Object { $_.Thumbprint -eq $certThumbprint }) + if ($match.Count -gt 0) { + # Prefer a copy that carries the private key. + $withKey = @($match | Where-Object { $_.HasPrivateKey } | Select-Object -First 1) + $certificate = if ($withKey.Count -gt 0) { $withKey[0] } else { $match[0] } + $resolvedLoc = $loc + break + } + } + catch { + # This store could not be opened/read (e.g. access denied to + # LocalMachine\My); skip it and try the other store. + } + finally { + $store.Close() + } + } + if (-not $certificate) { + Write-LogHost "ERROR: Certificate with thumbprint '$certThumbprint' not found in CurrentUser\My or LocalMachine\My." -ForegroundColor Red + throw "Certificate not found" + } + if (-not $certificate.HasPrivateKey) { + Write-LogHost "ERROR: Certificate '$certThumbprint' was found in $resolvedLoc\My but has no accessible private key (only the public .cer appears to be installed, or the running account cannot read the key)." -ForegroundColor Red + throw "Certificate has no usable private key" + } + if ($resolvedLoc -ne $requestedLoc) { + Write-LogHost " -> Certificate located in $resolvedLoc\My (requested store was $requestedLoc\My)." -ForegroundColor Gray + } + # Pin the resolved cert object for the whole run + token refreshes. Passing + # -Certificate (not -CertificateThumbprint) makes the lookup deterministic and + # independent of which stores the SDK searches; pinning prevents the + # SafeCertContext handle from being GC-invalidated on later token requests + # (same rationale as the -ClientCertificatePath branch). + $script:AuthConfig.CertThumbprint = $certThumbprint + $script:AuthConfig.CertStoreLocation = $resolvedLoc + $script:AuthConfig.CertObject = $certificate + $script:AuthConfig.CanReauthenticate = $true + Connect-MgGraph -TenantId $appTenantId -ClientId $appClientId -Certificate $certificate -NoWelcome -ErrorAction Stop + } + elseif (-not [string]::IsNullOrWhiteSpace($certPath)) { + Write-LogHost " -> Authenticating with certificate file $certPath" -ForegroundColor Gray + # CRITICAL: EphemeralKeySet keeps the private key in process memory only (no temp + # .pfx-key file in %APPDATA%\Microsoft\Crypto\) and ties the SafeCertContext lifetime + # to the X509Certificate2 object itself. Without this flag, default DefaultKeySet/UserKeySet + # storage causes Azure.Identity's ClientCertificateCredential to throw + # 'm_safeCertContext is an invalid handle' on every token request after the first. + # Exportable is required so the SDK can re-export the key for token signing. + $flags = [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet ` + -bor [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::Exportable + $cert = $null + try { + if ($certPasswordPlain) { + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($certPath, $certPasswordPlain, $flags) + } + else { + # 3-arg ctor with empty password is required so the explicit flags apply. + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2($certPath, [string]'', $flags) + } + # Store cert path and password for re-authentication + $script:AuthConfig.CertPath = $certPath + if ($certPasswordSecure) { $script:AuthConfig.CertPassword = $certPasswordSecure.Copy() } + # CRITICAL: Pin the cert object on $script: scope so it lives for the entire run. + # Azure.Identity's ClientCertificateCredential keeps a managed reference but does + # NOT clone the SafeCertContext - if the cert is GC'd or Disposed, every later + # token request fails with 'invalid handle'. The cert MUST NOT be disposed in + # the finally block; it is released at script exit when $script:AuthConfig is torn down. + $script:AuthConfig.CertObject = $cert + $script:AuthConfig.CanReauthenticate = $true + Connect-MgGraph -TenantId $appTenantId -ClientId $appClientId -Certificate $cert -NoWelcome -ErrorAction Stop + } + finally { + # Intentionally NOT disposing $cert here - it must outlive Connect-MgGraph for the + # duration of the script run. EphemeralKeySet ensures no on-disk artifacts to clean up; + # the cert is fully released at script exit when $script:AuthConfig is torn down. + if ($certPasswordPlain) { + Clear-Variable -Name certPasswordPlain -Force -ErrorAction SilentlyContinue + } + } + } + else { + Write-LogHost "ERROR: Provide either -ClientSecret, -ClientCertificateThumbprint, or -ClientCertificatePath for AppRegistration auth." -ForegroundColor Red + throw "No credential material supplied for AppRegistration" + } + + if ($certPasswordSecure) { + Clear-Variable -Name certPasswordSecure -Force -ErrorAction SilentlyContinue + } + } + } + + # Build a context-aware connection-success message + $connectMsg = if ($AuthMethod.ToLower() -eq 'appregistration') { + if ($IncludeAgent365Info -or $OnlyAgent365Info) { + # App-only + Agent 365 reuses this application context; the Agent 365 phase runs + # app-only against pre-consented application permissions with no interactive sign-in. + "Successfully connected to Microsoft Graph (app-only context). The Agent 365 phase reuses this application context - no interactive sign-in is required." + } else { + "Successfully connected to Microsoft Graph (app-only context)." + } + } elseif ($AuthMethod.ToLower() -eq 'managedidentity') { + if ($IncludeAgent365Info -or $OnlyAgent365Info) { + "Successfully connected to Microsoft Graph (managed-identity app-only context). The Agent 365 phase reuses this application context - no interactive sign-in is required." + } else { + "Successfully connected to Microsoft Graph (managed-identity app-only context)." + } + } else { + "Successfully connected to Microsoft Graph (delegated user context)." + } + Write-LogHost $connectMsg -ForegroundColor Green + + # Record token issue time for proactive refresh tracking + $script:AuthConfig.TokenIssueTime = Get-Date + + # Initialize shared auth state for thread jobs (enables proactive token refresh) + $tokenInfo = Get-GraphAccessTokenWithExpiry + if ($tokenInfo) { + $script:SharedAuthState.Token = $tokenInfo.Token + $script:SharedAuthState.ExpiresOn = $tokenInfo.ExpiresOn + $script:SharedAuthState.LastRefresh = Get-Date + $script:SharedAuthState.AuthMethod = $AuthMethod.ToLower() + Write-LogHost " Token expires: $($tokenInfo.ExpiresOn.ToString('yyyy-MM-dd HH:mm:ss')) UTC (source: $($tokenInfo.Source))" -ForegroundColor Gray + } + + # Get and display current context + $context = Get-MgContext + # Detect dual-context run: AppRegistration audit phase + delegated Agent 365 phase. + # In that case, defer the per-line Tenant/Account/Scopes display until AFTER Phase 2 + # sign-in completes, so the log shows BOTH phases honestly side-by-side. Capture the + # Phase 1 context now into script-scope vars so the combined emitter can use them. + # + # (app-only modes) - or the same delegated context (delegated modes) - so there is no + # separate Phase 2 context to combine and nothing to defer. Always emit the auth-context + # display inline below for every auth mode. + $script:DeferAuthContextDisplay = $false + if ($script:DeferAuthContextDisplay) { + $script:Phase1Context = [pscustomobject]@{ + TenantId = $context.TenantId + Account = $context.Account + GrantedRequired = @($RequiredScopes | Where-Object { $context.Scopes -contains $_ }) + RequiredScopes = @($RequiredScopes) + } + Write-LogHost " Phase 1 (audit) connected." -ForegroundColor Gray + } else { + Write-LogHost " Tenant ID: $($context.TenantId)" -ForegroundColor Gray + $maskedAccount = Get-MaskedUsername -Username $context.Account + if ([string]::IsNullOrWhiteSpace($maskedAccount)) { + # App-only contexts have no signed-in user; surface the auth mode instead of a blank value. + $maskedAccount = if ($AuthMethod -eq 'AppRegistration') { '(app-only / AppRegistration - no interactive user)' } + elseif ($AuthMethod -eq 'ManagedIdentity') { + if ($env:AZURE_CLIENT_ID) { "(app-only / ManagedIdentity - clientId $env:AZURE_CLIENT_ID)" } + else { '(app-only / ManagedIdentity - system-assigned)' } + } + else { '(no signed-in user)' } + } + Write-LogHost " Account: $maskedAccount" -ForegroundColor Gray + # Filter displayed scopes to those required by this script (avoids confusion when the + # user/app holds additional consented scopes unrelated to PAX). Granted required scopes + # are shown; any required scopes missing from the token are surfaced separately below. + $grantedRequired = @($RequiredScopes | Where-Object { $context.Scopes -contains $_ }) + Write-LogHost " Scopes: $($grantedRequired -join ', ')" -ForegroundColor Gray + } + + # Trigger Graph API version detection early (before queries start) + $null = Get-GraphAuditApiUri -Path 'queries' + + # Validate required scopes are present + # NOTE: For app-only (AppRegistration) contexts, the access token carries permissions + # in the 'roles' claim (application permissions), not the 'scp' claim. The Graph SDK + # only populates $context.Scopes from 'scp', so $context.Scopes is empty/partial for + # app-only auth even when admin-consented application permissions are fully granted. + # A scope-membership check would falsely flag every required permission as missing. + # App permissions are validated server-side at API call time, so we skip the warning. + $isAppOnlyContext = $false + if ($context) { + $isAppOnlyContext = ([string]::IsNullOrWhiteSpace($context.Account)) -or ($context.AuthType -eq 'AppOnly') + } + $missingScopes = @() + if (-not $isAppOnlyContext) { + foreach ($scope in $RequiredScopes) { + if ($context.Scopes -notcontains $scope) { + $missingScopes += $scope + } + } + } + + if ($missingScopes.Count -gt 0) { + Write-LogHost "" + Write-LogHost "WARNING: Missing required scope(s):" -ForegroundColor Yellow + foreach ($scope in $missingScopes) { + Write-LogHost " • $scope" -ForegroundColor Yellow + } + Write-LogHost "" + Write-LogHost "Script may fail when accessing audit logs." -ForegroundColor Yellow + Write-LogHost "Consider re-authenticating with full permissions." -ForegroundColor Yellow + Write-LogHost "" + } + + $script:Connected = $true + } + catch { + Write-LogHost "ERROR: Graph API authentication failed: $($_.Exception.Message)" -ForegroundColor Red + Write-LogHost "" + Write-LogHost "Troubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure you have AuditLogsQuery.Read.All permission" -ForegroundColor White + Write-LogHost " 2. Verify Azure AD role (Compliance/Security Administrator)" -ForegroundColor White + Write-LogHost " 3. Check network connectivity to Microsoft Graph API" -ForegroundColor White + Write-LogHost " 4. Try a different authentication method (-Auth parameter)" -ForegroundColor White + Write-LogHost " 5. Use -UseEOM switch to fall back to EOM mode" -ForegroundColor White + Write-LogHost "" + throw + } + } +} + +# ============================================== +# ACCESS TOKEN EXTRACTION HELPER +# ============================================== +function Get-GraphAccessToken { + <# + .SYNOPSIS + Extracts the current access token from an active Microsoft Graph session. + + .DESCRIPTION + Microsoft Graph PowerShell SDK 2.x does NOT expose AccessToken via Get-MgContext + for security reasons. This function reliably extracts the token by making a + lightweight request and extracting the Authorization header. + + Primary method: HTTP request header extraction (reliable in SDK 2.x) + Fallback method: Get-MgContext.AccessToken (for older SDK versions) + + .OUTPUTS + [string] The access token, or $null if extraction fails + #> + [CmdletBinding()] + param() + + # Primary method: Extract token from HTTP response headers (reliable in SDK 2.x) + try { + $response = Invoke-MgGraphRequest -Method GET -Uri 'https://graph.microsoft.com/v1.0/$metadata' -OutputType HttpResponseMessage -ErrorAction Stop + $token = $response.RequestMessage.Headers.Authorization.Parameter + if ($token) { + return $token + } + } + catch { + # HTTP method failed, try fallback + } + + # Fallback method: Get-MgContext.AccessToken (works in older SDK versions) + try { + $context = Get-MgContext -ErrorAction SilentlyContinue + if ($context -and $context.AccessToken) { + return $context.AccessToken + } + } + catch { + # Fallback also failed + } + + return $null +} + +# ============================================== +# ACCESS TOKEN WITH EXPIRY EXTRACTION (for shared auth state) +# ============================================== +function Get-GraphAccessTokenWithExpiry { + <# + .SYNOPSIS + Extracts access token AND expiry time from the active Microsoft Graph session. + + .DESCRIPTION + Decodes the JWT access token to extract the actual 'exp' (expiry) claim. + This is more reliable than Azure.Identity reflection and doesn't cause + extra authentication popups. + + JWT tokens have three base64-encoded parts: header.payload.signature + The payload contains the 'exp' claim as a Unix timestamp. + + Falls back to 50-minute estimated expiry if JWT decode fails. + + .OUTPUTS + [hashtable] with Token (string) and ExpiresOn (DateTime) properties + Returns $null if no token can be extracted + #> + [CmdletBinding()] + param() + + $result = @{ + Token = $null + ExpiresOn = $null + Source = 'unknown' + } + + # First, get the token using existing reliable method + $result.Token = Get-GraphAccessToken + if (-not $result.Token) { + return $null + } + + # Try to decode JWT to get actual expiry from 'exp' claim + # JWT format: base64url(header).base64url(payload).signature + try { + $tokenParts = $result.Token.Split('.') + if ($tokenParts.Count -ge 2) { + # Decode the payload (second part) + $payloadBase64 = $tokenParts[1] + + # Add padding if needed (base64url uses no padding) + $paddingNeeded = 4 - ($payloadBase64.Length % 4) + if ($paddingNeeded -lt 4) { + $payloadBase64 += ('=' * $paddingNeeded) + } + + # Convert base64url to standard base64 (replace - with +, _ with /) + $payloadBase64 = $payloadBase64.Replace('-', '+').Replace('_', '/') + + # Decode and parse JSON + $payloadBytes = [Convert]::FromBase64String($payloadBase64) + $payloadJson = [System.Text.Encoding]::UTF8.GetString($payloadBytes) + $payload = $payloadJson | ConvertFrom-Json + + if ($payload.exp) { + # 'exp' is Unix timestamp (seconds since 1970-01-01 UTC) + $unixEpoch = [DateTime]::new(1970, 1, 1, 0, 0, 0, [DateTimeKind]::Utc) + $result.ExpiresOn = $unixEpoch.AddSeconds($payload.exp) + $result.Source = 'JWT' + + # Calculate time remaining for logging + $timeRemaining = $result.ExpiresOn - (Get-Date).ToUniversalTime() + if ($timeRemaining.TotalMinutes -gt 0) { + Write-Verbose "Token expires in $([int]$timeRemaining.TotalMinutes) minutes (from JWT 'exp' claim)" + } + + return $result + } + } + } + catch { + # JWT decode failed, use fallback + Write-Verbose "JWT decode failed: $($_.Exception.Message)" + } + + # Fallback: estimate 50-minute expiry from now (observed token lifetime ~45-60 minutes) + # With 5-minute buffer, proactive refresh triggers at ~45 min mark + $result.ExpiresOn = (Get-Date).ToUniversalTime().AddMinutes(50) + $result.Source = 'estimated' + + return $result +} + +# ============================================== +# TOKEN REFRESH FUNCTION FOR LONG-RUNNING OPERATIONS +# ============================================== +function Invoke-TokenRefresh { + <# + .SYNOPSIS + Forces re-authentication for AppRegistration auth mode to get fresh access token. + + .DESCRIPTION + When using App Registration authentication (client secret or certificate), + this function reconnects to Microsoft Graph to obtain a fresh access token. + This is critical for long-running operations that exceed the default OAuth + token lifetime (~60-90 minutes). + + For interactive auth modes, this function returns $false as re-authentication + would require user interaction. + + .PARAMETER Force + Force re-authentication even if token doesn't appear expired. + + .OUTPUTS + [PSCustomObject] with Success ($true/$false) and NewToken properties + #> + [CmdletBinding()] + param( + [switch]$Force + ) + + $result = [PSCustomObject]@{ + Success = $false + NewToken = $null + Message = "" + AuthMethod = $script:AuthConfig.Method + } + + # Check if we can re-authenticate + if (-not $script:AuthConfig.CanReauthenticate) { + $result.Message = "Auth method '$($script:AuthConfig.Method)' does not support automatic re-authentication" + return $result + } + + # Validate we have stored config + if ($script:AuthConfig.Method -ne 'AppRegistration') { + $result.Message = "Only AppRegistration auth mode supports automatic token refresh" + return $result + } + + if ([string]::IsNullOrWhiteSpace($script:AuthConfig.TenantId) -or + [string]::IsNullOrWhiteSpace($script:AuthConfig.ClientId)) { + $result.Message = "Missing TenantId or ClientId in stored auth config" + return $result + } + + Write-LogHost " [TOKEN-REFRESH] Attempting re-authentication using AppRegistration..." -ForegroundColor Cyan + + try { + # Disconnect first to ensure clean state + try { + script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue + } catch { } + + # Re-authenticate based on stored credential type + $reconnected = $false + + # Try client secret first + if ($script:AuthConfig.ClientSecret) { + Write-LogHost " [TOKEN-REFRESH] Reconnecting with client secret..." -ForegroundColor Gray + $credential = New-Object System.Management.Automation.PSCredential( + $script:AuthConfig.ClientId, + $script:AuthConfig.ClientSecret + ) + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientSecretCredential $credential ` + -NoWelcome -ErrorAction Stop + $reconnected = $true + } + # Try certificate thumbprint + elseif ($script:AuthConfig.CertThumbprint) { + Write-LogHost " [TOKEN-REFRESH] Reconnecting with certificate thumbprint..." -ForegroundColor Gray + if ($script:AuthConfig.CertObject) { + # Reuse the cert object resolved + pinned at initial login (carries the + # CurrentUser/LocalMachine resolution; avoids re-searching the stores). + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientId $script:AuthConfig.ClientId ` + -Certificate $script:AuthConfig.CertObject ` + -NoWelcome -ErrorAction Stop + } + else { + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientId $script:AuthConfig.ClientId ` + -CertificateThumbprint $script:AuthConfig.CertThumbprint ` + -NoWelcome -ErrorAction Stop + } + $reconnected = $true + } + # Try certificate file + elseif ($script:AuthConfig.CertPath) { + Write-LogHost " [TOKEN-REFRESH] Reconnecting with certificate file..." -ForegroundColor Gray + # Prefer the cert object pinned at initial login - it already has the correct + # EphemeralKeySet+Exportable storage flags applied, and reusing it avoids any + # chance of leaking a second key file or invalidating the credential's handle. + $cert = $script:AuthConfig.CertObject + if (-not $cert) { + $flags = [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet ` + -bor [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::Exportable + if ($script:AuthConfig.CertPassword) { + $plainPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto( + [Runtime.InteropServices.Marshal]::SecureStringToBSTR($script:AuthConfig.CertPassword) + ) + try { + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2( + $script:AuthConfig.CertPath, $plainPassword, $flags + ) + } + finally { + Clear-Variable -Name plainPassword -Force -ErrorAction SilentlyContinue + } + } + else { + $cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2( + $script:AuthConfig.CertPath, [string]'', $flags + ) + } + # Re-pin so subsequent refreshes can reuse this same instance. + $script:AuthConfig.CertObject = $cert + } + Connect-MgGraph -TenantId $script:AuthConfig.TenantId ` + -ClientId $script:AuthConfig.ClientId ` + -Certificate $cert ` + -NoWelcome -ErrorAction Stop + # Intentionally NOT disposing $cert - must outlive Connect-MgGraph for the rest + # of the run. Cleanup happens at script exit when $script:AuthConfig is released. + $reconnected = $true + } + + if ($reconnected) { + # Use reliable token extraction helper (HTTP method primary) + $result.NewToken = Get-GraphAccessToken + if ($result.NewToken) { + $result.Success = $true + $result.Message = "Successfully refreshed token" + + # Update token timing and reset auth failure flags + $script:AuthConfig.TokenIssueTime = Get-Date + $script:TokenAcquiredTime = Get-Date + $script:AuthFailureDetected = $false + $script:Auth401MessageShown = $false # Reset for next auth failure cycle + + Write-LogHost " [TOKEN-REFRESH] Successfully obtained fresh access token" -ForegroundColor Green + Write-LogHost " [TOKEN-REFRESH] Token acquired at $(Get-Date -Format 'HH:mm:ss') - proactive refresh at 30-minute age" -ForegroundColor DarkGray + } + else { + $result.Message = "Reconnected but could not extract access token" + Write-LogHost " [TOKEN-REFRESH] ✗ $($result.Message)" -ForegroundColor Red + } + } + else { + $result.Message = "No valid credential found in stored auth config" + } + } + catch { + $result.Message = "Re-authentication failed: $($_.Exception.Message)" + Write-LogHost " [TOKEN-REFRESH] ✗ $($result.Message)" -ForegroundColor Red + } + + return $result +} + +# ============================================== +# PROACTIVE TOKEN REFRESH FOR LONG-RUNNING OPERATIONS +# ============================================== +# PAX-1E token-refresh lifecycle state (parent-owned; UTC). Defined early so every +# Refresh-GraphTokenIfNeeded call site can record truthful state. Holds ONLY lifecycle +# state + timing - never token or credential material. Set 'Refreshing' before a real +# synchronous refresh attempt; resolve on every outcome path and in a finally so the +# state can never stay stuck at 'Refreshing' after a throw. +function script:Initialize-PaxTokenRefreshState { + if (-not $script:PaxTokenRefreshState) { + $script:PaxTokenRefreshState = [pscustomobject]@{ + State = 'Idle'; LastReason = $null; LastAuthMode = $null; StartedUtc = $null + LastSuccessUtc = $null; LastElapsedSeconds = $null; LastError = $null + } + } +} +function script:Set-PaxTokenRefreshing { + param([string]$Reason, [string]$AuthMode, [Nullable[datetime]]$StartUtc = $null) + Initialize-PaxTokenRefreshState + $script:PaxTokenRefreshState.State = 'Refreshing' + $script:PaxTokenRefreshState.LastReason = $Reason + $script:PaxTokenRefreshState.LastAuthMode = $AuthMode + $script:PaxTokenRefreshState.StartedUtc = if ($StartUtc) { [datetime]$StartUtc } else { (Get-Date).ToUniversalTime() } + $script:PaxTokenRefreshState.LastError = $null +} +function script:Resolve-PaxTokenRefresh { + param([bool]$Success, [Nullable[datetime]]$CompletionUtc = $null, [string]$ErrorCategory = $null) + Initialize-PaxTokenRefreshState + $done = if ($CompletionUtc) { [datetime]$CompletionUtc } else { (Get-Date).ToUniversalTime() } + $started = $script:PaxTokenRefreshState.StartedUtc + if ($started) { $script:PaxTokenRefreshState.LastElapsedSeconds = [Math]::Round(($done - [datetime]$started).TotalSeconds, 1) } + if ($Success) { + $script:PaxTokenRefreshState.State = 'Succeeded' + $script:PaxTokenRefreshState.LastSuccessUtc = $done + $script:PaxTokenRefreshState.LastError = $null + } else { + $script:PaxTokenRefreshState.State = 'Failed' + $script:PaxTokenRefreshState.LastError = $ErrorCategory + } +} + +function Refresh-GraphTokenIfNeeded { + <# + .SYNOPSIS + Proactively refreshes the Graph access token if it's nearing expiry. + + .DESCRIPTION + Checks SharedAuthState.ExpiresOn and refreshes token if less than 10 minutes + remain before expiry. Uses Azure.Identity for interactive auth modes, or + Invoke-TokenRefresh for AppRegistration mode. + + This function is called from the main thread's job monitoring loop to ensure + thread jobs always have a valid token in SharedAuthState. + + IMPORTANT: Includes cooldown logic to prevent spam - only attempts refresh + once per 5 minutes. If silent refresh fails, sets AuthFailureDetected to + trigger interactive re-auth prompt. + + .PARAMETER BufferMinutes + Refresh if token expires within this many minutes. Default: 5. + + .OUTPUTS + $true - Token was refreshed successfully (silent or interactive) + $false - No refresh needed (token still valid) or within cooldown period + 'Quit' - User chose to quit at the re-auth prompt + + CRITICAL: Callers MUST check for 'Quit' return and handle gracefully! + #> + [CmdletBinding()] + param( + [int]$BufferMinutes = 5 + ) + + # Check if we have shared auth state + if (-not $script:SharedAuthState.ExpiresOn) { + return $false + } + + $now = (Get-Date).ToUniversalTime() + $expiresOn = $script:SharedAuthState.ExpiresOn + $minutesRemaining = ($expiresOn - $now).TotalMinutes + + # PROACTIVE REFRESH FOR APPREG: Refresh at 30-minute token age (not just near expiry) + # AppRegistration can refresh silently, so we do this proactively to avoid 401s + $needsProactiveRefresh = $false + if ($script:AuthConfig.Method -eq 'AppRegistration' -and $script:AuthConfig.CanReauthenticate) { + if ($script:AuthConfig.TokenIssueTime) { + $tokenAge = (Get-Date) - $script:AuthConfig.TokenIssueTime + if ($tokenAge.TotalMinutes -gt 30) { + $needsProactiveRefresh = $true + Write-LogHost " [TOKEN] Token age: $([Math]::Round($tokenAge.TotalMinutes, 1)) minutes - proactive refresh triggered" -ForegroundColor Yellow + } + } + } + + if ($minutesRemaining -gt $BufferMinutes -and -not $needsProactiveRefresh) { + return $false # Token still valid, no refresh needed + } + + # COOLDOWN CHECK: Auth-mode-aware cooldown between refresh attempts + # AppReg: 45 seconds (silent client_credentials grant - cheap and fast) + # Interactive: 5 minutes (avoids spamming browser/prompt windows) + $cooldownMinutes = if ($script:AuthConfig.Method -eq 'AppRegistration') { 0.75 } else { 5 } + if ($script:LastProactiveRefreshAttempt) { + $timeSinceLastAttempt = ((Get-Date) - $script:LastProactiveRefreshAttempt).TotalMinutes + if ($timeSinceLastAttempt -lt $cooldownMinutes) { + return $false + } + } + $script:LastProactiveRefreshAttempt = Get-Date + + # PAX-1E: mark token-refresh state 'Refreshing' for the duration of this REAL attempt + # (never on the no-refresh-needed / cooldown early-returns above). Resolved on every + # outcome path and in the finally so it can never stay stuck after a throw. + Set-PaxTokenRefreshing -Reason $(if ($needsProactiveRefresh) { 'proactive-age' } else { 'proactive-buffer' }) -AuthMode ("" + $script:AuthConfig.Method) -StartUtc (Get-Date).ToUniversalTime() + $script:PaxTokenRefreshResolved = $false + try { + + # Log appropriate message based on trigger reason + if (-not $needsProactiveRefresh) { + Write-LogHost " [TOKEN] Token expires in $([Math]::Round($minutesRemaining, 1)) minutes - attempting proactive refresh..." -ForegroundColor Yellow + } + + # Try to refresh using Azure.Identity (uses cached MSAL tokens, may prompt if needed) + $tokenInfo = Get-GraphAccessTokenWithExpiry + if ($tokenInfo -and $tokenInfo.Token -ne $script:SharedAuthState.Token) { + # Validate the new token is actually valid (ExpiresOn must be > 2 min in the future) + # Protects against stale MSAL cache returning already-expired tokens (e.g., after process suspension) + $tokenExpiresOn = $tokenInfo.ExpiresOn + $nowUtc = (Get-Date).ToUniversalTime() + $minutesUntilExpiry = ($tokenExpiresOn - $nowUtc).TotalMinutes + if ($minutesUntilExpiry -le 2) { + Write-LogHost " [TOKEN] WARNING: Refreshed token is already expired or near-expiry (expires in $([Math]::Round($minutesUntilExpiry, 1)) min) - forcing full re-authentication" -ForegroundColor Red + # Fall through to Invoke-TokenRefresh -Force below + } else { + # Got a genuinely valid new token + $script:SharedAuthState.Token = $tokenInfo.Token + $script:SharedAuthState.ExpiresOn = $tokenInfo.ExpiresOn + $script:SharedAuthState.LastRefresh = Get-Date + $script:SharedAuthState.RefreshCount++ + + Write-LogHost " [TOKEN] Token refreshed silently (expires: $($tokenInfo.ExpiresOn.ToString('HH:mm:ss')) UTC, refresh #$($script:SharedAuthState.RefreshCount))" -ForegroundColor Green + Write-LogHost " [TOKEN] Note: In-flight queries may still require re-auth before this expiration" -ForegroundColor DarkGray + Resolve-PaxTokenRefresh -Success $true -CompletionUtc (Get-Date).ToUniversalTime(); $script:PaxTokenRefreshResolved = $true + return $true + } + } + + # Azure.Identity didn't give us a new token (or it was stale), try AppRegistration refresh if available + if ($script:AuthConfig.CanReauthenticate) { + $refreshResult = Invoke-TokenRefresh -Force + if ($refreshResult.Success) { + $script:SharedAuthState.Token = $refreshResult.NewToken + $script:SharedAuthState.ExpiresOn = (Get-Date).ToUniversalTime().AddMinutes(50) + $script:SharedAuthState.LastRefresh = Get-Date + $script:SharedAuthState.RefreshCount++ + $script:AuthConfig.TokenIssueTime = Get-Date # Reset age timer for proactive refresh + + Write-LogHost " [TOKEN] Token refreshed via AppRegistration (refresh #$($script:SharedAuthState.RefreshCount))" -ForegroundColor Green + Resolve-PaxTokenRefresh -Success $true -CompletionUtc (Get-Date).ToUniversalTime(); $script:PaxTokenRefreshResolved = $true + return $true + } + } + + # SILENT REFRESH FAILED + # For AppRegistration + Force: FATAL exit (true headless operation) + # For AppRegistration without Force: Fall back to interactive prompt + # For interactive modes: Prompt user for re-authentication + Write-LogHost " [TOKEN] [!] Silent token refresh failed - interactive re-authentication required" -ForegroundColor Red + + # AppRegistration mode with -Force: Silent refresh failure is fatal (no interactive fallback for headless runs) + if ($script:AuthConfig.Method -eq 'AppRegistration' -and $Force) { + Write-LogHost " [TOKEN] FATAL: AppRegistration token refresh failed. Cannot continue headless (-Force mode)." -ForegroundColor Red + Write-LogHost " [TOKEN] Check: client secret expiration, certificate validity, or API permissions." -ForegroundColor Yellow + Resolve-PaxTokenRefresh -Success $false -CompletionUtc (Get-Date).ToUniversalTime() -ErrorCategory 'Fatal'; $script:PaxTokenRefreshResolved = $true + return 'Quit' + } + + # Interactive modes OR AppRegistration without -Force: prompt user for re-authentication + $refreshResult = Invoke-TokenRefreshPrompt + if ($refreshResult -eq 'Quit') { + # User chose to quit - return special value for callers to handle + Resolve-PaxTokenRefresh -Success $false -CompletionUtc (Get-Date).ToUniversalTime() -ErrorCategory 'UserQuit'; $script:PaxTokenRefreshResolved = $true + return 'Quit' + } + + # User pressed R and successfully re-authenticated + # Invoke-TokenRefreshPrompt already updated SharedAuthState and reset AuthFailureDetected + Resolve-PaxTokenRefresh -Success $true -CompletionUtc (Get-Date).ToUniversalTime(); $script:PaxTokenRefreshResolved = $true + return $true + } + finally { + if (-not $script:PaxTokenRefreshResolved) { Resolve-PaxTokenRefresh -Success $false -CompletionUtc (Get-Date).ToUniversalTime() -ErrorCategory 'RefreshError' } + } +} + +# ============================================== +# CHECKPOINT/RESUME FUNCTIONS FOR LONG-RUNNING OPERATIONS +# ============================================== + +function Initialize-CheckpointForNewRun { + <# + .SYNOPSIS + Creates new checkpoint structure for a fresh run (not resume mode). + .DESCRIPTION + Initializes checkpoint data structure with all processing parameters, + creates _PARTIAL output filename, and saves initial checkpoint file to disk. + On resume, all parameters are restored from checkpoint to ensure consistency. + #> + param( + [Parameter(Mandatory)] + [string]$OutputPath, + + [Parameter(Mandatory)] + [string]$BaseOutputFileName, + + [Parameter(Mandatory)] + [string]$RunTimestamp, + + [Parameter(Mandatory)] + [datetime]$StartDate, + + [Parameter(Mandatory)] + [datetime]$EndDate, + + [Parameter()] + [hashtable]$AllParameters + ) + + # Create _PARTIAL filename + $fileNameWithoutExt = [System.IO.Path]::GetFileNameWithoutExtension($BaseOutputFileName) + $fileExt = [System.IO.Path]::GetExtension($BaseOutputFileName) + $partialFileName = "${fileNameWithoutExt}_PARTIAL${fileExt}" + $script:PartialOutputPath = Join-Path $OutputPath $partialFileName + + # Create checkpoint file path (hidden file with dot prefix) + $script:CheckpointPath = Join-Path $OutputPath ".pax_checkpoint_${RunTimestamp}.json" + + # Acquire exclusive lock on this checkpoint so a second replica running against + # the same OutputPath cannot silently corrupt our state. Fails loudly if another + # live PAX process is already using this checkpoint. + script:Acquire-CheckpointLock -CheckpointPath $script:CheckpointPath + + # Initialize checkpoint data structure with comprehensive parameter snapshot + $script:CheckpointData = @{ + version = 2 # Bumped version for expanded parameter storage + # Structured schema/compatibility metadata. Read-Checkpoint + # uses these for forward/backward compatibility checks (Test-CheckpointCompatibility). + # `checkpointSchemaVersion` is a SemVer-style identifier for the file format. The + # legacy integer `version` field is preserved for older readers. + checkpointSchemaVersion = '2.1.0' + compatibilityMinimumVersion = '1.11.0' + createdByVersion = $ScriptVersion + createdUtc = (Get-Date).ToUniversalTime().ToString('o') + checkpointType = 'PurviewAudit' + runTimestamp = $RunTimestamp + created = (Get-Date).ToUniversalTime().ToString('o') + lastUpdated = (Get-Date).ToUniversalTime().ToString('o') + parameters = @{ + # Date range + startDate = $StartDate.ToUniversalTime().ToString('o') + endDate = $EndDate.ToUniversalTime().ToString('o') + + # Activity/Record filtering + activityTypes = if ($AllParameters.ActivityTypes) { @($AllParameters.ActivityTypes) } else { @() } + recordTypes = if ($AllParameters.RecordTypes) { @($AllParameters.RecordTypes) } else { @() } + serviceTypes = if ($AllParameters.ServiceTypes) { @($AllParameters.ServiceTypes) } else { @() } + userIds = if ($AllParameters.UserIds) { @($AllParameters.UserIds) } else { @() } + groupNames = if ($AllParameters.GroupNames) { @($AllParameters.GroupNames) } else { @() } + + # Agent filtering + agentId = if ($AllParameters.AgentId) { @($AllParameters.AgentId) } else { @() } + agentsOnly = [bool]$AllParameters.AgentsOnly + excludeAgents = [bool]$AllParameters.ExcludeAgents + + # Prompt filtering + promptFilter = if ($AllParameters.PromptFilter) { $AllParameters.PromptFilter } else { $null } + + # Schema/Explosion settings + explodeArrays = [bool]$AllParameters.ExplodeArrays + explodeDeep = [bool]$AllParameters.ExplodeDeep + explosionThreads = if ($AllParameters.ExplosionThreads) { $AllParameters.ExplosionThreads } else { 0 } + flatDepth = if ($AllParameters.FlatDepth) { $AllParameters.FlatDepth } else { 120 } + streamingSchemaSample = if ($AllParameters.StreamingSchemaSample) { $AllParameters.StreamingSchemaSample } else { 5000 } + streamingChunkSize = if ($AllParameters.StreamingChunkSize) { $AllParameters.StreamingChunkSize } else { 5000 } + + # M365/User info bundles + includeM365Usage = [bool]$AllParameters.IncludeM365Usage + includeUserInfo = [bool]$AllParameters.IncludeUserInfo + includeCopilotInteraction = [bool]$AllParameters.IncludeCopilotInteraction + excludeCopilotInteraction = [bool]$AllParameters.ExcludeCopilotInteraction + includeAgent365Info = [bool]$AllParameters.IncludeAgent365Info + onlyAgent365Info = [bool]$AllParameters.OnlyAgent365Info + + # Partitioning + blockHours = if ($AllParameters.BlockHours) { $AllParameters.BlockHours } else { 0.5 } + partitionHours = if ($AllParameters.PartitionHours) { $AllParameters.PartitionHours } else { 0 } + maxPartitions = if ($AllParameters.MaxPartitions) { $AllParameters.MaxPartitions } else { 160 } + + # Output settings + # Persist the USER-SUPPLIED -OutputPath (SharePoint URL / Fabric OneLake URL / + # local path), NOT the script-rewritten local scratch dir. At checkpoint-save + # time $OutputPath has already been redirected to $script:RemoteScratchDir for + # remote modes (the parse-time redirect block earlier in the script), so saving + # $OutputPath here would lose the destination URL and a -Resume run would + # re-resolve the tier as 'Local' — no upload sweep would ever fire. + # + # $script:DestRaw['Purview'] is the canonical source of truth. It is populated + # at parse-time from the ORIGINAL CLI-bound value (the SharePoint / Fabric URL + # or local path) BEFORE the scratch redirect runs, and is REPOPULATED on each + # resume from the restored value BEFORE the resume-side scratch redirect + # rewrites $OutputPath — so it carries the user-supplied destination across + # both fresh runs and resumes. Falls back to $OutputPath only for runs where + # -OutputPath was never bound and $script:DestRaw['Purview'] is unset + # (default PSScriptRoot). Tier-agnostic: identical behavior for Local, + # SharePoint, and Fabric (OneLake) modes. + outputPath = if ($script:DestRaw -and $script:DestRaw['Purview']) { [string]$script:DestRaw['Purview'] } else { $OutputPath } + exportWorkbook = [bool]$AllParameters.ExportWorkbook + combineOutput = [bool]$AllParameters.CombineOutput + # Deidentify: persisted so a resumed run keeps the original run's anonymization. + # -Deidentify cannot be passed on the resume command line (resume allow-list rejects + # it), so the checkpoint is the sole source of truth for it on resume. + deidentify = [bool]$AllParameters.Deidentify + + # FillerLabel: persisted (resolved hierarchy-filler mode + literal) so a resumed + # AIO/AIBV rollup keeps the original run's level-filler choice. -FillerLabel / + # -FillerLabelText cannot be passed on resume (allow-list), so the checkpoint is the + # sole source of truth for them on resume. + fillerLabelMode = [string]$AllParameters.FillerLabel + fillerLabelText = [string]$AllParameters.FillerLabelText + + # Append-mode targets. Persisted so a resumed run continues to merge + # against the same target file/dim, even if the resume command line omits the + # -Append* parameter. Empty string means "not set on this run". + appendFile = if ($AllParameters.AppendFile) { [string]$AllParameters.AppendFile } else { '' } + appendUserInfo = if ($AllParameters.AppendUserInfo) { [string]$AllParameters.AppendUserInfo } else { '' } + appendAgent365Info = if ($AllParameters.AppendAgent365Info) { [string]$AllParameters.AppendAgent365Info } else { '' } + + # Per-data-type destinations (captured so resume re-targets correctly). + # Storage tier is inferred from each value's form at resume time; no separate + # tier field is persisted. + outputPathUserInfo = if ($AllParameters.OutputPathUserInfo) { [string]$AllParameters.OutputPathUserInfo } else { '' } + outputPathAgent365Info = if ($AllParameters.OutputPathAgent365Info) { [string]$AllParameters.OutputPathAgent365Info } else { '' } + outputPathLog = if ($AllParameters.OutputPathLog) { [string]$AllParameters.OutputPathLog } else { '' } + outputDestinationType = if ($script:RemoteOutputMode) { [string]$script:RemoteOutputMode } else { 'None' } + + # Rollup post-processor mode (None|Rollup|RollupPlusRaw) and the resolved + # embedded-processor mode (None|CopilotInteraction|M365Bundle). Persisted so resume + # runs can restore the original rollup intent when the user does not pass a switch + # on the resume command line. If a rollup switch IS passed on resume, the resume + # value wins (last-write-wins) and these fields are overwritten on the next save. + rollupMode = if ([bool]$AllParameters.Rollup) { 'Rollup' } elseif ([bool]$AllParameters.RollupPlusRaw) { 'RollupPlusRaw' } else { 'None' } + processorMode = if ($script:RollupProcessorMode) { $script:RollupProcessorMode } else { 'None' } + # Resolved rollup dashboard (None|AIO|AIBV|M365). Persisted so a resumed + # CopilotInteraction run restores the correct --profile (AIO vs AIBV cannot + # be re-derived from -IncludeM365Usage alone). Last-write-wins on resume. + rollupDashboard = if ($script:RollupDashboard) { $script:RollupDashboard } else { 'None' } + # Dashboard value (AISID etc.), distinct from the embedded rollup processor + # profile above. Persisted so a resume restores the dashboard the run was launched with + # (an AISID run embeds the AIO processor, so rollupDashboard alone cannot recover it). + dashboard = if ([string]$AllParameters.Dashboard -ieq 'ValueLens') { 'ValueLens' } elseif ($AllParameters.Dashboard) { [string]$AllParameters.Dashboard } else { 'None' } + # AISID (Defender) output set destination + delta-cache switch - restored on resume so + # an AISID resume re-targets and re-caches exactly as the original run did. + outputPathDefenderUsage = if ($AllParameters.OutputPathDefenderUsage) { [string]$AllParameters.OutputPathDefenderUsage } else { '' } + appendDefenderUsage = if ($AllParameters.AppendDefenderUsage) { [string]$AllParameters.AppendDefenderUsage } else { '' } + disableAISIDDeltaCache = [bool]$AllParameters.DisableAISIDDeltaCache + # Resolved AISID (DefenderUsage) output folder — the same Resolve-DataTypePaths + # 'DefenderUsage' .EffectiveDir value shown in the parameter snapshot / metrics JSON, + # persisted so a resume checkpoint records where the AISID CSV set was routed. + resolvedDefenderUsageDir = (script:Get-AISIDResolvedOutputDir) + + # Auth (method only - no secrets) + auth = if ($AllParameters.Auth) { $AllParameters.Auth } else { 'WebLogin' } + tenantId = if ($AllParameters.TenantId) { $AllParameters.TenantId } else { $null } + clientId = if ($AllParameters.ClientId) { $AllParameters.ClientId } else { $null } + # Note: ClientSecret is NOT stored for security + + # Other settings + resultSize = if ($AllParameters.ResultSize) { $AllParameters.ResultSize } else { 10000 } + maxConcurrency = if ($AllParameters.MaxConcurrency) { $AllParameters.MaxConcurrency } else { 10 } + maxMemoryMB = if ($AllParameters.MaxMemoryMB) { $AllParameters.MaxMemoryMB } else { 0 } + useEOM = [bool]$AllParameters.UseEOM + autoCompleteness = [bool]$AllParameters.AutoCompleteness + includeTelemetry = [bool]$AllParameters.IncludeTelemetry + } + outputFiles = @{ + partialCsv = $partialFileName + finalCsv = $BaseOutputFileName + } + partitions = @{ + total = 0 + blockHours = if ($AllParameters.BlockHours) { $AllParameters.BlockHours } else { 0.5 } + completed = @() + queryCreated = @() + # Durable uncertain-create bucket: partitions whose first create POST may have + # been accepted but cannot be proven. Absent on legacy checkpoints; treated as + # empty on load. A partition here is reconciled read-only on resume and NEVER + # re-POSTed. + uncertainCreate = @() + } + statistics = @{ + totalRecordsSaved = 0 + partitionsComplete = 0 + partitionsQueryCreated = 0 + partitionsRemaining = 0 + } + explosion = @{ + status = 'NotStarted' # NotStarted, InProgress, Completed + recordsProcessed = 0 + rowsGenerated = 0 + lastUpdateTime = $null + } + # AISID rolling-window intersection state; present only for -Dashboard AISID runs, + # else $null. Persists the frozen ReferenceNowUtc plus requested/available/effective bounds and + # clip status so a resume replays the run's frozen window instead of the resume-time clock. + aisidWindow = if ($script:AISIDWindow) { + @{ + referenceNowUtc = $script:AISIDWindow.ReferenceNowUtc.ToString('o') + requestedStartUtc = $script:AISIDWindow.RequestedStartUtc.ToString('o') + requestedEndUtc = $script:AISIDWindow.RequestedEndUtc.ToString('o') + availableStartUtc = $script:AISIDWindow.AvailableStartUtc.ToString('o') + availableEndUtc = $script:AISIDWindow.AvailableEndUtc.ToString('o') + effectiveStartUtc = $script:AISIDWindow.EffectiveStartUtc.ToString('o') + effectiveEndUtc = $script:AISIDWindow.EffectiveEndUtc.ToString('o') + status = [string]$script:AISIDWindow.Status + wasClipped = [bool]$script:AISIDWindow.WasClipped + clipReason = [string]$script:AISIDWindow.ClipReason + retentionDays = [int]$script:AISIDWindow.RetentionDays + } + } else { $null } + # AISID resume-state version marker. Present (=1) only when this run resolved an AISID window + # AND persisted the dashboard + DefenderUsage destination + cache state above. Its presence + # marks a checkpoint that carries the full AISID resume state (dashboard + destination), versus + # one that stores only the collection window. + aisidResumeStateVersion = if ($script:AISIDWindow) { 1 } else { $null } + } + + # Save initial checkpoint + Save-CheckpointToDisk + + return $script:PartialOutputPath +} + +# Contract version stamped into every resume commit manifest. A manifest whose +# contractVersion does not match is treated as unverifiable and is never selected. +# Version 2 adds the mandatory nonce-bearing generationId, so remote generation and +# authority-record identity no longer depends on a numeric sequence alone. +# Version 3 adds the mandatory per-artifact CONTENT identity (sha256 + role). A +# self-digest proves the manifest is the complete object the writer intended; it says +# nothing about the bytes the manifest REFERS to. Recording the writer-computed digest +# of each artifact is what lets a reader re-derive the digest from the bytes it actually +# downloaded and prove the artifact set is the one that was committed. +$script:FabricResumeMirrorContractVersion = 3 + +function Get-FabricResumeArtifactSha256 { + <# + .SYNOPSIS + Lowercase-hex SHA256 of a local file, streamed so an arbitrarily large shard or + partial output is never materialised in memory. + .DESCRIPTION + This is the ONLY content-identity primitive the resume protocol uses. A + size/mtime stamp is a change HINT, not an identity: two different payloads can + share a length, and a stamp says nothing about the bytes a remote object holds. + The writer records this digest for every artifact it publishes and the reader + re-derives it from the bytes it downloaded, so a same-length substitution is + detected instead of silently resumed. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Path) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $stream = [System.IO.File]::Open($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite) + try { $hash = $sha.ComputeHash($stream) } + finally { $stream.Dispose() } + return (($hash | ForEach-Object { $_.ToString('x2') }) -join '') + } + finally { $sha.Dispose() } +} + +function Test-FabricResumeArtifactPath { + <# + .SYNOPSIS + Returns $true only for a safe run-relative resume artifact path. + .DESCRIPTION + An artifact path arrives from a remote manifest, so it is untrusted input that is + about to be joined onto a local directory. A rooted path, a drive qualifier, a + backslash or duplicated separator, a trailing separator, or any '.'/'..' segment + could place a hydrated file OUTSIDE the resume directory, so every one of them + fails closed. Only plain forward-slash-separated relative segments are accepted. + #> + [CmdletBinding()] + param([string] $Path) + $p = [string]$Path + if ([string]::IsNullOrWhiteSpace($p)) { return $false } + if ($p.Contains('\') -or $p.Contains(':')) { return $false } + if ($p.StartsWith('/') -or $p.EndsWith('/') -or $p.Contains('//')) { return $false } + foreach ($seg in $p.Split('/')) { + if ($seg -eq '' -or $seg -eq '.' -or $seg -eq '..') { return $false } + } + return $true +} + +function New-FabricResumeCommitIntegrity { + <# + .SYNOPSIS + Deterministic integrity digest over the identity-bearing fields of a resume + commit manifest. + .DESCRIPTION + The commit manifest is the single object whose successful write promotes a + staged generation to authoritative. OneLake file creation is a multi-request + PUT/PATCH/flush sequence, so a torn or truncated manifest object IS possible. + This digest lets a reader prove the manifest it downloaded is the complete + object the writer intended: it is computed over contractVersion, runTimestamp, + generation, and the ordered artifact tuples, and is embedded in the manifest. + A manifest whose recomputed digest disagrees is torn/ambiguous and is rejected. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Manifest) + $sb = [System.Text.StringBuilder]::new() + [void]$sb.Append([string]$Manifest.contractVersion) + [void]$sb.Append('|'); [void]$sb.Append([string]$Manifest.runTimestamp) + [void]$sb.Append('|'); [void]$sb.Append([string]$Manifest.generation) + # The nonce-bearing generation identity is part of the digested identity, so two + # authority records that share a sequence but name different generations can never + # produce the same digest and are always distinguishable on read. + [void]$sb.Append('|'); [void]$sb.Append([string]$Manifest.generationId) + foreach ($a in @($Manifest.artifacts)) { + [void]$sb.Append('|') + [void]$sb.Append([string]$a.localPath); [void]$sb.Append('>') + [void]$sb.Append([string]$a.remotePath); [void]$sb.Append('>') + [void]$sb.Append([string]$a.generation); [void]$sb.Append('>') + [void]$sb.Append([string]$a.length); [void]$sb.Append('>') + # The artifact role and its CONTENT digest are part of the digested identity, so a + # manifest cannot be re-pointed at different bytes (or have its checkpoint role + # moved) while still presenting a valid self-digest. + [void]$sb.Append([string]$a.role); [void]$sb.Append('>') + [void]$sb.Append([string]$a.sha256) + } + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + $hash = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($sb.ToString())) + return (($hash | ForEach-Object { $_.ToString('x2') }) -join '') + } + finally { $sha.Dispose() } +} + +function Test-FabricResumeCommitManifest { + <# + .SYNOPSIS + Validates a downloaded resume commit manifest. Returns $true only for a + complete, self-consistent, current-contract manifest. + .DESCRIPTION + FAIL CLOSED by design: any missing field, wrong contract version, non-positive + generation, malformed artifact entry, or integrity-digest mismatch returns + $false so the caller falls back to an OLDER verified generation instead of + guessing. Never throws - an unreadable manifest is simply not verified. + + The artifact list is validated as a SET, not merely element by element: every + local path and every remote path must be unique, every remote path must live in + the generation folder its own owning generation names, every artifact must carry + a well-formed content digest and a non-negative length, and exactly one artifact + may carry the checkpoint role. Structural validation here proves the manifest is + internally consistent and complete; it does NOT prove the referenced objects hold + those bytes. That proof is re-derived from the downloaded bytes in + Copy-FabricResumeCommitArtifacts, and a generation is usable only when BOTH pass. + #> + [CmdletBinding()] + param([object] $Manifest) + if (-not $Manifest) { return $false } + $names = @($Manifest.PSObject.Properties | ForEach-Object { $_.Name }) + foreach ($f in @('contractVersion', 'runTimestamp', 'generation', 'generationId', 'artifacts', 'integritySha256')) { + if ($names -notcontains $f) { return $false } + } + $contract = 0 + if (-not [int]::TryParse([string]$Manifest.contractVersion, [ref]$contract)) { return $false } + if ($contract -ne $script:FabricResumeMirrorContractVersion) { return $false } + if ([string]::IsNullOrWhiteSpace([string]$Manifest.runTimestamp)) { return $false } + $gen = 0 + if (-not [int]::TryParse([string]$Manifest.generation, [ref]$gen)) { return $false } + if ($gen -lt 1) { return $false } + # The generation identity must carry BOTH the ordering sequence and a nonce, and the + # sequence it carries must agree with the numeric generation field. A sequence-only + # name is rejected: object identity may never rely on a reusable sequence alone. + $genId = [string]$Manifest.generationId + if ($genId -notmatch '^gen-(\d+)-[0-9a-f]{16,}$') { return $false } + if ([int]$Matches[1] -ne $gen) { return $false } + if ($null -eq $Manifest.artifacts) { return $false } + $artifactList = @($Manifest.artifacts) + if ($artifactList.Count -lt 1) { return $false } + $seenLocal = @{} + $seenRemote = @{} + $checkpointCount = 0 + foreach ($a in $artifactList) { + if (-not $a) { return $false } + $an = @($a.PSObject.Properties | ForEach-Object { $_.Name }) + foreach ($af in @('localPath', 'remotePath', 'generation', 'length', 'sha256', 'role')) { + if ($an -notcontains $af) { return $false } + } + # The local path is untrusted input that is about to be joined onto a local + # directory, so traversal, rooted and alternate-separator forms fail closed. + $lp = [string]$a.localPath + if (-not (Test-FabricResumeArtifactPath -Path $lp)) { return $false } + if ($seenLocal.ContainsKey($lp)) { return $false } + $seenLocal[$lp] = $true + + $aGen = 0 + if (-not [int]::TryParse([string]$a.generation, [ref]$aGen)) { return $false } + # An artifact may be owned by this generation or INHERITED from an earlier one, + # but never from a generation that does not exist yet. + if ($aGen -lt 1 -or $aGen -gt $gen) { return $false } + + # An artifact must live inside the generation folder its OWN owning generation + # names, and the path inside that folder must be exactly its local path. A bare + # (non-generational) or disagreeing remote path would let a reader hydrate from + # unstaged, unknown, or another generation's state. + $rp = [string]$a.remotePath + if (-not $rp) { return $false } + $slash = $rp.IndexOf('/') + if ($slash -lt 1) { return $false } + $genFolder = $rp.Substring(0, $slash) + $genRest = $rp.Substring($slash + 1) + if ($genFolder -notmatch '^gen-(\d+)-[0-9a-f]{16,}$') { return $false } + if ([int]$Matches[1] -ne $aGen) { return $false } + if ($genRest -cne $lp) { return $false } + if ($seenRemote.ContainsKey($rp)) { return $false } + $seenRemote[$rp] = $true + + $len = [int64]0 + if (-not [int64]::TryParse([string]$a.length, [ref]$len)) { return $false } + if ($len -lt 0) { return $false } + + # Content identity is mandatory: without it a manifest can only prove its own + # shape, never that the objects it names hold the committed bytes. + if (([string]$a.sha256) -cnotmatch '^[0-9a-f]{64}$') { return $false } + + $role = [string]$a.role + if ($role -ceq 'checkpoint') { $checkpointCount++ } + elseif ($role -cne 'artifact') { return $false } + } + # A resume generation without exactly one checkpoint is not a resumable transaction. + if ($checkpointCount -ne 1) { return $false } + if ((New-FabricResumeCommitIntegrity -Manifest $Manifest) -cne [string]$Manifest.integritySha256) { return $false } + return $true +} + +function New-FabricResumeGenerationNonce { + <# + .SYNOPSIS + Collision-resistant lowercase-hex nonce appended to every resume generation + folder name and authority-record name. + .DESCRIPTION + A numeric sequence is USEFUL FOR ORDERING but is NOT an identity: it is derived + from a listing that may be stale or eventually consistent, and two processes (or + one process before and after a restart) can compute the same next sequence. The + nonce makes the remote object NAME unique regardless, so a stale listing can + never cause one generation to overwrite another. + #> + [CmdletBinding()] + param() + return ([guid]::NewGuid().ToString('n').Substring(0, 16)) +} + +function Get-FabricResumeNameSequence { + <# + .SYNOPSIS + Parses the ordering sequence out of a resume generation folder name or authority + record leaf name. Returns -1 when the name is not a resume protocol name. + .DESCRIPTION + Accepts BOTH the nonce-bearing current form (gen--, + commit--.json) and the earlier sequence-only form (gen-, + commit-.json). The earlier form is accepted HERE ONLY so that such a name + still RESERVES its sequence for high-water purposes; it is never written and, + because the manifest contract requires a nonce-bearing generationId, it is never + selected as authoritative. + #> + [CmdletBinding()] + param([string] $Name) + $n = [string]$Name + if ($n -match '^gen-(\d+)(-[0-9a-f]+)?$') { return [int]$Matches[1] } + if ($n -match '^commit-(\d+)(-[0-9a-f]+)?\.json$') { return [int]$Matches[1] } + return -1 +} + +function Get-FabricResumeRemoteHighWater { + <# + .SYNOPSIS + Highest sequence observed anywhere in a run's remote mirror listing. + .DESCRIPTION + Considers EVERY generation folder name and EVERY authority-record name, including + orphaned staging generations that carry no authority record and authority records + that fail verification. An orphan still RESERVES its name, so allocating strictly + above this value is what keeps a retry (or a restarted process) from re-staging + into a name that already exists remotely. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [AllowNull()] [AllowEmptyCollection()] [string[]] $RelativePaths) + $high = 0 + foreach ($p in @($RelativePaths)) { + $rest = [string]$p + if (-not $rest) { continue } + $seq = if ($rest.StartsWith('commits/')) { + Get-FabricResumeNameSequence -Name ($rest.Substring(8)) + } + else { + Get-FabricResumeNameSequence -Name ($rest.Split('/')[0]) + } + if ($seq -gt $high) { $high = $seq } + } + return $high +} + +function Get-FabricResumeRunListing { + <# + .SYNOPSIS + Lists everything under Files/.pax_resume// and returns the paths + RELATIVE to that run folder (directories included). + .DESCRIPTION + Directories are deliberately INCLUDED: a staging generation whose very first + artifact upload failed can exist as a folder with no files, and its name still + reserves a sequence. A 404 means the run has no mirror yet and yields an empty + listing; any other listing failure THROWS, because a publication that cannot + observe the remote high-water mark must fail closed rather than guess. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $RunTimestamp) + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume/$RunTimestamp" } else { ".pax_resume/$RunTimestamp" } + $dirParam = [System.Uri]::EscapeDataString("Files/$relInItem") + $listUri = "$($resolved.FilesystemBase)?resource=filesystem&recursive=true&directory=$($resolved.ItemFull)/$dirParam" + + $resp = $null + try { + $resp = Invoke-FabricWebRequest -Uri $listUri -Method GET + } + catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return @() } + throw + } + if (-not $resp -or -not $resp.Content) { return @() } + $parsed = $resp.Content | ConvertFrom-Json -ErrorAction Stop + if (-not $parsed -or -not $parsed.paths) { return @() } + + $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + $out = New-Object System.Collections.Generic.List[string] + foreach ($entry in $parsed.paths) { + $name = "$($entry.name)" + $rest = if ($name.StartsWith($prefix)) { $name.Substring($prefix.Length) } else { Split-Path -Leaf $name } + if ($rest) { [void]$out.Add($rest) } + } + return $out.ToArray() +} + +function ConvertTo-FabricResumeOwnership { + <# + .SYNOPSIS + Converts a VERIFIED commit manifest's artifact list into an ownership map. + .DESCRIPTION + Ownership reconstructed from a remote manifest ALONE is recorded with + InheritEligible = $false. The manifest states the digest the writer computed; it + is not proof that the remote object still holds those bytes, and a local + size/mtime stamp is not content identity either. Inheritance therefore requires a + digest that was RE-DERIVED from bytes actually downloaded and verified - that map + is produced by Copy-FabricResumeCommitArtifacts and recorded through + Register-FabricResumeRestoredState, never inferred here. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Manifest) + $map = @{} + foreach ($a in @($Manifest.artifacts)) { + $map[[string]$a.localPath] = @{ + RemotePath = [string]$a.remotePath + Generation = [int]$a.generation + Length = [int64]$a.length + Sha256 = [string]$a.sha256 + Role = [string]$a.role + InheritEligible = $false + } + } + return $map +} + +function Register-FabricResumeRestoredState { + <# + .SYNOPSIS + Records the verified authority manifest and artifact ownership produced by a + successful hydration, keyed by run timestamp. + .DESCRIPTION + Hydration and publication are separate concerns, and a restore that "succeeded" + while publication state stayed reset to zero is exactly the hidden coupling that + let a restarted process reuse generation 1. This record makes the reconstructed + state observable per run and never mixes ownership between runs. It is a RECORD, + not the allocation authority: Sync-FabricResumeMirror independently re-derives the + remote high-water mark before it stages anything. + + When the caller supplies VerifiedOwnership - the map produced by a hydration that + re-derived every artifact digest from the bytes it downloaded - that map is + recorded instead of the manifest-only reconstruction, because only a re-derived + digest is content identity. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [object] $Manifest, + [Parameter(Mandatory)] [AllowNull()] [AllowEmptyCollection()] [string[]] $RelativePaths, + [hashtable] $VerifiedOwnership + ) + if (-not $script:FabricResumeRestoredState) { $script:FabricResumeRestoredState = @{} } + $high = Get-FabricResumeRemoteHighWater -RelativePaths $RelativePaths + if ([int]$Manifest.generation -gt $high) { $high = [int]$Manifest.generation } + $ownership = if ($VerifiedOwnership -and $VerifiedOwnership.Count -gt 0) { $VerifiedOwnership } else { ConvertTo-FabricResumeOwnership -Manifest $Manifest } + $script:FabricResumeRestoredState[[string]$RunTimestamp] = @{ + Generation = [int]$Manifest.generation + GenerationId = [string]$Manifest.generationId + HighWater = [int]$high + Artifacts = $ownership + Manifest = $Manifest + } +} + +function Initialize-FabricResumeCommitStateFromRemote { + <# + .SYNOPSIS + Reconstructs remote generation state for a run from the REMOTE mirror before any + artifact is staged. Runs once per process per run. + .DESCRIPTION + $script:FabricResumeCommitState is an in-process CACHE, never the source of truth + for remote generation allocation: it does not survive a container restart, so a + restarted process previously computed generation 1 again and overwrote the + existing gen-1 objects and authority record. This function makes the remote store + the authority - it reads the run's listing, takes the high-water sequence across + committed generations, ORPHANED staging generations and every authority record, + and reconstructs the current authoritative generation plus artifact ownership from + the newest VERIFIED manifest. The next allocation is then strictly above every + remotely observed sequence. + + Fails closed: a listing failure other than 404 throws, because a publication that + cannot observe the remote high-water mark must not guess at a name. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $RunTimestamp) + if (-not $script:FabricResumeCommitState) { + $script:FabricResumeCommitState = @{ Generation = 0; Counter = 0; Artifacts = @{} } + } + if ($script:FabricResumeCommitState['RemoteReconciled'] -eq $true -and + ([string]$script:FabricResumeCommitState['RunTimestamp']) -ceq [string]$RunTimestamp) { + return + } + + $listing = @(Get-FabricResumeRunListing -RunTimestamp $RunTimestamp) + $high = Get-FabricResumeRemoteHighWater -RelativePaths $listing + $commitLeaves = New-Object System.Collections.Generic.List[string] + foreach ($rel in @($listing)) { + $r = [string]$rel + if ($r.StartsWith('commits/')) { + $leaf = $r.Substring(8) + if ($leaf -and -not $leaf.Contains('/')) { [void]$commitLeaves.Add($leaf) } + } + } + + $generation = 0 + $artifacts = @{} + if ($commitLeaves.Count -gt 0) { + $manifest = Get-FabricResumeAuthoritativeCommit -RunTimestamp $RunTimestamp -CommitLeafNames $commitLeaves.ToArray() + if ($manifest) { + $generation = [int]$manifest.generation + $artifacts = ConvertTo-FabricResumeOwnership -Manifest $manifest + # If THIS process hydrated exactly this generation and re-derived every + # artifact digest from the downloaded bytes, that verified ownership replaces + # the manifest-only reconstruction: it is the one map whose digests are content + # identity rather than the writer's claim. Anything else stays + # non-inherit-eligible and is re-staged into the fresh generation. + $restored = $null + if ($script:FabricResumeRestoredState) { $restored = $script:FabricResumeRestoredState[[string]$RunTimestamp] } + if ($restored -and ([string]$restored.GenerationId) -ceq ([string]$manifest.generationId) -and $restored.Artifacts) { + $artifacts = $restored.Artifacts + } + if ($generation -gt $high) { $high = $generation } + } + } + + $counter = [int]$script:FabricResumeCommitState['Counter'] + if ($high -gt $counter) { $counter = $high } + $script:FabricResumeCommitState = @{ + Generation = $generation + Counter = $counter + Artifacts = $artifacts + RemoteReconciled = $true + RunTimestamp = [string]$RunTimestamp + } +} + +function Get-FabricResumeAuthoritativeCommit { + <# + .SYNOPSIS + Resolves the authoritative (most recent VERIFIED) resume commit manifest for a + run from the list of commit object leaf names discovered under + .pax_resume//commits/. + .DESCRIPTION + Commit objects are IMMUTABLE and monotonically numbered (commit-.json); a + commit is never overwritten, so a torn write can only ever affect the NEWEST + object and can never destroy an already-established authority. Candidates are + evaluated newest-first and the first one that downloads, parses, matches its own + generation/run identity, and passes the integrity digest wins. A torn, missing, + or ambiguous newest commit is SKIPPED and the search falls back to the most + recent verified older generation. Returns $null when nothing verifies - the + caller must then hydrate nothing rather than guess. + + Staging generations (gen-/ folders with no committed manifest) are structurally + unreachable here: selection follows manifests only, never a directory listing. + + AMBIGUITY IS FATAL TO SELECTION. Candidates are grouped by sequence and evaluated + highest sequence first. If TWO OR MORE candidates at the SAME sequence both verify + but name different generation identities (or carry different integrity digests), + nothing is returned - the reader never resolves a conflict by listing order, and it + never falls back to a LOWER sequence to escape the conflict. A lower sequence is + reached only when every higher candidate is torn or unverifiable. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]] $CommitLeafNames + ) + $candidates = New-Object System.Collections.Generic.List[object] + foreach ($n in @($CommitLeafNames)) { + $leaf = [string]$n + $seq = Get-FabricResumeNameSequence -Name $leaf + if ($seq -ge 1) { + [void]$candidates.Add([pscustomobject]@{ Generation = $seq; Leaf = $leaf }) + } + } + foreach ($grp in @($candidates | Group-Object -Property Generation | Sort-Object -Property { [int]$_.Name } -Descending)) { + $seq = [int]$grp.Name + $verified = New-Object System.Collections.Generic.List[object] + foreach ($c in @($grp.Group | Sort-Object -Property Leaf)) { + $tmp = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_commit_read_{0}.json" -f [guid]::NewGuid().ToString('N')) + $parsed = $null + try { + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$RunTimestamp/commits/$($c.Leaf)" -DestinationPath $tmp + $parsed = (Get-Content -LiteralPath $tmp -Raw -ErrorAction Stop) | ConvertFrom-Json -ErrorAction Stop + } + catch { $parsed = $null } + finally { if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } } + if (-not (Test-FabricResumeCommitManifest -Manifest $parsed)) { + Write-Verbose ("Resume commit {0} for run {1} is torn or unverifiable; it cannot be selected." -f $c.Leaf, $RunTimestamp) + continue + } + if ([int]$parsed.generation -ne $seq) { continue } + if ([string]$parsed.runTimestamp -cne [string]$RunTimestamp) { continue } + [void]$verified.Add($parsed) + } + if ($verified.Count -eq 0) { continue } + if ($verified.Count -gt 1) { + $ids = @(@($verified | ForEach-Object { [string]$_.generationId }) | Sort-Object -Unique) + $digests = @(@($verified | ForEach-Object { [string]$_.integritySha256 }) | Sort-Object -Unique) + if ($ids.Count -gt 1 -or $digests.Count -gt 1) { + Write-Verbose ("Resume commit sequence {0} for run {1} is AMBIGUOUS - {2} verified authority records with differing identities or contents; hydrating nothing (fail closed)." -f $seq, $RunTimestamp, $verified.Count) + return $null + } + } + return $verified[0] + } + return $null +} + +function Copy-FabricResumeCommitArtifacts { + <# + .SYNOPSIS + Downloads, VERIFIES, and then atomically publishes exactly the artifact set named + by a VERIFIED resume commit manifest. + .DESCRIPTION + Each artifact carries the generation that owns it, so an artifact that was + unchanged (and therefore not re-uploaded) in the committed generation is + resolved from the earlier generation that does own it. Prior committed + generations are retained for the life of the run precisely so these inherited + references always resolve. Returns the number of files published. + + TWO-PHASE, ALL-OR-NOTHING. A manifest self-digest proves the manifest is the + complete object the writer intended; it proves NOTHING about the bytes the + manifest refers to, and a download that is hash-checked AFTER being written into + the live resume directory has already made a partial set discoverable. So: + + PHASE 1 (isolated) - every artifact is downloaded into a transaction-scoped + temporary hydration directory that auto-discovery cannot see. Each one must + exist, match its manifest LENGTH exactly, and match a SHA256 RE-DERIVED from the + bytes just downloaded (ordinal comparison; a same-length substitution therefore + fails). The checkpoint artifact must additionally parse as JSON. The staged set + must equal the manifest set exactly - no missing, extra, or duplicated entry. + The live LocalDir is not created, read, or written during this phase. + + PHASE 2 (promote) - only once the ENTIRE set has verified is the live directory + touched, and then as ONE exact-set replacement of the run's OWNED artifacts: + every owned path is SNAPSHOT to a transaction-scoped backup, the prior checkpoint + is WITHDRAWN FIRST (from that moment until the last step no checkpoint of this run + is discoverable, so a torn intermediate set can never be resumed), every owned + file the authority record does not name is withdrawn, the verified artifacts are + moved in, and the CHECKPOINT IS PUBLISHED LAST. Find-Checkpoints discovers a run + by its checkpoint file, so a candidate checkpoint becomes visible only once every + companion artifact is already in place. + + OWNED RESUME-ARTIFACT BOUNDARY. LocalDir is normally the customer's -OutputPath, so + a whole-directory replacement is NOT an option. Exactly three narrow, name-anchored + sets are owned, and nothing else in LocalDir is enumerated, moved or deleted: + 1. /.pax_checkpoint_.json - this run's canonical + checkpoint, the name this run publishes and Find-Checkpoints discovers. + 2. every file under /.pax_incremental/ - the resume protocol's own + private shard directory; nothing outside the protocol writes there. + 3. every *_PARTIAL* file directly in whose name carries this run's + timestamp - this run's partial companion. That timestamp scoping is what keeps + another run's partial companion OUT of the boundary. + Plus every localPath the authority record names, which is owned by definition. A + customer report, workbook, log, sub-folder, or another run's checkpoint is OUTSIDE + the boundary and is therefore untouchable. PreserveLocalPaths additionally exempts + paths an earlier run in the same multi-run sweep already published, so hydrating + run B never withdraws run A's verified shards from the shared shard directory. + + Any failure throws. On a phase-1 failure LocalDir is byte-for-byte untouched and + stays ABSENT if it did not already exist. On a phase-2 failure every candidate byte + this call placed is withdrawn and the snapshot is restored, so the prior verified + set - checkpoint AND every companion - is put back byte-for-byte and stays + discoverable; a previously verified artifact is NEVER deleted in the name of + cleanup. When LocalDir did not exist beforehand it is removed again. The temporary + hydration and snapshot directories are always removed. + + When the caller supplies VerifiedOwnership it is populated with the RE-DERIVED + per-artifact digests, which is the only form of ownership that may later be + inherited instead of re-staged. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [object] $Manifest, + [Parameter(Mandatory)] [string] $LocalDir, + [hashtable] $VerifiedOwnership, + [string[]] $PreserveLocalPaths + ) + $artifactList = @($Manifest.artifacts) + $generationId = [string]$Manifest.generationId + if ($artifactList.Count -lt 1) { + throw "Resume generation '$generationId' for run $RunTimestamp names no artifacts; hydrating nothing." + } + + $stageDir = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_hydrate_{0}" -f [guid]::NewGuid().ToString('N')) + New-Item -Path $stageDir -ItemType Directory -Force | Out-Null + try { + # ---- PHASE 1: download and verify in ISOLATION. LocalDir is never touched. ---- + $staged = New-Object System.Collections.Generic.List[object] + $verified = @{} + $seenLocal = @{} + foreach ($a in $artifactList) { + $lp = [string]$a.localPath + if (-not (Test-FabricResumeArtifactPath -Path $lp)) { + throw "Resume generation '$generationId' for run $RunTimestamp names an unsafe artifact path; hydrating nothing." + } + if ($seenLocal.ContainsKey($lp)) { + throw "Resume generation '$generationId' for run $RunTimestamp names artifact '$lp' more than once; hydrating nothing." + } + $seenLocal[$lp] = $true + + $dest = Join-Path $stageDir $lp + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$RunTimestamp/$([string]$a.remotePath)" -DestinationPath $dest + if (-not (Test-Path -LiteralPath $dest -PathType Leaf)) { + throw "Resume artifact '$lp' of generation '$generationId' for run $RunTimestamp did not download; hydrating nothing." + } + $actualLength = [int64](Get-Item -LiteralPath $dest).Length + if ($actualLength -ne [int64]$a.length) { + throw "Resume artifact '$lp' of generation '$generationId' for run $RunTimestamp has length $actualLength but the authority record declares $([int64]$a.length); hydrating nothing." + } + # Content identity is RE-DERIVED from the bytes that actually arrived - the one + # check a same-length substitution cannot survive. + $actualSha = Get-FabricResumeArtifactSha256 -Path $dest + if ($actualSha -cne ([string]$a.sha256)) { + throw "Resume artifact '$lp' of generation '$generationId' for run $RunTimestamp failed content verification; hydrating nothing." + } + if (([string]$a.role) -ceq 'checkpoint') { + try { $null = (Get-Content -LiteralPath $dest -Raw -ErrorAction Stop) | ConvertFrom-Json -ErrorAction Stop } + catch { throw "Resume checkpoint '$lp' of generation '$generationId' for run $RunTimestamp is not parseable JSON; hydrating nothing." } + } + [void]$staged.Add([pscustomobject]@{ LocalPath = $lp; StagePath = $dest; Role = [string]$a.role }) + $verified[$lp] = @{ + RemotePath = [string]$a.remotePath + Generation = [int]$a.generation + Length = $actualLength + Sha256 = $actualSha + Role = [string]$a.role + InheritEligible = $true + } + } + # The staged set must be exactly the manifest set - an extra file would mean the + # generation is not the one the authority record describes. + $stagedOnDisk = @(Get-ChildItem -LiteralPath $stageDir -Recurse -File -Force -ErrorAction SilentlyContinue) + if ($stagedOnDisk.Count -ne $artifactList.Count) { + throw "Resume generation '$generationId' for run $RunTimestamp staged $($stagedOnDisk.Count) files for $($artifactList.Count) declared artifacts; hydrating nothing." + } + + # ---- PHASE 2: promote as ONE exact-set replacement of the OWNED boundary. ---- + $localDirExisted = (Test-Path -LiteralPath $LocalDir -PathType Container) + $manifestPaths = @($staged | ForEach-Object { [string]$_.LocalPath }) + $checkpointPaths = @(@($staged | Where-Object { $_.Role -ceq 'checkpoint' } | ForEach-Object { [string]$_.LocalPath }) + ".pax_checkpoint_$RunTimestamp.json") + $preserved = @{} + foreach ($p in @($PreserveLocalPaths)) { if ($p) { $preserved[[string]$p] = $true } } + + # Enumerate the OWNED RESUME-ARTIFACT BOUNDARY (see .DESCRIPTION). Nothing outside + # it is listed here, so nothing outside it can be withdrawn - a customer report or + # sub-folder sitting in the same -OutputPath is structurally unreachable. + $owned = New-Object System.Collections.Generic.List[string] + $ownedSeen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($rel in @($manifestPaths + $checkpointPaths)) { + if ($rel -and $ownedSeen.Add([string]$rel)) { [void]$owned.Add([string]$rel) } + } + if ($localDirExisted) { + $incLive = Join-Path $LocalDir '.pax_incremental' + if (Test-Path -LiteralPath $incLive -PathType Container) { + foreach ($f in @(Get-ChildItem -LiteralPath $incLive -Recurse -File -Force -ErrorAction SilentlyContinue)) { + $relInc = '.pax_incremental/' + $f.FullName.Substring($incLive.Length + 1).Replace('\', '/') + if ($ownedSeen.Add($relInc)) { [void]$owned.Add($relInc) } + } + } + foreach ($f in @(Get-ChildItem -LiteralPath $LocalDir -File -Force -ErrorAction SilentlyContinue)) { + $leaf = [string]$f.Name + if ($leaf -like '*_PARTIAL*' -and $leaf -like "*$RunTimestamp*" -and $ownedSeen.Add($leaf)) { [void]$owned.Add($leaf) } + } + } + + $backupDir = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_prior_{0}" -f [guid]::NewGuid().ToString('N')) + $backedUp = New-Object System.Collections.Generic.List[string] + $published = New-Object System.Collections.Generic.List[string] + $createdDirs = New-Object System.Collections.Generic.List[string] + try { + # 2a) SNAPSHOT the prior owned set. This - not a delete-on-failure sweep - is + # what makes a failed promotion recoverable: deleting a target that already + # held a previously VERIFIED artifact destroys it instead of restoring it. + New-Item -Path $backupDir -ItemType Directory -Force | Out-Null + foreach ($rel in $owned) { + $live = Join-Path $LocalDir $rel + if (-not (Test-Path -LiteralPath $live -PathType Leaf)) { continue } + $bak = Join-Path $backupDir $rel + $bakDir = Split-Path -Parent $bak + if ($bakDir -and -not (Test-Path -LiteralPath $bakDir -PathType Container)) { + New-Item -Path $bakDir -ItemType Directory -Force | Out-Null + } + [System.IO.File]::Copy($live, $bak, $true) + [void]$backedUp.Add($rel) + } + + if (-not $localDirExisted) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + [void]$createdDirs.Add($LocalDir) + } + + # 2b) WITHDRAW the prior checkpoint FIRST. From here until 2d no checkpoint of + # this run is discoverable, so an interruption exposes no candidate set at + # all rather than a checkpoint whose companions are mid-replacement. + foreach ($rel in $owned) { + if ($preserved.ContainsKey($rel)) { continue } + if ($checkpointPaths -notcontains $rel) { continue } + $live = Join-Path $LocalDir $rel + if (Test-Path -LiteralPath $live -PathType Leaf) { [System.IO.File]::Delete($live) } + } + + # 2c) EXACT SET: withdraw every owned file the authority record does not name, + # so no stale shard, stale partial, or prior-generation artifact survives + # into the promoted set and downstream enumeration sees exactly one set. + foreach ($rel in $owned) { + if ($preserved.ContainsKey($rel)) { continue } + if ($manifestPaths -contains $rel) { continue } + $live = Join-Path $LocalDir $rel + if (Test-Path -LiteralPath $live -PathType Leaf) { [System.IO.File]::Delete($live) } + } + + # 2d) Publish the verified set, CHECKPOINT LAST. + $ordered = @(@($staged | Where-Object { $_.Role -cne 'checkpoint' }) + @($staged | Where-Object { $_.Role -ceq 'checkpoint' })) + foreach ($s in $ordered) { + $target = Join-Path $LocalDir ([string]$s.LocalPath) + $targetDir = Split-Path -Parent $target + if ($targetDir -and -not (Test-Path -LiteralPath $targetDir -PathType Container)) { + New-Item -Path $targetDir -ItemType Directory -Force | Out-Null + [void]$createdDirs.Add($targetDir) + } + Move-PaxFileOverwriteAtomic -SourcePath ([string]$s.StagePath) -DestinationPath $target + [void]$published.Add($target) + } + } + catch { + # ROLLBACK: withdraw every candidate byte this call ACTUALLY PLACED, then + # RESTORE the snapshot. Withdrawal names $published rather than the INTENDED + # set $manifestPaths, because withdrawal and restore must cover the SAME + # files: restore can only put back what the snapshot loop captured, so a + # manifest-wide withdrawal is broader than the restore and permanently loses + # a prior verified file that the snapshot loop had not reached when it threw. + # At a snapshot-window failure $published is empty, so the prior live set is + # left completely untouched. The prior verified set is put back - it is never + # deleted in the name of cleanup, which is what made a failed publication + # destructive before. + foreach ($live in $published) { + if (Test-Path -LiteralPath $live -PathType Leaf) { Remove-Item -LiteralPath $live -Force -ErrorAction SilentlyContinue } + } + foreach ($rel in $backedUp) { + $live = Join-Path $LocalDir $rel + $liveDir = Split-Path -Parent $live + if ($liveDir -and -not (Test-Path -LiteralPath $liveDir -PathType Container)) { + New-Item -Path $liveDir -ItemType Directory -Force -ErrorAction SilentlyContinue | Out-Null + } + [System.IO.File]::Copy((Join-Path $backupDir $rel), $live, $true) + } + for ($i = $createdDirs.Count - 1; $i -ge 0; $i--) { + $cd = $createdDirs[$i] + if (-not (Test-Path -LiteralPath $cd -PathType Container)) { continue } + if (-not $localDirExisted) { Remove-Item -LiteralPath $cd -Recurse -Force -ErrorAction SilentlyContinue; continue } + if (@(Get-ChildItem -LiteralPath $cd -Force -ErrorAction SilentlyContinue).Count -eq 0) { + Remove-Item -LiteralPath $cd -Force -ErrorAction SilentlyContinue + } + } + throw + } + finally { + if (Test-Path -LiteralPath $backupDir) { Remove-Item -LiteralPath $backupDir -Recurse -Force -ErrorAction SilentlyContinue } + } + + if ($null -ne $VerifiedOwnership) { + foreach ($k in @($verified.Keys)) { $VerifiedOwnership[$k] = $verified[$k] } + } + return $published.Count + } + finally { + if (Test-Path -LiteralPath $stageDir) { Remove-Item -LiteralPath $stageDir -Recurse -Force -ErrorAction SilentlyContinue } + } +} + +function Sync-FabricResumeMirror { + <# + .SYNOPSIS + Publishes the run's resume artifact set to OneLake as a TRANSACTIONAL, + generation-scoped commit under Files/.pax_resume//. + .DESCRIPTION + A sequential multi-file upload is NOT a transaction. Overwriting a single + remote checkpoint object in place meant a checkpoint upload that succeeded + before a later shard/partial upload failed left the NEW checkpoint remotely + authoritative while the local transaction rolled back - a container restart + then rehydrated the repudiated remote state and erased the safety the local + rollback achieved. This function replaces that with an explicit commit protocol: + + 1. STAGE - every artifact that is new or changed is uploaded into a fresh, + transaction-scoped generation folder gen--/. Nothing already committed + is overwritten, so a failure here cannot damage the prior generation. + 2. INHERIT - an unchanged artifact is NOT re-uploaded; the manifest records the + earlier generation that OWNS it. Prior committed generations are retained for + the life of the run (cleanup happens only in Remove-FabricResumeMirror after a + clean finish), so every inherited reference resolves at hydration time. + Inheritance requires ownership this PROCESS published; ownership reconstructed + from a remote manifest after a restart is not inherit-eligible, because a local + size/mtime stamp is not content identity. + 3. COMMIT - after the ENTIRE required set is staged successfully, ONE immutable + manifest object commits/commit--.json is written. That single write is + the ONLY authoritative-generation transition; there is no in-place pointer to + overwrite and therefore no way to expose a partially committed checkpoint. + + Failure semantics: + * any staging upload fails -> throws BEFORE the commit; generation is an + orphaned staging folder that hydration structurally cannot select, and the + prior committed generation remains authoritative. + * the commit write fails -> throws; the prior committed generation remains + authoritative (a torn commit-.json fails its integrity digest on read). + * on either failure the per-file change stamps are NOT advanced, so the next + transaction re-uploads everything the failed one staged. + + CONSISTENCY ASSUMPTIONS (stated plainly). OneLake/ADLS Gen2 offers no atomic + directory promotion through the API surface this script uses, so the commit point + is a single small object write. This protocol does NOT assume that write is atomic: + commit objects are immutable and monotonically numbered, and each carries an + integrity digest over its own identity-bearing fields, so a torn or truncated + commit object is DETECTED on read and the reader fails closed to the most recent + VERIFIED older generation. It does assume that a flushed object is subsequently + readable - a property this script already relies on for every customer-facing upload. + + IMMUTABILITY MECHANISM (what this actually relies on). Object names are NOT assumed + to be unique by convention, and the in-memory counter is NOT trusted to survive a + restart. Two independent mechanisms enforce immutability: + * NAME UNIQUENESS - the sequence is allocated strictly above the remote high-water + mark (Initialize-FabricResumeCommitStateFromRemote) AND every generation folder + and authority record carries a collision-resistant nonce, so identity never rests + on a sequence that a stale listing could cause two writers to compute. + * CREATE-IF-ABSENT - every resume-protocol upload uses + Send-FileToOneLake -FailIfExists, which issues the DFS Create File request with an + If-None-Match: * precondition. An existing target answers 409/412 and is raised as + a HARD COLLISION, never silently overwritten and never treated as success. A retry + then allocates a completely fresh identity rather than replacing the conflicted + object. If the storage surface ever stopped honouring the precondition, the nonce + alone still makes a name collision vanishingly unlikely, and any collision that + did occur would still fail closed. + Customer-facing output uploads are untouched by this: they keep the default + overwrite semantics because a customer output file is intentionally replaceable. + + ORDERING. Save-CheckpointStrict verifies the LOCAL publication (atomic replace + + re-read + parse) BEFORE calling this function, so a remote generation is never + promoted ahead of the local state it describes. + + No-op unless the Purview destination tier is Fabric. Local and SharePoint tiers + keep resume artifacts local-only. + + Per-file change tracking lives in $script:FabricResumeMirrorState keyed by full + local path with "{size}:{ticks}" stamps (advanced only on a committed + transaction); remote ownership lives in $script:FabricResumeCommitState. + #> + [CmdletBinding()] + param() + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return } + if (-not $script:CheckpointPath -or -not (Test-Path -LiteralPath $script:CheckpointPath)) { return } + if (-not $global:ScriptRunTimestamp) { return } + + if (-not $script:FabricResumeMirrorState) { $script:FabricResumeMirrorState = @{} } + # The in-memory commit state is a CACHE and does not survive a container restart, so + # remote generation identity is reconstructed from the REMOTE store before anything is + # staged. Without this a restarted process recomputed generation 1 and overwrote the + # existing gen-1 artifacts and authority record. + Initialize-FabricResumeCommitStateFromRemote -RunTimestamp $global:ScriptRunTimestamp + + $mirrorBase = ".pax_resume/$global:ScriptRunTimestamp" + # Every ATTEMPT claims a sequence strictly ABOVE every remotely observed sequence - + # committed generations, orphaned staging generations and authority records alike - + # AND a collision-resistant nonce. The sequence orders generations; the nonce is what + # makes the remote NAME unique, so neither a retry after a failed transaction, nor a + # process restart, nor a stale/eventually-consistent listing can re-stage into (or + # overwrite) an existing generation folder or authority record. + $generation = [int]$script:FabricResumeCommitState.Counter + 1 + $script:FabricResumeCommitState.Counter = $generation + $generationNonce = New-FabricResumeGenerationNonce + $generationId = "gen-$generation-$generationNonce" + $commitLeaf = "commit-$generation-$generationNonce.json" + + # Required artifact set for THIS transaction: the checkpoint JSON (always changed), + # every .pax_incremental/*.jsonl shard, and the *_PARTIAL.csv companion when present. + $required = New-Object System.Collections.Generic.List[object] + $cpItem = Get-Item -LiteralPath $script:CheckpointPath + [void]$required.Add([pscustomobject]@{ + LocalFullPath = $cpItem.FullName + LocalPath = $cpItem.Name + Length = [int64]$cpItem.Length + Stamp = ("{0}:{1}" -f $cpItem.Length, $cpItem.LastWriteTimeUtc.Ticks) + Sha256 = (Get-FabricResumeArtifactSha256 -Path $cpItem.FullName) + Role = 'checkpoint' + AlwaysUpload = $true + }) + + $cpDir = Split-Path -Parent $script:CheckpointPath + $incDir = Join-Path $cpDir ".pax_incremental" + if (Test-Path -LiteralPath $incDir -PathType Container) { + foreach ($f in (Get-ChildItem -LiteralPath $incDir -Filter '*.jsonl' -File -ErrorAction SilentlyContinue)) { + [void]$required.Add([pscustomobject]@{ + LocalFullPath = $f.FullName + LocalPath = ".pax_incremental/$($f.Name)" + Length = [int64]$f.Length + Stamp = ("{0}:{1}" -f $f.Length, $f.LastWriteTimeUtc.Ticks) + Sha256 = (Get-FabricResumeArtifactSha256 -Path $f.FullName) + Role = 'artifact' + AlwaysUpload = $false + }) + } + } + + if ($script:PartialOutputPath -and (Test-Path -LiteralPath $script:PartialOutputPath)) { + $pf = Get-Item -LiteralPath $script:PartialOutputPath + [void]$required.Add([pscustomobject]@{ + LocalFullPath = $pf.FullName + LocalPath = $pf.Name + Length = [int64]$pf.Length + Stamp = ("{0}:{1}" -f $pf.Length, $pf.LastWriteTimeUtc.Ticks) + Sha256 = (Get-FabricResumeArtifactSha256 -Path $pf.FullName) + Role = 'artifact' + AlwaysUpload = $false + }) + } + + # 1/2) STAGE + INHERIT. Any throw here aborts BEFORE the commit point. + $artifacts = New-Object System.Collections.Generic.List[object] + foreach ($item in $required) { + $prior = $script:FabricResumeCommitState.Artifacts[[string]$item.LocalPath] + # Inheritance is decided on CONTENT IDENTITY, never on a size/mtime stamp: the + # recorded digest must be one this process either published itself or re-derived + # from bytes it downloaded and verified, and it must equal the digest of the local + # file as it stands right now. Anything else is staged into the fresh generation. + $inheritEligible = ($prior -and ($prior['InheritEligible'] -eq $true) -and + ([string]$prior['Sha256']) -and (([string]$prior['Sha256']) -ceq ([string]$item.Sha256))) + if ((-not $item.AlwaysUpload) -and $inheritEligible) { + [void]$artifacts.Add([pscustomobject]@{ + localPath = [string]$item.LocalPath + remotePath = [string]$prior.RemotePath + generation = [int]$prior.Generation + length = [int64]$prior.Length + role = [string]$item.Role + sha256 = [string]$prior.Sha256 + }) + continue + } + $remotePath = "$generationId/$([string]$item.LocalPath)" + Send-FileToOneLake -LocalPath $item.LocalFullPath -RemoteFileName "$mirrorBase/$remotePath" -FailIfExists + [void]$artifacts.Add([pscustomobject]@{ + localPath = [string]$item.LocalPath + remotePath = $remotePath + generation = $generation + length = [int64]$item.Length + role = [string]$item.Role + sha256 = [string]$item.Sha256 + }) + } + + # 3) COMMIT. One immutable manifest object; its successful write - and nothing + # before it - promotes generation to authoritative. + $manifest = [pscustomobject]@{ + contractVersion = $script:FabricResumeMirrorContractVersion + runTimestamp = [string]$global:ScriptRunTimestamp + generation = $generation + generationId = $generationId + createdUtc = (Get-Date).ToUniversalTime().ToString('o') + artifacts = $artifacts.ToArray() + integritySha256 = '' + } + $manifest.integritySha256 = New-FabricResumeCommitIntegrity -Manifest $manifest + $commitLocal = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_resume_commit_{0}.json" -f [guid]::NewGuid().ToString('N')) + try { + ($manifest | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath $commitLocal -Encoding UTF8 -Force -ErrorAction Stop + Send-FileToOneLake -LocalPath $commitLocal -RemoteFileName "$mirrorBase/commits/$commitLeaf" -FailIfExists + } + finally { + if (Test-Path -LiteralPath $commitLocal) { Remove-Item -LiteralPath $commitLocal -Force -ErrorAction SilentlyContinue } + } + + # Promotion proven. ONLY now is the new generation recorded as authoritative and the + # per-file change stamps advanced. + $ownership = @{} + foreach ($a in $artifacts) { $ownership[[string]$a.localPath] = $a } + foreach ($item in $required) { + $a = $ownership[[string]$item.LocalPath] + $script:FabricResumeCommitState.Artifacts[[string]$item.LocalPath] = @{ + RemotePath = [string]$a.remotePath + Generation = [int]$a.generation + Length = [int64]$a.length + Sha256 = [string]$a.sha256 + Role = [string]$a.role + InheritEligible = $true + } + $script:FabricResumeMirrorState[[string]$item.LocalFullPath] = [string]$item.Stamp + } + $script:FabricResumeCommitState.Generation = $generation + $script:FabricResumeCommitState.GenerationId = $generationId +} + +function Restore-FabricResumeMirror { + <# + .SYNOPSIS + Hydrates local resume artifacts from the authoritative COMMITTED generation of + OneLake Files/.pax_resume//. + .DESCRIPTION + Used on startup of a Fabric run when a resume is requested and the local + working copy is missing or partial (typical after a container restart). + Lists the mirror folder via the ADLS Gen2 List Paths API, resolves the most + recent VERIFIED commit manifest, and downloads exactly the artifact set that + manifest names, resolving each artifact from the generation that owns it. + + Hydration NEVER enumerates gen-/ folders directly, so a staging generation + left behind by a failed transaction is structurally unreachable. A torn or + unverifiable newest commit falls back to the most recent verified older + generation; when nothing verifies, 0 is returned and nothing is hydrated + (fail closed) rather than guessing. + + A run mirror that carries NO commits/ folder at all predates the transactional + protocol; it is hydrated from its flat legacy layout, still excluding any + gen-/ or commits/ path. + + Throws on partial-download failure so callers can abort rather than + resume from a torn artifact set. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RunTimestamp, + [Parameter(Mandatory)] [string] $LocalDir + ) + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return 0 } + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume/$RunTimestamp" } else { ".pax_resume/$RunTimestamp" } + $dirParam = [System.Uri]::EscapeDataString("Files/$relInItem") + $listUri = "$($resolved.FilesystemBase)?resource=filesystem&recursive=true&directory=$($resolved.ItemFull)/$dirParam" + + $resp = $null + try { + $resp = Invoke-FabricWebRequest -Uri $listUri -Method GET + } catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return 0 } + throw + } + if (-not $resp -or -not $resp.Content) { return 0 } + $parsed = try { $resp.Content | ConvertFrom-Json -ErrorAction Stop } catch { $null } + if (-not $parsed -or -not $parsed.paths) { return 0 } + + # Classify the listing into commit objects vs legacy (pre-transactional) artifacts. + # gen-/ entries are deliberately ignored: only a verified manifest may name them. + $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + $commitLeaves = New-Object System.Collections.Generic.List[string] + $legacy = New-Object System.Collections.Generic.List[string] + # Every observed relative path, DIRECTORIES INCLUDED, is retained so the restored-state + # record carries the true remote high-water mark - an orphaned staging generation + # reserves its sequence even though it can never be selected as authoritative. + $allRel = New-Object System.Collections.Generic.List[string] + # Any gen-/ folder or commits/ object proves this run uses the transactional + # protocol. The flat legacy layout is then NEVER a fallback: doing so on a merely + # absent or not-yet-visible authority record would resume from unstaged state. + $transactional = $false + foreach ($entry in $parsed.paths) { + $isDir = $entry.isDirectory + $name = "$($entry.name)" + $rest = if ($name.StartsWith($prefix)) { $name.Substring($prefix.Length) } else { Split-Path -Leaf $name } + if ($rest) { [void]$allRel.Add($rest) } + if ($rest -and ($rest.StartsWith('gen-') -or $rest.StartsWith('commits/') -or $rest -eq 'commits')) { $transactional = $true } + if ($isDir -eq 'true' -or $isDir -eq $true) { continue } + if (-not $rest) { continue } + if ($rest.StartsWith('commits/')) { [void]$commitLeaves.Add($rest.Substring(8)); continue } + if ($rest.StartsWith('gen-')) { continue } + [void]$legacy.Add($rest) + } + + if ($commitLeaves.Count -gt 0) { + $manifest = Get-FabricResumeAuthoritativeCommit -RunTimestamp $RunTimestamp -CommitLeafNames $commitLeaves + if (-not $manifest) { + Write-Verbose ("No verified resume commit for run {0}; hydrating nothing (fail closed)." -f $RunTimestamp) + return 0 + } + $verifiedOwnership = @{} + $downloaded = Copy-FabricResumeCommitArtifacts -RunTimestamp $RunTimestamp -Manifest $manifest -LocalDir $LocalDir -VerifiedOwnership $verifiedOwnership + # Record the verified authority and its artifact ownership for THIS run only, so a + # successful hydration never leaves publication state silently reset to zero. The + # ownership carries RE-DERIVED digests, so it is the one form that may be inherited. + Register-FabricResumeRestoredState -RunTimestamp $RunTimestamp -Manifest $manifest -RelativePaths $allRel -VerifiedOwnership $verifiedOwnership + return $downloaded + } + + if ($transactional) { + # Generation folders exist but no authority record does: a first transaction that + # has not committed, or a listing that has not caught up. Either way there is no + # authoritative state to hydrate, and the flat layout is not a substitute. + Write-Verbose ("Run {0} uses the transactional resume protocol but exposes no authority record; hydrating nothing (fail closed)." -f $RunTimestamp) + return 0 + } + + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + } + + $downloaded = 0 + foreach ($rest in $legacy) { + $dest = Join-Path $LocalDir $rest + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$RunTimestamp/$rest" -DestinationPath $dest + $downloaded++ + } + return $downloaded +} + +function Remove-FabricResumeMirror { + <# + .SYNOPSIS + Deletes the OneLake Files/.pax_resume// mirror after a successful run. + .DESCRIPTION + Best-effort cleanup. Failures are logged at Verbose only — they do not + abort the run because the customer-visible outputs are already durable + and a leftover mirror folder is purely cosmetic. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $RunTimestamp) + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return } + if (-not $script:FabricResolved) { return } + $resolved = $script:FabricResolved + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume/$RunTimestamp" } else { ".pax_resume/$RunTimestamp" } + $dfsPath = "$($resolved.FilesystemBase)/$($resolved.ItemFull)/Files/$relInItem`?recursive=true" + try { + $null = Invoke-FabricWebRequest -Uri $dfsPath -Method DELETE + if ($script:FabricResumeMirrorState) { $script:FabricResumeMirrorState.Clear() } + # Every committed generation AND any orphaned staging generation lives under the + # folder just deleted, so the recorded remote ownership is now meaningless. This is + # the ONLY place resume generations are removed: staging orphans are deliberately + # retained during the run so a failed transaction never destroys recovery evidence. + $script:FabricResumeCommitState = @{ Generation = 0; Counter = 0; Artifacts = @{} } + # The recorded hydration state describes objects that no longer exist. + $script:FabricResumeRestoredState = @{} + } catch { + Write-Verbose ("Resume mirror cleanup failed: {0}" -f $_.Exception.Message) + } +} + +function Restore-AllFabricResumeMirrors { + <# + .SYNOPSIS + Discovers all in-progress run mirrors under OneLake Files/.pax_resume/ and + downloads their artifacts to a local directory for auto-discover resume. + .DESCRIPTION + Lists Files/.pax_resume/ recursively, then for each / folder resolves + that run's most recent VERIFIED commit manifest and downloads exactly the artifact + set it names (checkpoint JSON, *_PARTIAL.csv, .pax_incremental/*.jsonl) into + LocalDir. Artifact local paths are already relative to the run folder, so the + / prefix is naturally flattened - every artifact filename already + embeds the run timestamp. Returns the number of files downloaded. + + A staging generation left behind by a failed transaction is never selected: only a + verified commit manifest may name a generation. A run whose commits are all torn or + unverifiable contributes NOTHING (fail closed). A run mirror with no commits/ folder + at all predates the transactional protocol and is hydrated from its flat legacy + layout, still excluding any gen-/ or commits/ path. No-op on Local/SharePoint tiers. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $LocalDir) + if (-not $script:DestTier -or $script:DestTier['Purview'] -ne 'Fabric') { return 0 } + if (-not $script:FabricResolved) { $script:FabricResolved = Resolve-FabricTarget -Url $script:RemoteOutputUrl } + $resolved = $script:FabricResolved + + $relInItem = if ($resolved.FilesPath) { "$($resolved.FilesPath)/.pax_resume" } else { ".pax_resume" } + $dirParam = [System.Uri]::EscapeDataString("Files/$relInItem") + $listUri = "$($resolved.FilesystemBase)?resource=filesystem&recursive=true&directory=$($resolved.ItemFull)/$dirParam" + + $resp = $null + try { + $resp = Invoke-FabricWebRequest -Uri $listUri -Method GET + } catch { + $status = try { $_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 404) { return 0 } + throw + } + if (-not $resp -or -not $resp.Content) { return 0 } + $parsed = try { $resp.Content | ConvertFrom-Json -ErrorAction Stop } catch { $null } + if (-not $parsed -or -not $parsed.paths) { return 0 } + + if (-not (Test-Path -LiteralPath $LocalDir -PathType Container)) { + New-Item -Path $LocalDir -ItemType Directory -Force | Out-Null + } + + # Group the flat listing by run timestamp, separating commit objects from legacy + # (pre-transactional) artifacts. gen-/ entries are deliberately ignored here: + # only a verified commit manifest is allowed to name a generation. + $prefix = "$($resolved.ItemFull)/Files/$relInItem/" + $byRun = @{} + foreach ($e in $parsed.paths) { + $isDir = $e.isDirectory + $n = "$($e.name)" + if (-not $n.StartsWith($prefix)) { continue } + $rest = $n.Substring($prefix.Length) + if (-not $rest) { continue } + # rest = "/<...>". + $slash = $rest.IndexOf('/') + if ($slash -lt 0) { continue } + $run = $rest.Substring(0, $slash) + $sub = $rest.Substring($slash + 1) + if (-not $sub) { continue } + if (-not $byRun.ContainsKey($run)) { + $byRun[$run] = [pscustomobject]@{ + Commits = (New-Object System.Collections.Generic.List[string]) + Legacy = (New-Object System.Collections.Generic.List[string]) + All = (New-Object System.Collections.Generic.List[string]) + Transactional = $false + } + } + # Retained PER RUN (directories included) so each run's high-water mark is derived + # only from its own remote state - ownership is never mixed between runs. + [void]$byRun[$run].All.Add($sub) + if ($sub.StartsWith('gen-') -or $sub.StartsWith('commits/') -or $sub -eq 'commits') { $byRun[$run].Transactional = $true } + if ($isDir -eq 'true' -or $isDir -eq $true) { continue } + if ($sub.StartsWith('commits/')) { [void]$byRun[$run].Commits.Add($sub.Substring(8)); continue } + if ($sub.StartsWith('gen-')) { continue } + [void]$byRun[$run].Legacy.Add($sub) + } + + $count = 0 + # Every path an earlier run in THIS sweep already published. Exact-set promotion is + # scoped to ONE run's owned boundary and the .pax_incremental shard directory is + # shared across runs, so run B is told explicitly which files run A already verified + # and must therefore never withdraw. + $sweepPublished = New-Object System.Collections.Generic.List[string] + foreach ($run in @($byRun.Keys | Sort-Object)) { + $bucket = $byRun[$run] + if ($bucket.Commits.Count -gt 0) { + $manifest = Get-FabricResumeAuthoritativeCommit -RunTimestamp $run -CommitLeafNames $bucket.Commits + if (-not $manifest) { + Write-Verbose ("No verified resume commit for run {0}; skipping it (fail closed)." -f $run) + continue + } + # Hydration is all-or-nothing PER RUN: a run whose artifact set fails content + # verification throws and contributes nothing, and the runs already restored + # keep exactly the verified sets they were given. + $verifiedOwnership = @{} + $count += (Copy-FabricResumeCommitArtifacts -RunTimestamp $run -Manifest $manifest -LocalDir $LocalDir -VerifiedOwnership $verifiedOwnership -PreserveLocalPaths $sweepPublished.ToArray()) + foreach ($a in @($manifest.artifacts)) { [void]$sweepPublished.Add([string]$a.localPath) } + Register-FabricResumeRestoredState -RunTimestamp $run -Manifest $manifest -RelativePaths $bucket.All -VerifiedOwnership $verifiedOwnership + continue + } + if ($bucket.Transactional) { + # Staging generations with no authority record: nothing is authoritative yet and + # the flat legacy layout is never a fallback for a transactional run. + Write-Verbose ("Run {0} uses the transactional resume protocol but exposes no authority record; skipping it (fail closed)." -f $run) + continue + } + foreach ($localRel in $bucket.Legacy) { + $dest = Join-Path $LocalDir $localRel + $destDir = Split-Path -Parent $dest + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + Get-RemoteFile-OneLake -RelativeName ".pax_resume/$run/$localRel" -DestinationPath $dest + [void]$sweepPublished.Add([string]$localRel) + $count++ + } + } + return $count +} + +function Save-CsvAtomic { + <# + .SYNOPSIS + Atomic CSV write helper using temp+rename, mirroring Save-CheckpointToDisk. + .DESCRIPTION + Replaces direct `Export-Csv -Path $target` calls at output sites that previously + risked a half-written file on Ctrl+C / power loss / container OOM-kill. + Writes to "$Path.tmp" first, then Move-Item -Force overwrites the target in a + single filesystem operation. On any failure the temp file is removed and the + original $Path (if any) is left untouched. + + For remote tiers (SharePoint / Fabric) the caller still passes a LOCAL scratch + path; remote upload happens later via the existing upload pipeline. So this + helper only needs to handle the local atomic-rename case. + + Encoding/quote handling mirrors what the existing call sites used (UTF-8, + NoTypeInformation) so output bytes are unchanged for successful writes. + .PARAMETER InputObject + Rows to export. Accepts arrays, lists, or single objects via pipeline. + .PARAMETER Path + Final destination path. Will be replaced atomically when the write succeeds. + .PARAMETER NoTypeInformation + Forwarded to Export-Csv. Default $true (matches all current call sites). + .PARAMETER Encoding + Forwarded to Export-Csv. Default 'UTF8' (matches all current call sites). + #> + [CmdletBinding()] + param( + [Parameter(Mandatory, ValueFromPipeline)] + [AllowNull()] + $InputObject, + + [Parameter(Mandatory)] + [string]$Path, + + [switch]$NoTypeInformation = $true, + [string]$Encoding = 'UTF8' + ) + begin { + $rows = New-Object System.Collections.Generic.List[object] + } + process { + if ($null -ne $InputObject) { + # Accept arrays piped as a single object, single objects, or per-pipeline-element streams. + if ($InputObject -is [System.Collections.IEnumerable] -and -not ($InputObject -is [string])) { + foreach ($r in $InputObject) { if ($null -ne $r) { [void]$rows.Add($r) } } + } else { + [void]$rows.Add($InputObject) + } + } + } + end { + $tmp = "$Path.tmp" + try { + # Ensure destination directory exists (caller may have skipped this). + $destDir = Split-Path -Path $Path -Parent + if ($destDir -and -not (Test-Path -LiteralPath $destDir -PathType Container)) { + New-Item -Path $destDir -ItemType Directory -Force | Out-Null + } + + if ($rows.Count -eq 0) { + # Header-only fallback: write an empty file (best-effort header derived from $InputObject's first row would be ambiguous here). + # Match the behavior of `Export-Csv -InputObject @()`: produce an empty file with no header. + Set-Content -LiteralPath $tmp -Value '' -Encoding $Encoding -NoNewline + } else { + $rows | Export-Csv -Path $tmp -NoTypeInformation:$NoTypeInformation -Encoding $Encoding -Force -ErrorAction Stop + } + + # Atomic replace. On Windows, Move-Item -Force will overwrite an existing file. + Move-Item -LiteralPath $tmp -Destination $Path -Force -ErrorAction Stop + } + catch { + # Best-effort cleanup of temp; rethrow so callers see the failure. + try { if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue } } catch {} + throw + } + } +} + +# ========================================================================== +# CHECKPOINT LOCKING (G-9) +# ========================================================================== +# Defends against two replicas of the same PAX run racing on the same checkpoint +# file. The race surface is real only when an operator deliberately sets ACA Job +# parallelism > 1 (or runs two PAX invocations against the same OutputPath in +# the same wall-clock second). The default ACA Job replicaCompletionCount=1 +# path is safe; this is defense-in-depth for the misconfigured case. +# +# Lock file: ".lock" +# Contents: JSON { pid, host, started, runTimestamp } +# Stale detection: +# - If the lock file's pid is dead on this machine -> treat as stale, take over. +# - If pid is alive on this machine -> reject (loud error, exit non-zero). +# - If the lock file is on a different host (different "host" field) AND was +# last touched more than $script:CheckpointLockStaleTtl ago -> treat as +# stale, take over (cannot verify a foreign PID). +# The exclusive FileStream we open on the lock file is held for the lifetime of +# the run, so a concurrent acquire on the same Windows host will fail at the +# filesystem level even if the JSON-based check races. +$script:CheckpointLockStream = $null +$script:CheckpointLockPath = $null +$script:CheckpointLockStaleTtl = [TimeSpan]::FromMinutes(10) + +function script:Acquire-CheckpointLock { + param( + [Parameter(Mandatory)][string]$CheckpointPath + ) + $lockPath = "$CheckpointPath.lock" + + # Inspect any existing lock for stale-takeover eligibility BEFORE attempting open. + if (Test-Path -LiteralPath $lockPath -PathType Leaf) { + $existing = $null + try { $existing = Get-Content -LiteralPath $lockPath -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop } catch {} + $stale = $false + $reason = '' + if (-not $existing) { + $stale = $true; $reason = 'unreadable lock file (corrupt or empty)' + } + elseif ($existing.host -eq [System.Net.Dns]::GetHostName()) { + # Same-host: definitive PID check. + $alive = $false + try { $alive = [bool](Get-Process -Id ([int]$existing.pid) -ErrorAction Stop) } catch { $alive = $false } + if (-not $alive) { $stale = $true; $reason = "owning PID $($existing.pid) is no longer running on this host" } + } + else { + # Foreign host: fall back to age-based TTL. + try { + $age = (Get-Date) - ([datetime]$existing.started) + if ($age -gt $script:CheckpointLockStaleTtl) { + $stale = $true + $reason = "foreign-host lock from $($existing.host) is older than $($script:CheckpointLockStaleTtl.TotalMinutes) min (age $([int]$age.TotalMinutes) min)" + } + } catch { $stale = $true; $reason = 'foreign-host lock has unparseable timestamp' } + } + + if ($stale) { + Write-LogHost " Checkpoint lock at $lockPath is stale ($reason); taking over." -ForegroundColor Yellow + try { Remove-Item -LiteralPath $lockPath -Force -ErrorAction Stop } catch { + throw "G-9: Could not remove stale checkpoint lock at $lockPath`: $($_.Exception.Message)" + } + } + else { + $ownerDesc = if ($existing) { "pid=$($existing.pid) host=$($existing.host) started=$($existing.started)" } else { '(unknown owner)' } + throw "G-9: Another PAX run is using this checkpoint ($ownerDesc). Refusing to start. If you know the other run is dead, delete the lock file manually: $lockPath" + } + } + + # Open the lock file exclusively (CreateNew + FileShare.None). Hold the stream for the lifetime of the run. + try { + $stream = [System.IO.File]::Open($lockPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None) + } + catch { + throw "G-9: Failed to acquire checkpoint lock at $lockPath`: $($_.Exception.Message). Another process may have raced in." + } + + # Stamp the lock with our identity so other replicas can do stale detection later. + $payload = @{ + pid = $PID + host = [System.Net.Dns]::GetHostName() + started = (Get-Date).ToUniversalTime().ToString('o') + runTimestamp = $global:ScriptRunTimestamp + } | ConvertTo-Json -Compress + $bytes = [System.Text.Encoding]::UTF8.GetBytes($payload) + $stream.Write($bytes, 0, $bytes.Length) + $stream.Flush() + + $script:CheckpointLockStream = $stream + $script:CheckpointLockPath = $lockPath +} + +function script:Release-CheckpointLock { + if ($script:CheckpointLockStream) { + try { $script:CheckpointLockStream.Dispose() } catch {} + $script:CheckpointLockStream = $null + } + if ($script:CheckpointLockPath -and (Test-Path -LiteralPath $script:CheckpointLockPath)) { + try { Remove-Item -LiteralPath $script:CheckpointLockPath -Force -ErrorAction SilentlyContinue } catch {} + } + $script:CheckpointLockPath = $null +} + +function Save-CheckpointToDisk { + <# + .SYNOPSIS + Writes current checkpoint data to disk atomically. + .DESCRIPTION + Uses temp file + rename pattern for atomic writes to prevent corruption. + + This function SELF-GATES on $script:CheckpointEnabled. Callers are still + expected to gate at the call site, but if a future caller forgets, this + guard turns "silently writes a checkpoint when disabled" into a safe no-op. + #> + + # Self-gate. Do NOT remove without revisiting the corresponding static tests. + if (-not $script:CheckpointEnabled) { + return + } + + if (-not $script:CheckpointPath -or -not $script:CheckpointData) { + return + } + + try { + # Update timestamp + $script:CheckpointData.lastUpdated = (Get-Date).ToUniversalTime().ToString('o') + + # Update statistics + $script:CheckpointData.statistics.partitionsComplete = $script:CheckpointData.partitions.completed.Count + $script:CheckpointData.statistics.partitionsQueryCreated = $script:CheckpointData.partitions.queryCreated.Count + $script:CheckpointData.statistics.partitionsRemaining = $script:CheckpointData.partitions.total - + $script:CheckpointData.statistics.partitionsComplete - + $script:CheckpointData.statistics.partitionsQueryCreated + + # Write to temp file first (atomic write pattern) + $tempPath = "$($script:CheckpointPath).tmp" + $script:CheckpointData | ConvertTo-Json -Depth 10 | Set-Content -Path $tempPath -Encoding UTF8 -Force + + # Remove destination first if it exists (Move-Item -Force doesn't always work on Windows) + if (Test-Path $script:CheckpointPath) { + Remove-Item -Path $script:CheckpointPath -Force -ErrorAction SilentlyContinue + } + + # Rename to final (atomic on most filesystems) + Move-Item -Path $tempPath -Destination $script:CheckpointPath -Force + + # Fabric tier: mirror the full resume artifact set (checkpoint JSON, + # .pax_incremental/*.jsonl, *_PARTIAL.csv) to a durable OneLake path inside + # the same try block so either both the local write and the mirror upload + # succeed or the run aborts. Local and SharePoint tiers keep resume artifacts + # local-only (those hosts are not ephemeral). See Sync-FabricResumeMirror. + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + Sync-FabricResumeMirror + } + } + catch { + # Fabric-tier mirror failure must abort: a torn artifact set on the durable + # mirror would silently corrupt a future resume. Local checkpoint write + # failures remain a soft warning. + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + throw ("Checkpoint mirror to OneLake failed; aborting to preserve resume integrity: {0}" -f $_.Exception.Message) + } + Write-LogHost " Warning: Failed to save checkpoint: $($_.Exception.Message)" -ForegroundColor Yellow + } +} + +function Move-PaxFileOverwriteAtomic { + <# + .SYNOPSIS + Same-directory overwrite-rename that keeps the DESTINATION path continuously + present, with a bounded retry for transient Windows sharing failures. + .DESCRIPTION + [System.IO.File]::Move(..., overwrite) maps to MoveFileEx with + MOVEFILE_REPLACE_EXISTING - one directory-entry replacement, so the destination + path never disappears. That is the property a checkpoint swap needs and that + neither remove-then-move nor [System.IO.File]::Replace provides: both rename or + delete the destination BEFORE putting the replacement in place, leaving a window + in which an interruption exposes no file at all. + + MoveFileEx can transiently fail while another agent (indexer, antivirus scan, a + just-closed reader) still holds the destination open, so a bounded retry with a + short backoff is applied. Every attempt is a COMPLETE replacement: a failed + attempt leaves the destination holding its previous content - never nothing - so + retrying is always safe. The last failure is rethrown when the budget is + exhausted so callers fail closed instead of proceeding on an unproven swap. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $SourcePath, + [Parameter(Mandatory)] [string] $DestinationPath, + [int] $MaxAttempts = 10, + [int] $BackoffMilliseconds = 25 + ) + $attempt = 0 + while ($true) { + try { + [System.IO.File]::Move($SourcePath, $DestinationPath, $true) + return + } + catch { + $attempt++ + if ($attempt -ge $MaxAttempts) { throw } + Start-Sleep -Milliseconds $BackoffMilliseconds + } + } +} + +function Save-CheckpointStrict { + <# + .SYNOPSIS + Durable, verify-after-write checkpoint publication for the operator-cleared + uncertain-create transition. Unlike Save-CheckpointToDisk, a local write or + replace failure is NEVER downgraded to a soft warning: any failure THROWS so + the caller can fail closed and refuse to authorize a fresh create. + .DESCRIPTION + Serializes the current checkpoint to a temp file with terminating error + semantics (-ErrorAction Stop, so a nonterminating write error cannot be + silently ignored), then atomically replaces the destination WITHOUT a + remove-then-move loss window (a retained backup copy plus a single + overwrite-rename when the destination already exists, else a plain move into + place), and finally re-reads the + destination from disk and confirms it parses before returning $true. Any + failure in write, replace, or re-read verification throws. The Fabric-tier + durable-or-abort mirror contract of Save-CheckpointToDisk is preserved. + + ORDERING (local publication vs remote transaction). These are two separate + systems, so the order is fixed and explicit: the LOCAL publication is completed + and verified FIRST (atomic replace, then re-read + parse), and only then is the + Fabric-tier remote transaction committed by Sync-FabricResumeMirror. A remote + generation therefore never becomes authoritative before the local publication it + describes has been proven, so a local post-publication failure can never require + "un-publishing" a remotely authoritative checkpoint. Conversely, a remote + transaction that fails before its commit point leaves the PRIOR remote generation + authoritative, so the local rollback below has nothing remote to undo. + + ROLLBACK ATOMICITY. The rollback restores the retained prior copy with a single + overwrite-rename via Move-PaxFileOverwriteAtomic (MoveFileEx / + MOVEFILE_REPLACE_EXISTING), so the canonical checkpoint path stays continuously + present for the whole rollback - the earlier remove-then-move sequence opened a + window in which an interruption left NO checkpoint at the canonical path, and + System.IO.File::Replace opens a smaller but real window of the same kind because + ReplaceFile renames the destination aside before moving the replacement in. The + repudiated candidate is retained as recovery material until the restoration has been + proven byte-exact. + .OUTPUTS + $true on a proven durable publication; otherwise throws. + #> + + # Self-gate identically to Save-CheckpointToDisk. A disabled or unconfigured + # checkpoint is not a durable-write failure; it is simply nothing to persist. + if (-not $script:CheckpointEnabled) { return $false } + if (-not $script:CheckpointPath -or -not $script:CheckpointData) { return $false } + + # Update timestamp + statistics (same shape as Save-CheckpointToDisk). + $script:CheckpointData.lastUpdated = (Get-Date).ToUniversalTime().ToString('o') + $script:CheckpointData.statistics.partitionsComplete = $script:CheckpointData.partitions.completed.Count + $script:CheckpointData.statistics.partitionsQueryCreated = $script:CheckpointData.partitions.queryCreated.Count + $script:CheckpointData.statistics.partitionsRemaining = $script:CheckpointData.partitions.total - + $script:CheckpointData.statistics.partitionsComplete - + $script:CheckpointData.statistics.partitionsQueryCreated + + $json = $script:CheckpointData | ConvertTo-Json -Depth 10 + $tempPath = "$($script:CheckpointPath).strict.tmp" + + # 1) Write the temp file with terminating semantics so a nonterminating write + # error surfaces as a failure instead of being silently ignored. + Set-Content -Path $tempPath -Value $json -Encoding UTF8 -Force -ErrorAction Stop + + # 2) Publish as a REAL transaction. The full post-publication verification (local + # re-read + parse, then the Fabric-tier durable-or-abort mirror) runs INSIDE the + # transaction, and the prior copy is RETAINED until that verification has fully + # succeeded. A post-publication failure is NOT allowed to leave the destination in + # the new state while memory rolls back (a split-brain): the prior bytes are + # restored atomically from the retained backup (dest-exists) or the newly published + # candidate is removed (new-destination), the restore is itself proven, and only + # then is the original failure re-thrown so the caller fails closed with the durable + # checkpoint byte-for-byte unchanged. A restore that itself fails raises the DISTINCT + # PAX-FATAL-ROLLBACK category so an indeterminate disk state is never hidden as a + # normal fail-closed success. Never bare-remove an existing destination before the + # swap (that opens a data-loss window). + try { + if (Test-Path -LiteralPath $script:CheckpointPath) { + # Capture the exact prior bytes BEFORE the swap so the restore can be proven to + # reproduce them byte-for-byte. A locked/unreadable destination throws here, + # BEFORE any mutation, leaving the prior checkpoint untouched. + $priorBytes = [System.IO.File]::ReadAllBytes($script:CheckpointPath) + $backupPath = "$($script:CheckpointPath).strict.bak" + # RETAIN the prior copy first, then swap with a SINGLE overwrite-rename so the + # canonical path is continuously present. [System.IO.File]::Replace is deliberately + # NOT used: the underlying ReplaceFile renames the destination ASIDE before renaming + # the replacement into place, which leaves an observable window in which the + # canonical path does not exist. Move-PaxFileOverwriteAtomic uses MoveFileEx with + # MOVEFILE_REPLACE_EXISTING - one directory-entry replacement, no absence window. + # The prior copy is RETAINED in the backup until the whole post-publication + # verification has committed. + [System.IO.File]::Copy($script:CheckpointPath, $backupPath, $true) + Move-PaxFileOverwriteAtomic -SourcePath $tempPath -DestinationPath $script:CheckpointPath + try { + # 3) Durability proof: re-read the destination and confirm it exists and parses. + if (-not (Test-Path -LiteralPath $script:CheckpointPath)) { + throw "Strict checkpoint publish failed: destination missing after replace." + } + $roundTrip = Get-Content -LiteralPath $script:CheckpointPath -Raw -ErrorAction Stop + $null = $roundTrip | ConvertFrom-Json -ErrorAction Stop + # Fabric tier: mirror inside the same durable-or-abort contract as + # Save-CheckpointToDisk (a mirror failure aborts to preserve resume integrity). + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + Sync-FabricResumeMirror + } + } + catch { + $__pubErr = $_ + # Post-publication failure: restore the prior bytes from the retained backup and + # PROVE the restore reproduced them exactly, then re-throw so the caller fails + # closed with the durable checkpoint byte-for-byte unchanged. A restore that + # cannot be proven raises the distinct rollback-failure category. + $rollbackBackupPath = "$($script:CheckpointPath).strict.rbk" + try { + if (Test-Path -LiteralPath $backupPath) { + # Restore the prior bytes from the retained backup WITHOUT ever removing the + # canonical destination first. The repudiated candidate is copied aside as + # recovery material (BEST EFFORT - failing to take that extra copy must never + # block the restoration itself), then a single overwrite-rename puts the prior + # bytes back. Move-PaxFileOverwriteAtomic is one directory-entry replacement, so + # the canonical checkpoint path is CONTINUOUSLY PRESENT for the whole rollback: + # a process termination, host restart, or storage fault can never observe an + # absent checkpoint. (The prior remove-then-move sequence had exactly that loss + # window, and [System.IO.File]::Replace has a smaller but real one because + # ReplaceFile renames the destination aside first.) + if (Test-Path -LiteralPath $script:CheckpointPath) { + try { [System.IO.File]::Copy($script:CheckpointPath, $rollbackBackupPath, $true) } + catch { Write-Verbose ("Repudiated checkpoint candidate could not be retained: {0}" -f $_.Exception.Message) } + } + Move-PaxFileOverwriteAtomic -SourcePath $backupPath -DestinationPath $script:CheckpointPath + } + else { + throw "retained backup is missing; the prior checkpoint bytes cannot be restored." + } + $restoredBytes = [System.IO.File]::ReadAllBytes($script:CheckpointPath) + $restoredMatches = ($restoredBytes.Length -eq $priorBytes.Length) + if ($restoredMatches) { + for ($__i = 0; $__i -lt $priorBytes.Length; $__i++) { + if ($restoredBytes[$__i] -ne $priorBytes[$__i]) { $restoredMatches = $false; break } + } + } + if (-not $restoredMatches) { + throw "the restored destination does not byte-match the pre-transaction checkpoint." + } + # Proven byte-exact restoration: the repudiated candidate retained by the atomic + # swap is no longer recovery material, so it is discarded HERE (success path only). + if (Test-Path -LiteralPath $rollbackBackupPath) { Remove-Item -LiteralPath $rollbackBackupPath -Force -ErrorAction SilentlyContinue } + } + catch { + $script:GenericFatal = $true + throw "PAX-FATAL-ROLLBACK: strict checkpoint publish failed and the prior checkpoint could NOT be restored for '$($script:CheckpointPath)'; disk state is indeterminate: $($_.Exception.Message)" + } + throw $__pubErr + } + # Proven durable publication: the prior copy is no longer needed. + Remove-Item -LiteralPath $backupPath -Force -ErrorAction SilentlyContinue + } + else { + # New destination (no prior checkpoint to preserve). Publish, then verify + mirror + # inside the transaction; a post-publication failure REMOVES the newly published + # candidate so no false durable state survives, then fails closed. + [System.IO.File]::Move($tempPath, $script:CheckpointPath) + try { + if (-not (Test-Path -LiteralPath $script:CheckpointPath)) { + throw "Strict checkpoint publish failed: destination missing after move." + } + $roundTrip = Get-Content -LiteralPath $script:CheckpointPath -Raw -ErrorAction Stop + $null = $roundTrip | ConvertFrom-Json -ErrorAction Stop + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + Sync-FabricResumeMirror + } + } + catch { + $__pubErr = $_ + # No prior checkpoint existed: remove the newly published candidate to restore + # the no-checkpoint pre-state, then fail closed. A removal that itself fails + # raises the distinct rollback-failure category (disk state indeterminate). + try { + if (Test-Path -LiteralPath $script:CheckpointPath) { + [System.IO.File]::Delete($script:CheckpointPath) + } + } + catch { + $script:GenericFatal = $true + throw "PAX-FATAL-ROLLBACK: strict checkpoint publish failed and the newly published candidate could NOT be removed for new destination '$($script:CheckpointPath)'; disk state is indeterminate: $($_.Exception.Message)" + } + throw $__pubErr + } + } + } + finally { + # Clean the strict temp without ever touching the prior valid checkpoint. After a + # successful Replace/Move the temp is already consumed; this only removes a leftover + # temp from a pre-publication failure. + if (Test-Path -LiteralPath $tempPath) { Remove-Item -LiteralPath $tempPath -Force -ErrorAction SilentlyContinue } + } + return $true +} + +function Save-Checkpoint { + <# + .SYNOPSIS + Updates checkpoint with partition state change and saves to disk. + .PARAMETER PartitionIndex + The partition index (1-based). + .PARAMETER State + 'QueryCreated', 'Completed', or 'UncertainCreate' + .PARAMETER QueryId + The server-assigned query ID. + .PARAMETER PartitionStart + Partition start time (optional - looked up from partitionStatus if not provided). + .PARAMETER PartitionEnd + Partition end time (optional - looked up from partitionStatus if not provided). + .PARAMETER RecordCount + Number of records (only for Completed state). + .PARAMETER DeferDiskWrite + Perform the in-memory bucket transition ONLY and skip the warning-only disk write. + Reserved for a caller that owns a STRICT single-publication transaction: it stages the + COMPLETE post-transition state here, then publishes it exactly once through + Save-CheckpointStrict and fails closed unless that publication is proven durable. + Ignored by -Force. + .PARAMETER Force + Just save current checkpoint state to disk without updating partition info. + #> + param( + [Parameter()] + [int]$PartitionIndex, + + [Parameter()] + [ValidateSet('QueryCreated', 'Completed', 'UncertainCreate')] + [string]$State, + + [Parameter()] + [string]$QueryId, + + [Parameter()] + [datetime]$PartitionStart, + + [Parameter()] + [datetime]$PartitionEnd, + + [Parameter()] + [int]$RecordCount = 0, + + [Parameter()] + [hashtable]$LifecycleState, + + [Parameter()] + [string]$Fingerprint, + + [Parameter()] + [switch]$DeferDiskWrite, + + [Parameter()] + [switch]$Force + ) + + if (-not $script:CheckpointData) { + return + } + + # If -Force is specified, just save current state to disk without updating partition info + if ($Force) { + Save-CheckpointToDisk + return + } + + # For normal calls, require the mandatory parameters. An UncertainCreate entry is + # durable state for a partition that has NO queryId yet (a possibly-accepted create + # whose acceptance is unproven), so it is exempt from the QueryId requirement. + if (-not $PartitionIndex -or -not $State -or (-not $QueryId -and $State -ne 'UncertainCreate')) { + Write-Verbose "Save-Checkpoint: Missing required parameters (PartitionIndex, State, QueryId) - skipping" + return + } + + # Look up partition times from partitionStatus if not provided + if (-not $PartitionStart -or -not $PartitionEnd) { + $partitionInfo = $script:partitionStatus[$PartitionIndex] + if ($partitionInfo -and $partitionInfo.Partition) { + # The Partition object has PStart and PEnd properties + if (-not $PartitionStart -and $partitionInfo.Partition.PStart) { $PartitionStart = $partitionInfo.Partition.PStart } + if (-not $PartitionEnd -and $partitionInfo.Partition.PEnd) { $PartitionEnd = $partitionInfo.Partition.PEnd } + } + + # If still missing, we can't proceed with checkpoint save + if (-not $PartitionStart -or -not $PartitionEnd) { + Write-Verbose "Save-Checkpoint: Could not determine partition times for index $PartitionIndex - skipping checkpoint update" + return + } + } + + $partitionEntry = @{ + index = $PartitionIndex + start = $PartitionStart.ToUniversalTime().ToString('o') + end = $PartitionEnd.ToUniversalTime().ToString('o') + queryId = $QueryId + } + # Optionally persist structured per-query lifecycle state. These + # fields are ABSENT on legacy checkpoints; when absent on load the resume path + # treats each field as its default and NEVER discards or recreates the stored + # queryId. Only stable, resume-relevant fields are persisted. + if ($LifecycleState) { + foreach ($lk in @('QueryApiVersion', 'FailureCategory', 'AlternateVersionChecked', 'AlternateVersionResult', 'QueryConfirmedAbsent', 'ReplacementAttempted', 'ReplacementQueryId', 'ZeroRecordComplete', 'UncertainCreate', 'UncertainCreateInfo')) { + if ($LifecycleState.ContainsKey($lk)) { $partitionEntry[$lk] = $LifecycleState[$lk] } + } + } + + # Durable per-contract identity. The fingerprint (New-GraphAuditQueryFingerprint over the + # canonical creation contract) is the STABLE identity of this entry across workload passes + # that reuse the same partition index. Prefer an explicitly supplied fingerprint; else + # derive it from the partition contract carried on partitionStatus; else inherit one already + # present on this entry (an uncertain-create entry carries it via UncertainCreateInfo). + # Absent (a legacy caller with no contract in hand) it is simply not stamped and identity + # falls back to the historical index-only behavior. + $entryFingerprint = $null + if (-not [string]::IsNullOrWhiteSpace($Fingerprint)) { + $entryFingerprint = $Fingerprint + } + else { + $pInfo = if ($null -ne $script:partitionStatus) { $script:partitionStatus[$PartitionIndex] } else { $null } + if ($pInfo -and $pInfo.Partition -and $pInfo.Partition.PStart -and $pInfo.Partition.PEnd) { + try { + $entryFingerprint = New-GraphAuditQueryFingerprint -PStart $pInfo.Partition.PStart -PEnd $pInfo.Partition.PEnd -Activities $pInfo.Partition.Activities -RecordTypes $pInfo.Partition.RecordTypes -ServiceFilter $pInfo.Partition.ServiceFilter + } catch { $entryFingerprint = $null } + } + if ([string]::IsNullOrWhiteSpace($entryFingerprint)) { + $inherited = Get-PaxCheckpointEntryFingerprint $partitionEntry + if (-not [string]::IsNullOrWhiteSpace($inherited)) { $entryFingerprint = $inherited } + } + } + if (-not [string]::IsNullOrWhiteSpace($entryFingerprint)) { $partitionEntry.fingerprint = $entryFingerprint } + + if ($State -eq 'QueryCreated') { + $partitionEntry.createdAt = (Get-Date).ToUniversalTime().ToString('o') + + # A confirmed QueryCreated (an adopted or freshly created query) SUPERSEDES any + # durable uncertain-create entry for the SAME contract identity: vacate it here, in the + # SAME in-memory mutation, BEFORE the atomic disk write, so an adopted partition is never + # persisted in both queryCreated and uncertainCreate at once (a possibly-accepted create + # that is now proven created is no longer uncertain). Identity is the fingerprint when + # present so a different workload contract reusing this index is NOT wrongly vacated. + $script:CheckpointData.partitions.uncertainCreate = @( + $script:CheckpointData.partitions.uncertainCreate | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint) } + ) + + # Add to queryCreated list (if not already there) + $existing = $script:CheckpointData.partitions.queryCreated | Where-Object { Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint } + if (-not $existing) { + $script:CheckpointData.partitions.queryCreated += $partitionEntry + } + } + elseif ($State -eq 'Completed') { + $partitionEntry.records = $RecordCount + + # Remove from queryCreated if present (same contract identity) + $script:CheckpointData.partitions.queryCreated = @( + $script:CheckpointData.partitions.queryCreated | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint) } + ) + # A Completed partition also SUPERSEDES any durable uncertain-create entry for the + # SAME contract identity (same atomic mutation, before the disk write) so completion + # never leaves a stale uncertain duplicate behind. + $script:CheckpointData.partitions.uncertainCreate = @( + $script:CheckpointData.partitions.uncertainCreate | Where-Object { -not (Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint) } + ) + + # Add to completed list (if not already there) + $existing = $script:CheckpointData.partitions.completed | Where-Object { Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint } + if (-not $existing) { + $script:CheckpointData.partitions.completed += $partitionEntry + $script:CheckpointData.statistics.totalRecordsSaved += $RecordCount + } + } + elseif ($State -eq 'UncertainCreate') { + # Durable uncertain-create state: the first create POST may have been accepted but + # cannot be proven. Persist the fingerprinted display name + contract (carried on + # UncertainCreateInfo via the lifecycle allow-list above) so ANY later attempt + # reconciles read-only and never re-POSTs. No real queryId is stored. + if ($partitionEntry.ContainsKey('queryId')) { $partitionEntry.Remove('queryId') } + $partitionEntry.UncertainCreate = $true + $partitionEntry.uncertainAt = (Get-Date).ToUniversalTime().ToString('o') + # Dedupe by contract identity (fingerprint when present, else index). + $existing = $script:CheckpointData.partitions.uncertainCreate | Where-Object { Test-PaxCheckpointIdentityMatch $_ $PartitionIndex $entryFingerprint } + if (-not $existing) { + $script:CheckpointData.partitions.uncertainCreate += $partitionEntry + } + } + + # Final normalization guard: a single partition IDENTITY must never be persisted in more + # than one bucket. Identity is the durable fingerprint when present (so two workload-pass + # contracts that reuse the same index remain distinct and both survive) and the integer + # index for legacy entries. This guard now legitimately CALLS the shared identity helpers + # (Get-PaxEntryIdentityKey / Get-PaxCheckpointEntryFingerprint), so an isolated dot-source + # of Save-Checkpoint must ALSO load those helpers or the calls throw at runtime. + # Enforce precedence completed > queryCreated > uncertainCreate BEFORE the atomic disk + # write, removing any duplicated identity from the lower-precedence bucket(s) in this same + # in-memory mutation. + if ($script:CheckpointData -and $script:CheckpointData.partitions) { + $bxCompletedKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($bxC in @($script:CheckpointData.partitions.completed)) { + $bxk = Get-PaxEntryIdentityKey $bxC + if ($null -ne $bxk) { [void]$bxCompletedKeys.Add($bxk) } + } + # completed supersedes queryCreated + $script:CheckpointData.partitions.queryCreated = @(@($script:CheckpointData.partitions.queryCreated) | Where-Object { $null -eq $_ -or $null -eq (Get-PaxEntryIdentityKey $_) -or -not $bxCompletedKeys.Contains((Get-PaxEntryIdentityKey $_)) }) + # completed OR queryCreated supersedes uncertainCreate + $bxQueryCreatedKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($bxQ in @($script:CheckpointData.partitions.queryCreated)) { + $bxk2 = Get-PaxEntryIdentityKey $bxQ + if ($null -ne $bxk2) { [void]$bxQueryCreatedKeys.Add($bxk2) } + } + $script:CheckpointData.partitions.uncertainCreate = @(@($script:CheckpointData.partitions.uncertainCreate) | Where-Object { $null -eq $_ -or $null -eq (Get-PaxEntryIdentityKey $_) -or -not ($bxCompletedKeys.Contains((Get-PaxEntryIdentityKey $_)) -or $bxQueryCreatedKeys.Contains((Get-PaxEntryIdentityKey $_))) }) + } + + # Save to disk. A caller that owns a STRICT single-publication transaction stages the + # COMPLETE bucket transition in memory (-DeferDiskWrite) and publishes it itself, so the + # warning-only writer is never allowed to turn an unproven write into a silent success and + # the transition is never published twice. + if (-not $DeferDiskWrite) { + Save-CheckpointToDisk + } +} + +function Test-CheckpointCompatibility { + <# + .SYNOPSIS + Validates that a parsed checkpoint can safely be resumed by this script version. + .DESCRIPTION + Defends against three failure modes: + 1. Legacy integer `version` outside the supported range (1..2). + 2. `checkpointSchemaVersion` from a FUTURE major release — warned but allowed + when within the same major (forward-compat best effort), rejected on major + mismatch. + 3. `compatibilityMinimumVersion` higher than the running script version — hard + reject (the checkpoint explicitly says this script is too old). + The check is intentionally conservative: it never silently up/down-converts the + structure on disk, and it always emits actionable recovery guidance. + .OUTPUTS + $true -> safe to resume. + $false -> incompatible; caller must abort the resume. + #> + param( + [Parameter(Mandatory)] $CheckpointData, + [Parameter(Mandatory)] [string]$RunningScriptVersion + ) + + # 1) Legacy integer version range (preserves prior behaviour). + if (-not $CheckpointData.version -or [int]$CheckpointData.version -lt 1 -or [int]$CheckpointData.version -gt 2) { + Write-LogHost "ERROR: Unsupported checkpoint legacy version: $($CheckpointData.version). Supported range is 1..2." -ForegroundColor Red + Write-LogHost " Recovery: start a fresh run without -Resume, or use a checkpoint produced by PAX 1.11.x." -ForegroundColor Yellow + return $false + } + + # 2/3) New structured fields are OPTIONAL (older checkpoints predate them). When + # present, they are authoritative. + $schemaVer = [string]$CheckpointData.checkpointSchemaVersion + $minVerStr = [string]$CheckpointData.compatibilityMinimumVersion + $createdBy = [string]$CheckpointData.createdByVersion + + try { $running = [version]$RunningScriptVersion } catch { $running = $null } + try { $minVer = if ($minVerStr) { [version]$minVerStr } else { $null } } catch { $minVer = $null } + try { $created = if ($createdBy) { [version]$createdBy } else { $null } } catch { $created = $null } + + if ($minVer -and $running -and $running -lt $minVer) { + Write-LogHost "ERROR: Checkpoint requires PAX >= $minVerStr but running version is $RunningScriptVersion." -ForegroundColor Red + Write-LogHost " Recovery: upgrade PAX to a version >= $minVerStr or start a fresh run without -Resume." -ForegroundColor Yellow + return $false + } + + if ($schemaVer) { + # Parse the schema version. Only the leading "major.minor" pair gates compatibility. + $parts = $schemaVer.Split('.') + $schemaMajor = 0; [void][int]::TryParse($parts[0], [ref]$schemaMajor) + if ($schemaMajor -gt 2) { + Write-LogHost "ERROR: Checkpoint schema major version $schemaMajor is newer than this script supports (max major 2)." -ForegroundColor Red + Write-LogHost " Recovery: upgrade PAX or start a fresh run without -Resume." -ForegroundColor Yellow + return $false + } + } + + # Created-by drift is informational only (not all script edits change the on-disk shape). + if ($created -and $running -and $created -gt $running) { + Write-LogHost " [WARN] Checkpoint was produced by PAX $createdBy; running PAX $RunningScriptVersion is older. Resume will proceed but newer fields may be ignored." -ForegroundColor Yellow + } + + return $true +} + +function Read-Checkpoint { + <# + .SYNOPSIS + Loads and validates a checkpoint file. + .PARAMETER CheckpointPath + Path to the checkpoint JSON file. + .OUTPUTS + $true if valid and loaded, $false if invalid. + #> + param( + [Parameter(Mandatory)] + [string]$CheckpointPath + ) + + if (-not (Test-Path $CheckpointPath)) { + Write-LogHost "ERROR: Checkpoint file not found: $CheckpointPath" -ForegroundColor Red + return $false + } + + try { + $data = Get-Content -Path $CheckpointPath -Raw | ConvertFrom-Json -AsHashtable + + # Validate version (supports version 1 and 2) + if (-not $data.version -or $data.version -gt 2) { + Write-LogHost "ERROR: Unsupported checkpoint version: $($data.version). This script supports versions 1-2." -ForegroundColor Red + return $false + } + + # Structured compatibility check (schema version, + # compatibilityMinimumVersion, createdByVersion). Hard-rejects checkpoints from + # the future or from releases that explicitly require a newer PAX. + if (-not (Test-CheckpointCompatibility -CheckpointData $data -RunningScriptVersion $ScriptVersion)) { + return $false + } + + # Validate required fields + if (-not $data.runTimestamp -or -not $data.outputFiles -or -not $data.partitions) { + Write-LogHost "ERROR: Checkpoint file is missing required fields" -ForegroundColor Red + return $false + } + + # Get output directory from checkpoint path + $outputDir = Split-Path $CheckpointPath -Parent + $partialCsvPath = Join-Path $outputDir $data.outputFiles.partialCsv + + # Note: We don't require _PARTIAL.csv to exist - the actual data is in .pax_incremental/*.jsonl files + # The _PARTIAL.csv is only created when merging at completion, or may not exist yet + + # Check for incremental save data if there are completed partitions + $completedPartitions = @($data.partitionStates.PSObject.Properties | Where-Object { $_.Value.state -eq 'Completed' }) + if ($completedPartitions.Count -gt 0) { + $incrementalDir = Join-Path $outputDir ".pax_incremental" + $hasIncrementalData = $false + $incrementalRecordCount = 0 + + if (Test-Path $incrementalDir) { + $jsonlFiles = Get-ChildItem -Path $incrementalDir -Filter "*.jsonl" -ErrorAction SilentlyContinue + if ($jsonlFiles -and $jsonlFiles.Count -gt 0) { + $hasIncrementalData = $true + # Count records in files + foreach ($file in $jsonlFiles) { + $incrementalRecordCount += (Get-Content $file.FullName | Measure-Object -Line).Lines + } + } + } + + $expectedRecords = ($completedPartitions | ForEach-Object { $_.Value.recordCount } | Measure-Object -Sum).Sum + + if (-not $hasIncrementalData) { + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost " WARNING: INCREMENTAL DATA MISSING" -ForegroundColor Red + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + Write-LogHost " Checkpoint shows $($completedPartitions.Count) completed partition(s) with ~$expectedRecords records," -ForegroundColor Yellow + Write-LogHost " but the .pax_incremental folder is missing or empty." -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost " Expected location: $incrementalDir" -ForegroundColor White + Write-LogHost "" + Write-LogHost " If you continue, data from completed partitions will be LOST." -ForegroundColor Red + Write-LogHost " The remaining partitions will be re-queried, but previous data cannot be recovered." -ForegroundColor Red + Write-LogHost "" + Write-LogHost " OPTIONS:" -ForegroundColor Cyan + Write-LogHost " 1. Restore the .pax_incremental folder from backup (if available)" -ForegroundColor White + Write-LogHost " 2. Start a fresh run without -Resume (will re-query all partitions)" -ForegroundColor White + Write-LogHost " 3. Continue anyway and accept data loss from completed partitions" -ForegroundColor White + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + + # G-6 noninteractive guard: a noninteractive host (container, ACA Job, scheduled + # task, CI runner) cannot answer a Read-Host. The original code would hang + # forever on stdin. Fail-fast with a clear message that points at the only + # two correct recoveries (restore the incremental folder, or start fresh). + if (script:Test-IsNonInteractive) { + Write-LogHost " Noninteractive host detected (no console available to confirm data loss)." -ForegroundColor Red + Write-LogHost " Resume aborted to prevent silent data loss." -ForegroundColor Red + Write-LogHost " Recovery: restore the .pax_incremental folder from backup OR re-run without -Resume." -ForegroundColor Yellow + return $false + } + + $response = Read-Host "Continue with potential data loss? (yes/no)" + if ($response -notmatch '^y(es)?$') { + Write-LogHost "Resume cancelled. Consider starting a fresh run." -ForegroundColor Yellow + return $false + } + Write-LogHost "Continuing with resume despite missing incremental data..." -ForegroundColor Yellow + } elseif ($incrementalRecordCount -lt ($expectedRecords * 0.9)) { + # Warn if incremental count is significantly less than expected (allow 10% variance for counting differences) + Write-LogHost "" + Write-LogHost " [WARN] Incremental data may be incomplete:" -ForegroundColor Yellow + Write-LogHost " Checkpoint expects ~$expectedRecords records from completed partitions" -ForegroundColor Yellow + Write-LogHost " Found $incrementalRecordCount records in .pax_incremental" -ForegroundColor Yellow + Write-LogHost "" + } + } + + # Load into script scope + $script:CheckpointPath = $CheckpointPath + $script:CheckpointData = $data + $script:PartialOutputPath = $partialCsvPath + $script:IsResumeMode = $true + + return $true + } + catch { + Write-LogHost "ERROR: Failed to parse checkpoint file: $($_.Exception.Message)" -ForegroundColor Red + return $false + } +} + +function Find-Checkpoints { + <# + .SYNOPSIS + Discovers checkpoint files in the specified output directory. + .PARAMETER OutputPath + Directory to search for checkpoint files. + .OUTPUTS + Array of checkpoint info objects sorted by LastUpdated (newest first). + #> + param( + [Parameter(Mandatory)] + [string]$OutputPath + ) + + if (-not (Test-Path $OutputPath)) { + return @() + } + + $checkpointFiles = Get-ChildItem -Path $OutputPath -Filter ".pax_checkpoint_*.json" -Force -ErrorAction SilentlyContinue + + if (-not $checkpointFiles -or $checkpointFiles.Count -eq 0) { + return @() + } + + $checkpoints = @() + + foreach ($file in $checkpointFiles) { + try { + $data = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json -AsHashtable + + $checkpoints += [PSCustomObject]@{ + Path = $file.FullName + FileName = $file.Name + RunTimestamp = $data.runTimestamp + LastUpdated = script:Parse-DateSafe $data.lastUpdated + StartDate = if ($data.parameters.startDate) { $d = script:Parse-DateSafe $data.parameters.startDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + EndDate = if ($data.parameters.endDate) { $d = script:Parse-DateSafe $data.parameters.endDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + PartitionsComplete = $data.statistics.partitionsComplete + PartitionsTotal = $data.partitions.total + RecordsSaved = $data.statistics.totalRecordsSaved + } + } + catch { + # Skip invalid checkpoint files + continue + } + } + + # Sort by LastUpdated descending (newest first) + return $checkpoints | Sort-Object -Property LastUpdated -Descending +} + +function Select-Checkpoint { + <# + .SYNOPSIS + Prompts user to select from multiple checkpoint files. + .PARAMETER Checkpoints + Array of checkpoint info objects from Find-Checkpoints. + .OUTPUTS + Selected checkpoint path, or $null if user quits. + #> + param( + [Parameter(Mandatory)] + [array]$Checkpoints + ) + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Multiple checkpoint files found. Select one to resume:" -ForegroundColor Cyan + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host "" + + for ($i = 0; $i -lt $Checkpoints.Count; $i++) { + $cp = $Checkpoints[$i] + $num = $i + 1 + Write-Host " [$num] $($cp.LastUpdated.ToString('yyyy-MM-dd HH:mm')) | $($cp.StartDate) to $($cp.EndDate) | $($cp.PartitionsComplete)/$($cp.PartitionsTotal) partitions | $($cp.RecordsSaved.ToString('N0')) records" -ForegroundColor White + Write-Host " $($cp.FileName)" -ForegroundColor DarkGray + Write-Host "" + } + + Write-Host " [Q] Quit (do not resume)" -ForegroundColor Yellow + Write-Host "" + + # G-6 noninteractive guard: cannot prompt for a checkpoint selection on a + # container/ACA Job/CI runner. Reject explicitly with a recovery hint instead + # of letting Read-Host hang on stdin. Operators who actually want resume in + # noninteractive contexts must pass -Resume . + if (script:Test-IsNonInteractive) { + Write-Host " Noninteractive host detected: cannot prompt to choose among $($Checkpoints.Count) checkpoint(s)." -ForegroundColor Red + Write-Host " Re-run with an explicit path: -Resume `"`"" -ForegroundColor Yellow + Write-Host " Or run on an interactive host to use the selection menu." -ForegroundColor Yellow + return $null + } + + Send-PromptNotification + while ($true) { + $choice = Read-Host " Enter selection (1-$($Checkpoints.Count)) or 'Q' to quit" + + if ($choice -eq 'Q' -or $choice -eq 'q') { + return $null + } + + $selection = 0 + if ([int]::TryParse($choice, [ref]$selection)) { + if ($selection -ge 1 -and $selection -le $Checkpoints.Count) { + return $Checkpoints[$selection - 1] + } + } + + Write-Host " Invalid selection. Please enter a number 1-$($Checkpoints.Count) or 'Q' to quit." -ForegroundColor Red + } +} + +function Remove-Checkpoint { + <# + .SYNOPSIS + Deletes checkpoint file after successful completion. + #> + + if ($script:CheckpointPath -and (Test-Path $script:CheckpointPath)) { + try { + Remove-Item -Path $script:CheckpointPath -Force + } + catch { + Write-LogHost " Warning: Could not delete checkpoint file: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Release the exclusive checkpoint lock acquired at run start. + script:Release-CheckpointLock + + $script:CheckpointPath = $null + $script:CheckpointData = $null +} + +function Get-PartitionsToProcess { + <# + .SYNOPSIS + Categorizes partitions based on checkpoint state for resume. + .PARAMETER AllPartitions + Array of all partition objects for the date range. + .OUTPUTS + Hashtable with ToSkip, ToFetchOnly, and ToCreateAndFetch arrays. + #> + param( + [Parameter(Mandatory)] + [array]$AllPartitions + ) + + $result = @{ + ToSkip = @() # Already completed - skip entirely + ToFetchOnly = @() # Query exists on server - just fetch records + ToCreateAndFetch = @() # Start fresh - create query then fetch + ToReconcileOnly = @() # Durable uncertain-create - reconcile READ-ONLY, never POST + } + + if (-not $script:CheckpointData) { + # No checkpoint - all partitions need full processing + $result.ToCreateAndFetch = $AllPartitions + return $result + } + + # Categorize each partition by DURABLE CONTRACT IDENTITY (the fingerprint when the entry + # carries one, else the partition index for a legacy checkpoint). Because Graph workload + # passes RESTART partition indexes at 1, a bucket can now hold MULTIPLE entries that share + # an index but denote different contracts; each partition is matched to the entry whose + # fingerprint equals its OWN, so it resumes against its own stored query and a + # possibly-accepted first POST is never repeated across a resume boundary. A durable + # uncertain-create partition is routed to a READ-ONLY reconcile bucket BEFORE the + # fresh-create fallback. + $cpCompleted = @($script:CheckpointData.partitions.completed | Where-Object { $null -ne $_ }) + $cpQueryCreated = @($script:CheckpointData.partitions.queryCreated | Where-Object { $null -ne $_ }) + $cpUncertain = @($script:CheckpointData.partitions.uncertainCreate | Where-Object { $null -ne $_ }) + + foreach ($partition in $AllPartitions) { + $pIdx = [int]$partition.Index + $pFp = $null + if ($partition.PStart -and $partition.PEnd) { + try { + $pFp = New-GraphAuditQueryFingerprint -PStart $partition.PStart -PEnd $partition.PEnd -Activities $partition.Activities -RecordTypes $partition.RecordTypes -ServiceFilter $partition.ServiceFilter + } + catch { $pFp = $null } + } + + $matchCompleted = @($cpCompleted | Where-Object { Test-PaxCheckpointIdentityMatch $_ $pIdx $pFp })[0] + if ($matchCompleted) { + $result.ToSkip += $partition + continue + } + $matchQuery = @($cpQueryCreated | Where-Object { Test-PaxCheckpointIdentityMatch $_ $pIdx $pFp })[0] + if ($matchQuery) { + # Add QueryId to partition for fetch-only processing + $partition | Add-Member -NotePropertyName 'StoredQueryId' -NotePropertyValue $matchQuery.queryId -Force + $result.ToFetchOnly += $partition + continue + } + $matchUncertain = @($cpUncertain | Where-Object { Test-PaxCheckpointIdentityMatch $_ $pIdx $pFp })[0] + if ($matchUncertain) { + # Attach the durable uncertain state so the read-only reconcile path can rebuild + # the fingerprinted exact name; this partition is NEVER routed to create. + $partition | Add-Member -NotePropertyName 'UncertainCreateState' -NotePropertyValue $matchUncertain -Force + $result.ToReconcileOnly += $partition + continue + } + $result.ToCreateAndFetch += $partition + } + + return $result +} + +function Test-ShouldPromptTokenRefresh { + <# + .SYNOPSIS + Checks if token refresh handling is needed. + .DESCRIPTION + Reactive detection: Returns true only when a 401 Unauthorized error + has been detected, indicating the token has actually expired. + No proactive time-based prompts - only triggers on real auth failures. + + For AppRegistration mode, this still returns true on 401 so the + auth handling block can perform automatic silent token refresh. + For interactive modes, this triggers a user prompt. + .OUTPUTS + $true if auth failure detected and refresh handling is needed, $false otherwise. + #> + + # Reactive detection: Return true when 401 error detected (for ALL auth methods) + # The auth handling block will determine whether to auto-refresh (AppRegistration) + # or prompt the user (interactive modes) + return $script:AuthFailureDetected +} + +function Invoke-TokenRefreshPrompt { + <# + .SYNOPSIS + Handles token refresh for interactive auth modes (WebLogin/DeviceCode). + .DESCRIPTION + Triggered reactively when a 401 Unauthorized error is detected. + First attempts silent token refresh (SDK may have valid refresh token cached). + If silent refresh fails, prompts user to re-authenticate or quit. + .OUTPUTS + 'Refreshed' - Token refreshed successfully (silent or interactive) + 'Quit' - User chose to exit + #> + + $tokenAge = if ($script:TokenAcquiredTime) { (Get-Date) - $script:TokenAcquiredTime } else { $null } + + # ═══════════════════════════════════════════════════════════════════════════ + # NOTE: We intentionally do NOT attempt automatic token refresh here. + # On Windows, Connect-MgGraph with InteractiveBrowserCredential ALWAYS opens + # a browser popup. If the user is away, this popup sits waiting, and when + # they return and press 'R' to re-auth, a SECOND popup appears. + # By going straight to the user prompt, we ensure only ONE popup when ready. + # ═══════════════════════════════════════════════════════════════════════════ + + # ═══════════════════════════════════════════════════════════════════════════ + # Prompt user for interactive re-authentication (single popup, user-initiated) + # ═══════════════════════════════════════════════════════════════════════════ + + # Get progress info for display + $completedCount = if ($script:CheckpointData) { $script:CheckpointData.statistics.partitionsComplete } else { 0 } + $totalCount = if ($script:CheckpointData) { $script:CheckpointData.partitions.total } else { 0 } + $recordsSaved = if ($script:CheckpointData) { $script:CheckpointData.statistics.totalRecordsSaved } else { 0 } + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " [!] AUTHENTICATION EXPIRED - Re-authentication Required" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host " Authentication failure detected (401 Unauthorized)." -ForegroundColor White + Write-Host " Your session token has expired. Please re-authenticate to continue." -ForegroundColor White + Write-Host "" + if ($tokenAge) { + Write-Host " Session duration: $([Math]::Round($tokenAge.TotalMinutes, 0)) minutes" -ForegroundColor Gray + } + Write-Host " Progress: $completedCount/$totalCount partitions complete | $($recordsSaved.ToString('N0')) records saved to disk" -ForegroundColor White + Write-Host " Auth method: $($Auth) (interactive)" -ForegroundColor Gray + Write-Host "" + Write-Host " [R] Re-authenticate now (recommended)" -ForegroundColor Green + Write-Host " [Q] Quit and save progress (resume later with -Resume)" -ForegroundColor Cyan + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + + # G-6 noninteractive guard: on a container/ACA Job/CI runner there is nobody at + # the keyboard to choose R/Q. Auto-choose Q ('quit and save progress'); the + # operator can resume the checkpoint later from an interactive host. Logging + # the auto-choice prevents the silent stdin hang the original code would have + # produced. + if (script:Test-IsNonInteractive) { + Write-Host "" -ForegroundColor Yellow + Write-Host " Noninteractive host detected: cannot prompt for re-authentication." -ForegroundColor Yellow + Write-Host " Auto-selecting [Q] - progress will be saved to the checkpoint." -ForegroundColor Yellow + Write-Host " Resume from an interactive host with: -Resume `"$($script:CheckpointPath)`"" -ForegroundColor Yellow + return 'Quit' + } + + Send-PromptNotification + while ($true) { + $choice = Read-Host " Enter choice (R/Q)" + + switch ($choice.ToUpper()) { + 'R' { + Write-Host "" + Write-Host " Re-authenticating..." -ForegroundColor Cyan + + try { + # Disconnect and reconnect + script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue + Connect-PurviewAudit -AuthMethod $Auth -UseEOMMode $false + + # Update token timing and reset auth failure flags + $script:TokenAcquiredTime = Get-Date + $script:AuthFailureDetected = $false + $script:Auth401MessageShown = $false # Reset for next auth failure cycle + + Write-Host " Re-authentication successful. Resuming execution..." -ForegroundColor Green + Write-Host " Failed partitions will be retried with fresh token." -ForegroundColor Green + Write-Host "" + + return 'Refreshed' + } + catch { + Write-Host " ✗ Re-authentication failed: $($_.Exception.Message)" -ForegroundColor Red + Write-Host " Please try again or quit." -ForegroundColor Yellow + } + } + 'Q' { + Write-Host "" + Show-CheckpointExitMessage + return 'Quit' + } + default { + Write-Host " Invalid choice. Please enter R or Q." -ForegroundColor Red + } + } + } +} + +function Merge-IncrementalSaves { + <# + .SYNOPSIS + Merges incremental JSON save files into the main allLogs collection. + .DESCRIPTION + Called at the end of execution to consolidate partition data that was + saved incrementally during execution. In resume mode, only merges data + from partitions that were skipped (already completed at start of run), + since partitions completed during this run already have data in memory. + .PARAMETER AllLogs + Reference to the synchronized ArrayList to add records to. + .PARAMETER OutputDirectory + The output directory containing the .pax_incremental folder. + .PARAMETER CleanupAfterMerge + If true, deletes the incremental files after successful merge. + .PARAMETER OnlyPartitionIndices + If specified, only merge files for these partition indices. + Used in resume mode to avoid double-counting data from partitions + that completed during this run. + #> + param( + [Parameter(Mandatory = $true)] + [System.Collections.ArrayList]$AllLogs, + + [Parameter(Mandatory = $true)] + [string]$OutputDirectory, + + [Parameter(Mandatory = $false)] + [bool]$CleanupAfterMerge = $true, + + [Parameter(Mandatory = $false)] + [int[]]$OnlyPartitionIndices = $null + ) + + $incrementalDir = Join-Path $OutputDirectory ".pax_incremental" + + if (-not (Test-Path $incrementalDir)) { + return 0 + } + + $incrementalFiles = Get-ChildItem -Path $incrementalDir -Filter "*.jsonl" -ErrorAction SilentlyContinue + + if (-not $incrementalFiles -or $incrementalFiles.Count -eq 0) { + return 0 + } + + $mergedCount = 0 + $filesProcessed = 0 + $filesSkipped = 0 + + $filterMsg = if ($OnlyPartitionIndices) { " (filtering for partitions: $($OnlyPartitionIndices -join ', '))" } else { "" } + Write-LogHost " [MERGE] Found $($incrementalFiles.Count) incremental save files$filterMsg..." -ForegroundColor Cyan + + foreach ($file in $incrementalFiles) { + try { + # If filtering by partition indices, check if this file matches + # Filename format: Part{N}_timestamp_qid-{QueryId}_Nrecords.jsonl (recovery files use qid-recovery) + if ($OnlyPartitionIndices) { + $partMatch = [regex]::Match($file.Name, '^Part(\d+)_') + if ($partMatch.Success) { + $filePartitionIndex = [int]$partMatch.Groups[1].Value + if ($filePartitionIndex -notin $OnlyPartitionIndices) { + $filesSkipped++ + # Don't delete - this file is for a partition completed in this run + continue + } + } + } + + # Read JSON Lines (NDJSON) format - one record per line + $lines = Get-Content -Path $file.FullName -Encoding utf8 + $fileRecordCount = 0 + + foreach ($line in $lines) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + + try { + $record = $line | ConvertFrom-Json + [void]$AllLogs.Add($record) + $fileRecordCount++ + } catch { + # Skip malformed lines but continue processing + Write-Verbose "Skipped malformed line in $($file.Name)" + } + } + + $mergedCount += $fileRecordCount + if ($fileRecordCount -gt 0) { $filesProcessed++ } + + if ($CleanupAfterMerge) { + Remove-Item -Path $file.FullName -Force -ErrorAction SilentlyContinue + } + } + catch { + Write-LogHost " [WARN] Failed to merge $($file.Name): $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Clean up directory if empty + if ($CleanupAfterMerge) { + $remainingFiles = Get-ChildItem -Path $incrementalDir -ErrorAction SilentlyContinue + if (-not $remainingFiles -or $remainingFiles.Count -eq 0) { + Remove-Item -Path $incrementalDir -Force -ErrorAction SilentlyContinue + } + } + + if ($mergedCount -gt 0) { + Write-LogHost " [MERGE] Merged $($mergedCount.ToString('N0')) records from $filesProcessed incremental files" -ForegroundColor Green + } + + return $mergedCount +} + +function Merge-IncrementalSaves-Streaming { + <# + .SYNOPSIS + Memory-efficient streaming merge of incremental JSONL files directly to CSV. + .DESCRIPTION + Instead of loading all records into memory, this function streams records + from incremental JSONL files directly to the final CSV output using batched + writes and explicit garbage collection between files. This prevents memory + exhaustion when merging millions of records. + .PARAMETER OutputFile + The final CSV file path to write merged data to. + .PARAMETER OutputDirectory + The output directory containing the .pax_incremental folder. + .PARAMETER OnlyPartitionIndices + If specified, only merge files for these partition indices. + .PARAMETER Columns + The column schema to use for CSV output. If not specified, uses default 7-column schema. + .PARAMETER RunTimestamp + The script run timestamp used to filter incremental files to only those from the current run. + Prevents stale files from prior runs being merged into the output. + .RETURNS + The total number of records merged. + #> + param( + [Parameter(Mandatory = $true)] + [string]$OutputFile, + + [Parameter(Mandatory = $true)] + [string]$OutputDirectory, + + [Parameter(Mandatory = $false)] + [int[]]$OnlyPartitionIndices = $null, + + [Parameter(Mandatory = $false)] + [string[]]$Columns = $null, + + [Parameter(Mandatory = $false)] + [System.Collections.Generic.HashSet[string]]$ExcludeRecordIds = $null, + + [Parameter(Mandatory = $false)] + [ref]$ActivityCounts = $null, + + [Parameter(Mandatory = $false)] + [string]$RunTimestamp = $null + ) + + $incrementalDir = Join-Path $OutputDirectory ".pax_incremental" + + if (-not (Test-Path $incrementalDir)) { + Write-LogHost " [MERGE-STREAM] No incremental directory found" -ForegroundColor Yellow + return 0 + } + + # Filter by run timestamp to avoid merging stale files from prior runs + $jsonlFilter = if ($RunTimestamp) { "*_${RunTimestamp}_*.jsonl" } else { "*.jsonl" } + $allFiles = Get-ChildItem -Path $incrementalDir -Filter $jsonlFilter -ErrorAction SilentlyContinue + if (-not $allFiles -or $allFiles.Count -eq 0) { + Write-LogHost " [MERGE-STREAM] No incremental files found$(if ($RunTimestamp) { " for run $RunTimestamp" })" -ForegroundColor Yellow + return 0 + } + + # Sort files by partition number for consistent output ordering + $files = $allFiles | Sort-Object { + if ($_.Name -match 'Part(\d+)_') { [int]$Matches[1] } else { 999999 } + } + + # Filter by partition indices if specified + if ($OnlyPartitionIndices) { + $files = $files | Where-Object { + $partMatch = [regex]::Match($_.Name, '^Part(\d+)_') + if ($partMatch.Success) { + [int]$partMatch.Groups[1].Value -in $OnlyPartitionIndices + } else { + $false + } + } + } + + if (-not $files -or @($files).Count -eq 0) { + Write-LogHost " [MERGE-STREAM] No matching incremental files for specified partitions" -ForegroundColor Yellow + return 0 + } + + # When multiple JSONL files exist for the same partition (from retries with different QueryIds), + # keep only the largest file per partition. This prevents duplicate records from partial first attempts + # being merged alongside the full retry result. + $filesByPartition = @{} + foreach ($f in @($files)) { + if ($f.Name -match '^Part(\d+)_') { + $pIdx = [int]$Matches[1] + if (-not $filesByPartition.ContainsKey($pIdx) -or $f.Length -gt $filesByPartition[$pIdx].Length) { + $filesByPartition[$pIdx] = $f + } + } + } + $deduplicatedFiles = @($filesByPartition.Values | Sort-Object { if ($_.Name -match 'Part(\d+)_') { [int]$Matches[1] } else { 999999 } }) + $removedFileCount = @($files).Count - $deduplicatedFiles.Count + if ($removedFileCount -gt 0) { + Write-LogHost " [MERGE-STREAM] Removed $removedFileCount duplicate partition file(s) from prior retry attempts — keeping largest per partition" -ForegroundColor DarkYellow + } + $files = $deduplicatedFiles + + $fileCount = @($files).Count + Write-LogHost " [MERGE-STREAM] Streaming $fileCount incremental files to CSV..." -ForegroundColor Cyan + + # Use default 8-column schema if not specified (non-explosion mode) + # Column names match Purview UI audit export: RecordId, CreationDate, RecordType, Operation, UserId, AuditData, AssociatedAdminUnits, AssociatedAdminUnitsNames + if (-not $Columns) { + $Columns = @('RecordId', 'CreationDate', 'RecordType', 'Operation', 'UserId', 'AuditData', 'AssociatedAdminUnits', 'AssociatedAdminUnitsNames') + } + + $totalMerged = 0 + $filesProcessed = 0 + $headerWritten = $false + $batchSize = 5000 + $startTime = Get-Date + $lastProgressTime = Get-Date + + # Track seen RecordIds for deduplication (seed with any in-memory RecordIds already written to CSV) + $seenIds = if ($ExcludeRecordIds) { New-Object System.Collections.Generic.HashSet[string] ($ExcludeRecordIds) } else { New-Object System.Collections.Generic.HashSet[string] } + $duplicatesSkipped = 0 + + foreach ($file in $files) { + $filesProcessed++ + $partNum = if ($file.Name -match 'Part(\d+)_') { $Matches[1] } else { "?" } + + try { + # Open CSV writer on first file with data + if (-not $headerWritten) { + Open-CsvWriter -Path $OutputFile -Columns $Columns + $headerWritten = $true + } + + # Stream records from this file in batches + $batch = New-Object System.Collections.Generic.List[object] + $fileRecords = 0 + + # Use StreamReader instead of Get-Content pipeline for ~5-10x faster file reading + $reader = [System.IO.StreamReader]::new($file.FullName, [System.Text.Encoding]::UTF8) + while ($null -ne ($line = $reader.ReadLine())) { + if (-not [string]::IsNullOrWhiteSpace($line)) { + try { + $record = $line | ConvertFrom-Json + + # Deduplicate by RecordId + $recordId = $null + if ($record.Identity) { $recordId = $record.Identity } + elseif ($record.Id) { $recordId = $record.Id } + elseif ($record.RecordId) { $recordId = $record.RecordId } + + if ($recordId -and $seenIds.Contains($recordId)) { + $script:StreamingMergeDuplicatesSkipped++ + continue # Skip duplicate — 'continue' works in while loop (was 'return' in ForEach-Object) + } + if ($recordId) { [void]$seenIds.Add($recordId) } + + # Client-side date-range trimming for streaming path + if ($script:TrimStartDateUTC -or $script:TrimEndDateUTC) { + $recDate = script:Parse-DateSafe $record.CreationDate + if ($recDate) { + $recDateUtc = $recDate.ToUniversalTime() + if ($script:TrimStartDateUTC -and $recDateUtc -lt $script:TrimStartDateUTC) { $script:DateTrimCount++; continue } + if ($script:TrimEndDateUTC -and $recDateUtc -ge $script:TrimEndDateUTC) { $script:DateTrimCount++; continue } + } + } + + # Parse AuditData for Operation if needed + $auditData = $record.AuditData + $parsedAudit = if ($record.PSObject.Properties['_ParsedAuditData']) { + $record._ParsedAuditData + } else { + try { $auditData | ConvertFrom-Json -ErrorAction SilentlyContinue } catch { $null } + } + $opValue = if ($parsedAudit -and $parsedAudit.Operation) { $parsedAudit.Operation } else { $record.Operations } + + # Track per-activity counts for Activity Type Breakdown + if ($ActivityCounts -and $opValue) { + if (-not $ActivityCounts.Value.ContainsKey($opValue)) { $ActivityCounts.Value[$opValue] = 0 } + $ActivityCounts.Value[$opValue]++ + } + + # Create normalized record matching expected schema (column names match Purview UI export) + $normalizedRecord = [pscustomobject]@{ + RecordId = if ($record.RecordId) { $record.RecordId } elseif ($record.Identity) { $record.Identity } elseif ($record.Id) { $record.Id } elseif ($parsedAudit -and $parsedAudit.Id) { $parsedAudit.Id } else { $null } + CreationDate = if ($record.CreationDate) { + $dt = script:Parse-DateSafe $record.CreationDate; if ($dt) { $dt.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) } else { $record.CreationDate } + } else { '' } + RecordType = $record.RecordType + Operation = $opValue + UserId = if ($record.UserId) { $record.UserId } elseif ($record.UserIds) { $record.UserIds } else { '' } + AuditData = $auditData + AssociatedAdminUnits = $(try { if ($parsedAudit.AssociatedAdminUnits) { $parsedAudit.AssociatedAdminUnits } elseif ($record.AssociatedAdminUnits) { $record.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($parsedAudit.AssociatedAdminUnitsNames) { $parsedAudit.AssociatedAdminUnitsNames } elseif ($record.AssociatedAdminUnitsNames) { $record.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + } + + $batch.Add($normalizedRecord) + $fileRecords++ + + # Write batch when full + if ($batch.Count -ge $batchSize) { + Write-CsvRows -Rows $batch -Columns $Columns + $totalMerged += $batch.Count + $batch.Clear() + } + } catch { + # Skip malformed lines + Write-Verbose "Skipped malformed line in $($file.Name): $($_.Exception.Message)" + } + } + } + # Dispose StreamReader to release file handle + if ($reader) { $reader.Dispose(); $reader = $null } + + # Flush remaining batch for this file + if ($batch.Count -gt 0) { + Write-CsvRows -Rows $batch -Columns $Columns + $totalMerged += $batch.Count + $batch.Clear() + } + + # Progress reporting every 30 seconds + $now = Get-Date + if (($now - $lastProgressTime).TotalSeconds -ge 30) { + $elapsed = ($now - $startTime).TotalSeconds + $rate = if ($elapsed -gt 0) { [int]($totalMerged / $elapsed) } else { 0 } + Write-LogHost " [MERGE-STREAM] Progress: $filesProcessed/$fileCount files | $($totalMerged.ToString('N0')) records | ~$rate rec/sec" -ForegroundColor DarkCyan + $lastProgressTime = $now + } + + # Reduced GC frequency from every file to every 5th file for better throughput + $batch = $null + if ($filesProcessed % 5 -eq 0) { + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + } + + } catch { + # Ensure StreamReader is disposed on error to release file handle + if ($reader) { try { $reader.Dispose() } catch {} ; $reader = $null } + Write-LogHost " [WARN] Failed to stream merge $($file.Name): $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Close CSV writer + if ($headerWritten) { + Close-CsvWriter + } + + # Final stats + $totalElapsed = (Get-Date) - $startTime + $finalRate = if ($totalElapsed.TotalSeconds -gt 0) { [int]($totalMerged / $totalElapsed.TotalSeconds) } else { 0 } + + Write-LogHost " [MERGE-STREAM] Streaming merge complete: $($totalMerged.ToString('N0')) records from $filesProcessed files" -ForegroundColor Green + Write-LogHost " [MERGE-STREAM] Time: $([Math]::Round($totalElapsed.TotalSeconds, 1))s | Rate: $finalRate rec/sec" -ForegroundColor DarkGray + if ($duplicatesSkipped -gt 0 -or $script:StreamingMergeDuplicatesSkipped -gt 0) { + $totalDupes = $duplicatesSkipped + $script:StreamingMergeDuplicatesSkipped + Write-LogHost " [MERGE-STREAM] Duplicates skipped: $totalDupes" -ForegroundColor DarkGray + } + + # Clear the HashSet to free memory + $seenIds.Clear() + $seenIds = $null + [GC]::Collect() + + return $totalMerged +} + +function Show-CheckpointExitMessage { + <# + .SYNOPSIS + Displays checkpoint save confirmation and resume instructions. + #> + + if (-not $script:CheckpointData -or -not $script:CheckpointPath) { + return + } + + $completedCount = if ($script:CheckpointData.statistics.partitionsComplete) { $script:CheckpointData.statistics.partitionsComplete } else { 0 } + $queryCreatedCount = if ($script:CheckpointData.statistics.partitionsQueryCreated) { $script:CheckpointData.statistics.partitionsQueryCreated } else { 0 } + $totalCount = if ($script:CheckpointData.partitions.total) { $script:CheckpointData.partitions.total } else { 0 } + $remaining = $totalCount - $completedCount - $queryCreatedCount + $recordsSaved = if ($script:CheckpointData.statistics.totalRecordsSaved) { $script:CheckpointData.statistics.totalRecordsSaved } else { 0 } + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Green + Write-Host " PROGRESS SAVED" -ForegroundColor Green + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Green + Write-Host "" + # The checkpoint file and the local _PARTIAL file always live on the local + # scratch path (they are required for -Resume; the checkpoint isn't uploaded + # until the run completes successfully). Display them as full local paths so + # the user can copy/paste them into -Resume verbatim. + # + # The run log is written to local scratch throughout the run. On successful + # completion the end-of-run upload sweep pushes it to the remote destination + # (SP/Fabric); on an interrupted run (Ctrl+C / failure) the upload sweep does + # NOT fire, so the log lives only at the local scratch path until the next + # -Resume completes successfully. Always display the literal local path here + # (consistent with Checkpoint: and Partial data:) so the user can inspect / + # attach the in-progress log immediately. When a remote destination is bound, + # print a follow-up "Uploads to:" note so the user knows where the log will + # land on successful completion. + Write-Host " Checkpoint: $script:CheckpointPath" -ForegroundColor White + # Live in-progress records stream into /.pax_incremental/*.jsonl + # shards as each partition completes. The companion _PARTIAL.csv ($script:PartialOutputPath) + # is only materialized at end-of-run merge or on successful completion, so it + # generally does NOT exist on an interrupted Ctrl+C exit. Point the user at + # the .pax_incremental folder (the actual durable source of partial data); if + # a _PARTIAL.csv already exists alongside it, list that on a follow-up line. + $_partialBaseDir = $null + # Defensive: Split-Path -LiteralPath throws "Parameter set cannot be resolved" + # when handed $null, an empty string, or a non-string (array / hashtable from a + # malformed checkpoint outputFiles entry). Coerce to a non-empty string before + # splitting and swallow any residual parsing failure so the exit banner never + # crashes after a real upstream error has already been reported. + $_pop = if ($script:PartialOutputPath) { [string]$script:PartialOutputPath } else { '' } + if ($_pop -and $_pop.Trim()) { + try { $_partialBaseDir = Split-Path -LiteralPath $_pop -Parent -ErrorAction Stop } catch { $_partialBaseDir = $null } + } + if (-not $_partialBaseDir -and $script:CheckpointPath) { + $_cpp = [string]$script:CheckpointPath + if ($_cpp -and $_cpp.Trim()) { + try { $_partialBaseDir = Split-Path -LiteralPath $_cpp -Parent -ErrorAction Stop } catch { $_partialBaseDir = $null } + } + } + $_incDir = if ($_partialBaseDir) { Join-Path $_partialBaseDir '.pax_incremental' } else { $null } + if ($_incDir) { + Write-Host " Partial data: $_incDir\*.jsonl" -ForegroundColor White + } else { + Write-Host " Partial data: (incremental .jsonl shards under scratch .pax_incremental/)" -ForegroundColor White + } + if ($script:PartialOutputPath -and (Test-Path -LiteralPath $script:PartialOutputPath)) { + Write-Host " Merged CSV (so far): $script:PartialOutputPath" -ForegroundColor DarkGray + } + if ($script:LogFile) { + Write-Host " Log file: $script:LogFile" -ForegroundColor White + $_logRemoteTarget = $null + try { + if ($script:DestTier -and $script:DestTier.ContainsKey('Log') -and $script:DestTier['Log'] -and $script:DestTier['Log'] -ne 'Local') { + $_logFolder = if ($script:DestParentUrl -and $script:DestParentUrl.ContainsKey('Log') -and $script:DestParentUrl['Log']) { + $script:DestParentUrl['Log'] + } elseif ($script:DestRaw -and $script:DestRaw.ContainsKey('Log') -and $script:DestRaw['Log']) { + $script:DestRaw['Log'] + } else { $null } + if ($_logFolder) { + $_logLeaf = [System.IO.Path]::GetFileName($script:LogFile) + $_logRemoteTarget = ($_logFolder.TrimEnd('/','\') + '/' + $_logLeaf) + } + } + } catch {} + if ($_logRemoteTarget) { + Write-Host " (uploads to $_logRemoteTarget on successful completion)" -ForegroundColor DarkGray + } + } + Write-Host " Records saved: $($recordsSaved.ToString('N0'))" -ForegroundColor White + # Build the Partitions status line as a single string before emission so the + # global Write-Host proxy (which mirrors every Write-Host call to the log + # file as ONE timestamped [INFO] entry regardless of -NoNewline) writes one + # log line instead of 2-3 fragmented entries. The console output is the same + # either way — both forms produce an identical + # "Partitions: X/Y complete[, N queries pending][, M not started]" line — + # but a multi-call -NoNewline approach produces orphan log lines like + # ", 1 queries pending" on their own timestamp because -NoNewline only + # concatenates on the host stream, not in the proxy's Add-Content path. + $partitionLine = " Partitions: $completedCount/$totalCount complete" + if ($queryCreatedCount -gt 0) { $partitionLine += ", $queryCreatedCount queries pending" } + if ($remaining -gt 0) { $partitionLine += ", $remaining not started" } + Write-Host $partitionLine -ForegroundColor White + Write-Host "" + Write-Host " To resume later:" -ForegroundColor Cyan + # Resume credential hint. -Auth / -TenantId / -ClientId are restored from the + # checkpoint on -Resume (see the resume restore block), so they are NOT + # re-supplied on the resume command line. Only AppRegistration needs credential + # material on every invocation, because the secret / certificate is never + # persisted to the checkpoint by design — and the hint must reflect the SAME + # credential kind the operator originally used (client secret vs. certificate + # thumbprint vs. PFX path), detected from the promoted $script:Client* values. + # Interactive modes (WebLogin / DeviceCode / Credential / Silent) and + # ManagedIdentity need nothing beyond -Resume. + if ($Auth -eq 'AppRegistration') { + $_resumeCred = + if (-not [string]::IsNullOrWhiteSpace([string]$script:ClientCertificateThumbprint)) { "-ClientCertificateThumbprint ''" } + elseif (-not [string]::IsNullOrWhiteSpace([string]$script:ClientCertificatePath)) { "-ClientCertificatePath '' [-ClientCertificatePassword '']" } + else { "-ClientSecret ''" } + Write-Host " -Resume `"$($script:CheckpointPath)`" $_resumeCred" -ForegroundColor White + } else { + Write-Host " -Resume `"$($script:CheckpointPath)`"" -ForegroundColor White + } + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Green +} + +function Complete-CheckpointRun { + <# + .SYNOPSIS + Finalizes successful run: renames _PARTIAL file, deletes checkpoint. + .PARAMETER FinalOutputPath + The final output path (without _PARTIAL). + #> + param( + [Parameter(Mandatory)] + [string]$FinalOutputPath + ) + + # Rename _PARTIAL output (and log) to final names ONLY if the _PARTIAL file still exists. + # In CSV-split / ExportWorkbook modes the intermediate _PARTIAL.csv may already have + # been deleted by upstream code paths; in that case skip the rename but STILL proceed + # with checkpoint deletion below so the checkpoint file is not orphaned. + if ($script:PartialOutputPath -and (Test-Path $script:PartialOutputPath)) { + try { + # Rename _PARTIAL to final + if (Test-Path $FinalOutputPath) { + # Final file already exists - add timestamp to avoid overwrite + $dir = Split-Path $FinalOutputPath -Parent + $name = [System.IO.Path]::GetFileNameWithoutExtension($FinalOutputPath) + $ext = [System.IO.Path]::GetExtension($FinalOutputPath) + $timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + $FinalOutputPath = Join-Path $dir "${name}_${timestamp}${ext}" + } + + Move-Item -Path $script:PartialOutputPath -Destination $FinalOutputPath -Force + + # Rename log file (remove _PARTIAL suffix) + $partialLogPath = $script:LogFile + if ($partialLogPath -and (Test-Path $partialLogPath) -and $partialLogPath -match '_PARTIAL\.log$') { + $finalLogPath = $partialLogPath -replace '_PARTIAL\.log$', '.log' + if (Test-Path $finalLogPath) { + $logDir = Split-Path $finalLogPath -Parent + $logName = [System.IO.Path]::GetFileNameWithoutExtension($finalLogPath) + $timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' + $finalLogPath = Join-Path $logDir "${logName}_${timestamp}.log" + } + Move-Item -Path $partialLogPath -Destination $finalLogPath -Force + $script:LogFile = $finalLogPath + } + + $script:PartialOutputPath = $null + } + catch { + Write-LogHost " Warning: Could not finalize output file: $($_.Exception.Message)" -ForegroundColor Yellow + } + } + + # Delete checkpoint (always — caller has determined the run succeeded). + Remove-Checkpoint +} + +function Get-UserLicenseData { + <# + .SYNOPSIS + Fetches user license information from Microsoft Graph API. + + .DESCRIPTION + Queries Graph API for subscribedSkus and builds lookup hashtables for: + - User license assignments (userId -> list of SKU names) + - Copilot license detection (userId -> has Copilot license) + + Dynamic Copilot license detection (no hardcoded SKU GUIDs): + 1. From /subscribedSkus, collect every servicePlanId whose servicePlanName matches '*COPILOT*'. + 2. For each user, scan assignedPlans for entries where capabilityStatus == 'Enabled' + AND servicePlanId is in the discovered Copilot plan set. + This honors capability state (admin-disabled service plans are correctly excluded) + and auto-adapts to any current or future Copilot SKU (M365, EDU, Sales, Finance, GCC, etc.). + Requires only User.Read.All + Organization.Read.All (no additional permissions). + + .OUTPUTS + Hashtable with two keys: + - UserLicenses: @{userId = @('SKU1', 'SKU2')} + - UserHasCopilot: @{userId = $true/$false} + #> + + Write-LogHost "" + Write-LogHost "Fetching license data from Microsoft Graph API..." -ForegroundColor Cyan + + try { + # Fetch all subscribed SKUs with assigned user data + $uri = "https://graph.microsoft.com/v1.0/subscribedSkus" + $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + $skus = $response.value + + Write-LogHost " Found $($skus.Count) SKU(s) in tenant" -ForegroundColor Gray + + # Build SKU lookup: skuId -> skuPartNumber + # Also dynamically discover Copilot service plan IDs by name pattern (no hardcoded GUIDs). + $skuLookup = @{} + $copilotPlanIds = New-Object System.Collections.Generic.HashSet[string] + + foreach ($sku in $skus) { + $skuId = $sku.skuId + $skuName = $sku.skuPartNumber + $skuLookup[$skuId] = $skuName + if ($sku.servicePlans) { + foreach ($plan in $sku.servicePlans) { + if ($plan.servicePlanName -and $plan.servicePlanName -like '*COPILOT*' -and $plan.servicePlanId) { + [void]$copilotPlanIds.Add([string]$plan.servicePlanId) + } + } + } + } + Write-LogHost " Discovered $($copilotPlanIds.Count) Copilot service plan(s) in tenant" -ForegroundColor Gray + + # Build user license hashtables + $userLicenses = @{} # userId -> @('SKU1', 'SKU2') + $userHasCopilot = @{} # userId -> $true/$false + + # Fetch all users with licenses in batches + # assignedPlans is required for capability-aware Copilot detection (still under User.Read.All). + Write-LogHost " Fetching user license assignments..." -ForegroundColor Gray + $userUri = "https://graph.microsoft.com/v1.0/users?`$select=id,userPrincipalName,assignedLicenses,assignedPlans&`$top=999" + $userCount = 0 + $copilotUserCount = 0 + + do { + $userResponse = Invoke-MgGraphRequest -Method GET -Uri $userUri -ErrorAction Stop + $users = $userResponse.value + + foreach ($user in $users) { + $userId = $user.id + $upn = $user.userPrincipalName + + if (-not $userId -or -not $upn) { continue } + + $userCount++ + $licenses = @() + $hasCopilot = $false + + # Friendly SKU name list (unchanged): map assignedLicenses[].skuId -> skuPartNumber via $skuLookup + foreach ($license in $user.assignedLicenses) { + $skuId = $license.skuId + $skuName = if ($skuLookup.ContainsKey($skuId)) { $skuLookup[$skuId] } else { $skuId } + $licenses += $skuName + } + + # Capability-aware Copilot detection: any assignedPlan that is Enabled AND a Copilot service plan. + if ($copilotPlanIds.Count -gt 0 -and $user.assignedPlans) { + foreach ($plan in $user.assignedPlans) { + if ($plan.capabilityStatus -eq 'Enabled' -and $plan.servicePlanId -and $copilotPlanIds.Contains([string]$plan.servicePlanId)) { + $hasCopilot = $true + break + } + } + } + + # Store license data keyed by both ID and UPN for flexible lookup + if ($licenses.Count -gt 0) { + $userLicenses[$userId] = $licenses + $userLicenses[$upn] = $licenses + } + + $userHasCopilot[$userId] = $hasCopilot + $userHasCopilot[$upn] = $hasCopilot + + if ($hasCopilot) { + $copilotUserCount++ + } + } + + $userUri = $userResponse.'@odata.nextLink' + } while ($userUri) + + Write-LogHost " Processed $userCount user(s) with license assignments" -ForegroundColor Gray + Write-LogHost " Detected $copilotUserCount user(s) with Copilot licenses" -ForegroundColor Green + Write-LogHost "" + + return @{ + UserLicenses = $userLicenses + UserHasCopilot = $userHasCopilot + } + } + catch { + Write-LogHost "WARNING: Failed to fetch license data: $($_.Exception.Message)" -ForegroundColor Yellow + Write-LogHost " License columns will be empty in export" -ForegroundColor Yellow + Write-LogHost "" + + # Return empty hashtables on failure + return @{ + UserLicenses = @{} + UserHasCopilot = @{} + } + } +} + +function ConvertTo-FlatEntraUsers { + <# + .SYNOPSIS + Flattens Entra user objects into CSV-friendly format. + + .DESCRIPTION + Converts Entra ID user objects with nested properties into flat tabular format. + Filters out non-user accounts (rooms, resources) based on userType validation. + Explodes arrays (proxyAddresses, manager) into individual columns. + + NOTE: This version excludes the 9 granular Entra license columns from the Graph script. + License data is added separately via Get-UserLicenseData() in MAC format (assignedLicenses, hasLicense). + + .PARAMETER Users + Array of user objects from Microsoft Graph API (with 35 properties + manager expansion). + + .OUTPUTS + Array of PSCustomObjects with 37 flattened columns (35 user properties + 5 manager columns, no license columns yet). + #> + param( + [Parameter(Mandatory = $true)] + [array]$Users + ) + + $flattenedUsers = @() + + foreach ($user in $Users) { + # Filter: Only include real user accounts (exclude rooms, resources, shared mailboxes) + # Room and resource mailboxes have specific characteristics: + # - userType is often null or not "Member"/"Guest" + # - They typically lack givenName and surname + # - They often have mail but no userPrincipalName with typical user format + + $userTypeValue = $user.userType + + # Skip if userType is null/empty (likely a room or resource) + if ([string]::IsNullOrWhiteSpace($userTypeValue)) { + continue + } + + # Only include users with userType = "Member" or "Guest" + # Rooms/resources typically have different userType values or null + if ($userTypeValue -ne 'Member' -and $userTypeValue -ne 'Guest') { + continue + } + + # Additional heuristic: Real users typically have either givenName or surname + # Room mailboxes typically have neither (only displayName) + # This is not foolproof but combined with userType check, it's quite reliable + $hasGivenName = -not [string]::IsNullOrWhiteSpace($user.givenName) + $hasSurname = -not [string]::IsNullOrWhiteSpace($user.surname) + + # If user has Member/Guest type but no name components, might be a shared resource + # Allow through if they have at least givenName OR surname OR if account is enabled + # (most room mailboxes are enabled but lack name components) + if (-not $hasGivenName -and -not $hasSurname -and $user.accountEnabled) { + # Additional check: if they have licenses assigned, likely a real user + if (-not $user.assignedLicenses -or $user.assignedLicenses.Count -eq 0) { + # No licenses and no name components - likely a room/resource + continue + } + } + + $flatUser = [ordered]@{} + + # Core Identity Properties (simple strings) + $flatUser['userPrincipalName'] = $user.userPrincipalName + $flatUser['displayName'] = $user.displayName + $flatUser['id'] = $user.id + $flatUser['mail'] = $user.mail + $flatUser['givenName'] = $user.givenName + $flatUser['surname'] = $user.surname + + # Job Properties + $flatUser['jobTitle'] = $user.jobTitle + $flatUser['department'] = $user.department + $flatUser['employeeType'] = $user.employeeType + $flatUser['employeeId'] = $user.employeeId + $flatUser['employeeHireDate'] = $user.employeeHireDate + + # Location Properties + $flatUser['officeLocation'] = $user.officeLocation + $flatUser['city'] = $user.city + $flatUser['state'] = $user.state + $flatUser['country'] = $user.country + $flatUser['postalCode'] = $user.postalCode + $flatUser['companyName'] = $user.companyName + + # Organizational Properties + $flatUser['employeeOrgData_division'] = if ($user.employeeOrgData) { $user.employeeOrgData.division } else { $null } + $flatUser['employeeOrgData_costCenter'] = if ($user.employeeOrgData) { $user.employeeOrgData.costCenter } else { $null } + + # Status Properties + $flatUser['accountEnabled'] = $user.accountEnabled + $flatUser['userType'] = $user.userType + $flatUser['createdDateTime'] = $user.createdDateTime + + # Usage Properties + $flatUser['usageLocation'] = $user.usageLocation + $flatUser['preferredLanguage'] = $user.preferredLanguage + + # Sync Properties + $flatUser['onPremisesSyncEnabled'] = $user.onPremisesSyncEnabled + $flatUser['onPremisesImmutableId'] = $user.onPremisesImmutableId + $flatUser['externalUserState'] = $user.externalUserState + + # Explode proxyAddresses array (Email aliases) + if ($user.proxyAddresses -and $user.proxyAddresses.Count -gt 0) { + $primarySMTP = $user.proxyAddresses | Where-Object { $_ -like 'SMTP:*' } | Select-Object -First 1 + $flatUser['proxyAddresses_Primary'] = if ($primarySMTP) { $primarySMTP -replace '^SMTP:', '' } else { $null } + $flatUser['proxyAddresses_Count'] = $user.proxyAddresses.Count + $flatUser['proxyAddresses_All'] = ($user.proxyAddresses -join '; ') + } + else { + $flatUser['proxyAddresses_Primary'] = $null + $flatUser['proxyAddresses_Count'] = 0 + $flatUser['proxyAddresses_All'] = $null + } + + # Handle manager object separately (flatten to individual columns) + if ($user.manager) { + $flatUser['manager_id'] = $user.manager.id + $flatUser['manager_displayName'] = $user.manager.displayName + $flatUser['manager_userPrincipalName'] = $user.manager.userPrincipalName + $flatUser['manager_mail'] = $user.manager.mail + $flatUser['manager_jobTitle'] = $user.manager.jobTitle + } + else { + $flatUser['manager_id'] = $null + $flatUser['manager_displayName'] = $null + $flatUser['manager_userPrincipalName'] = $null + $flatUser['manager_mail'] = $null + $flatUser['manager_jobTitle'] = $null + } + + # License columns will be added separately by Get-EntraUsersData() + # using Get-UserLicenseData() to provide MAC-format columns: + # - assignedLicenses (semicolon-separated SKU names) + # - hasLicense (Copilot detection boolean) + + # ===================================================================== + # Power BI AI-in-One Dashboard 2701 Template Compatibility Columns + # These alias columns map existing Graph API data to 2701 template column names + # ===================================================================== + $flatUser['ManagerID'] = $flatUser['manager_id'] + $flatUser['BusinessAreaLabel'] = $flatUser['employeeOrgData_division'] + $flatUser['CountryofEmployment'] = $flatUser['country'] + $flatUser['CompanyCodeLabel'] = $flatUser['companyName'] + $flatUser['CostCentreLabel'] = $flatUser['employeeOrgData_costCenter'] + $flatUser['UserName'] = $flatUser['displayName'] + + # Viva Insights-specific columns (not available from Microsoft Graph API) + # These are placeholders for template compatibility - data must come from HR systems + $flatUser['EffectiveDate'] = $null + $flatUser['FunctionType'] = $null + $flatUser['BusinessAreaCode'] = $null + $flatUser['OrgLevel_3Label'] = $null + + # Convert ordered hashtable to PSCustomObject for proper CSV export + $flattenedUsers += [PSCustomObject]$flatUser + } + + return $flattenedUsers +} + +function Get-EntraUsersData { + <#! + .SYNOPSIS + Collects and flattens Entra ID (Azure AD) user directory data and enriches with MAC-format license info. + + .DESCRIPTION + # New naming: Purview_Audit_CombinedUsageActivity[_EntraUsers]_timestamp.xlsx + $baseName = "Purview_Audit_CombinedUsageActivity" + if ($IncludeUserInfo -and -not $UseEOM) { $baseName += "_EntraUsers" } + $excelDescriptor = if ($IncludeUserInfo -and -not $UseEOM) { 'multi-tab workbook (CombinedActivity + EntraUsers)' } else { 'single-tab workbook' } + Write-LogHost "Output File: ${outputDir}${baseName}_.xlsx ($excelDescriptor)" -ForegroundColor White + Filters out non-user principals (rooms/resources) using userType + name heuristics identical to ConvertTo-FlatEntraUsers. + Flattens users via ConvertTo-FlatEntraUsers, then appends two MAC-aligned columns: + • assignedLicenses (semicolon-separated SKU `skuPartNumber` names) + • hasLicense (Copilot license boolean; renamed from UserHasCopilotLicense) + + License enrichment uses existing Get-UserLicenseData() hashtables (UserLicenses, UserHasCopilot). + Only called when -IncludeUserInfo is specified (Graph API mode). + + .OUTPUTS + Array[psobject] of flattened users (30 core columns + 5 manager columns + 2 license columns = 37 total). + + .NOTES + If Graph API call fails, returns empty array with warning. License enrichment silently skips if lookup missing. + #> + param( + [switch]$Quiet + ) + + $entraUsers = @() + try { + if (-not $Quiet) { Write-LogHost "Fetching Entra user directory (35 properties + manager)..." -ForegroundColor Cyan } + + # Properties mirrored from Graph script (excluding license arrays we purposefully omit) + $entraUserSelect = @( + 'userPrincipalName','displayName','id','mail','givenName','surname','jobTitle','department','employeeType','employeeId','employeeHireDate', + 'officeLocation','city','state','country','postalCode','companyName','accountEnabled','userType','createdDateTime','usageLocation', + 'preferredLanguage','onPremisesSyncEnabled','onPremisesImmutableId','externalUserState','employeeOrgData','proxyAddresses' + ) -join ',' + + $baseUri = "https://graph.microsoft.com/v1.0/users?`$select=$entraUserSelect&`$expand=manager&`$top=999" + $nextLink = $baseUri + # Generic List accumulator (was $rawUsers = @() with +=). On a mega-tenant + # (~400K+ users) the array-append pattern is O(n^2) — every += reallocates and + # copies the whole backing array — which is both slow and memory-heavy. A + # List[object] with AddRange appends amortized O(1). + $rawUsers = [System.Collections.Generic.List[object]]::new() + $loops = 0 + # Scale-aware paging ceiling. With $expand=manager, Graph caps each page far + # below the requested $top=999 (often ~100 rows/page), so a mega-tenant needs + # thousands of pages: at the ~100/page floor, 8000 pages covers ~800K users. + # The old fixed >2000 guard tripped mid-fetch on a ~412K-user tenant. This + # ceiling still catches a true runaway (e.g. a non-advancing @odata.nextLink). + $maxPages = 8000 + # Inner try/catch around paging ONLY: on a throttling/network error or the + # safety-abort throw, keep whatever pages were already collected and fall + # through to flatten/enrich so a PARTIAL directory is still emitted. The + # shortfall is flagged so the end-of-run summary and exit code report it + # instead of masking it behind "Enterprise Export Complete". + try { + while ($nextLink) { + $loops++ + $resp = Invoke-GraphRequest -Uri $nextLink -Method GET -ErrorAction Stop + if ($resp.value) { $rawUsers.AddRange([object[]]$resp.value) } + $nextLink = $resp.'@odata.nextLink' + # Progress heartbeat: large tenants page for several minutes; emit a count + # every 25 pages so the run is not mistaken for a hang. + if (-not $Quiet -and ($loops % 25 -eq 0)) { + Write-LogHost " ... $($rawUsers.Count) users retrieved so far ($loops pages)" -ForegroundColor DarkGray + } + if ($loops -gt $maxPages) { throw "Safety abort: excessive paging (>$maxPages pages)" } + } + } + catch { + $script:EntraUsersFetchFailed = $true + $script:EntraUsersFetchError = $_.Exception.Message + Write-LogHost "WARNING: Entra user directory paging did not complete: $($_.Exception.Message)" -ForegroundColor Yellow + } + + # If paging collected nothing there is no directory to emit: flag (if not + # already) and return empty so the writers skip and the summary/exit report it. + if ($rawUsers.Count -eq 0) { + if (-not $script:EntraUsersFetchFailed) { + $script:EntraUsersFetchFailed = $true + $script:EntraUsersFetchError = 'Graph returned no users for the directory query.' + } + return $entraUsers + } + # Partial directory: some pages succeeded before a paging failure. Emit what we + # have (schema-complete via the flatten/enrich below) but mark it partial. + if ($script:EntraUsersFetchFailed) { + $script:EntraUsersPartial = $true + if (-not $Quiet) { Write-LogHost " Emitting PARTIAL directory: $($rawUsers.Count) users collected before the failure." -ForegroundColor Yellow } + } + + if (-not $Quiet) { Write-LogHost " Retrieved $($rawUsers.Count) raw user objects" -ForegroundColor Gray } + $flattened = ConvertTo-FlatEntraUsers -Users $rawUsers + if (-not $Quiet) { Write-LogHost " Flattened to $($flattened.Count) user rows (filtered)" -ForegroundColor Gray } + + # License enrichment (MAC-format columns) + $licenseData = $script:LicenseData + foreach ($u in $flattened) { + $upn = $u.userPrincipalName + $assignedNames = $null + $hasCopilot = $false + if ($licenseData) { + # lookup by UPN then id for flexibility + if ($licenseData.UserLicenses.ContainsKey($upn)) { + $assignedNames = ($licenseData.UserLicenses[$upn] -join ';') + } elseif ($licenseData.UserLicenses.ContainsKey($u.id)) { + $assignedNames = ($licenseData.UserLicenses[$u.id] -join ';') + } + if ($licenseData.UserHasCopilot.ContainsKey($upn)) { + $hasCopilot = [bool]$licenseData.UserHasCopilot[$upn] + } elseif ($licenseData.UserHasCopilot.ContainsKey($u.id)) { + $hasCopilot = [bool]$licenseData.UserHasCopilot[$u.id] + } + } + Add-Member -InputObject $u -NotePropertyName 'assignedLicenses' -NotePropertyValue $assignedNames -Force + Add-Member -InputObject $u -NotePropertyName 'hasLicense' -NotePropertyValue $hasCopilot -Force + } + $entraUsers = $flattened + # Validate schema (non-fatal) + try { Test-EntraUsersSchema -Users $entraUsers -Quiet:$Quiet } catch { } + } + catch { + # Flatten/enrich (or any non-paging) failure. Flag it so the summary + exit + # code report the shortfall; preserve an inner paging-error reason if one was + # already captured. + $script:EntraUsersFetchFailed = $true + if (-not $script:EntraUsersFetchError) { $script:EntraUsersFetchError = $_.Exception.Message } + Write-LogHost "WARNING: Failed to collect Entra user directory: $($_.Exception.Message)" -ForegroundColor Yellow + } + return $entraUsers +} + +# ============================================== +# GRAPH API QUERY FUNCTIONS +# ============================================== +# REST-based audit log query functions for Microsoft Graph Security API + +function Test-Is429 { + <# + .SYNOPSIS + Safely detects 429 (Too Many Requests) throttling errors. + + .DESCRIPTION + Provides null-safe detection of 429 throttling responses from Graph API. + Handles PowerShell 7+ variations where .Response property may be null. + + Three-layer fallback strategy: + 1. Check .Response.StatusCode (when Response object exists) + 2. Check .Exception.Response.StatusCode directly (PS7+ pattern) + 3. Parse error message for '429' string (final fallback) + + .PARAMETER Exception + The caught exception object from try/catch block + + .OUTPUTS + $true if 429 throttling detected, $false otherwise + + .EXAMPLE + try { + Invoke-RestMethod -Uri $uri -Headers $headers + } + catch { + if (Test-Is429 -Exception $_) { + Start-Sleep -Seconds 60 + } + } + #> + + param( + [Parameter(Mandatory = $true)] + [System.Management.Automation.ErrorRecord]$Exception + ) + + # Layer 1: Check .Response.StatusCode (traditional method) + if ($Exception.Exception.Response -and $Exception.Exception.Response.StatusCode) { + if ($Exception.Exception.Response.StatusCode -eq 429 -or $Exception.Exception.Response.StatusCode -eq 'TooManyRequests') { + return $true + } + } + + # Layer 2: Check .Exception.Response.StatusCode directly (PS7+ sometimes skips wrapper) + if ($Exception.Exception.Response.StatusCode) { + if ($Exception.Exception.Response.StatusCode.value__ -eq 429) { + return $true + } + } + + # Layer 3: Parse error message as final fallback + $errorMessage = $Exception.Exception.Message + if ($errorMessage -match '429' -or $errorMessage -match 'Too Many Requests' -or $errorMessage -match 'TooManyRequests') { + return $true + } + + return $false +} + +function Invoke-GraphAuditQuery { + <# + .SYNOPSIS + Creates a new audit log query in Microsoft Graph Security API. + + .DESCRIPTION + Submits an audit log query request to Microsoft Graph Security API. + Returns a query ID that can be used to poll for status and retrieve results. + + The Graph API uses an asynchronous query model: + 1. Submit query (this function) - returns queryId + 2. Poll query status - wait for "succeeded" state + 3. Retrieve records - paginated results + + .PARAMETER DisplayName + Friendly name for the query (for tracking purposes) + + .PARAMETER StartDate + Start date/time for audit log query (ISO 8601 format) + + .PARAMETER EndDate + End date/time for audit log query (ISO 8601 format) + + .PARAMETER Operations + Array of operation types to query (e.g., 'CopilotInteraction') + + .PARAMETER RecordTypes + Optional record type filters to include in the Graph query body (passthrough). + + .PARAMETER ServiceTypes + Optional service/workload filters to include in the Graph query body (passthrough). + + .OUTPUTS + Query ID string if successful, $null if failed + #> + + param( + [Parameter(Mandatory = $true)] + [string]$DisplayName, + + [Parameter(Mandatory = $true)] + [Alias('FilterStartDateTime')] + [datetime]$StartDate, + + [Parameter(Mandatory = $true)] + [Alias('FilterEndDateTime')] + [datetime]$EndDate, + + [Parameter(Mandatory = $false)] + [Alias('OperationFilters')] + [string[]]$Operations, + + [Parameter(Mandatory = $false)] + [Alias('RecordTypeFilters')] + [string[]]$RecordTypes, + + [Parameter(Mandatory = $false)] + [Alias('ServiceFilter')] + [string[]]$ServiceTypes + ) + + try { + # Format dates to ISO 8601 format required by Graph API + $startDateStr = $StartDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + $endDateStr = $EndDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + + # Build request body + $body = @{ + displayName = $DisplayName + filterStartDateTime = $startDateStr + filterEndDateTime = $endDateStr + } + + # Fail-safe sanitizer: If operations include M365 usage ops, drop record/service filters + try { + $usageOps = $script:m365UsageActivityBundle + if (-not $usageOps) { $usageOps = $m365UsageActivityBundle } + $hasUsageOps = $false + if ($Operations -and $usageOps) { + $opsLower = @($Operations | ForEach-Object { $_.ToLowerInvariant() }) + $usageLower = @($usageOps | ForEach-Object { $_.ToLowerInvariant() }) + $hasUsageOps = ($opsLower | Where-Object { $usageLower -contains $_ }) | Select-Object -First 1 + } + if ($hasUsageOps) { + $RecordTypes = $null + $ServiceTypes = $null + } + } catch { } + + # Add operation filters if specified + if ($Operations -and $Operations.Count -gt 0) { + $body.operationFilters = @($Operations) + } + + # Add optional record/service filters (passthrough from caller) + if ($RecordTypes -and $RecordTypes.Count -gt 0) { + $body.recordTypeFilters = @($RecordTypes) + } + + if ($ServiceTypes -and $ServiceTypes.Count -gt 0) { + $body.serviceFilter = $ServiceTypes[0] + } + + # Log query details for troubleshooting (persisted to run log) + Write-LogHost "[INFO] Graph API Query Body:" -ForegroundColor Magenta + if ($Operations -and $Operations.Count -gt 0) { + Write-LogHost " operationFilters: $($Operations -join ', ')" -ForegroundColor DarkGray + } + if ($RecordTypes -and $RecordTypes.Count -gt 0) { + Write-LogHost " recordTypeFilters: $($RecordTypes -join ', ')" -ForegroundColor DarkGray + } + if ($ServiceTypes -and $ServiceTypes.Count -gt 0) { + Write-LogHost " serviceFilter: $($ServiceTypes[0])" -ForegroundColor DarkGray + } + $bodyJson = $body | ConvertTo-Json -Depth 10 + Write-LogHost $bodyJson -ForegroundColor DarkGray + + # Submit query via Graph API through the duplicate-safe decision layer. + # A fully-paginated exact-name lookup runs BEFORE any POST: an existing + # identically-named query is adopted (no duplicate), multiple exact matches + # fail closed (no POST), otherwise a single beta creation proceeds under the + # ambiguous-POST guard. Version pinning is honoured by the decision layer. + $__createState = New-GraphAuditLifecycleState + $__listQueries = { + $__items = New-Object System.Collections.Generic.List[object] + $__lu = Get-GraphAuditApiUri -Path 'queries' + while ($__lu) { + $__page = Invoke-MgGraphRequest -Method GET -Uri $__lu -ErrorAction Stop + if ($__page -and $__page.value) { + foreach ($__q in $__page.value) { + $__items.Add(@{ id = $__q.id; displayName = $__q.displayName }) + } + } + $__lu = $null + if ($__page) { $__lu = $__page.'@odata.nextLink' } + } + return $__items.ToArray() + } + $__createQuery = { + param($Version, $Name) + try { + $__cu = "https://graph.microsoft.com/$Version/security/auditLog/queries" + $__resp = Invoke-MgGraphRequest -Method POST -Uri $__cu -Body $body -ErrorAction Stop + $__id = $null + if ($__resp) { $__id = $__resp.id } + return @{ HttpStatus = 201; QueryId = $__id; Ambiguous = $false } + } + catch { + $__sc = $null + if ($_.Exception.Response) { + try { $__sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $__sc) { try { $__sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + Write-LogHost "ERROR: Failed to submit Graph audit query: $($_.Exception.Message)" -ForegroundColor Red + return @{ HttpStatus = $__sc; QueryId = $null; Ambiguous = ($__sc -eq 409) } + } + } + $__resolve = Resolve-GraphAuditDuplicateSafeQuery -DisplayName $DisplayName -State $__createState -ListQueries $__listQueries -CreateQuery $__createQuery + if ($__resolve -and $__resolve.QueryId) { + return $__resolve.QueryId + } + else { + Write-LogHost "WARNING: Graph API query not created (Action=$($__resolve.Action); Reason=$($__resolve.Reason))" -ForegroundColor Yellow + return $null + } + } + catch { + Write-LogHost "ERROR: Failed to submit Graph audit query: $($_.Exception.Message)" -ForegroundColor Red + try { + if ($_.Exception.Response) { + $respStream = $_.Exception.Response.GetResponseStream() + if ($respStream) { + $reader = New-Object System.IO.StreamReader($respStream) + $body = $reader.ReadToEnd() + $reader.Dispose() + if ($body) { Write-LogHost "GRAPH response body: $body" -ForegroundColor DarkGray } + } + } + } catch {} + return $null + } +} + +function Get-GraphAuditQueryStatus { + <# + .SYNOPSIS + Checks the status of a Graph API audit log query. + + .DESCRIPTION + Polls the Microsoft Graph Security API to check query execution status. + + Possible status values: + - notStarted: Query submitted but not yet processing + - queued: Query waiting in backend queue for available execution slot + - running: Query is executing + - succeeded: Query completed successfully, records ready + - failed: Query failed + - cancelled: Query was cancelled + + .PARAMETER QueryId + The query ID returned by Invoke-GraphAuditQuery + + .OUTPUTS + Hashtable with status information: @{ Status='succeeded'; RecordCount=1234 } + Returns $null if query check fails + #> + + param( + [Parameter(Mandatory = $true)] + [string]$QueryId + ) + + try { + # Route the status GET through the shared decision layer so a transient 404 gets a + # bounded same-id eventual-consistency retry and then a same-id GET on the alternate + # version (read-only; never a POST) before status is judged unavailable. Status and + # records are decided independently. + $__lcState = New-GraphAuditLifecycleState + $__statusInvoke = { + param($Version, $Qid, $Op) + try { + $u = "https://graph.microsoft.com/$Version/security/auditLog/queries/$Qid" + $resp = Invoke-MgGraphRequest -Method GET -Uri $u -ErrorAction Stop + return @{ HttpStatus = 200; Data = $resp; ExceptionType = $null } + } + catch { + $sc = $null + if ($_.Exception.Response) { + try { $sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $sc) { try { $sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + if ($null -eq $sc -and ($_.Exception.Message -match '404|Not Found')) { $sc = 404 } + return @{ HttpStatus = $sc; Data = $null; ExceptionType = $_.Exception.GetType().Name } + } + } + $__lc = Invoke-GraphAuditLifecycleOperation -Operation 'Status' -QueryId $QueryId -Invoke $__statusInvoke -State $__lcState + if (-not $__lc.Success) { + Write-LogHost "ERROR: Failed to get Graph query status (category $($__lc.Category), HTTP $($__lc.HttpStatus))" -ForegroundColor Red + return $null + } + $response = $__lc.Data + + $result = @{ + QueryId = $QueryId + Status = $response.status + RecordCount = 0 + } + + # Some status responses include record count + if ($response.PSObject.Properties.Name -contains 'recordCount') { + $result.RecordCount = $response.recordCount + } + + return $result + } + catch { + Write-LogHost "ERROR: Failed to get Graph query status: $($_.Exception.Message)" -ForegroundColor Red + return $null + } +} + +function Get-GraphAuditRecords { + <# + .SYNOPSIS + Retrieves audit log records from a completed Graph API query. + + .DESCRIPTION + Fetches audit log records from Microsoft Graph Security API for a completed query. + Handles pagination automatically using @odata.nextLink. + + Only call this function after confirming query status is "succeeded". + + .PARAMETER QueryId + The query ID returned by Invoke-GraphAuditQuery + + .PARAMETER MaxRecords + Maximum number of records to retrieve (default: unlimited) + + .OUTPUTS + Array of audit log record objects, or empty array if none found + #> + + param( + [Parameter(Mandatory = $true)] + [string]$QueryId, + + [Parameter(Mandatory = $false)] + [int]$MaxRecords = 0 + ) + + try { + $allRecords = @() + # Route the first records GET through the shared decision layer (independent of the + # status decision): a transient 404 gets a bounded same-id retry then a same-id GET on + # the alternate version (read-only; never a POST). A successful ZERO-record page is a + # complete result, never a failure. Remaining pages follow @odata.nextLink verbatim on + # whichever version served the first page (no $top / cap / truncation added). + $__recState = New-GraphAuditLifecycleState + $__recInvoke = { + param($Version, $Qid, $Op) + try { + $u = "https://graph.microsoft.com/$Version/security/auditLog/queries/$Qid/records" + $resp = Invoke-MgGraphRequest -Method GET -Uri $u -ErrorAction Stop + return @{ HttpStatus = 200; Data = $resp; ExceptionType = $null } + } + catch { + $sc = $null + if ($_.Exception.Response) { + try { $sc = [int]$_.Exception.Response.StatusCode.value__ } catch {} + if ($null -eq $sc) { try { $sc = [int]$_.Exception.Response.StatusCode } catch {} } + } + if ($null -eq $sc -and ($_.Exception.Message -match '404|Not Found')) { $sc = 404 } + return @{ HttpStatus = $sc; Data = $null; ExceptionType = $_.Exception.GetType().Name } + } + } + $__lc = Invoke-GraphAuditLifecycleOperation -Operation 'Records' -QueryId $QueryId -Invoke $__recInvoke -State $__recState + if (-not $__lc.Success) { + Write-LogHost "ERROR: Failed to retrieve Graph audit records (category $($__lc.Category), HTTP $($__lc.HttpStatus))" -ForegroundColor Red + return @() + } + $response = $__lc.Data + if ($response -and $response.value) { + $allRecords += $response.value + if ($MaxRecords -gt 0 -and $allRecords.Count -ge $MaxRecords) { + return ($allRecords | Select-Object -First $MaxRecords) + } + } + $uri = $response.'@odata.nextLink' + + while ($uri) { + $response = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + + if ($response -and $response.value) { + $allRecords += $response.value + + # Check if we've hit the max records limit + if ($MaxRecords -gt 0 -and $allRecords.Count -ge $MaxRecords) { + $allRecords = $allRecords | Select-Object -First $MaxRecords + break + } + } + + # Check for pagination + $uri = $response.'@odata.nextLink' + } + + return $allRecords + } + catch { + Write-LogHost "ERROR: Failed to retrieve Graph audit records: $($_.Exception.Message)" -ForegroundColor Red + return @() + } +} + +# ============================================== +# DATA NORMALIZATION FUNCTION +# ============================================== +# Converts Graph API audit records to EOM-compatible schema + +function ConvertFrom-GraphAuditRecord { + <# + .SYNOPSIS + Normalizes Graph API audit records to match EOM cmdlet output schema. + + .DESCRIPTION + Transforms Microsoft Graph Security API audit log records into the same + structure as Search-UnifiedAuditLog cmdlet output. This ensures the + existing explosion logic works identically regardless of data source. + + Graph API Schema → EOM Schema Mapping: + - auditLogRecordType → RecordType + - operation → Operations + - createdDateTime → CreationDate + - auditData → AuditData (JSON string) + - userPrincipalName → UserIds + - id → Identity (unique record identifier) + + .PARAMETER GraphRecords + Array of audit log records from Graph API (Get-GraphAuditRecords output) + + .OUTPUTS + Array of normalized records matching EOM schema structure + #> + + param( + [Parameter(Mandatory = $true)] + [AllowEmptyCollection()] + [array]$GraphRecords + ) + + if (-not $GraphRecords -or $GraphRecords.Count -eq 0) { + return @() + } + + $normalized = @() + + foreach ($record in $GraphRecords) { + try { + # Create EOM-compatible object structure + $eomRecord = [PSCustomObject]@{ + RecordType = $null + CreationDate = $null + UserIds = $null + Operations = $null + AuditData = '{}' + } + # Map: auditLogRecordType → RecordType + if ($record.PSObject.Properties.Name -contains 'auditLogRecordType') { + $eomRecord.RecordType = $record.auditLogRecordType + } + + # Map: createdDateTime → CreationDate + if ($record.PSObject.Properties.Name -contains 'createdDateTime') { + try { + $eomRecord.CreationDate = script:Parse-DateSafe $record.createdDateTime + } + catch { + $eomRecord.CreationDate = $record.createdDateTime + } + } + + # Map: userPrincipalName → UserIds + if ($record.PSObject.Properties.Name -contains 'userPrincipalName') { + $eomRecord.UserIds = $record.userPrincipalName + } + + # Map: operation → Operations + if ($record.PSObject.Properties.Name -contains 'operation') { + $eomRecord.Operations = $record.operation + } + + # Map: id → Identity (unique identifier) + if ($record.PSObject.Properties.Name -contains 'id') { + $eomRecord.Identity = $record.id + } + + # Map: auditData → AuditData (must be JSON string for explosion logic) + # PERF: Also store _ParsedAuditData to avoid re-parsing during explosion + if ($record.PSObject.Properties.Name -contains 'auditData') { + $auditDataObj = $record.auditData + + # Store the already-parsed object for explosion optimization + $eomRecord | Add-Member -NotePropertyName '_ParsedAuditData' -NotePropertyValue $auditDataObj -Force + + # If auditData is already an object, convert to JSON string + if ($auditDataObj -is [string]) { + $eomRecord.AuditData = $auditDataObj + # String means it wasn't pre-parsed, clear _ParsedAuditData + $eomRecord._ParsedAuditData = $null + } + else { + # Convert object to JSON string (explosion logic expects string) + try { + $eomRecord.AuditData = ($auditDataObj | ConvertTo-Json -Depth 100 -Compress) + } + catch { + Write-LogHost "WARNING: Failed to serialize auditData for record $($eomRecord.Identity)" -ForegroundColor Yellow + $eomRecord.AuditData = '{}' + $eomRecord._ParsedAuditData = $null + } + } + } + else { + # No auditData present - create minimal valid JSON + $eomRecord.AuditData = '{}' + } + + $normalized += $eomRecord + } + catch { + Write-LogHost "WARNING: Failed to normalize Graph record: $($_.Exception.Message)" -ForegroundColor Yellow + # Continue processing remaining records + } + } + + return $normalized +} + +# ============================================================================= +# MICROSOFT AGENT 365 ENRICHMENT FUNCTIONS +# ============================================================================= +# Self-contained section. All functions are no-ops unless -IncludeAgent365Info +# or -OnlyAgent365Info is set. Reuses existing Graph SDK plumbing +# (Invoke-MgGraphRequest, Connect-MgGraph, token-refresh helpers, retry helpers). +# Output schema must exactly match temp/Synthetic_Agent365_May2026.csv (28 cols). +# Empty cells are intentional when the API does not return a value (no fabrication). +# Reference: https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api +# ============================================================================= + +# Module-scope caches populated during the Agent 365 phase. +$script:Agent365FrontierAvailable = $null # $true / $false / $null = untested +$script:Agent365DeveloperCache = @{} # appId/displayName -> resolved name +$script:Agent365AuditEnrichment = @{} # titleId/appId -> @{ Created=...; CreatedBy=... } +$script:Agent365InteractiveCtx = $false # set $true after secondary interactive sign-in (AppReg path) +$script:Agent365PreAuthCompleted = $false # set $true after eager up-front Agent 365 sign-in (AppReg + Agent365 combo) + +function Get-Agent365PackagesUri { + param([Parameter(Mandatory = $false)][string]$PackageId = '') + # Microsoft Agent 365 / Copilot package management API is currently published at /beta only. + # When/if the endpoint is also published at /v1.0, switch this constant. + $base = 'https://graph.microsoft.com/beta/copilot/admin/catalog/packages' + if ([string]::IsNullOrWhiteSpace($PackageId)) { return $base } + return ($base + '/' + [System.Uri]::EscapeDataString($PackageId)) +} + +function Connect-Agent365InteractiveContext { + <# + .SYNOPSIS + Establishes the Microsoft Graph context for the Agent 365 phase. + + Delegated auth modes (WebLogin / DeviceCode / Credential / Silent) already carry the + Agent 365 scopes consented at their initial interactive sign-in, so this is a no-op + for them (early return $true). + + App-only auth modes (AppRegistration certificate, AppRegistration client secret, + ManagedIdentity) reuse the EXISTING application Graph context. The app-only token must + already carry the APPLICATION app-role CopilotPackages.Read.All (plus Application.Read.All + for developer-name resolution via /applications), granted and admin-consented on the app + registration / managed-identity service principal. NO interactive sign-in and NO + Connect-MgGraph -Scopes call is performed - the catalog GET runs directly on the active + app-only context. Idempotent: safe to call multiple times. + #> + # Delegated modes: their initial sign-in already consented the Agent 365 scopes. + if ($Auth -ne 'AppRegistration' -and $Auth -ne 'ManagedIdentity') { return $true } + + # App-only modes: surface an APP-ONLY Phase 2 banner (no delegated user, no prompt) so the + # log is honest about the auth surface, then reuse the existing application context as-is. + $appOnlyLabel = if ($Auth -eq 'ManagedIdentity') { 'managed identity service principal' } else { 'app registration service principal' } + Write-LogHost "" + Write-LogHost "=== Phase 2: Agent 365 - APP-ONLY context (no interactive sign-in) ===" -ForegroundColor Cyan + Write-LogHost (" Reusing the existing app-only Graph context ({0})." -f $appOnlyLabel) -ForegroundColor Gray + Write-LogHost " Required APPLICATION permissions (granted + admin-consented on the SP):" -ForegroundColor White + Write-LogHost " [App-only] CopilotPackages.Read.All (read /copilot/admin/catalog/packages)" -ForegroundColor Yellow + Write-LogHost " [App-only] Application.Read.All (resolve developer/owner via /applications)" -ForegroundColor Yellow + Write-LogHost " Note: the catalog API is published at /beta only; the tenant must hold a" -ForegroundColor Gray + Write-LogHost " Microsoft Agent 365 license / program enrollment." -ForegroundColor Gray + Write-LogHost "" + return $true +} + + +function Invoke-Agent365EarlyInteractiveSignIn { + <# + .SYNOPSIS + No-op retained for call-site stability. + + .DESCRIPTION + The Agent Package Management API is read with + the APPLICATION app-role CopilotPackages.Read.All (+ Application.Read.All) on the app / + managed-identity service principal, so NO interactive sign-in is required in ANY auth mode: + - app-only modes (AppRegistration cert/secret, ManagedIdentity) reuse the existing + application context (see Connect-Agent365InteractiveContext); + - delegated modes (WebLogin / DeviceCode / Credential / Silent) already consented the + Agent 365 scopes at their initial sign-in. + This function is a no-op. + + .OUTPUTS + $true always (no early sign-in is ever needed). + #> + return $true +} + +function Test-Agent365FrontierAccess { + <# + .SYNOPSIS + Probes the Agent Package Management API with a minimal GET to confirm + tenant enrollment in the Frontier program and adequate caller role. + .OUTPUTS + $true - access confirmed; agent phase may proceed + $false - access denied (403/404 or unsupported); banner 7d printed; phase skipped + #> + if ($null -ne $script:Agent365FrontierAvailable) { return $script:Agent365FrontierAvailable } + + $uri = (Get-Agent365PackagesUri) + '?$top=1' + try { + $null = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + $script:Agent365FrontierAvailable = $true + return $true + } catch { + $status = $null + try { if ($_.Exception.Response) { $status = [int]$_.Exception.Response.StatusCode } } catch {} + if ($status -in @(401,403,404)) { + # Message branches by auth surface: an app-only 403 means the APP/MI service + # principal lacks the app-role (or the tenant is unlicensed/not-enrolled), NOT that + # a signed-in user needs a directory role. Delegated text is kept byte-identical. + $isAppOnlyMode = ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity') + Write-LogHost "" + Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-LogHost "| Microsoft Agent 365 - catalog unavailable for this tenant |" -ForegroundColor Yellow + Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-LogHost ("| The Agent Package Management API returned HTTP {0,-3}, indicating |" -f $status) -ForegroundColor Yellow + if ($isAppOnlyMode) { + Write-LogHost "| this tenant is not enrolled in the Microsoft Agent 365 program (or |" -ForegroundColor Yellow + Write-LogHost "| lacks a Microsoft Agent 365 license), OR the app registration / |" -ForegroundColor Yellow + Write-LogHost "| managed-identity service principal has NOT been granted the |" -ForegroundColor Yellow + Write-LogHost "| CopilotPackages.Read.All APPLICATION permission (admin-consented). |" -ForegroundColor Yellow + Write-LogHost "| App-only auth needs the app-role, not a user directory role. The |" -ForegroundColor Yellow + Write-LogHost "| Agent 365 CSV will be skipped for this run. |" -ForegroundColor Yellow + } else { + Write-LogHost "| this tenant is not enrolled in the Microsoft Agent 365 program (or |" -ForegroundColor Yellow + Write-LogHost "| lacks a Microsoft Agent 365 license), or the signed-in user lacks |" -ForegroundColor Yellow + Write-LogHost "| the AI Administrator / Global Administrator role. The Agent 365 CSV |" -ForegroundColor Yellow + Write-LogHost "| will be skipped for this run. |" -ForegroundColor Yellow + } + Write-LogHost "| |" -ForegroundColor Yellow + Write-LogHost "| Reference: |" -ForegroundColor Yellow + Write-LogHost "| https://learn.microsoft.com/en-us/microsoft-agent-365/admin/graph-api|" -ForegroundColor Yellow + Write-LogHost "+----------------------------------------------------------------------+" -ForegroundColor Yellow + Write-LogHost "" + } else { + Write-LogHost (" Agent 365 probe failed (HTTP {0}): {1}" -f ($status, $_.Exception.Message)) -ForegroundColor Red + } + $script:Agent365FrontierAvailable = $false + return $false + } +} + +function Get-Agent365Packages { + <# + .SYNOPSIS + Returns the Agent 365 catalog packages (list view) together with an explicit + completeness verdict. + .DESCRIPTION + The walk follows every @odata.nextLink the service supplies and ends only when + the service stops supplying one. There is no page, package, or row cap. + + Completeness fails closed. A repeated next link (a paging cycle), a page that + carries no value collection, or a page request that fails all end the walk with + Complete = $false and a reason, instead of silently returning a partial array. + Callers must refuse to publish the canonical catalog from an incomplete listing: + with an unknown universe, departed-agent classification cannot be trusted. + .OUTPUTS + PSCustomObject with .Packages, .Complete, .Reason and .PageCount. + #> + $results = New-Object System.Collections.Generic.List[object] + $uri = Get-Agent365PackagesUri + $pageNum = 0 + $complete = $true + $reason = '' + $seenLinks = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + while ($uri) { + if (-not $seenLinks.Add([string]$uri)) { + $complete = $false + $reason = "the catalog repeated a paging link after page $pageNum (paging cycle)" + break + } + $pageNum++ + try { + # Suppress the return value ($true/$false/'Quit') so it does not leak into this + # function's output pipeline and pollute the returned package array. + $null = Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue + } catch {} + $resp = $null + try { + $resp = Invoke-MgGraphRequest -Method GET -Uri $uri -ErrorAction Stop + } catch { + $complete = $false + $reason = ("catalog page {0} could not be retrieved: {1}" -f $pageNum, $_.Exception.Message) + break + } + $hasValue = $false + if ($null -ne $resp) { + try { + if ($resp -is [System.Collections.IDictionary]) { $hasValue = $resp.Contains('value') } + else { $hasValue = [bool](@($resp.PSObject.Properties.Name) -contains 'value') } + } catch { $hasValue = $false } + } + if (-not $hasValue) { + $complete = $false + $reason = ("catalog page {0} was malformed (no result collection)" -f $pageNum) + break + } + $pageValue = $resp.value + if ($null -ne $pageValue) { + foreach ($p in $pageValue) { [void]$results.Add($p) } + } + $next = $null + try { $next = $resp.'@odata.nextLink' } catch { $next = $null } + $uri = $next + } + return [PSCustomObject]@{ + Packages = $results.ToArray() + Complete = $complete + Reason = $reason + PageCount = $pageNum + } +} + +function Invoke-Agent365GraphWithRetry { + <# + .SYNOPSIS + Throttle-aware Graph GET for the Agent 365 catalog read path. Retries on HTTP 429 + and 5xx with exponential backoff (up to 5 attempts, min(60, 2^attempt) seconds), + honoring a Retry-After header when present. Non-throttle / non-5xx errors rethrow + immediately so callers keep their existing skip-with-warning behavior. Mirrors the + backoff convention used by the chunked remote-upload path. + #> + param([Parameter(Mandatory = $true)][string]$Uri) + $attempt = 0 + while ($true) { + try { + return Invoke-MgGraphRequest -Method GET -Uri $Uri -ErrorAction Stop + } catch { + $status = try { [int]$_.Exception.Response.StatusCode.value__ } catch { 0 } + if ($status -eq 0) { if ($_.Exception.Message -match '429|Too Many Requests|TooManyRequests') { $status = 429 } } + $attempt++ + if (($status -eq 429 -or $status -ge 500) -and $attempt -lt 5) { + $retryAfter = try { [int]$_.Exception.Response.Headers['Retry-After'] } catch { 0 } + $wait = if ($retryAfter -gt 0) { $retryAfter } else { [Math]::Min(60, [Math]::Pow(2, $attempt)) } + Start-Sleep -Seconds $wait + continue + } + throw + } + } +} + +function Get-Agent365PackageDetail { + <# + .SYNOPSIS + Returns a structured detail outcome for a single agent package. + .DESCRIPTION + The result always distinguishes the outcome rather than collapsing every failure to + $null, so the catalog phase can reconcile every listed package exactly. Outcome is + 'Success' (Detail carries the package detail including elementDetails), + 'FailedDependency' (the service reported the FailedDependency reason phrase), or + 'DetailFailed' (any other retrieval failure). Per-package diagnostics are written to + the run log at real severity; the console carries only the end-of-phase aggregate so + package identifiers never flood it. + .OUTPUTS + PSCustomObject with .Outcome, .Detail and .Reason. + #> + param([Parameter(Mandatory = $true)][string]$PackageId) + $uri = Get-Agent365PackagesUri -PackageId $PackageId + try { + # Suppress the return value ($true/$false/'Quit') so it does not leak into this + # function's output pipeline and turn the returned detail object into an array. + $null = Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue + } catch {} + try { + $a365Detail = Invoke-Agent365GraphWithRetry -Uri $uri + return [PSCustomObject]@{ Outcome = 'Success'; Detail = $a365Detail; Reason = '' } + } catch { + $a365Reason = [string]$_.Exception.Message + $a365StatusName = '' + try { $a365StatusName = [string]$_.Exception.Response.StatusCode } catch { $a365StatusName = '' } + $a365Outcome = if ($a365StatusName -eq 'FailedDependency' -or $a365Reason -match 'FailedDependency') { 'FailedDependency' } else { 'DetailFailed' } + Write-LogFile ("Agent 365 detail retrieval failed for package '{0}' (outcome {1}): {2}" -f $PackageId, $a365Outcome, $a365Reason) -Level 'WARNING' + return [PSCustomObject]@{ Outcome = $a365Outcome; Detail = $null; Reason = $a365Reason } + } +} + +function Resolve-Agent365DeveloperName { + <# + .SYNOPSIS + Resolves a developer/publisher name. Tries package's developer.name first, + falls back to /applications + /owners lookup keyed on appId. Cached. + #> + param( + [Parameter(Mandatory = $false)][string]$DeveloperName, + [Parameter(Mandatory = $false)][string]$AppId + ) + if ($DeveloperName) { return $DeveloperName } + if (-not $AppId) { return '' } + if ($script:Agent365DeveloperCache.ContainsKey($AppId)) { + return $script:Agent365DeveloperCache[$AppId] + } + $resolved = '' + try { + $appUri = "https://graph.microsoft.com/v1.0/applications?`$filter=appId eq '$AppId'&`$select=id,displayName,publisherDomain" + $appResp = Invoke-Agent365GraphWithRetry -Uri $appUri + if ($appResp -and $appResp.value -and $appResp.value.Count -gt 0) { + $app = $appResp.value[0] + if ($app.publisherDomain) { $resolved = $app.publisherDomain } + elseif ($app.displayName) { $resolved = $app.displayName } + # Optional owner lookup (first owner UPN) only if still blank + if (-not $resolved -and $app.id) { + try { + $ownerUri = "https://graph.microsoft.com/v1.0/applications/$($app.id)/owners?`$select=userPrincipalName,displayName" + $ownerResp = Invoke-Agent365GraphWithRetry -Uri $ownerUri + if ($ownerResp -and $ownerResp.value -and $ownerResp.value.Count -gt 0) { + $resolved = $ownerResp.value[0].displayName + if (-not $resolved) { $resolved = $ownerResp.value[0].userPrincipalName } + } + } catch { } + } + } + } catch { + # Application.Read.All may not be granted; degrade gracefully (no fabrication). + } + $script:Agent365DeveloperCache[$AppId] = $resolved + return $resolved +} + +function Get-Agent365AuditEnrichment { + <# + .SYNOPSIS + Runs a single narrow Graph audit query to retrieve agent create/publish events + and returns a hashtable keyed on agent id (titleId/appId/lower-cased displayName) + with @{ Created = [datetime]; CreatedBy = [string] }. Skipped when -OnlyAgent365Info. + .NOTES + Best-effort enrichment. If the audit query fails or returns no records, both + fields stay blank in the output (no fabrication). + #> + $enrichment = @{} + if ($OnlyAgent365Info) { return $enrichment } + if (-not $script:GraphConnected -and -not (Get-MgContext -ErrorAction SilentlyContinue)) { return $enrichment } + + # Resolve time window: prefer main-run StartDate/EndDate; else last 30 days. + $rangeStart = $null; $rangeEnd = $null + try { + if ($StartDate) { $rangeStart = [datetime]::ParseExact($StartDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } + if ($EndDate) { $rangeEnd = [datetime]::ParseExact($EndDate, 'yyyy-MM-dd', [System.Globalization.CultureInfo]::InvariantCulture) } + } catch {} + if (-not $rangeStart) { $rangeStart = (Get-Date).ToUniversalTime().AddDays(-30) } + if (-not $rangeEnd) { $rangeEnd = (Get-Date).ToUniversalTime() } + + # Operation set covering agent/app create/publish/install. Conservative; missing + # operations simply mean blank columns for affected agents. + $ops = @( + 'AppCatalogPublishedAppCreated', + 'AppCatalogPublishedAppUpdated', + 'AgentCreated', + 'AgentPublished', + 'CopilotAgentInstalled' + ) + + Write-LogHost " Running narrow audit query for Agent 365 enrichment (Date created / Created by)..." -ForegroundColor DarkGray + $queryId = $null + try { + $queryId = Invoke-GraphAuditQuery ` + -DisplayName ("PAX_Agents365_Enrichment_{0}" -f (Get-Date -Format 'yyyyMMddHHmmss')) ` + -StartDate $rangeStart ` + -EndDate $rangeEnd ` + -Operations $ops + } catch { + Write-LogHost (" WARNING: Agent 365 enrichment query submit failed: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + return $enrichment + } + if (-not $queryId) { + Write-LogHost " WARNING: Agent 365 enrichment query did not return a query id; columns will be blank." -ForegroundColor Yellow + return $enrichment + } + + # Poll for query completion. Fortune-500 tenants routinely take 30-90 minutes + # for audit queries even on narrow operation/time filters. Use the same + # tolerance window the main audit phase uses (-MaxNetworkOutageMinutes is for + # transport errors only; query-completion timeout is a separate, longer cap). + # 4 hours is the documented Graph audit query SLA upper bound. + $pollTimeoutMinutes = 240 + $pollDeadline = (Get-Date).AddMinutes($pollTimeoutMinutes) + $pollIntervalSeconds = 15 + $lastLoggedMinute = -1 + $status = $null + Write-LogHost (" Polling enrichment query (timeout {0} min, refresh-token aware)..." -f $pollTimeoutMinutes) -ForegroundColor DarkGray + while ((Get-Date) -lt $pollDeadline) { + Start-Sleep -Seconds $pollIntervalSeconds + # Suppress the return value ($true/$false/'Quit') so it does not leak into this + # function's output pipeline and make the returned enrichment map an array. + try { $null = Refresh-GraphTokenIfNeeded -ErrorAction SilentlyContinue } catch {} + try { $status = Get-GraphAuditQueryStatus -QueryId $queryId } catch { $status = $null } + if ($status -and $status.Status -in @('succeeded','failed','cancelled')) { break } + # Heartbeat once per minute to keep long polls visible in logs. + $elapsedMin = [int]((Get-Date) - $pollDeadline.AddMinutes(-$pollTimeoutMinutes)).TotalMinutes + if ($elapsedMin -ne $lastLoggedMinute -and ($elapsedMin % 5) -eq 0) { + $lastLoggedMinute = $elapsedMin + $st = if ($status -and $status.Status) { $status.Status } else { 'pending' } + Write-LogHost (" ... {0} min elapsed, status={1}" -f $elapsedMin, $st) -ForegroundColor DarkGray + } + # Gentle backoff: 15s -> 30s -> 60s after 2 and 10 minutes respectively. + if ($elapsedMin -ge 10 -and $pollIntervalSeconds -lt 60) { $pollIntervalSeconds = 60 } + elseif ($elapsedMin -ge 2 -and $pollIntervalSeconds -lt 30) { $pollIntervalSeconds = 30 } + } + if (-not $status -or $status.Status -ne 'succeeded') { + Write-LogHost (" WARNING: Agent 365 enrichment query did not succeed (status={0}); columns will be blank." -f ($status.Status)) -ForegroundColor Yellow + return $enrichment + } + + $records = @() + try { $records = Get-GraphAuditRecords -QueryId $queryId } catch { $records = @() } + if (-not $records -or $records.Count -eq 0) { return $enrichment } + + foreach ($rec in $records) { + try { + $auditObj = $null + if ($rec.PSObject.Properties.Name -contains 'auditData') { $auditObj = $rec.auditData } + if ($auditObj -is [string]) { try { $auditObj = $auditObj | ConvertFrom-Json -ErrorAction Stop } catch { $auditObj = $null } } + $created = $null + if ($rec.PSObject.Properties.Name -contains 'createdDateTime' -and $rec.createdDateTime) { + try { $created = [datetime]$rec.createdDateTime } catch {} + } + $createdBy = $null + if ($rec.PSObject.Properties.Name -contains 'userPrincipalName' -and $rec.userPrincipalName) { + $createdBy = [string]$rec.userPrincipalName + } + # Pull keys from auditData defensively (preview schema may shift). + $keys = New-Object System.Collections.Generic.List[string] + if ($auditObj) { + foreach ($p in 'TitleId','titleId','AppId','appId','PackageId','packageId','TeamsAppId','teamsAppId','DisplayName','displayName','Name','name') { + try { if ($auditObj.$p) { [void]$keys.Add(([string]$auditObj.$p).ToLowerInvariant()) } } catch {} + } + } + foreach ($k in $keys) { + if (-not $enrichment.ContainsKey($k)) { + $enrichment[$k] = @{ Created = $created; CreatedBy = $createdBy } + } + } + } catch { continue } + } + Write-LogHost (" Audit enrichment matched {0} agent identifier key(s)." -f $enrichment.Keys.Count) -ForegroundColor DarkGray + return $enrichment +} + +function ConvertTo-Agent365Row { + <# + .SYNOPSIS + Maps one package detail object to the exact 28-column synthetic schema. + Empty cells stay empty when fields are absent (no fabrication). + #> + param( + [Parameter(Mandatory = $true)][object]$Package, + [Parameter(Mandatory = $false)][hashtable]$AuditEnrichment + ) + function _g { param($obj, [string[]]$names) foreach ($n in $names) { try { if ($null -ne $obj.$n -and "$($obj.$n)" -ne '') { return $obj.$n } } catch {} } return '' } + function _gb { param($obj, [string[]]$names) foreach ($n in $names) { try { if ($null -ne $obj.$n) { return [bool]$obj.$n } } catch {} } return '' } + function _join { param($v, [string]$sep) if ($null -eq $v) { return '' }; if ($v -is [System.Collections.IEnumerable] -and -not ($v -is [string])) { return (@($v) -join $sep) } else { return [string]$v } } + function _fmtDate { param($v) if (-not $v) { return '' }; try { return ([datetime]$v).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ssZ') } catch { return [string]$v } } + + $elem = $null + try { $elem = $Package.elementDetails } catch {} + + $titleIdRaw = (_g $Package @('id','titleId','packageId')) + $titleId = '' + if ($titleIdRaw) { $titleId = if ($titleIdRaw -match '^T_') { [string]$titleIdRaw } else { 'T_' + [string]$titleIdRaw } } + + # Audit enrichment lookup (lowercase keys). Try titleId raw, appId, displayName. + $dateCreated = '' + $createdBy = '' + if ($AuditEnrichment) { + $probeKeys = @() + if ($titleIdRaw) { $probeKeys += ([string]$titleIdRaw).ToLowerInvariant() } + $appIdProbe = (_g $Package @('appId','applicationId')) + if ($appIdProbe) { $probeKeys += ([string]$appIdProbe).ToLowerInvariant() } + $dispProbe = (_g $Package @('displayName','name')) + if ($dispProbe) { $probeKeys += ([string]$dispProbe).ToLowerInvariant() } + foreach ($k in $probeKeys) { + if ($AuditEnrichment.ContainsKey($k)) { + $hit = $AuditEnrichment[$k] + if ($hit.Created) { $dateCreated = (_fmtDate $hit.Created) } + if ($hit.CreatedBy) { $createdBy = [string]$hit.CreatedBy } + break + } + } + } + # Fallback to package's own createdDateTime if audit didn't supply one + if (-not $dateCreated) { + $pkgCreated = (_g $Package @('createdDateTime','createdDate')) + if ($pkgCreated) { $dateCreated = (_fmtDate $pkgCreated) } + } + + $developer = Resolve-Agent365DeveloperName -DeveloperName (_g $Package @('developer.name')) -AppId (_g $Package @('appId','applicationId')) + # fallback: try direct nested object access + if (-not $developer) { + try { if ($Package.developer -and $Package.developer.name) { $developer = [string]$Package.developer.name } } catch {} + } + + [PSCustomObject][ordered]@{ + 'Name' = (_g $Package @('displayName','name')) + 'Supported in' = (_join (_g $Package @('supportedHosts','supportedClients')) ';') + 'Date created' = $dateCreated + 'Developer Name' = $developer + 'Type' = (_g $Package @('agentType','type')) + 'Version' = (_g $Package @('version')) + 'Availability' = (_g $Package @('availability','allowedUsersAndGroups')) + 'Created by' = $createdBy + 'Description' = (_g $Package @('description')) + 'Created in' = (_g $Package @('source','origin','createdIn')) + 'Last updated' = (_fmtDate (_g $Package @('lastModifiedDateTime','lastUpdatedDateTime'))) + 'Custom actions' = (_g $elem @('customActions')) + 'Title ID' = $titleId + 'Sensitivity' = (_g $Package @('sensitivity')) + 'Can read OneDrive and Sharepoint items'= (_gb $elem @('canReadOneDriveAndSharepointItems')) + 'OneDrive and Sharepoint items' = (_g $elem @('oneDriveAndSharepointItems')) + 'Can read OneDrive files' = (_gb $elem @('canReadOneDriveFiles')) + 'OneDrive files' = (_g $elem @('oneDriveFiles')) + 'OneDrive sites' = (_g $elem @('oneDriveSites')) + 'Can read Sharepoint sites and files' = (_gb $elem @('canReadSharepointSitesAndFiles')) + 'Sharepoint files' = (_g $elem @('sharepointFiles')) + 'Sharepoint sites' = (_g $elem @('sharepointSites')) + 'Can extend to Graph connector' = (_gb $elem @('canExtendToGraphConnector')) + 'Graph connector details' = (_g $elem @('graphConnectorDetails')) + 'Can generate images using user prompt' = (_gb $elem @('canGenerateImagesUsingUserPrompt')) + 'Can use code interpreter' = (_gb $elem @('canUseCodeInterpreter')) + 'Contains uploaded files' = (_gb $elem @('containsUploadedFiles')) + 'Uploaded files' = (_g $elem @('uploadedFiles')) + } +} + +function Export-Agent365Csv { + <# + .SYNOPSIS + Writes the Agent 365 CSV (UTF-8 BOM) to OutputPath using the run timestamp. + .DESCRIPTION + When -AppendAgent365Info is in effect, this function pre-reads the + target CSV and union-merges its rows with the current API-derived rows: the + current run's row wins on AgentId conflicts (freshest metadata) and any + target-only AgentId is carried forward (departed agents are not dropped). + .OUTPUTS + Full path to written file, or $null if no rows. + #> + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Rows + ) + if (-not $Rows -or $Rows.Count -eq 0) { + Write-LogHost " Agent 365: no rows to write." -ForegroundColor Yellow + return $null + } + + # Union-merge with -AppendAgent365Info target when set. Failures here are + # non-fatal — fall back to current-run rows only. + $rowsToWrite = [System.Collections.Generic.List[object]]::new() + foreach ($r in $Rows) { [void]$rowsToWrite.Add($r) } + if ($AppendAgent365Info -and (Test-Path -LiteralPath $AppendAgent365Info -PathType Leaf)) { + try { + $currentAgentIds = New-Object System.Collections.Generic.HashSet[string] + foreach ($r in $Rows) { + $aid = $null + try { $aid = [string]$r.AgentId } catch {} + if ($aid) { [void]$currentAgentIds.Add($aid) } + } + $targetRows = @(Import-Csv -LiteralPath $AppendAgent365Info -Encoding UTF8) + $carried = 0 + foreach ($tr in $targetRows) { + $aid = $null + try { $aid = [string]$tr.AgentId } catch {} + if (-not $aid) { continue } + if (-not $currentAgentIds.Contains($aid)) { + [void]$rowsToWrite.Add($tr) + $carried++ + } + } + Write-LogHost (" -AppendAgent365Info: carried {0:N0} departed agent(s) from '{1}'; current run contributed {2:N0}." -f $carried, $AppendAgent365Info, $Rows.Count) -ForegroundColor DarkCyan + } + catch { + Write-LogHost (" WARNING: -AppendAgent365Info merge failed ({0}); writing current-run rows only." -f $_.Exception.Message) -ForegroundColor Yellow + } + } + + $ts = $global:ScriptRunTimestamp + if (-not $ts) { $ts = (Get-Date).ToString('yyyyMMdd_HHmmss') } + # Per-data-type destination. If -OutputPathAgent365Info resolves to a Local + # folder or file form, route the catalog there; otherwise inherit $OutputPath. Remote + # tiers (SharePoint/Fabric) continue to write to the scratch dir and upload from there. + $agentOutDir = $OutputPath + # Effective basename (honors -AppendAgent365Info target leaf so the merged file + # produced this run can be uploaded back over the original remote URL \u2014 matched + # by the upload sweep's Append-leaf filter). + $agentBasename = (& $script:GetEffectiveAgent365Basename) + try { + $dt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename $agentBasename + if ($dt -and $dt.IsBound -and $dt.Tier -eq 'Local') { + $agentOutDir = $dt.EffectiveDir + if ($dt.Basename) { $agentBasename = $dt.Basename } + if (-not (Test-Path -LiteralPath $agentOutDir -PathType Container)) { + New-Item -Path $agentOutDir -ItemType Directory -Force | Out-Null + } + } + } catch {} + $outFile = Join-Path $agentOutDir $agentBasename + try { + # Atomic write — Save-CsvAtomic uses temp+rename so a Ctrl+C / OOM-kill mid-write + # cannot leave a half-formed Agent365 CSV in the operator's output folder. + Save-CsvAtomic -InputObject ($rowsToWrite.ToArray()) -Path $outFile -NoTypeInformation -Encoding UTF8 + # In remote (SharePoint/Fabric) mode the file lands in LOCAL SCRATCH and is uploaded + # by the end-of-run artifact sweep; say so, rather than implying it is already at the + # displayed remote destination. + if ($script:RemoteOutputMode -ne 'None') { + Write-LogHost (" Agent 365 CSV written to scratch, queued for upload: {0} ({1} rows)" -f (Get-DisplayPath -LocalPath $outFile), $rowsToWrite.Count) -ForegroundColor Green + } else { + Write-LogHost (" Agent 365 CSV written: {0} ({1} rows)" -f (Get-DisplayPath -LocalPath $outFile), $rowsToWrite.Count) -ForegroundColor Green + } + return $outFile + } catch { + Write-LogHost (" ERROR: Failed to write Agent 365 CSV: {0}" -f $_.Exception.Message) -ForegroundColor Red + return $null + } +} + +function Save-Agent365RecoveryCsv { + <# + .SYNOPSIS + Writes the rows retrieved during an INCOMPLETE catalog listing to a clearly-named + recovery file that is never treated as the canonical Agent 365 catalog. + .DESCRIPTION + An incomplete listing means the catalog universe is unknown, so the canonical target + (and any -AppendAgent365Info union) must be left byte-for-byte unchanged. The rows + that WERE retrieved are still valuable, so they are preserved locally under a + distinct leaf. That leaf is registered as recovery material so the end-of-run local + listing and the remote upload sweep both exclude it: recovery material is never + published as a completed catalog. + .OUTPUTS + Full path to the recovery file, or $null when there was nothing to preserve. + #> + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Rows + ) + if (-not $Rows -or $Rows.Count -eq 0) { return $null } + $ts = $global:ScriptRunTimestamp + if (-not $ts) { $ts = (Get-Date).ToString('yyyyMMdd_HHmmss') } + $recoveryLeaf = "Agent365_IncompleteListing_${ts}_recovery.csv" + $recoveryPath = Join-Path $OutputPath $recoveryLeaf + try { + Save-CsvAtomic -InputObject $Rows -Path $recoveryPath -NoTypeInformation -Encoding UTF8 + if (-not $script:Agent365RecoveryLeafs) { $script:Agent365RecoveryLeafs = New-Object System.Collections.Generic.List[string] } + if (-not $script:Agent365RecoveryLeafs.Contains($recoveryLeaf)) { [void]$script:Agent365RecoveryLeafs.Add($recoveryLeaf) } + Write-LogHost (" Agent 365: {0:N0} retrieved row(s) preserved as recovery material (excluded from upload): {1}" -f $Rows.Count, (Get-DisplayPath -LocalPath $recoveryPath)) -ForegroundColor Yellow + return $recoveryPath + } + catch { + Write-LogHost (" WARNING: Agent 365 recovery file could not be written: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + return $null + } +} + +function Add-Agent365WorkbookTab { + <# + .SYNOPSIS + Appends an 'Agents365' worksheet to the existing workbook (after EntraUsers tab). + No-op when -ExportWorkbook is not set or ImportExcel is unavailable. + #> + param( + [Parameter(Mandatory = $true)][string]$WorkbookPath, + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Rows + ) + if (-not $ExportWorkbook) { return } + if (-not $Rows -or $Rows.Count -eq 0) { return } + if (-not (Get-Module -ListAvailable -Name ImportExcel)) { + Write-LogHost " Agent 365: ImportExcel not available; skipping workbook tab." -ForegroundColor Yellow + return + } + try { + Import-Module ImportExcel -Force -ErrorAction Stop + $Rows | Export-Excel -Path $WorkbookPath -WorksheetName 'Agents365' -AutoSize -BoldTopRow -FreezeTopRow -ErrorAction Stop + Write-LogHost (" Agent 365 worksheet appended to: {0}" -f (Get-DisplayPath -LocalPath $WorkbookPath)) -ForegroundColor Green + } catch { + Write-LogHost (" WARNING: Could not append Agents365 worksheet: {0}" -f $_.Exception.Message) -ForegroundColor Yellow + } +} + +function Invoke-Agent365Phase { + <# + .SYNOPSIS + Top-level orchestrator for the Agent 365 phase. Called once from the main + flow when -IncludeAgent365Info or -OnlyAgent365Info is set. No-op otherwise. + .OUTPUTS + Hashtable: @{ CsvPath = ; Rows = } + #> + if (-not ($IncludeAgent365Info -or $OnlyAgent365Info)) { return @{ CsvPath = $null; Rows = @() } } + + Write-LogHost "" + Write-LogHost "============================================================" -ForegroundColor Cyan + Write-LogHost " Microsoft Agent 365 enrichment phase" -ForegroundColor Cyan + Write-LogHost "============================================================" -ForegroundColor Cyan + + # App-only modes (AppRegistration certificate/secret, ManagedIdentity): surface the app-only + # Phase 2 banner and reuse the EXISTING application Graph context (no interactive sign-in). + # Delegated modes skip this block and go straight to the Frontier probe using the Agent 365 + # scopes already consented at their initial sign-in. + if ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity') { + if (-not (Connect-Agent365InteractiveContext)) { + Write-LogHost " Agent 365 phase aborted (app-only Graph context unavailable)." -ForegroundColor Red + return @{ CsvPath = $null; Rows = @() } + } + } + + # Frontier probe (prints banner 7d on 401/403/404 and returns false). + if (-not (Test-Agent365FrontierAccess)) { + return @{ CsvPath = $null; Rows = @() } + } + + # Audit enrichment (skipped when -OnlyAgent365Info). + $script:Agent365AuditEnrichment = Get-Agent365AuditEnrichment + # Defensive safety net (NOT a substitute for the pipeline-leak suppression in the + # enrichment/list/detail helpers): the enrichment lookup must resolve to a single + # hashtable. If it is ever anything else, coerce to an empty map and warn loudly so + # row building degrades to blank enrichment columns instead of throwing a raw + # type-coercion error on every package. + if ($script:Agent365AuditEnrichment -isnot [hashtable]) { + $a365EnrType = if ($null -eq $script:Agent365AuditEnrichment) { 'null' } else { $script:Agent365AuditEnrichment.GetType().Name } + Write-LogHost (" WARNING: Agent 365 audit enrichment resolved to an unexpected type ({0}); continuing with blank enrichment columns." -f $a365EnrType) -ForegroundColor Yellow + $script:Agent365AuditEnrichment = @{} + } + + # List + per-package detail + Write-LogHost " Listing Agent 365 packages..." -ForegroundColor DarkGray + $listResult = Get-Agent365Packages + $listed = @($listResult.Packages) + $a365ListComplete = [bool]$listResult.Complete + if (-not $a365ListComplete) { + Write-LogHost (" WARNING: Agent 365 catalog listing is INCOMPLETE after {0:N0} page(s): {1}" -f $listResult.PageCount, $listResult.Reason) -ForegroundColor Yellow + Write-LogFile ("Agent 365 catalog listing incomplete after {0} page(s): {1}" -f $listResult.PageCount, $listResult.Reason) -Level 'ERROR' + } + if (-not $listed -or $listed.Count -eq 0) { + if (-not $a365ListComplete) { + $script:Agent365HadGaps = $true + Write-LogHost " Agent 365: no packages could be listed and the listing did not complete; the existing catalog target is left unchanged (completed with gaps, exit 40)." -ForegroundColor Red + } else { + Write-LogHost " No Agent 365 packages returned by the catalog." -ForegroundColor Yellow + } + return @{ CsvPath = $null; Rows = @(); Listed = 0; Emitted = 0; DetailFailed = 0; FailedDependency = 0; RowBuildFailed = 0; SkippedNoId = 0; Reconciled = $a365ListComplete; ListComplete = $a365ListComplete; RecoveryPath = $null } + } + Write-LogHost (" {0} package(s) listed; fetching details..." -f $listed.Count) -ForegroundColor DarkGray + + $rows = New-Object System.Collections.Generic.List[object] + $idx = 0 + # Complete outcome accounting. Every listed package resolves to exactly one bucket, so + # listed == emitted + detailFailed + rowBuildFailed + skippedNoId holds for the walk. + $a365Emitted = 0 + $a365DetailFailed = 0 + $a365FailedDependency = 0 + $a365RowBuildFailed = 0 + $a365SkippedNoId = 0 + foreach ($p in $listed) { + $idx++ + $pkgId = $null + try { $pkgId = $p.id } catch {} + if (-not $pkgId) { try { $pkgId = $p.titleId } catch {} } + if (-not $pkgId) { + $a365SkippedNoId++ + Write-LogFile ("Agent 365: listed entry {0} carries no usable package identifier; no row emitted." -f $idx) -Level 'WARNING' + continue + } + $detailResult = Get-Agent365PackageDetail -PackageId $pkgId + if ($null -eq $detailResult -or $detailResult.Outcome -ne 'Success') { + $a365DetailFailed++ + if ($detailResult -and $detailResult.Outcome -eq 'FailedDependency') { $a365FailedDependency++ } + continue + } + try { + $row = ConvertTo-Agent365Row -Package $detailResult.Detail -AuditEnrichment $script:Agent365AuditEnrichment + [void]$rows.Add($row) + $a365Emitted++ + } catch { + $a365RowBuildFailed++ + Write-LogFile ("Agent 365: row build failed for package '{0}': {1}" -f $pkgId, $_.Exception.Message) -Level 'ERROR' + } + if (($idx % 25) -eq 0) { + Write-LogHost (" ... {0}/{1} packages processed" -f $idx, $listed.Count) -ForegroundColor DarkGray + } + } + + $a365Failed = $a365DetailFailed + $a365RowBuildFailed + $a365Accounted = $a365Emitted + $a365DetailFailed + $a365RowBuildFailed + $a365SkippedNoId + $a365Reconciled = ($a365Accounted -eq $listed.Count) + # One concise completion summary. Per-package diagnostics stay in the run log. + Write-LogHost (" Agent 365: {0:N0} listed, {1:N0} emitted, {2:N0} failed, {3:N0} skipped." -f $listed.Count, $a365Emitted, $a365Failed, $a365SkippedNoId) -ForegroundColor $(if ($a365Failed -gt 0 -or $a365SkippedNoId -gt 0) { 'Yellow' } else { 'Green' }) + Write-LogFile ("Agent 365 accounting: listed={0} emitted={1} detailFailed={2} (FailedDependency={3}) rowBuildFailed={4} skippedNoId={5} reconciled={6}" -f $listed.Count, $a365Emitted, $a365DetailFailed, $a365FailedDependency, $a365RowBuildFailed, $a365SkippedNoId, $a365Reconciled) -Level $(if ($a365Failed -gt 0 -or $a365SkippedNoId -gt 0 -or -not $a365Reconciled) { 'ERROR' } else { 'INFO' }) + if (-not $a365Reconciled) { + Write-LogFile ("Agent 365 accounting did not reconcile: {0} listed vs {1} accounted." -f $listed.Count, $a365Accounted) -Level 'ERROR' + } + # Requested Agent 365 output is incomplete whenever any listed package produced no row. + if ($a365Failed -gt 0 -or $a365SkippedNoId -gt 0 -or -not $a365Reconciled -or -not $a365ListComplete) { + $script:Agent365HadGaps = $true + Write-LogHost " Agent 365: requested catalog output is INCOMPLETE (completed with gaps, exit 40)." -ForegroundColor Yellow + } + + if (-not $a365ListComplete) { + # The catalog universe is unknown, so departed-agent classification cannot be trusted. + # Never publish or overwrite the canonical catalog target from a partial listing: leave + # any existing target byte-for-byte unchanged and keep this run's rows as recovery material. + Write-LogHost " Agent 365: the catalog file is NOT written or overwritten this run because the listing did not complete; any existing target is left unchanged." -ForegroundColor Red + $a365RecoveryPath = Save-Agent365RecoveryCsv -Rows $rows.ToArray() + return @{ + CsvPath = $null + Rows = @() + Listed = $listed.Count + Emitted = $a365Emitted + DetailFailed = $a365DetailFailed + FailedDependency = $a365FailedDependency + RowBuildFailed = $a365RowBuildFailed + SkippedNoId = $a365SkippedNoId + Reconciled = $a365Reconciled + ListComplete = $false + RecoveryPath = $a365RecoveryPath + } + } + + $csvPath = Export-Agent365Csv -Rows $rows.ToArray() + return @{ + CsvPath = $csvPath + Rows = $rows.ToArray() + Listed = $listed.Count + Emitted = $a365Emitted + DetailFailed = $a365DetailFailed + FailedDependency = $a365FailedDependency + RowBuildFailed = $a365RowBuildFailed + SkippedNoId = $a365SkippedNoId + Reconciled = $a365Reconciled + ListComplete = $true + RecoveryPath = $null + } +} + +# ============================================================================= +# END MICROSOFT AGENT 365 ENRICHMENT FUNCTIONS +# ============================================================================= + +# +# Core live-mode functions providing connectivity and paged audit retrieval. +# NOTE: This function is now wrapped by Connect-PurviewAudit for EOM mode compatibility + +function Connect-ToComplianceCenter { + param() + if ($script:Connected) { return } + Write-LogHost "Connecting to Microsoft 365 Security & Compliance Center..." -ForegroundColor Cyan + # Ensure ExchangeOnlineManagement module is available + try { + $existingEOM = Get-Module -ListAvailable -Name ExchangeOnlineManagement | Sort-Object Version -Descending | Select-Object -First 1 + if (-not $existingEOM) { + Write-LogHost "Installing ExchangeOnlineManagement module (CurrentUser scope)..." -ForegroundColor Yellow + try { Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser -Force -AllowClobber -ErrorAction Stop } catch { Write-LogHost "Failed to install module: $($_.Exception.Message)" -ForegroundColor Red; throw } + } + Import-Module ExchangeOnlineManagement -Force -ErrorAction Stop + } catch { + Write-LogHost "Module load/install failure: $($_.Exception.Message)" -ForegroundColor Red + throw + } + + # Authentication modes (subset retained for stability) + try { + switch ($Auth.ToLower()) { + 'weblogin' { + try { + $exoCmd = Get-Command Connect-ExchangeOnline -ErrorAction Stop + $hasUseWeb = $exoCmd.Parameters.ContainsKey('UseWebLogin') + if ($hasUseWeb) { + Write-LogHost 'Using Connect-ExchangeOnline -UseWebLogin (parameter present).' -ForegroundColor DarkGray + Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null + } + else { + Write-LogHost 'UseWebLogin parameter not available in this host/module; invoking standard interactive Connect-ExchangeOnline.' -ForegroundColor Yellow + Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null + } + } + catch { Write-LogHost "WebLogin flow failed: $($_.Exception.Message)" -ForegroundColor Red; throw } + } + 'devicecode' { + Connect-ExchangeOnline -ShowBanner:$false -Device | Out-Null + } + 'credential' { + $cred = Get-Credential -Message 'Enter admin credentials for Exchange Online' + Connect-ExchangeOnline -ShowBanner:$false -Credential $cred | Out-Null + } + default { + # Silent first, fallback to WebLogin + $silentOk = $true + try { Connect-ExchangeOnline -ShowBanner:$false -ErrorAction Stop | Out-Null } catch { $silentOk = $false } + if (-not $silentOk) { + try { Connect-ExchangeOnline -ShowBanner:$false -UseWebLogin -ErrorAction Stop | Out-Null } catch { Write-LogHost "Silent + fallback auth failed: $($_.Exception.Message)" -ForegroundColor Red; throw } + } + } + } + $script:Connected = $true + Write-LogHost "Connected successfully." -ForegroundColor Green + } + catch { + Write-LogHost "Connection failure: $($_.Exception.Message)" -ForegroundColor Red + throw + } +} + +# ============================================== +# DUAL-MODE DIAGNOSTICS FUNCTION +# ============================================== +# Unified capability check for both EOM and Graph API modes + +function Test-PurviewAuditCapability { + <# + .SYNOPSIS + Tests audit log query capability for either EOM or Graph API mode. + + .DESCRIPTION + Performs connectivity and permission checks based on active mode. + + EOM Mode: + - Verifies Search-UnifiedAuditLog cmdlet availability + - Performs probe query to test permissions + - Checks for proper role assignments + + Graph API Mode: + - Tests Graph API connectivity + - Verifies AuditLogsQuery.Read.All permissions + - Performs lightweight endpoint check + + .PARAMETER UseEOMMode + If true, test EOM capabilities. If false, test Graph API. + + .PARAMETER SkipChecks + If true, skip all diagnostic checks (for advanced scenarios) + + .OUTPUTS + $true if capability check passes, $false otherwise + #> + + param( + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false, + + [Parameter(Mandatory = $false)] + [bool]$SkipChecks = $false + ) + + if ($SkipChecks) { + Write-LogHost "Diagnostics: Skipped (per user request)" -ForegroundColor Gray + return $true + } + + if ($UseEOMMode) { + # ======================================== + # EOM MODE DIAGNOSTICS + # ======================================== + + Write-LogHost "Running EOM capability diagnostics..." -ForegroundColor Cyan + + # Check if cmdlet is available + $cmd = Get-Command Search-UnifiedAuditLog -ErrorAction SilentlyContinue + if (-not $cmd) { + Write-LogHost " ✗ DIAGNOSTIC FAILED: 'Search-UnifiedAuditLog' cmdlet not found" -ForegroundColor Red + Write-LogHost "" + Write-LogHost "Troubleshooting:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure ExchangeOnlineManagement module v3+ is installed" -ForegroundColor White + Write-LogHost " 2. Try: Install-Module ExchangeOnlineManagement -Scope CurrentUser" -ForegroundColor White + Write-LogHost " 3. Verify authentication completed successfully" -ForegroundColor White + Write-LogHost "" + Write-LogHost "Role Requirements:" -ForegroundColor Yellow + Write-LogHost " • View-Only Audit Logs role (minimum)" -ForegroundColor White + Write-LogHost " • Compliance Management role group" -ForegroundColor White + Write-LogHost " • Organization Management role group" -ForegroundColor White + return $false + } + + # Perform probe query to test permissions + try { + $now = (Get-Date).ToUniversalTime() + $probeStart = $now.AddMinutes(-7) + $probeEnd = $now.AddMinutes(-6) + + # Lightweight probe with unlikely operation + $null = Search-UnifiedAuditLog -StartDate $probeStart -EndDate $probeEnd -Operations 'UserLoggedIn' -ResultSize 1 -ErrorAction Stop + + Write-LogHost " EOM capability check passed" -ForegroundColor Green + return $true + } + catch { + $msg = $_.Exception.Message + Write-LogHost " ✗ DIAGNOSTIC FAILED: Probe query failed" -ForegroundColor Red + Write-LogHost " Error: $msg" -ForegroundColor Yellow + Write-LogHost "" + + if ($msg -match 'is not within the current user|Access denied|not authorized|insufficient') { + Write-LogHost "Likely Cause: Missing required roles" -ForegroundColor Yellow + Write-LogHost " Add account to 'Audit Logs' role group in Microsoft Purview" -ForegroundColor White + } + elseif ($msg -match 'The term .*Search-UnifiedAuditLog.* is not recognized') { + Write-LogHost "Likely Cause: Module not loaded properly" -ForegroundColor Yellow + Write-LogHost " Try: Import-Module ExchangeOnlineManagement -Force" -ForegroundColor White + } + else { + Write-LogHost "General Guidance:" -ForegroundColor Yellow + Write-LogHost " 1. Ensure Unified Audit Log is enabled tenant-wide" -ForegroundColor White + Write-LogHost " 2. Verify role assignments are properly configured" -ForegroundColor White + Write-LogHost " 3. Check for conditional access policies blocking access" -ForegroundColor White + } + + return $false + } + } + else { + # ======================================== + # GRAPH API MODE DIAGNOSTICS + # ======================================== + + Write-LogHost "Running Graph API capability diagnostics..." -ForegroundColor Cyan + + # Verify connected to Graph + try { + $context = Get-MgContext -ErrorAction Stop + + if (-not $context) { + Write-LogHost " ✗ DIAGNOSTIC FAILED: Not connected to Microsoft Graph" -ForegroundColor Red + Write-LogHost " Run Connect-PurviewAudit first to establish connection" -ForegroundColor Yellow + return $false + } + + # Check for required scopes + $requiredScope = 'AuditLogsQuery.Read.All' + if ($context.Scopes -notcontains $requiredScope) { + Write-LogHost " [!] WARNING: Missing required scope: $requiredScope" -ForegroundColor Yellow + Write-LogHost " Queries may fail without this permission" -ForegroundColor Yellow + } + } + catch { + Write-LogHost " ✗ DIAGNOSTIC FAILED: Unable to get Graph context" -ForegroundColor Red + Write-LogHost " Error: $($_.Exception.Message)" -ForegroundColor Yellow + return $false + } + + # Test Graph API endpoint connectivity + try { + # Test actual query endpoint with minimal test query + $testQueryBody = @{ + displayName = "PAX-Diagnostic-Test-$(Get-Date -Format 'HHmmss')" + filterStartDateTime = (Get-Date).AddMinutes(-1).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + filterEndDateTime = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + operationFilters = @('UserLoggedIn') # Common activity type for quick test + } + + $createUri = Get-GraphAuditApiUri -Path 'queries' + $createResponse = Invoke-MgGraphRequest -Method POST -Uri $createUri -Body $testQueryBody -ErrorAction Stop if ($createResponse.id) { + Write-LogHost " Graph API capability check passed" -ForegroundColor Green + Write-LogHost " Successfully created test query (ID: $($createResponse.id))" -ForegroundColor Green + return $true + } + else { + Write-LogHost " ✗ DIAGNOSTIC WARNING: Query created but no ID returned" -ForegroundColor Yellow + return $false + } + } + catch { + $msg = $_.Exception.Message + + # Check if this is a throttling error (429 TooManyRequests) + $isThrottling = $msg -match 'TooManyRequests|429|Too many requests|throttl' + + if ($isThrottling) { + # Set flag so we don't show scary warning message later + $script:ThrottlingDetected = $true + + # Throttling detected - friendly terminal message, full details to log only + Write-Host "" + Write-Host "============================================================================================================" -ForegroundColor DarkYellow + Write-Host " [!] Graph API Throttling Detected (429 - Too Many Requests)" -ForegroundColor DarkYellow + Write-Host "============================================================================================================" -ForegroundColor DarkYellow + Write-Host "" + Write-Host " Microsoft Graph is currently rate-limiting requests to your tenant." -ForegroundColor White + Write-Host "" + Write-Host " How PAX handles throttling:" -ForegroundColor Cyan + Write-Host " • Automatic exponential backoff with retries" -ForegroundColor Gray + Write-Host " • Circuit breaker protection (pauses after repeated failures)" -ForegroundColor Gray + Write-Host " • Adaptive concurrency (reduces parallel requests)" -ForegroundColor Gray + Write-Host " • Real-time notifications (you'll see throttle events as they occur)" -ForegroundColor Gray + Write-Host "" + Write-Host " Recommendation: Graph API throttling typically clears within 5-10 minutes." -ForegroundColor Yellow + Write-Host "" # Interactive prompt (unless -Force is used for headless runs) + if (-not $Force) { + Write-Host " Options:" -ForegroundColor Cyan + Write-Host " [C] CONTINUE - Proceed with automatic throttling handling (may be slow)" -ForegroundColor Green + Write-Host " [E] EXIT - Stop gracefully and retry later (recommended if heavily throttled)" -ForegroundColor Red + Write-Host "" + + # Strict noninteractive guard: a noninteractive + # host cannot choose C/E. Auto-select C (CONTINUE) so the run progresses + # with PAX's existing throttling controls (exponential backoff, circuit + # breaker, adaptive concurrency). Operators who want a hard exit on throttling + # should run interactively or pre-empt via run-orchestration retry policy. + if (script:Test-IsNonInteractive) { + Write-Host " Noninteractive host detected: auto-selecting [C] CONTINUE." -ForegroundColor DarkGray + Write-Host "" + $choice = 'C' + } else { + Send-PromptNotification + $choice = Read-Host " Enter your choice [C/E]" + } + if ($choice -match '^E$|^Exit$') { + Write-Host "" + Write-Host " Exiting gracefully..." -ForegroundColor Yellow + Write-Host " Disconnecting from Microsoft Graph..." -ForegroundColor Gray + + try { + script:Invoke-PaxGraphDisconnect -ErrorAction SilentlyContinue + Write-Host " Disconnected successfully" -ForegroundColor Green + } + catch { + Write-Host " (Graph connection cleanup completed)" -ForegroundColor Gray + } + + Write-Host "" + Write-Host " Please wait 5-10 minutes before retrying." -ForegroundColor Cyan + Write-Host "" + + # Log the graceful exit + Write-Output "[DIAGNOSTIC] User chose to exit due to throttling. Will retry later." | Out-File -FilePath $LogFile -Append -Encoding utf8 + + exit 0 + } + else { + Write-Host "" + Write-Host " Proceeding with automatic throttling handling..." -ForegroundColor Green + Write-Host " Expect slower execution times while Graph API recovers." -ForegroundColor Gray + Write-Host "" + } + } + else { + # -Force flag present (headless/automation mode) - proceed automatically + Write-Host " -Force flag detected: Proceeding automatically with throttling handling..." -ForegroundColor Green + Write-Host " Expect slower execution times while Graph API recovers." -ForegroundColor Gray + Write-Host "" + } + + # Log full error details to log file only (not terminal) + Write-Output "[DIAGNOSTIC] Graph API throttling detected during capability check" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[DIAGNOSTIC] Full error details: $msg" | Out-File -FilePath $LogFile -Append -Encoding utf8 + Write-Output "[DIAGNOSTIC] Continuing with automatic throttling handling enabled" | Out-File -FilePath $LogFile -Append -Encoding utf8 + } + else { + # Non-throttling error - show full details + Write-LogHost " ✗ DIAGNOSTIC FAILED: Graph API endpoint test failed" -ForegroundColor Red + Write-LogHost " Error: $msg" -ForegroundColor Yellow + Write-LogHost "" + + if ($msg -match 'Forbidden|403|Access.*denied|Insufficient privileges') { + Write-LogHost "Likely Cause: Missing required permissions" -ForegroundColor Yellow + Write-LogHost " Required: AuditLogsQuery.Read.All Graph API scope" -ForegroundColor White + Write-LogHost " Required: Azure AD role (Compliance/Security Administrator)" -ForegroundColor White + } + elseif ($msg -match 'Unauthorized|401') { + Write-LogHost "Likely Cause: Authentication issue" -ForegroundColor Yellow + Write-LogHost " Try disconnecting and reconnecting: Disconnect-MgGraph; Connect-PurviewAudit" -ForegroundColor White + } + else { + Write-LogHost "General Guidance:" -ForegroundColor Yellow + Write-LogHost " 1. Verify admin has consented to AuditLogsQuery.Read.All scope" -ForegroundColor White + Write-LogHost " 2. Check Azure AD role assignments" -ForegroundColor White + Write-LogHost " 3. Ensure network connectivity to graph.microsoft.com" -ForegroundColor White + } + } + + return $false + } + } +} + +# ============================================== +# USER-SCOPE RESOLUTION FUNCTION +# ============================================== +# Resolve explicit UserIds and group memberships into one fail-closed target-user union + +function Resolve-PaxUserScope { + <# + .SYNOPSIS + Resolves explicit UserIds and group memberships into a single deduplicated user scope. + + .DESCRIPTION + Treats -UserIds and -GroupNames as symmetrical user-scope inputs and returns one + case-insensitive deduplicated union of explicit user principal names and the members + of every requested group. Group membership is read through injectable seams so the + function runs under unit test without a live directory: Graph mode reads group identity + and transitive user membership through the authenticated Invoke-MgGraphRequest REST seam; + Exchange Online mode reads distribution-group membership through Get-DistributionGroupMember. + + The resolution fails closed. When any requested group cannot be resolved, matches more than + one directory object, has no user members, or errors, the whole result is marked Failed so a + caller never degrades a requested scope to an unfiltered query. On success with any requested + scope the returned FinalTargetUsers is always non-empty. + + .PARAMETER UserIds + Explicit user principal names to include in the scope. + + .PARAMETER GroupNames + Group display names, mail addresses, or object ids whose user members join the scope. + + .PARAMETER UseEOMMode + When true, read group membership through Exchange Online (direct members only); otherwise + read transitive user membership through Graph REST. + + .PARAMETER GraphRequest + Seam that issues a Graph REST GET for a relative uri and returns the parsed response. + + .PARAMETER EomGroupMemberOp + Seam that returns the members of an Exchange Online distribution group. + + .PARAMETER RosterValidator + Optional seam that reports whether an explicit user principal name exists in the directory. + Unmatched explicit users are reported but still included in the scope. When omitted, every + explicit user is treated as matched. + + .OUTPUTS + A single object describing the requested inputs, per-group resolution, the FinalTargetUsers + union, an Outcome of 'Succeeded' or 'Failed', and a FailureStage when Failed. + #> + + param( + [string[]]$UserIds = @(), + [string[]]$GroupNames = @(), + [bool]$UseEOMMode = $false, + [scriptblock]$GraphRequest = { param($Uri) Invoke-MgGraphRequest -Method GET -Uri $Uri -OutputType PSObject -ErrorAction Stop }, + [scriptblock]$EomGroupMemberOp = { param($GroupIdentity) Get-DistributionGroupMember -Identity $GroupIdentity -ErrorAction Stop }, + [scriptblock]$RosterValidator = $null + ) + + # Clean and case-insensitively deduplicate explicit UserIds, preserving first-seen casing. + $seenUsers = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $requestedUserIds = New-Object System.Collections.Generic.List[string] + foreach ($u in @($UserIds)) { + if ($null -eq $u) { continue } + $t = ([string]$u).Trim() + if ($t.Length -eq 0) { continue } + if ($seenUsers.Add($t)) { [void]$requestedUserIds.Add($t) } + } + + # Clean and case-insensitively deduplicate group names, preserving first-seen casing. + $seenGroups = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $requestedGroups = New-Object System.Collections.Generic.List[string] + foreach ($g in @($GroupNames)) { + if ($null -eq $g) { continue } + $t = ([string]$g).Trim() + if ($t.Length -eq 0) { continue } + if ($seenGroups.Add($t)) { [void]$requestedGroups.Add($t) } + } + + # Split explicit users into matched/unmatched when a roster validator is supplied; unmatched + # users are still included in the effective scope. + $matchedUsers = New-Object System.Collections.Generic.List[string] + $unmatchedUsers = New-Object System.Collections.Generic.List[string] + foreach ($u in $requestedUserIds) { + if ($RosterValidator) { + $exists = $false + try { $exists = [bool](& $RosterValidator $u) } catch { $exists = $false } + if ($exists) { [void]$matchedUsers.Add($u) } else { [void]$unmatchedUsers.Add($u) } + } + else { + [void]$matchedUsers.Add($u) + } + } + + # Seed the effective union with the explicit UserIds (matched and unmatched both count as + # requested scope; an unmatched user simply matches zero rows downstream). + $finalSeen = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $finalUsers = New-Object System.Collections.Generic.List[string] + foreach ($u in $requestedUserIds) { + if ($finalSeen.Add($u)) { [void]$finalUsers.Add($u) } + } + + $resolvedGroups = New-Object System.Collections.Generic.List[object] + $failedGroups = New-Object System.Collections.Generic.List[string] + $ambiguousGroups = New-Object System.Collections.Generic.List[string] + $zeroMemberGroups = New-Object System.Collections.Generic.List[string] + $unauthorizedGroups = New-Object System.Collections.Generic.List[string] + $transportErrorGroups = New-Object System.Collections.Generic.List[string] + $resolutionErrorGroups = New-Object System.Collections.Generic.List[string] + $directMembers = New-Object System.Collections.Generic.List[string] + $transitiveMembers = New-Object System.Collections.Generic.List[string] + + $outcome = 'Succeeded' + $failureStage = $null + $guidPattern = '^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$' + + foreach ($g in $requestedGroups) { + try { + if ($UseEOMMode) { + # Exchange Online: direct members only (nested groups are not expanded). + $raw = @(& $EomGroupMemberOp $g) + $eomMembers = New-Object System.Collections.Generic.List[string] + foreach ($m in $raw) { + $smtp = if ($m -is [string]) { $m } elseif ($null -ne $m.PrimarySmtpAddress) { [string]$m.PrimarySmtpAddress } else { $null } + if ($smtp) { $st = $smtp.Trim(); if ($st.Length -gt 0) { [void]$eomMembers.Add($st) } } + } + if ($eomMembers.Count -eq 0) { + [void]$zeroMemberGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupZeroMembers' } + continue + } + foreach ($m in $eomMembers) { + [void]$directMembers.Add($m); [void]$transitiveMembers.Add($m) + if ($finalSeen.Add($m)) { [void]$finalUsers.Add($m) } + } + [void]$resolvedGroups.Add([pscustomobject]@{ Requested = $g; Id = $null; DisplayName = $g; DirectMembers = $eomMembers.Count; TransitiveMembers = $eomMembers.Count; MembershipScope = 'DirectOnly (Exchange Online; nested groups not expanded)' }) + continue + } + + # Graph: resolve the group identity, then read transitive user membership. + $groupId = $null; $groupDisplay = $null + if ($g -match $guidPattern) { + $resp = & $GraphRequest ("/v1.0/groups/{0}?`$select=id,displayName" -f $g) + if ($resp -and $resp.id) { $groupId = [string]$resp.id; $groupDisplay = [string]$resp.displayName } + else { [void]$failedGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupNotFound' }; continue } + } + else { + $escaped = ($g -replace "'", "''") + $grpHits = @() + $r1 = & $GraphRequest ("/v1.0/groups?`$filter=displayName eq '{0}'&`$select=id,displayName" -f $escaped) + if ($r1 -and $r1.value) { $grpHits += @($r1.value) } + if ($grpHits.Count -eq 0) { + $r2 = & $GraphRequest ("/v1.0/groups?`$filter=mail eq '{0}' or mailNickname eq '{0}'&`$select=id,displayName" -f $escaped) + if ($r2 -and $r2.value) { $grpHits += @($r2.value) } + } + if ($grpHits.Count -eq 0) { [void]$failedGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupNotFound' }; continue } + if ($grpHits.Count -gt 1) { [void]$ambiguousGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupAmbiguous' }; continue } + $groupId = [string]$grpHits[0].id; $groupDisplay = [string]$grpHits[0].displayName + } + + # Transitive user members (paginated); nested-group users are included. + $transSet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $grpTrans = New-Object System.Collections.Generic.List[string] + $uri = ("/v1.0/groups/{0}/transitiveMembers/microsoft.graph.user?`$select=userPrincipalName" -f $groupId) + while ($uri) { + $page = & $GraphRequest $uri + if ($page -and $page.value) { + foreach ($m in @($page.value)) { + $upn = if ($null -ne $m.userPrincipalName) { ([string]$m.userPrincipalName).Trim() } else { $null } + if ($upn) { if ($transSet.Add($upn)) { [void]$grpTrans.Add($upn) } } + } + } + $uri = if ($page -and $page.'@odata.nextLink') { [string]$page.'@odata.nextLink' } else { $null } + } + if ($grpTrans.Count -eq 0) { [void]$zeroMemberGroups.Add($g); $outcome = 'Failed'; if (-not $failureStage) { $failureStage = 'GroupZeroMembers' }; continue } + + # Direct user members (paginated) for the direct/transitive split. + $dirSet = New-Object System.Collections.Generic.HashSet[string] ([System.StringComparer]::OrdinalIgnoreCase) + $grpDirectCount = 0 + $duri = ("/v1.0/groups/{0}/members/microsoft.graph.user?`$select=userPrincipalName" -f $groupId) + while ($duri) { + $dpage = & $GraphRequest $duri + if ($dpage -and $dpage.value) { + foreach ($m in @($dpage.value)) { + $upn = if ($null -ne $m.userPrincipalName) { ([string]$m.userPrincipalName).Trim() } else { $null } + if ($upn) { if ($dirSet.Add($upn)) { $grpDirectCount++; [void]$directMembers.Add($upn) } } + } + } + $duri = if ($dpage -and $dpage.'@odata.nextLink') { [string]$dpage.'@odata.nextLink' } else { $null } + } + + foreach ($upn in $grpTrans) { + [void]$transitiveMembers.Add($upn) + if ($finalSeen.Add($upn)) { [void]$finalUsers.Add($upn) } + } + [void]$resolvedGroups.Add([pscustomobject]@{ Requested = $g; Id = $groupId; DisplayName = $groupDisplay; DirectMembers = $grpDirectCount; TransitiveMembers = $grpTrans.Count; MembershipScope = 'Transitive' }) + } + catch { + $ex = $_.Exception + $sc = $null + try { + if ($ex.Response -and $ex.Response.StatusCode) { $sc = [int]$ex.Response.StatusCode } + elseif ($ex.PSObject.Properties['StatusCode'] -and $null -ne $ex.StatusCode) { $sc = [int]$ex.StatusCode } + } catch { $sc = $null } + $emsg = [string]$ex.Message + if ($sc -eq 401 -or $sc -eq 403 -or $emsg -match '(?i)Authorization_RequestDenied|Forbidden|Unauthorized|Access is denied|Insufficient privileges|\b40[13]\b') { + [void]$unauthorizedGroups.Add($g); $stg = 'GroupAuthorizationError' + } + elseif (($null -ne $sc -and ($sc -ge 500 -or $sc -eq 408 -or $sc -eq 429)) -or $emsg -match '(?i)timed out|timeout|connection|network|remote server|temporarily unavailable|ServiceUnavailable|Gateway|throttl') { + [void]$transportErrorGroups.Add($g); $stg = 'GroupTransportError' + } + else { + [void]$resolutionErrorGroups.Add($g); $stg = 'GroupResolutionError' + } + $outcome = 'Failed'; if (-not $failureStage) { $failureStage = $stg } + continue + } + } + + if ($outcome -eq 'Succeeded') { $failureStage = $null } + + return [pscustomobject]@{ + RequestedUserIds = $requestedUserIds.ToArray() + MatchedExplicitUserIds = $matchedUsers.ToArray() + UnmatchedExplicitUserIds = $unmatchedUsers.ToArray() + RequestedGroups = $requestedGroups.ToArray() + ResolvedGroups = $resolvedGroups.ToArray() + FailedGroups = $failedGroups.ToArray() + AmbiguousGroups = $ambiguousGroups.ToArray() + ZeroMemberGroups = $zeroMemberGroups.ToArray() + UnauthorizedGroups = $unauthorizedGroups.ToArray() + TransportErrorGroups = $transportErrorGroups.ToArray() + ResolutionErrorGroups = $resolutionErrorGroups.ToArray() + ResolvedDirectMembers = $directMembers.ToArray() + ResolvedTransitiveMembers = $transitiveMembers.ToArray() + FinalTargetUsers = $finalUsers.ToArray() + Outcome = $outcome + FailureStage = $failureStage + } +} + +# ============================================== +# DUAL-MODE GROUP EXPANSION FUNCTION +# ============================================== +# Expand distribution/security groups to individual user principal names + +function Expand-GroupToUsers { + <# + .SYNOPSIS + Expands a distribution or security group to individual user principal names. + + .DESCRIPTION + Delegates to Resolve-PaxUserScope for a single group and returns its resolved user + principal names. Graph mode reads group membership through the authenticated + Invoke-MgGraphRequest REST seam; Exchange Online mode reads distribution-group + membership through Get-DistributionGroupMember. A group that cannot be resolved, is + ambiguous, or has no user members throws so the caller never receives a silent empty + set that would widen the effective query scope. + + .PARAMETER GroupIdentity + The group identifier: display name, mail address, or object id. + + .PARAMETER UseEOMMode + When true, read membership through Exchange Online; otherwise through Graph REST. + + .OUTPUTS + Array of user principal names. + #> + + param( + [Parameter(Mandatory = $true)] + [string]$GroupIdentity, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + $resolved = Resolve-PaxUserScope -GroupNames @($GroupIdentity) -UseEOMMode $UseEOMMode + if ($resolved.Outcome -ne 'Succeeded') { + throw "Group '$GroupIdentity' could not be resolved to any user members (stage: $($resolved.FailureStage)); refusing to return an empty membership set." + } + return @($resolved.FinalTargetUsers) +} + +# ============================================== +# DUAL-MODE QUERY EXECUTION WRAPPER +# ============================================== +# Unified query function that routes to either EOM or Graph API + +function Invoke-PurviewAuditQuery { + <# + .SYNOPSIS + Executes an audit log query using either EOM or Graph API. + + .DESCRIPTION + Routes audit log queries to the appropriate backend: + - EOM Mode: Uses Search-UnifiedAuditLog cmdlet + - Graph API Mode: Uses async query pattern (create → poll → retrieve) + + Returns audit records in a normalized format compatible with downstream processing. + + .PARAMETER StartDate + Query start date (inclusive) + + .PARAMETER EndDate + Query end date (exclusive) + + .PARAMETER Operations + Activity type(s) to query + + .PARAMETER UserIds + Optional array of user principal names to filter by + + .PARAMETER ResultSize + Maximum number of records to retrieve (EOM mode) + + .PARAMETER UseEOMMode + If true, use EOM cmdlets. If false, use Graph API. + + .OUTPUTS + Array of audit log records in normalized schema + #> + + param( + [Parameter(Mandatory = $true)] + [datetime]$StartDate, + + [Parameter(Mandatory = $true)] + [datetime]$EndDate, + + [Parameter(Mandatory = $true)] + [string]$Operations, + + [Parameter(Mandatory = $false)] + [string[]]$UserIds, + + [Parameter(Mandatory = $false)] + [int]$ResultSize = 5000, + + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Use existing Search-UnifiedAuditLog logic + # ======================================== + + # Call existing retry wrapper (preserves all the sophisticated logic) + $results = Invoke-SearchUnifiedAuditLogWithRetry ` + -Start $StartDate ` + -End $EndDate ` + -Operation $Operations ` + -ResultSize $ResultSize ` + -UserIds $UserIds ` + -AutoSubdivide $true + + return $results + } + else { + # ======================================== + # GRAPH API MODE: Async query pattern + # ======================================== + + try { + # Step 1: Create async query + Write-Host " [Graph API] Creating async query for $Operations..." -ForegroundColor DarkGray + + # Use last included minute (EndDate - 1 minute) since end date is exclusive + $endDisplay = $EndDate.AddMinutes(-1) + $displayName = "PAX_Query_$($StartDate.ToString('yyyyMMdd_HHmm'))-$($endDisplay.ToString('yyyyMMdd_HHmm'))" + + # M365 usage mode requires operationFilters only (no recordType/service filters) + $recordTypesArg = $null + $serviceFilterArg = $null + if (-not $script:IncludeM365Usage) { + # Only populate filters when NOT in M365 usage mode + $recordTypesArg = $RecordTypes + $serviceFilterArg = $script:CurrentServiceFilter + if (-not $serviceFilterArg -and $ServiceTypes -and $ServiceTypes.Count -gt 0) { + $serviceFilterArg = $ServiceTypes[0] + } + } + + $queryId = Invoke-GraphAuditQuery ` + -DisplayName $displayName ` + -FilterStartDateTime $StartDate ` + -FilterEndDateTime $EndDate ` + -OperationFilters @($Operations) ` + -RecordTypeFilters $recordTypesArg ` + -ServiceFilter $serviceFilterArg + + if (-not $queryId) { + Write-Host " [Graph API] Failed to create query" -ForegroundColor Red + return @() + } + + Write-Host " [Graph API] Query created: $queryId" -ForegroundColor DarkGray + + # Step 2: Poll for completion + # Replaced fixed-count polling with time-budget model supporting extended outages (up to 30 minutes) + $effectiveOutageMinutes = if ($MaxNetworkOutageMinutes -and $MaxNetworkOutageMinutes -gt 0) { $MaxNetworkOutageMinutes } else { 30 } + $maxPollDurationSeconds = $effectiveOutageMinutes * 60 # Absolute cap for network outage tolerance + $pollInterval = 5 # Base interval (seconds) when healthy + $maxHealthyInterval = 15 # Cap interval when status retrieval succeeds + $pollStart = Get-Date + $pollCount = 0 + $queryComplete = $false + $networkErrorStreak = 0 + $networkOutageStart = $null + $lastNetMessage = $null # Throttle repetitive network messages + + Write-Host " [Graph API] Polling for query completion..." -ForegroundColor DarkGray + + # Transient network resilience variables + $transientPatterns = @('timed out','unable to connect','connection','remote name could not be resolved','temporarily unavailable') + + while (-not $queryComplete) { + # Network outage guard: only abort if we've been in a CONTINUOUS network outage + # exceeding the user's MaxNetworkOutageMinutes threshold. Normal polling + # continues indefinitely until the query succeeds, fails, or the user cancels. + if ($networkOutageStart) { + $outageElapsed = (Get-Date) - $networkOutageStart + if ($outageElapsed.TotalSeconds -ge $maxPollDurationSeconds) { + Write-Host " [NET] Polling aborted after $effectiveOutageMinutes minutes of continuous network outage" -ForegroundColor Red + break + } + } + Start-Sleep -Seconds $pollInterval + $pollCount++ + + $status = $null + try { + $status = Get-GraphAuditQueryStatus -QueryId $queryId -ErrorAction Stop + # Successful status retrieval resets outage tracking + $networkErrorStreak = 0 + if ($networkOutageStart) { + $outageDuration = (Get-Date) - $networkOutageStart + # Only log recovery if outage lasted > 1 minute (ignore brief connection blips) + if ($outageDuration.TotalMinutes -ge 1) { + Write-Host " [NET] Network recovered after $([Math]::Round($outageDuration.TotalMinutes,1)) minutes" -ForegroundColor Green + } + $networkOutageStart = $null + $lastNetMessage = $null + } + # Query status retrieved successfully; interval adjusted + # Tighten interval gradually back to healthy baseline + $pollInterval = [Math]::Max(5, [Math]::Min($pollInterval - 2, $maxHealthyInterval)) + } + catch { + $errMsg = $_.Exception.Message + if ($transientPatterns | Where-Object { $errMsg.ToLower().Contains($_) }) { + $networkErrorStreak++ + if (-not $networkOutageStart) { $networkOutageStart = Get-Date } + $outageElapsed = (Get-Date) - $networkOutageStart + # Throttle messages: only show if outage > 1 min OR first error with no recent message + if ($outageElapsed.TotalMinutes -ge 1 -or ($networkErrorStreak -eq 1 -and (-not $lastNetMessage -or ((Get-Date) - $lastNetMessage).TotalSeconds -ge 60))) { + if (-not $lastNetMessage -or ((Get-Date) - $lastNetMessage).TotalSeconds -ge 60) { + Write-Host " [NET] Poll $pollCount`: transient network issue (streak $networkErrorStreak, outage $([Math]::Round($outageElapsed.TotalMinutes,1))m)" -ForegroundColor Yellow + $lastNetMessage = Get-Date + } + } + # Dynamic backoff growth with ceiling (to avoid hammering during outage) + $pollInterval = [Math]::Min(90, [Math]::Round($pollInterval * 1.6 + (Get-Random -Minimum 2 -Maximum 6))) + continue + } else { + Write-Host " [Graph API] Non-transient status error: $errMsg" -ForegroundColor Red + break + } + } + + if (-not $status) { continue } + + Write-Host " [Graph API] Poll $pollCount`: Status=$($status.Status), RecordCount=$($status.RecordCount)" -ForegroundColor DarkGray + + switch ($status.Status) { + 'succeeded' { + $queryComplete = $true + Write-Host " [Graph API] Query completed: $($status.RecordCount) records available" -ForegroundColor Green + break + } + 'failed' { + Write-Host " [Graph API] Query failed" -ForegroundColor Red + return @() + } + 'cancelled' { + Write-Host " [Graph API] Query was cancelled" -ForegroundColor Yellow + return @() + } + default { continue } + } + } + + if (-not $queryComplete) { + Write-Host " [Graph API] Query polling aborted (network outage or non-transient error after $pollCount polls)" -ForegroundColor Yellow + return @() + } + + # Step 3: Retrieve records + Write-Host " [Graph API] Retrieving records..." -ForegroundColor DarkGray + + # Retrieve records with transient retry resilience + $graphRecords = $null + $recordStart = Get-Date + $recordAttempt = 0 + $maxRecordDurationSeconds = $effectiveOutageMinutes * 60 + $retrieveInterval = 4 + while (-not $graphRecords) { + $recordAttempt++ + if (((Get-Date) - $recordStart).TotalSeconds -ge $maxRecordDurationSeconds) { + Write-Host " [NET] Record retrieval aborted after $effectiveOutageMinutes minutes of network instability" -ForegroundColor Red + break + } + try { + # Graph API: MaxRecords=0 (unlimited) - 10K limit only applies to EOM mode + $graphRecords = Get-GraphAuditRecords -QueryId $queryId -MaxRecords 0 -ErrorAction Stop + } + catch { + $err = $_.Exception.Message + if ($transientPatterns | Where-Object { $err.ToLower().Contains($_) }) { + Write-Host " [NET] Transient record fetch issue (attempt $recordAttempt, elapsed $([Math]::Round(((Get-Date)-$recordStart).TotalMinutes,2))m): $err" -ForegroundColor Yellow + $retrieveInterval = [Math]::Min(90, [Math]::Round($retrieveInterval * 1.5 + (Get-Random -Minimum 1 -Maximum 5))) + Start-Sleep -Seconds $retrieveInterval + continue + } else { + Write-Host " [Graph API] Non-transient record fetch error: $err" -ForegroundColor Red + break + } + } + } + if (-not $graphRecords) { Write-Host " [NET] Retrieval failed after extended retry window" -ForegroundColor Red } + + if (-not $graphRecords -or $graphRecords.Count -eq 0) { + Write-Host " [Graph API] No records returned" -ForegroundColor Gray + return @() + } + + Write-Host " [Graph API] Retrieved $($graphRecords.Count) records, normalizing..." -ForegroundColor DarkGray + + # Step 4: Normalize to EOM-compatible schema + $normalized = @() + foreach ($record in $graphRecords) { + $normalizedRecord = ConvertFrom-GraphAuditRecord -GraphRecord $record + if ($normalizedRecord) { + $normalized += $normalizedRecord + } + } + + # Filter by UserIds if specified (Graph API doesn't support UPN filtering in query) + if ($UserIds -and $UserIds.Count -gt 0 -and $normalized.Count -gt 0) { + Write-Host " [Graph API] Applying client-side UserIds filter..." -ForegroundColor DarkGray + $beforeFilter = $normalized.Count + $normalized = $normalized | Where-Object { $UserIds -contains $_.UserIds } + Write-Host " [Graph API] Filtered: $beforeFilter → $($normalized.Count) records" -ForegroundColor DarkGray + } + + Write-Host " [Graph API] Normalization complete: $($normalized.Count) records ready" -ForegroundColor Green + + return $normalized + } + catch { + Write-Host " [Graph API] Query error: $($_.Exception.Message)" -ForegroundColor Red + Write-Host " [Graph API] Falling back to empty result set" -ForegroundColor Yellow + return @() + } + } +} + +# ============================================== +# DUAL-MODE DISCONNECTION FUNCTION +# ============================================== +# Unified disconnection for both EOM and Graph API modes + +function Disconnect-PurviewAudit { + <# + .SYNOPSIS + Disconnects from either Exchange Online or Microsoft Graph. + + .DESCRIPTION + Cleanly disconnects active sessions based on mode. + + EOM Mode: + - Calls Disconnect-ExchangeOnline + - No confirmation prompt + + Graph API Mode: + - Calls Disconnect-MgGraph + - Clears Graph context + + .PARAMETER UseEOMMode + If true, disconnect from EOM. If false, disconnect from Graph. + + .OUTPUTS + None + #> + + param( + [Parameter(Mandatory = $false)] + [bool]$UseEOMMode = $false + ) + + if ($UseEOMMode) { + # ======================================== + # EOM MODE: Disconnect-ExchangeOnline + # ======================================== + + try { + Write-LogHost "Disconnecting from Exchange Online..." -ForegroundColor Gray + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction Stop | Out-Null + Write-LogHost " Disconnected from Exchange Online" -ForegroundColor Green + } + catch { + # Silently handle - may not be connected or already disconnected + Write-LogHost " (Exchange Online disconnection skipped or already disconnected)" -ForegroundColor DarkGray + } + } + else { + # ======================================== + # GRAPH API MODE: Disconnect-MgGraph + # ======================================== + + try { + # Check if connected first + $context = Get-MgContext -ErrorAction SilentlyContinue + + if ($context) { + Write-LogHost "Disconnecting from Microsoft Graph..." -ForegroundColor Gray + script:Invoke-PaxGraphDisconnect -ErrorAction Stop + Write-LogHost " Disconnected from Microsoft Graph" -ForegroundColor Green + } + else { + Write-LogHost " (Not connected to Microsoft Graph)" -ForegroundColor DarkGray + } + } + catch { + # Silently handle - may not be connected or already disconnected + Write-LogHost " (Microsoft Graph disconnection skipped or already disconnected)" -ForegroundColor DarkGray + } + } +} + +# Pre-query diagnostic: verify Search-UnifiedAuditLog availability & likely permission coverage. +# NOTE: This function is now wrapped by Test-PurviewAuditCapability for EOM mode compatibility +function Invoke-AuditCapabilityDiagnostics { + param() + if ($SkipDiagnostics) { return $true } + $cmd = Get-Command Search-UnifiedAuditLog -ErrorAction SilentlyContinue + if (-not $cmd) { + Write-LogHost "DIAGNOSTIC: 'Search-UnifiedAuditLog' cmdlet not found in this session." -ForegroundColor Red + Write-LogHost "Guidance: Ensure ExchangeOnlineManagement module (v3+) is installed and imported. Try: Install-Module ExchangeOnlineManagement -Scope CurrentUser" -ForegroundColor Yellow + Write-LogHost "Role Requirements: Membership in 'Audit Logs' (preferred) or 'View-Only Audit Logs' / appropriate Compliance role group." -ForegroundColor Yellow + return $false + } + # Attempt a minimal, very narrow harmless probe query (empty expected results) + try { + $now = (Get-Date).ToUniversalTime() + $probeStart = $now.AddMinutes(-7) + $probeEnd = $now.AddMinutes(-6) + # Use an operation that is unlikely to appear but valid syntactically + $null = Search-UnifiedAuditLog -StartDate $probeStart -EndDate $probeEnd -Operations 'UserLoggedIn' -ResultSize 1 -ErrorAction Stop + Write-LogHost "Diagnostics: Audit search cmdlet available (probe succeeded/no error)." -ForegroundColor DarkGray + return $true + } + catch { + $msg = $_.Exception.Message + Write-LogHost "DIAGNOSTIC: Probe audit search failed: $msg" -ForegroundColor Yellow + if ($msg -match 'is not within the current user' -or $msg -match 'Access denied' -or $msg -match 'not authorized' -or $msg -match 'insufficient') { + Write-LogHost "Likely Missing Roles: Add the account to 'Audit Logs' (Microsoft Purview) or at minimum 'View-Only Audit Logs'." -ForegroundColor Red + } + elseif ($msg -match 'The term .*Search-UnifiedAuditLog.* is not recognized') { + Write-LogHost "Module Issue: Cmdlet not loaded. Import-Module ExchangeOnlineManagement or update module version." -ForegroundColor Red + } + else { + Write-LogHost "General Guidance: Ensure Unified Audit Log is enabled tenant-wide & correct role assignments are in place." -ForegroundColor Yellow + } + return $false + } +} + +function Invoke-SearchUnifiedAuditLogWithRetry { + <# + Provides pagination & early 10K detection. + Adjustments: + * Honors $PacingMs but leaves adaptive / circuit breaker to caller. + * Maintains metrics.PagesFetched & global limit flags used by higher layers. + #> + param( + [Parameter(Mandatory)][datetime]$Start, + [Parameter(Mandatory)][datetime]$End, + [Parameter(Mandatory)][string]$Operation, + [Parameter(Mandatory)][int]$ResultSize, + [string[]]$UserIds, + [int]$MaxRetries = 3, + [bool]$AutoSubdivide = $true + ) + + $script:Hit10KLimit = $false + $script:LimitTimeWindow = "" + $allResults = New-Object System.Collections.ArrayList + $totalFetched = 0 + $pageNumber = 1 + $maxPages = 50 + $pageSize = [Math]::Min($ResultSize, 5000) + $useSessionPagination = $ResultSize -gt 5000 + $sessionId = if ($useSessionPagination) { [guid]::NewGuid().ToString() } else { $null } + + Write-LogHost (" Using {0} pagination (page size {1})" -f ($(if ($useSessionPagination){'session'} else {'standard'}), $pageSize)) -ForegroundColor DarkCyan + + try { + while ($totalFetched -lt $ResultSize -and $pageNumber -le $maxPages) { + $remainingNeeded = $ResultSize - $totalFetched + $currentPageSize = [Math]::Min($pageSize, $remainingNeeded) + $attempt = 0; $pageResults = $null + while ($attempt -le $MaxRetries) { + try { + $params = @{ StartDate = $Start; EndDate = $End; Operations = $Operation; ResultSize = $currentPageSize; ErrorAction = 'Stop' } + if ($UserIds) { $params['UserIds'] = $UserIds } + if ($useSessionPagination) { + $params['SessionId'] = $sessionId + $params['SessionCommand'] = if ($pageNumber -eq 1) { 'ReturnLargeSet' } else { 'ReturnNextPreviewPage' } + } + if ($PacingMs -gt 0) { Start-Sleep -Milliseconds $PacingMs } + if ($attempt -gt 0) { Write-LogHost " Retrying page $pageNumber (attempt $($attempt+1))" -ForegroundColor Yellow } + $pageResults = Search-UnifiedAuditLog @params + break + } + catch { + $attempt++ + if ($attempt -le $MaxRetries) { + $delay = [Math]::Min(30, [Math]::Pow(2, $attempt)) + Write-LogHost " Page $pageNumber failed: $($_.Exception.Message). Backoff ${delay}s" -ForegroundColor DarkYellow + Start-Sleep -Seconds $delay + if ($useSessionPagination -and $attempt -gt 1) { $sessionId = [guid]::NewGuid().ToString(); Write-LogHost " New session id for retry: $sessionId" -ForegroundColor DarkGray } + } else { + Write-LogHost " Page $pageNumber permanently failed after $attempt attempts" -ForegroundColor Red + throw + } + } + } + + if ($pageResults -and $pageResults.Count -gt 0) { + # Early 10K detection (first page result count meta) + if ($pageNumber -eq 1 -and $AutoSubdivide) { + try { + $est = $pageResults[0].ResultCount + if ($null -ne $est -and $est -ge 10000) { + Write-LogHost " [!] Estimated >=10K records in window – consider subdivision" -ForegroundColor Yellow + } + } catch {} + } + # Safe add - handle both array and single object returns + if ($pageResults -is [Array]) { + foreach ($item in $pageResults) { [void]$allResults.Add($item) } + } else { + [void]$allResults.Add($pageResults) + } + $totalFetched += $pageResults.Count + # Hard stop enforcement: never return more than requested -ResultSize + if ($totalFetched -ge $ResultSize) { + if ($totalFetched -gt $ResultSize) { + $excess = $totalFetched - $ResultSize + # Trim excess items from tail + for ($trim = 0; $trim -lt $excess; $trim++) { [void]$allResults.RemoveAt($allResults.Count - 1) } + $totalFetched = $ResultSize + } + Write-LogHost " Requested result size $ResultSize reached (cumulative: $totalFetched) – stopping" -ForegroundColor DarkCyan + try { $script:metrics.PagesFetched += 1 } catch {} + break + } + try { $script:metrics.PagesFetched += 1 } catch {} + Write-LogHost " Page $pageNumber returned $($pageResults.Count) (cumulative: $totalFetched)" -ForegroundColor DarkCyan + if ($pageResults.Count -lt $currentPageSize) { break } + if ($totalFetched -ge 10000) { + $script:Hit10KLimit = $true + $script:LimitTimeWindow = "$(($Start).ToString('yyyy-MM-dd HH:mm')) to $(($End).ToString('yyyy-MM-dd HH:mm'))" + # SMART SUBDIVISION for EOM: Analyze timestamp distribution + if ($AutoSubdivide -and $allResults.Count -ge 10000) { + try { + $timestamps = @() + foreach ($rec in $allResults) { + if ($rec.CreationDate) { + $ts = script:Parse-DateSafe $rec.CreationDate; if ($ts) { $timestamps += $ts } + } + } + if ($timestamps.Count -gt 100) { + $sorted = $timestamps | Sort-Object + $coveredHours = ($sorted[-1] - $sorted[0]).TotalHours + $totalHours = ($End - $Start).TotalHours + if ($coveredHours -gt 0 -and $coveredHours -lt $totalHours) { + $recordsPerHour = 10000 / $coveredHours + $targetHours = 8000 / $recordsPerHour + $subdivFactor = [Math]::Max(2, [Math]::Ceiling($totalHours / $targetHours)) + Write-LogHost " [SMART SUBDIVISION] EOM: $([Math]::Round($coveredHours,2))h of $([Math]::Round($totalHours,2))h → suggest dividing by $subdivFactor" -ForegroundColor Cyan + } + } + } catch {} + } + Write-LogHost " 10K server limit reached in this window" -ForegroundColor Yellow + break + } + } else { + Write-LogHost " Page $pageNumber empty – stopping" -ForegroundColor DarkCyan + break + } + $pageNumber++ + } + + if ($pageNumber -gt $maxPages) { + Write-LogHost " Reached max page limit ($maxPages)" -ForegroundColor Yellow + } + Write-LogHost " Pagination complete: $($allResults.Count) records" -ForegroundColor Green + return $allResults.ToArray() + } + catch { + Write-LogHost " Pagination failed: $($_.Exception.Message)" -ForegroundColor Red + throw + } +} + +# Wrapper for main processing (kept minimal for clarity) +function Invoke-PAXProcessingCore { + param() + try { + # Existing core logic already executed above in previous top-level scope. + # This wrapper intentionally left minimal to avoid structural parse issues. + } + catch { + Write-LogHost "Core processing error: $($_.Exception.Message)" -ForegroundColor Red + throw + } +} + +$script:adaptiveThroughputBaseline = $null +$script:adaptiveLowLatencyStreak = 0 +$script:consecutiveBlockFailures = 0 +$script:circuitBreakerOpen = $false +$script:circuitBreakerOpenUntil = $null + +function Get-BackoffDelaySeconds { + param( + [Parameter(Mandatory)][int]$Attempt, + [Parameter(Mandatory)][double]$BaseSeconds, + [Parameter(Mandatory)][int]$MaxSeconds + ) + if ($Attempt -lt 1) { return 0 } + $raw = $BaseSeconds * [math]::Pow(2, ($Attempt - 1)) + return [math]::Min($MaxSeconds, $raw) +} + +function Test-CircuitBreakerTrip { + param( + [Parameter(Mandatory)][int]$ConsecutiveFailures, + [Parameter(Mandatory)][int]$Threshold + ) + return ($ConsecutiveFailures -ge $Threshold) +} + +$JsonDepth = 60 +$FlatDepthStandard = 6 +$FlatDepthDeep = 120 +$ExplosionPerRecordRowCap = 1000 +$script:TenantPrimaryDomain = $null +if (-not $script:TenantId) { $script:TenantId = $null } +$script:TenantIndicators = @() +$ForcedRawInputCsvExplosion = $false + +# Auth config storage for token refresh (AppRegistration mode) +$script:AuthConfig = @{ + Method = $null + TenantId = $null + ClientId = $null + ClientSecret = $null # SecureString + CertThumbprint = $null + CertPath = $null + CertPassword = $null # SecureString + CertStoreLocation = 'CurrentUser' + TokenIssueTime = $null + CanReauthenticate = $false +} + +# Shared auth state for thread job token refresh (synchronized hashtable for cross-thread access) +# Thread jobs read Token from this hashtable; main thread updates it proactively before expiry +$script:SharedAuthState = [hashtable]::Synchronized(@{ + Token = $null + ExpiresOn = $null + LastRefresh = $null + RefreshCount = 0 + AuthMethod = $null +}) + +# Azure (Fabric/OneLake) storage-audience token state. Independent of Graph because +# OneLake DFS uses a separate audience (https://storage.azure.com/) and a separate SDK +# (Az.Accounts), with its own MSAL token cache. Mirrors $script:SharedAuthState shape so +# the refresh code paths read uniformly. Single-thread access (uploads run on main thread), +# so a regular hashtable is sufficient. +$script:AzAuthState = @{ + Token = $null + ExpiresOn = $null # UTC DateTime + AcquiredAt = $null # local DateTime, used for age-cap proactive refresh + LastRefresh = $null + LastRefreshAttempt = $null # cooldown anchor + RefreshCount = 0 + AuthMethod = $null # 'ManagedIdentity' | 'AppRegistration' | 'Interactive' +} + +# Checkpoint/Resume state for long-running operations +$script:CheckpointPath = $null # Path to checkpoint JSON file +$script:CheckpointData = $null # Loaded/active checkpoint object +$script:IsResumeMode = $false # Whether we're resuming from checkpoint +$script:PartialOutputPath = $null # Path to _PARTIAL.csv file during execution +$script:OriginallySkippedPartitionIndices = @() # Partition indices that were already completed before this run (for resume mode) +$script:StreamingMergeDuplicatesSkipped = 0 # Count of duplicate records removed during streaming merge +$script:StreamingMergeDataLoss = $false # Whether streaming merge detected missing partition data + +# Token expiration detection (reactive - triggers on 401 Unauthorized) +$script:TokenAcquiredTime = $null # When current token was obtained +$script:AuthFailureDetected = $false # Set to $true when 401 error detected - triggers reauth prompt +$script:Auth401MessageShown = $false # Suppresses duplicate 401 error messages (reset after successful reauth) +$script:AuthPromptInProgress = $false # Debounce flag - prevents multiple auth prompts from triggering simultaneously + +# PowerShell version detection for parallel processing features +$script:IsPS7 = ($PSVersionTable.PSVersion.Major -ge 7) + +if ($RAWInputCSV) { $ForcedRawInputCsvExplosion = $true } + +$script:RegexTrueFalse = [regex]::new('^(?i:true|false)$', [System.Text.RegularExpressions.RegexOptions]::Compiled) +$script:RegexYes1 = [regex]::new('^(?i:yes|1)$', [System.Text.RegularExpressions.RegexOptions]::Compiled) +$script:RegexNo0 = [regex]::new('^(?i:no|0)$', [System.Text.RegularExpressions.RegexOptions]::Compiled) +$script:LocaleDateParsingNotified = $false + +function script:Parse-DateSafe { + <# + .SYNOPSIS + Culture-invariant date parsing that handles Purview API date formats. + .DESCRIPTION + Purview API returns dates in US format (M/d/yyyy HH:mm:ss) regardless of client locale. + This function safely parses such dates on systems with non-US regional settings (e.g., UK). + #> + param([Parameter(Mandatory=$false)][AllowNull()][AllowEmptyString()]$DateValue) + + # Log once when running under non-US locale + if (-not $script:LocaleDateParsingNotified) { + $script:LocaleDateParsingNotified = $true + $currentCulture = [System.Threading.Thread]::CurrentThread.CurrentCulture.Name + if ($currentCulture -and $currentCulture -ne 'en-US') { + Write-LogHost " [DATE] Locale-aware date parsing active (Culture: $currentCulture)" -ForegroundColor DarkCyan + } + } + + # Already a DateTime? Return as-is + if ($DateValue -is [datetime]) { return $DateValue } + + # Null or empty? Return null + if ([string]::IsNullOrWhiteSpace($DateValue)) { return $null } + + $dateStr = [string]$DateValue + + # Try ISO 8601 formats first (most common from properly-formatted API responses) + $isoFormats = @( + 'yyyy-MM-ddTHH:mm:ss.fffffffK', + 'yyyy-MM-ddTHH:mm:ss.fffK', + 'yyyy-MM-ddTHH:mm:ssK', + 'yyyy-MM-ddTHH:mm:ss.fffffff', + 'yyyy-MM-ddTHH:mm:ss.fffZ', + 'yyyy-MM-ddTHH:mm:ssZ', + 'yyyy-MM-ddTHH:mm:ss.fff', + 'yyyy-MM-ddTHH:mm:ss', + 'yyyy-MM-dd HH:mm:ss.fff', + 'yyyy-MM-dd HH:mm:ss', + 'yyyy-MM-dd' + ) + + foreach ($fmt in $isoFormats) { + try { + return [datetime]::ParseExact($dateStr, $fmt, [System.Globalization.CultureInfo]::InvariantCulture, [System.Globalization.DateTimeStyles]::AdjustToUniversal) + } + catch { } + } + + # Try US formats explicitly (what Purview actually returns - causes UK locale issues) + $usFormats = @( + 'M/d/yyyy HH:mm:ss', + 'M/d/yyyy h:mm:ss tt', + 'M/d/yyyy H:mm:ss', + 'MM/dd/yyyy HH:mm:ss', + 'M/d/yyyy', + 'MM/dd/yyyy' + ) + + foreach ($fmt in $usFormats) { + try { + return [datetime]::ParseExact($dateStr, $fmt, [System.Globalization.CultureInfo]::InvariantCulture) + } + catch { } + } + + # Last resort: use InvariantCulture with Parse + try { + return [datetime]::Parse($dateStr, [System.Globalization.CultureInfo]::InvariantCulture) + } + catch { + return $null + } +} + +function script:Format-DatePurviewFast($dt) { + if (-not $dt) { return '' } + try { + if ($dt -is [datetime]) { + return $dt.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + } + else { + $p = script:Parse-DateSafe $dt + if ($null -eq $p) { return '' } + return $p.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + } + } + catch { return '' } +} + +function script:BoolTFFast($v) { + if ($null -eq $v) { return '' } + if ($v -is [bool]) { return $v.ToString().ToUpper() } + $vStr = [string]$v + if ($script:RegexTrueFalse.IsMatch($vStr)) { return $vStr.ToUpper() } + if ($script:RegexYes1.IsMatch($vStr)) { return 'TRUE' } + if ($script:RegexNo0.IsMatch($vStr)) { return 'FALSE' } + return $vStr +} + +# Apply FlatDepth override (if provided) +try { + if ($PSBoundParameters.ContainsKey('FlatDepth')) { + $FlatDepthDeep = $FlatDepth + $FlatDepthStandard = [int][Math]::Min($FlatDepth, $FlatDepthStandard) + } +} catch {} + +function script:ToJsonIfObjectFast($v) { + if ($null -eq $v) { return '' } + if (Test-ScalarValue $v) { return $v } + try { return ($v | ConvertTo-Json -Depth $JsonDepth -Compress) } + catch { return [string]$v } +} + +function script:GetArrayFast($parent, [string]$name) { + $val = Get-SafeProperty $parent $name + if ($null -eq $val) { return @() } + if ($val -is [System.Collections.IEnumerable] -and -not ($val -is [string])) { + return @($val) + } + return @($val) +} + +$effectiveExplodeForProgress = ($ExplodeDeep -or $ExplodeArrays -or $ForcedRawInputCsvExplosion) + +# MEMORY MANAGEMENT: Resolve MaxMemoryMB (-1 = auto 75% of system RAM, 0 = disabled, >0 = explicit limit) +$script:ResolvedMaxMemoryMB = $MaxMemoryMB +if ($MaxMemoryMB -eq -1) { + # Auto-detect: use 75% of total physical memory + try { + $totalRAM = [math]::Round((Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction SilentlyContinue).TotalPhysicalMemory / 1MB, 0) + $script:ResolvedMaxMemoryMB = [math]::Round($totalRAM * 0.75, 0) + Write-LogHost "Memory management: Auto-detected ${totalRAM}MB total RAM -> limit set to $($script:ResolvedMaxMemoryMB)MB (75%)" -ForegroundColor Cyan + } catch { + # Fallback if CIM fails (e.g., Linux/macOS) + $script:ResolvedMaxMemoryMB = 4096 + Write-LogHost "Memory management: Could not detect system RAM, defaulting to 4096MB limit" -ForegroundColor Yellow + } +} elseif ($MaxMemoryMB -eq 0) { + $script:ResolvedMaxMemoryMB = 0 + Write-LogHost "Memory management: DISABLED (-MaxMemoryMB 0)" -ForegroundColor DarkGray +} + +# Memory flush mode: enabled when ResolvedMaxMemoryMB > 0 AND explosion is disabled (explosion needs full $allLogs in memory) +$script:memoryFlushEnabled = ($script:ResolvedMaxMemoryMB -gt 0) -and (-not $ExplodeDeep) -and (-not $ExplodeArrays) -and (-not $ForcedRawInputCsvExplosion) +$script:memoryFlushed = $false # Track if we've flushed $allLogs during this run (affects export path) +$enableParallelSwitchUsed = $EnableParallel.IsPresent +if ($enableParallelSwitchUsed) { $ParallelMode = 'On' } + +function Get-ParallelActivationDecision { + param( + [array]$QueryPlan, + [string]$ParallelMode, + [int]$MaxParallelGroups, + [int]$MaxConcurrency + ) + $ps7 = ($PSVersionTable.PSVersion.Major -ge 7) + $totalGroups = $QueryPlan.Count + $totalActivities = ($QueryPlan | ForEach-Object { $_.Activities.Count } | Measure-Object -Sum).Sum + # Auto parallel eligibility heuristic: allow auto parallel when there's at least one group + # AND either >1 group OR a single group whose planned concurrency would yield >1 partition. + # (Single-activity multi-partition scenarios — e.g., CopilotInteraction with 3 partitions — + # also benefit from parallelism.) + $singleGroupMultiPartition = ($totalGroups -eq 1) -and ($QueryPlan[0].Concurrency -gt 1) + $autoEligible = $ps7 -and ($MaxParallelGroups -gt 0) -and ($MaxConcurrency -gt 1) -and ($totalGroups -ge 1) -and (($totalGroups -gt 1) -or $singleGroupMultiPartition) + + switch ($ParallelMode) { + 'On' { + return @{ Enabled = ($ps7 -and $MaxParallelGroups -gt 0 -and $MaxConcurrency -gt 0); Reason = if ($ps7) { 'Forced On' } else { 'PS < 7 (cannot parallel)' }; AutoEligible = $autoEligible } + } + 'Auto' { + return @{ Enabled = $autoEligible; Reason = if ($autoEligible) { 'Auto criteria met' } else { 'Auto criteria not met' }; AutoEligible = $autoEligible } + } + default { + return @{ Enabled = $false; Reason = 'Mode Off'; AutoEligible = $autoEligible } + } + } +} + +$weights = if ($effectiveExplodeForProgress) { @{ Query = 0.30; Explosion = 0.60; Export = 0.10 } } else { @{ Query = 0.80; Explosion = 0.00; Export = 0.20 } } +if ($RAWInputCSV) { + try { + $weights = @{ Parsing = 0.10; Query = 0.0; Explosion = 0.80; Export = 0.10 } + } + catch {} +} +$script:originalWeights = $weights.Clone() +$script:progressState = @{ Weights = $weights; Phase = 'Query'; Parsing = @{Current = 0; Total = 0 }; Query = @{Current = 0; Total = 0 }; Explode = @{Current = 0; Total = 0 }; Export = @{Current = 0; Total = 1 } } +function Set-ProgressPhase { param([ValidateSet('Parsing', 'Query', 'Explosion', 'Export', 'Complete')] [string]$Phase, [string]$Status = ''); $script:progressState.Phase = $Phase; Update-Progress -Status $Status } +function Update-Progress { + param( + [string]$Status = '', + [int]$BatchCurrent = 0, + [int]$BatchTotal = 0, + [int]$BatchRangeStart = 0, + [int]$BatchRangeEnd = 0, + [int]$BatchStartPercent = 0, + [int]$BatchEndPercent = 0, + [bool]$BatchTotalIsEstimate = $false + ) + $w = $script:progressState.Weights; $ps = $script:progressState.Parsing; $qs = $script:progressState.Query; $es = $script:progressState.Explode; $xs = $script:progressState.Export + $pPct = if ($ps.Total -gt 0 -and $w.ContainsKey('Parsing') -and $w.Parsing -gt 0) { [double]$ps.Current / [double]$ps.Total } else { 0.0 } + $qPct = if ($qs.Total -gt 0) { [double]$qs.Current / [double]$qs.Total } else { 0.0 } + $ePct = if ($es.Total -gt 0 -and $w.Explosion -gt 0) { [double]$es.Current / [double]$es.Total } else { 0.0 } + $xPct = if ($xs.Total -gt 0) { [double]$xs.Current / [double]$xs.Total } else { 0.0 } + # Zero-record weighting: emphasize Query progression when no records retrieved yet + if ($script:progressState.Phase -eq 'Query' -and ($script:metrics.TotalRecordsFetched -eq 0)) { + $w.Query = 1.0; $w.Explosion = 0.0; $w.Export = 0.0; if ($w.ContainsKey('Parsing')) { $w.Parsing = 0.0 } + } + # Restoration: Once at least one record has been fetched, revert weights if they were temporarily overridden. + elseif ($script:progressState.Phase -eq 'Query' -and ($script:metrics.TotalRecordsFetched -gt 0)) { + if ($script:originalWeights -and $w.Query -eq 1.0 -and $w.Explosion -eq 0.0 -and $w.Export -eq 0.0) { + foreach ($key in $script:originalWeights.Keys) { $w[$key] = $script:originalWeights[$key] } + } + } + # Calculate phase-specific progress details + $phase = $script:progressState.Phase + $pDetail = if ($w.ContainsKey('Parsing') -and $w.Parsing -gt 0 -and $ps.Total -gt 0) { "{0}/{1}({2}%)" -f $ps.Current, $ps.Total, ([int]([Math]::Round($pPct * 100))) } else { '' } + $qDetail = if ($w.Query -gt 0 -and $qs.Total -gt 0) { "{0}/{1}({2}%)" -f $qs.Current, $qs.Total, ([int]([Math]::Round($qPct * 100))) } else { '' } + if ($BatchRangeStart -ge 1 -and $BatchRangeEnd -ge 1 -and $es.Total -gt 0) { + if ($BatchStartPercent -ge 0 -and $BatchEndPercent -gt 0) { + $batchTotalDisplay = if ($BatchTotalIsEstimate) { "~$BatchTotal" } else { "$BatchTotal" } + $batchInfo = if ($BatchTotal -ge 1) { " Batch: {0}/{1}({2}%-{3}%)" -f $BatchCurrent, $batchTotalDisplay, $BatchStartPercent, $BatchEndPercent } else { '' } + } + else { + $batchPct = if ($BatchTotal -gt 0 -and $BatchCurrent -gt 0) { [int]([Math]::Round(([double]$BatchCurrent / [double]$BatchTotal) * 100)) } else { 0 } + $batchTotalDisplay = if ($BatchTotalIsEstimate) { "~$BatchTotal" } else { "$BatchTotal" } + $batchInfo = if ($BatchTotal -ge 1) { " Batch: {0}/{1}({2}%)" -f $BatchCurrent, $batchTotalDisplay, $batchPct } else { '' } + } + $explosionCounts = "Records {0}-{1}/{2}{3}" -f $BatchRangeStart, $BatchRangeEnd, $es.Total, $batchInfo + } + elseif ($BatchTotal -ge 1) { + $batchPct = if ($BatchTotal -gt 0 -and $BatchCurrent -gt 0) { [int]([Math]::Round(([double]$BatchCurrent / [double]$BatchTotal) * 100)) } else { 0 } + $batchTotalDisplay = if ($BatchTotalIsEstimate) { "~$BatchTotal" } else { "$BatchTotal" } + $batchInfo = " Batch: {0}/{1}({2}%)" -f $BatchCurrent, $batchTotalDisplay, $batchPct + $explosionCounts = if ($es.Total -gt 0) { "Records {0}/{1}{2}" -f $es.Current, $es.Total, $batchInfo } else { "0/0" } + } + else { + $explosionCounts = if ($es.Total -gt 0) { "{0}/{1}({2}%)" -f $es.Current, $es.Total, ([int]([Math]::Round($ePct * 100))) } else { '0/0' } + } + $eDetail = if ($w.Explosion -gt 0) { + if ($phase -eq 'Explosion') { + " | $explosionCounts" + } + else { + " | Explosion: $explosionCounts" + } + } + else { '' } + $batchDetail = '' + $xDetail = if ($xs.Total -gt 0) { " | Export: {0}/{1}({2}%)" -f $xs.Current, $xs.Total, ([int]([Math]::Round($xPct * 100))) } else { ' | Export: 0/0' } + $parsingLabel = 'Pre-parsing JSON' + if (($AgentId -or $AgentsOnly -or $ExcludeAgents -or $PromptFilter) -and $phase -eq 'Parsing') { + $parsingLabel = 'Pre-parsing + Filtering' + } + $phasePrefix = switch ($phase) { 'Parsing' { $parsingLabel } 'Query' { 'Query' } 'Explosion' { 'Explosion' } 'Export' { 'Export' } 'Complete' { 'Complete' } default { $phase } } + if ($phase -eq 'Parsing' -and $pDetail) { + $composite = "${phasePrefix}: $pDetail$eDetail$batchDetail$xDetail" + } + elseif ($phase -eq 'Explosion' -and -not $qDetail) { + $composite = "Explosion: $explosionCounts$batchDetail$xDetail" + } + else { + $composite = if ($qDetail) { "${phasePrefix}: $qDetail$eDetail$batchDetail$xDetail" } else { "${phasePrefix}:$eDetail$batchDetail$xDetail" } + } + $statusText = if ($Status) { "$Status :: $composite" } else { $composite } + if ($statusText.Length -gt 180) { $statusText = $statusText.Substring(0, 177) + '...' } + # Placeholder for progress display compatibility +} +function Complete-Progress { + # Placeholder for progress display compatibility +} + +# Lightweight explicit progress tick to ensure visual movement in long zero-record scenarios. +function Write-ProgressTick { + # Placeholder for progress display compatibility +} + +$script:learnedActivityBlockSize = @{} +$script:globalLearnedBlockSize = $BlockHours + +function Get-QueryPlan { + param([string[]]$RequestedActivities) + # Normalize and deduplicate (DSPM logic already handled fallback, so no default here) + $normalized = @(); foreach ($a in $RequestedActivities) { if ($a -and -not ($normalized -contains $a)) { $normalized += $a } } + # If still empty after DSPM logic, something went wrong - but DSPM validation should prevent this + if ($normalized.Count -eq 0) { + Write-Host "ERROR: No activity types provided to Get-QueryPlan. This should not happen after DSPM validation." -ForegroundColor Red + exit 1 + } + $plan = @(); $i = 0 + + # DUAL-MODE QUERY PLANNING: + # Graph API mode: Combine all activity types into single group (Graph API accepts multiple operationFilters) + # EOM mode: Separate groups per activity type (Search-UnifiedAuditLog performs better with single activity) + if (-not $UseEOM) { + # Graph API mode: Single group with all activities combined + $plan += @{ + Name = "Combined: $($normalized -join ', ')"; + Group = 'GraphCombined'; + Activities = $normalized; + Concurrency = $MaxConcurrency + } + } + else { + # EOM mode: One group per activity type (sequential processing) + foreach ($a in $normalized) { + $i++ + $plan += @{ + Name = "Activity: $a"; + Group = 'EOM_Sequential'; + Activities = @($a); + Concurrency = $MaxConcurrency + } + } + } + return $plan +} + +function Update-LearnedBlockSize { + param([string]$ActivityType, [double]$BlockHours, [int]$RecordCount, [bool]$Success) + if ($Success) { + if ($RecordCount -eq $ResultSize) { + $newSize = [Math]::Max(0.083333, $BlockHours * 0.5) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + $script:globalLearnedBlockSize = [Math]::Min($script:globalLearnedBlockSize, $newSize) + Write-LogHost " → Learned: Reducing block size to $([math]::Round($newSize,2))h due to limit hit" -ForegroundColor Magenta + } + elseif ($RecordCount -gt ($ResultSize * 0.8)) { + $newSize = [Math]::Max(0.083333, $BlockHours * 0.7) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + Write-LogHost " → Learned: Reducing block size to $([math]::Round($newSize,2))h (high volume: $RecordCount records)" -ForegroundColor Magenta + } + elseif ($RecordCount -lt ($ResultSize * 0.1)) { + $newSize = [Math]::Min(24.0, $BlockHours * 1.5) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + Write-LogHost " → Learned: Increasing block size to $([math]::Round($newSize,2))h (low volume: $RecordCount records)" -ForegroundColor Magenta + } + elseif ($RecordCount -lt ($ResultSize * 0.05)) { + $newSize = [Math]::Min(24.0, $BlockHours * 2.0) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + Write-LogHost " → Learned: Increasing block size to $([math]::Round($newSize,2))h (very low volume: $RecordCount records)" -ForegroundColor Magenta + } + } else { + $newSize = [Math]::Max(0.083333, $BlockHours * 0.5) + $script:learnedActivityBlockSize[$ActivityType] = $newSize + $script:globalLearnedBlockSize = [Math]::Min($script:globalLearnedBlockSize, $newSize) + Write-LogHost " → Learned: Reducing block size to $([math]::Round($newSize,2))h due to failure" -ForegroundColor Magenta + } +} +function Get-NextSmallerBlockSize { param([double]$CurrentSize) return [Math]::Max(0.001389, $CurrentSize / 2) } # Min 2 minutes + +function Get-OptimalBlockSize { param([string]$ActivityType) if ($script:learnedActivityBlockSize.ContainsKey($ActivityType)) { return $script:learnedActivityBlockSize[$ActivityType] } elseif ($script:globalLearnedBlockSize -ne $BlockHours) { return $script:globalLearnedBlockSize } else { return $BlockHours } } + +function Invoke-ActivityTimeWindowProcessing { + param( + [Parameter(Mandatory = $true)][string]$ActivityType, + [Parameter(Mandatory = $true)][datetime]$StartDate, + [Parameter(Mandatory = $true)][datetime]$EndDate, + [int]$PartitionIndex = 1, + [int]$TotalPartitions = 1, + [bool]$UseEOMMode = $false + ) + + Write-Host "Processing $ActivityType (partition $PartitionIndex/$TotalPartitions) from $($StartDate.ToString('yyyy-MM-dd HH:mm')) to $($EndDate.ToString('yyyy-MM-dd HH:mm'))..." -ForegroundColor White + $blockHours = Get-OptimalBlockSize -ActivityType $ActivityType + Write-Host " Using initial block size: $blockHours hours" -ForegroundColor DarkCyan + + $allResults = New-Object System.Collections.ArrayList + $current = $StartDate + $blockNumber = 1 + + while ($current -lt $EndDate) { + # Show progress BEFORE block processing to ensure visibility (before log output clears it) + Write-ProgressTick + + if ($script:circuitBreakerOpen) { + if ($script:circuitBreakerOpenUntil -and (Get-Date) -lt $script:circuitBreakerOpenUntil) { + Write-LogHost " Circuit breaker OPEN until $($script:circuitBreakerOpenUntil.ToString('HH:mm:ss')) – skipping remaining blocks for $ActivityType" -ForegroundColor Red + break + } else { + $script:circuitBreakerOpen = $false + $script:consecutiveBlockFailures = 0 + Write-LogHost " Circuit breaker cooldown elapsed – resuming block processing" -ForegroundColor DarkGreen + } + } + if ($script:learnedActivityBlockSize.ContainsKey($ActivityType)) { + $blockHours = $script:learnedActivityBlockSize[$ActivityType] + } + + $blockEnd = $current.AddHours($blockHours) + if ($blockEnd -gt $EndDate) { $blockEnd = $EndDate } + + $actualBlockHours = [math]::Round(($blockEnd - $current).TotalHours, 2) + Write-Host " Block $blockNumber`: $($current.ToString('yyyy-MM-dd HH:mm')) to $($blockEnd.ToString('yyyy-MM-dd HH:mm')) ($($actualBlockHours)h)" -ForegroundColor Yellow + + try { + $results = Invoke-PurviewAuditQuery -StartDate $current -EndDate $blockEnd -Operations $ActivityType -ResultSize $ResultSize -UserIds $script:targetUsers -UseEOMMode $UseEOMMode + + if ($results -and $results.Count -gt 0) { + # Safe add - handle both array and single object + if ($results -is [Array]) { + foreach ($item in $results) { [void]$allResults.Add($item) } + } else { + [void]$allResults.Add($results) + } + Write-Host " Added $($results.Count) records (total: $($allResults.Count))" -ForegroundColor Green + Update-LearnedBlockSize -ActivityType $ActivityType -BlockHours $actualBlockHours -RecordCount $results.Count -Success $true + $script:consecutiveBlockFailures = 0 + } + else { + Write-Host " No records found in this block" -ForegroundColor Gray + $script:consecutiveBlockFailures = 0 + } + } + catch { + Write-Host " Block failed: $($_.Exception.Message)" -ForegroundColor Red + Update-LearnedBlockSize -ActivityType $ActivityType -BlockHours $actualBlockHours -RecordCount 0 -Success $false + $script:consecutiveBlockFailures++ + $attemptNum = $script:consecutiveBlockFailures + $expDelay = [math]::Min($BackoffMaxSeconds, $BackoffBaseSeconds * [math]::Pow(2, ($attemptNum - 1))) + $jitterMs = Get-Random -Minimum 150 -Maximum 750 + $totalDelaySec = [math]::Round($expDelay,2) + [math]::Round($jitterMs/1000,2) + try { $script:metrics.BackoffTotalDelaySeconds += $totalDelaySec } catch {} + Write-LogHost " Reliability: Backoff delay $([math]::Round($expDelay,2))s + jitter $([math]::Round($jitterMs/1000,2))s (attempt $attemptNum)" -ForegroundColor DarkYellow + Start-Sleep -Seconds ([int][math]::Ceiling($expDelay)) + Start-Sleep -Milliseconds $jitterMs + if ($script:consecutiveBlockFailures -ge $CircuitBreakerThreshold) { + $script:circuitBreakerOpen = $true + $script:circuitBreakerOpenUntil = (Get-Date).AddSeconds($CircuitBreakerCooldownSeconds) + try { $script:metrics.CircuitBreakerTrips++ } catch {} + Write-LogHost " CIRCUIT BREAKER TRIPPED after $script:consecutiveBlockFailures consecutive block failures – cooling down for $CircuitBreakerCooldownSeconds seconds (until $($script:circuitBreakerOpenUntil.ToString('HH:mm:ss')))" -ForegroundColor Magenta + break + } + if ($blockHours -gt 0.5) { + $smallerBlockHours = Get-NextSmallerBlockSize -CurrentSize $blockHours + Write-Host " Retrying with smaller $smallerBlockHours hour block..." -ForegroundColor Yellow + + try { + $blockEnd = $current.AddHours($smallerBlockHours) + if ($blockEnd -gt $EndDate) { $blockEnd = $EndDate } + + $results = Invoke-PurviewAuditQuery -StartDate $current -EndDate $blockEnd -Operations $ActivityType -ResultSize $ResultSize -UserIds $script:targetUsers -UseEOMMode $UseEOMMode + + if ($results -and $results.Count -gt 0) { + # Safe add - handle both array and single object + if ($results -is [Array]) { + foreach ($item in $results) { [void]$allResults.Add($item) } + } else { + [void]$allResults.Add($results) + } + Write-Host " Smaller block succeeded: $($results.Count) records" -ForegroundColor Green + Update-LearnedBlockSize -ActivityType $ActivityType -BlockHours $smallerBlockHours -RecordCount $results.Count -Success $true + $blockHours = $smallerBlockHours + $script:consecutiveBlockFailures = 0 + } + } + catch { + Write-Host " Smaller block also failed: $($_.Exception.Message)" -ForegroundColor Red + $script:consecutiveBlockFailures++ + $attemptNum = $script:consecutiveBlockFailures + $expDelay = [math]::Min($BackoffMaxSeconds, $BackoffBaseSeconds * [math]::Pow(2, ($attemptNum - 1))) + $jitterMs = Get-Random -Minimum 150 -Maximum 750 + $totalDelaySec = [math]::Round($expDelay,2) + [math]::Round($jitterMs/1000,2) + try { $script:metrics.BackoffTotalDelaySeconds += $totalDelaySec } catch {} + Write-LogHost " Reliability: Backoff delay $([math]::Round($expDelay,2))s + jitter $([math]::Round($jitterMs/1000,2))s (attempt $attemptNum)" -ForegroundColor DarkYellow + Start-Sleep -Seconds ([int][math]::Ceiling($expDelay)) + Start-Sleep -Milliseconds $jitterMs + if ($script:consecutiveBlockFailures -ge $CircuitBreakerThreshold) { + $script:circuitBreakerOpen = $true + $script:circuitBreakerOpenUntil = (Get-Date).AddSeconds($CircuitBreakerCooldownSeconds) + try { $script:metrics.CircuitBreakerTrips++ } catch {} + Write-LogHost " CIRCUIT BREAKER TRIPPED after $script:consecutiveBlockFailures consecutive block failures – cooling down for $CircuitBreakerCooldownSeconds seconds (until $($script:circuitBreakerOpenUntil.ToString('HH:mm:ss')))" -ForegroundColor Magenta + break + } + } + } + } + + try { + if ($script:progressState.Query.Current -ge $script:progressState.Query.Total) { + $script:progressState.Query.Total += 1 + } + $script:progressState.Query.Current += 1 + $script:progressBlocksCompleted = ($script:progressBlocksCompleted + 1) + $script:progressBlockHoursSum = ($script:progressBlockHoursSum + $actualBlockHours) + if ($script:progressBlocksCompleted -gt 0) { + # --- Progress Estimation Logic (multi-partition accurate) --- + # 1. Estimate remaining blocks in the CURRENT partition. + # 2. Add an estimate for yet-to-start partitions based on the average blocks/partition so far. + # 3. Enforce a monotonic (non-decreasing) Query.Total so percent cannot jump to 100% early. + # This prevents Query.Total from shrinking between partitions and reporting premature 100% + # completion when later partitions have not yet started. + $avgBlock = $script:progressBlockHoursSum / $script:progressBlocksCompleted + $elapsedHours = $script:progressBlockHoursSum + $currentPartitionRangeHours = ($EndDate - $StartDate).TotalHours + $remainingHoursCurrentPartition = [Math]::Max(0.0, $currentPartitionRangeHours - $elapsedHours) + $remainingBlocksEstCurrent = if ($avgBlock -gt 0) { [Math]::Ceiling($remainingHoursCurrentPartition / $avgBlock) } else { 0 } + $remainingPartitions = if ($TotalPartitions -gt $PartitionIndex) { $TotalPartitions - $PartitionIndex } else { 0 } + $avgBlocksPerCompletedPartition = if ($PartitionIndex -gt 0) { [double]$script:progressBlocksCompleted / [double]$PartitionIndex } else { [double]$script:progressBlocksCompleted } + $futurePartitionBlocksEst = if ($remainingPartitions -gt 0 -and $avgBlocksPerCompletedPartition -gt 0) { [int][Math]::Ceiling($avgBlocksPerCompletedPartition * $remainingPartitions) } else { 0 } + $newCalcGlobal = $script:progressBlocksCompleted + $remainingBlocksEstCurrent + $futurePartitionBlocksEst + # Apply optional smoothing but NEVER allow total to decrease (monotonic total). + if ($ProgressSmoothingAlpha -gt 0 -and $script:progressState.Query.Total -gt 0) { + $smoothed = [int]([Math]::Round(($ProgressSmoothingAlpha * $newCalcGlobal) + ((1 - $ProgressSmoothingAlpha) * $script:progressState.Query.Total))) + $newTotalCandidate = [Math]::Max($script:progressState.Query.Total, $smoothed, $newCalcGlobal) + } else { + $newTotalCandidate = [Math]::Max($script:progressState.Query.Total, $newCalcGlobal) + } + $script:progressState.Query.Total = [Math]::Max($script:progressState.Query.Total, $newTotalCandidate, $script:progressBlocksCompleted) + } + Update-Progress + # Explicit tick for visibility even if Update-Progress weighting collapses. + Write-ProgressTick + } + catch {} + + $current = $blockEnd + $blockNumber++ + } + + Write-Host " Completed $ActivityType (partition $PartitionIndex/$TotalPartitions)`: $($allResults.Count) total records" -ForegroundColor Green + return $allResults.ToArray() +} + +# Note: Write-Log and Write-LogHost are defined earlier in the script (near line 670) + +function Open-CsvWriter { + param([string]$Path, [string[]]$Columns) + $enc = New-Object System.Text.UTF8Encoding($false) + # OPTIMIZATION: Use 1MB StreamWriter buffer (default is 1KB) to reduce write syscalls + $script:PAX_CsvWriter = [System.IO.StreamWriter]::new($Path, $false, $enc, 1048576) + $escapedCols = New-Object System.Collections.Generic.List[string] + foreach ($col in $Columns) { + $c = [string]$col + $needsQuote = ($c -match '[",\r\n]') -or $c.StartsWith(' ') -or $c.EndsWith(' ') + $escaped = $c -replace '"', '""' + if ($needsQuote) { $escaped = '"' + $escaped + '"' } + $escapedCols.Add($escaped) | Out-Null + } + $script:PAX_CsvWriter.WriteLine(($escapedCols -join ',')) +} +function Close-CsvWriter { if ($script:PAX_CsvWriter) { try { $script:PAX_CsvWriter.Flush(); $script:PAX_CsvWriter.Dispose() } catch {}; Remove-Variable PAX_CsvWriter -Scope Script -ErrorAction SilentlyContinue } } +function Write-CsvRows { + param([System.Collections.IEnumerable]$Rows, [string[]]$Columns) + if (-not $Rows) { return } + if (-not $script:PAX_CsvWriter) { throw "CSV writer not initialized" } + + # Pre-compile regex once (not per-cell) for significant performance gain + $needsQuotePattern = [regex]::new('[",\r\n]', [System.Text.RegularExpressions.RegexOptions]::Compiled) + + # OPTIMIZATION: Pre-allocate larger buffer (4MB) to reduce write syscalls + $sb = New-Object System.Text.StringBuilder(4194304) + $colCount = $Columns.Count + $fieldValues = New-Object string[] $colCount # Reuse array instead of creating List per row + + # OPTIMIZATION: Build column index lookup table for O(1) access by name + # This eliminates per-cell string lookups which were the main bottleneck + $columnIndex = @{} + for ($i = 0; $i -lt $colCount; $i++) { + $columnIndex[$Columns[$i]] = $i + } + + foreach ($row in $Rows) { + if ($null -eq $row) { continue } + + # Reset field values (faster than creating new array) + for ($i = 0; $i -lt $colCount; $i++) { $fieldValues[$i] = "" } + + # For hashtables, iterate keys and map to column index (much faster than iterating columns) + if ($row -is [hashtable]) { + foreach ($key in $row.Keys) { + if ($columnIndex.ContainsKey($key)) { + $idx = $columnIndex[$key] + $val = $row[$key] + + if ($null -eq $val) { continue } + + # Handle arrays/collections + if ($val -is [System.Collections.IEnumerable] -and -not ($val -is [string])) { + try { $val = ($val | ForEach-Object { if ($_ -ne $null) { [string]$_ } else { '' } }) -join ';' } catch { $val = [string]$val } + } + + $s = [string]$val + # Use pre-compiled regex and avoid method calls where possible + if ($s.Length -gt 0 -and ($needsQuotePattern.IsMatch($s) -or $s[0] -eq ' ' -or $s[$s.Length - 1] -eq ' ')) { + $s = '"' + ($s -replace '"', '""') + '"' + } + $fieldValues[$idx] = $s + } + } + } else { + # OPTIMIZED: For PSObjects, iterate only populated properties (not all columns) + # This reduces iterations from ~163 columns to ~50 actual properties per row + foreach ($prop in $row.PSObject.Properties) { + $key = $prop.Name + if (-not $columnIndex.ContainsKey($key)) { continue } + + $idx = $columnIndex[$key] + $val = $prop.Value + + if ($null -eq $val) { continue } + + # Handle arrays/collections + if ($val -is [System.Collections.IEnumerable] -and -not ($val -is [string])) { + try { $val = ($val | ForEach-Object { if ($_ -ne $null) { [string]$_ } else { '' } }) -join ';' } catch { $val = [string]$val } + } + + $s = [string]$val + if ($s.Length -gt 0 -and ($needsQuotePattern.IsMatch($s) -or $s[0] -eq ' ' -or $s[$s.Length - 1] -eq ' ')) { + $s = '"' + ($s -replace '"', '""') + '"' + } + $fieldValues[$idx] = $s + } + } + + [void]$sb.AppendLine(($fieldValues -join ',')) + # Flush at 4MB (increased from 1MB to reduce write syscalls) + if ($sb.Length -gt 4194304) { + $script:PAX_CsvWriter.Write($sb.ToString()) + [void]$sb.Clear() + } + } + if ($sb.Length -gt 0) { $script:PAX_CsvWriter.Write($sb.ToString()) } +} + +function Test-AgentFilter { + param( + [Parameter(Mandatory = $true)] + $ParsedAuditData, + [string[]]$AgentIdFilter, + [bool]$AgentsOnlyFilter + ) + if (-not $AgentIdFilter -and -not $AgentsOnlyFilter) { + return $true + } + $recordAgentId = $null + try { + if ($ParsedAuditData.AgentId) { + $recordAgentId = [string]$ParsedAuditData.AgentId + } + } + catch { + return $false + } + if ($AgentsOnlyFilter) { + if ([string]::IsNullOrWhiteSpace($recordAgentId)) { + return $false + } + if (-not $AgentIdFilter) { + return $true + } + } + if ($AgentIdFilter) { + if ([string]::IsNullOrWhiteSpace($recordAgentId)) { + return $false + } + foreach ($filterId in $AgentIdFilter) { + if ($recordAgentId -eq $filterId) { + return $true + } + } + return $false + } + return $true +} + +# Ensure output directory exists +if (-not (Test-Path $OutputPath)) { New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null } + +# Generate output filename with proper extension +$fileExtension = if ($ExportWorkbook) { "xlsx" } else { "csv" } +$filePrefix = "Purview_Audit" + +# Determine initial combine mode (needed early for filename decision) +$isCsv = (-not $ExportWorkbook) +$initialCsvCombine = if ($RAWInputCSV -and $isCsv) { $true } elseif ($isCsv) { $CombineOutput.IsPresent } else { $false } + +# Predict the combined-audit CSV leaf name upfront. When the run is gated to a +# single activity type ($ActivityTypes count == 1 and -IncludeM365Usage is NOT +# expanding the set), name the file Purview_Audit_UsageActivity__.csv +# directly so the log file (derived from $OutputFile basename) matches. The +# post-export dynamic single-activity downgrade is a secondary safety net for +# runs where the resulting data happens to contain only one activity type. +$_combinedAuditLeaf = "Purview_Audit_UsageActivity_CombinedActivityTypes_$global:ScriptRunTimestamp.csv" +if ($ActivityTypes -and @($ActivityTypes).Count -eq 1 -and -not $IncludeM365Usage) { + $_onlyActivityType = [string]$ActivityTypes[0] + if (-not [string]::IsNullOrWhiteSpace($_onlyActivityType)) { + $_safeActivityType = $_onlyActivityType -replace '[\\/:*?"<>|]', '_' + $_combinedAuditLeaf = "Purview_Audit_UsageActivity_${_safeActivityType}_$global:ScriptRunTimestamp.csv" + } +} + +# Determine script mode for logging and validation +$scriptMode = if ($RAWInputCSV) { + "Replay (RAWInputCSV)" +} elseif ($ExplodeDeep) { + "Deep Column Explosion (-ExplodeDeep)" +} elseif ($ExplodeArrays -or $ForcedRawInputCsvExplosion) { + if ($ForcedRawInputCsvExplosion -and -not $ExplodeArrays.IsPresent -and -not $ExplodeDeep.IsPresent) { "Array Explosion (-ExplodeArrays, RAWInput implied)" } else { "Array Explosion (-ExplodeArrays)" } +} else { + "Standard (1:1, no explosion)" +} + +# Resolve OutputFile path +# OnlyUserInfo mode: Output file is EntraUsers_MAClicensing (log file will match) +if ($OnlyUserInfo) { + $fileExtension = if ($ExportWorkbook) { "xlsx" } else { "csv" } + $OutputFile = Join-Path $OutputPath "EntraUsers_MAClicensing_$global:ScriptRunTimestamp.$fileExtension" +} +elseif ($AppendFile) { + # User provided filename, full local path, or remote (SharePoint/Fabric) URL. + # Classify the form BEFORE doing any path join — IsPathRooted returns $false + # for "https://..." which would cause Join-Path to mangle the URL into the + # local scratch dir. The remote pre-flight downstream pulls the remote file + # into local scratch and then redirects $AppendFile to that local copy. + $appendInputTier = if (Get-Command -Name Get-PathTier -ErrorAction SilentlyContinue) { + # Suppress Get-PathTier's exit-on-invalid behaviour for relative filenames + # by short-circuiting based on form first. + if ($AppendFile -match '^https?://') { Get-PathTier $AppendFile -SwitchName 'AppendFile' } + elseif ([System.IO.Path]::IsPathRooted($AppendFile)) { 'Local' } + else { 'Local' } # relative filename → resolves under $OutputPath (Local) + } else { 'Local' } + + # Rollup-output detection. The embedded Python processors emit rollup outputs + # with one of the following suffixes on the original raw stem (the M365Bundle + # processor appends a current-run timestamp; the CopilotInteraction processor + # does not): + # CopilotInteraction processor : '_Interactions.csv' + # '_Users.csv' + # M365Bundle processor (rollup) : '_Rollup_.csv' + # '_UserStats_.csv' + # '_SessionCohort_.csv' + # '_SessionStats_.csv' + # M365Bundle processor (event) : '_Exploded.csv' + # When the user passes any such file as -AppendFile UNDER -Rollup / -RollupPlusRaw + # (or under -IncludeM365Usage event-level mode), they are pointing at the previous + # run's ROLLUP/EXPLOSION target — NOT at a raw Purview audit CSV. The raw audit + # writer must NOT be re-pointed to the rollup leaf, or it will overwrite the + # rollup target with raw audit rows and produce zero actual raw output (defeating + # the whole point of -RollupPlusRaw). Instead: keep $OutputFile on its natural + # timestamped raw-audit name, and let -AppendFile flow through downstream: + # • CopilotInteraction: seed-map source merged inside Python. + # • M365Bundle (rollup): union-merged PS-side via Merge-M365RollupCsv after + # the processor emits its timestamped Rollup; sidecars are then regenerated + # from the merged union via --rebuild-sidecars-from-rollup. + # • M365Bundle (event): --exclude-record-ids source for de-duplication. + $appendLeaf = [System.IO.Path]::GetFileName($AppendFile) + # Rollup-shape match accepts both the anchored '_.csv' shape + # (typical for customer-renamed long-lived targets) and the timestamped + # '__.csv' shape (when the customer points + # AppendFile at a previous run's raw output without renaming). + $appendIsRollupShaped = ($appendLeaf -match '_(Interactions|Rollup|UserStats|SessionCohort|Exploded)(?:_\d{8}_\d{6})?\.csv$') + # The embedded Python processor only runs under -Rollup / -RollupPlusRaw. + # Outside those switches a rollup-shaped leaf is just a coincidence (the user + # is presumably appending to a real raw-audit CSV that happens to share the + # suffix), so leave the legacy raw-append behaviour intact in that case. + $rollupActive = ($Rollup -or $RollupPlusRaw) + $treatAppendAsRollupTarget = ($appendIsRollupShaped -and $rollupActive) + + # (c) Visibility NOTE: a -Rollup / -RollupPlusRaw run whose -AppendFile target leaf does + # not match the rollup-shaped naming convention is still handled correctly (a fresh + # raw-audit input is written this run and merged into the target), but log a NOTE so this + # class of target naming is visible in the log rather than silent. The M365 rollup path + # enforces its own leaf rules below with a hard error, so it is excluded here. + if ($rollupActive -and -not $appendIsRollupShaped -and -not $IncludeM365Usage) { + Write-LogHost (" NOTE: -AppendFile target leaf '{0}' does not match the rollup-shaped naming convention (expected an '_Interactions' / '_Rollup' / '_UserStats' / '_SessionCohort' / '_Exploded' suffix, optionally with a _YYYYMMDD_HHMMSS stamp). The rollup writes a fresh raw-audit input this run and merges into the target; consider naming rollup append targets accordingly." -f $appendLeaf) -ForegroundColor Yellow + } + + # M365 rollup-mode leaf validation. When -IncludeM365Usage is set with -Rollup + # or -RollupPlusRaw, the embedded M365Bundle processor produces 4 outputs + # ('_Rollup_.csv', '_UserStats_.csv', '_SessionCohort_.csv', + # '_SessionStats_.csv'). Only the Rollup file supports the union-merge + # semantic as the AppendFile anchor (PS-side via Merge-M365RollupCsv); + # UserStats, SessionCohort, and SessionStats are recomputed/merged sidecars + # anchored off the AppendFile leaf stem at the post-processor regen step. + # Pointing -AppendFile at a sidecar (or at a CopilotInteraction _Interactions + # leaf, or at an M365 event-level _Exploded leaf) would either silently feed + # the processor the wrong schema or anchor sidecars at the wrong stem, so we + # reject upfront with a leaf-specific error. + if ($IncludeM365Usage -and $rollupActive) { + $m365LeafErr = $null + if ($appendLeaf -match '_UserStats(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_UserStats.csv' is a recomputed sidecar of the M365 rollup; it cannot be the merge anchor." } + elseif ($appendLeaf -match '_SessionCohort(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_SessionCohort.csv' is a recomputed sidecar of the M365 rollup; it cannot be the merge anchor." } + elseif ($appendLeaf -match '_SessionStats(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_SessionStats.csv' is a recomputed sidecar of the M365 rollup; it cannot be the merge anchor." } + elseif ($appendLeaf -match '_Interactions(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_Interactions.csv' is the CopilotInteraction rollup shape, not M365. Remove -IncludeM365Usage or supply a '_Rollup.csv' leaf." } + elseif ($appendLeaf -match '_Exploded(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "'_Exploded.csv' is the M365 event-level output. To append under -Rollup/-RollupPlusRaw you need a '_Rollup.csv' anchor (event-level append via --exclude-record-ids is not currently exposed through -AppendFile)." } + elseif ($appendLeaf -notmatch '_Rollup(?:_\d{8}_\d{6})?\.csv$') { $m365LeafErr = "Under -IncludeM365Usage with -Rollup/-RollupPlusRaw the -AppendFile leaf must end in '_Rollup.csv' or '_Rollup_.csv' (the M365 rollup anchor). The UserStats, SessionCohort, and SessionStats sidecars are derived from the anchor stem and overwritten in-place." } + if ($m365LeafErr) { + Write-Host "" -ForegroundColor Red + Write-Host ("ERROR: -AppendFile leaf '{0}' is not valid in this mode: {1}" -f $appendLeaf, $m365LeafErr) -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "Expected: a '_Rollup.csv' or '_Rollup_.csv' leaf (filename, full local path, or remote URL)." -ForegroundColor Yellow + Write-Host "Example: -AppendFile 'Purview_Audit_UsageActivity_CombinedActivityTypes_20260101_120000_Rollup.csv'" -ForegroundColor Yellow + Write-Host "" -ForegroundColor Yellow + Write-Host "If you do not yet have a Rollup target on the destination, run once WITHOUT -AppendFile to produce the initial timestamped Rollup (plus its UserStats / SessionCohort / SessionStats sidecars)," -ForegroundColor Gray + Write-Host "then point -AppendFile at that Rollup on subsequent runs (keep the timestamp or rename — both shapes are accepted)." -ForegroundColor Gray + exit 1 + } + } + + if ($rollupActive) { + # Under -Rollup / -RollupPlusRaw the writer always produces a FRESH raw-audit CSV + # (the rollup processor's --purview input); the -AppendFile target is a separate + # rollup artifact consumed only for seeding + merge. Force $OutputFile to the + # precomputed combined-audit leaf (single-activity name when $ActivityTypes is + # exactly one type, otherwise CombinedActivityTypes) REGARDLESS of whether the target + # leaf matches the rollup-shaped naming convention. This prevents a non-rollup-shaped + # target leaf from routing $OutputFile to the target's own scratch path — where the + # remote pre-flight download of the target would share that path and the rollup would + # ingest the already-rolled-up target instead of this run's fresh records. The + # post-export dynamic downgrade is suppressed under -AppendFile, so the upfront + # prediction is what the log file (and the final raw artifact) inherit. + $OutputFile = Join-Path $OutputPath $_combinedAuditLeaf + } + elseif ($appendInputTier -eq 'SharePoint' -or $appendInputTier -eq 'Fabric') { + # Remote URL: the final $OutputFile is the local scratch path the remote + # pre-flight will populate (Join-Path $OutputPath (GetFileName URL)). + $remoteName = [System.IO.Path]::GetFileName($AppendFile) + $OutputFile = Join-Path $OutputPath $remoteName + } + elseif ([System.IO.Path]::IsPathRooted($AppendFile)) { + # Full local path provided + $OutputFile = $AppendFile + } else { + # Relative filename - combine with OutputPath + $OutputFile = Join-Path $OutputPath $AppendFile + } + + # Validate file exists. For Local-tier outputs this is a direct Test-Path. For + # SharePoint / Fabric URLs, defer the existence check to the remote helpers + # downstream (the remote pre-flight at Connect time will pull the file into + # scratch; if the source does not exist there, the pull will raise a clear + # remote error). + # Skip the existence check entirely under -Rollup / -RollupPlusRaw — $OutputFile is + # now a fresh raw-audit path the writer will create, and the AppendFile target's + # existence is verified by the rollup pre-seed block (Test-Path -LiteralPath + # $AppendFile in the rollup setup downstream). + if (-not $rollupActive -and $appendInputTier -eq 'Local' -and -not (Test-Path $OutputFile)) { + Write-Host "ERROR: Cannot append to file - file does not exist: $OutputFile" -ForegroundColor Red + Write-Host "" -ForegroundColor Yellow + Write-Host "The file must exist before you can append to it." -ForegroundColor Yellow + Write-Host "Either:" -ForegroundColor Green + Write-Host " 1. Create the file first by running without -AppendFile" -ForegroundColor Green + Write-Host " 2. Verify the path and filename are correct" -ForegroundColor Green + exit 1 + } + + # Note: Column validation happens after banner display +} +elseif ($ExportWorkbook) { + # Excel workbook mode - determine final filename upfront so log file matches + if ($CombineOutput) { + # Single-tab workbook + if ($IncludeUserInfo -and -not $UseEOM) { + $OutputFile = Join-Path $OutputPath "Purview_Audit_CombinedUsageActivity_EntraUsers_MAClicensing_$global:ScriptRunTimestamp.xlsx" + } else { + $OutputFile = Join-Path $OutputPath "Purview_Audit_CombinedUsageActivity_$global:ScriptRunTimestamp.xlsx" + } + } else { + # Multi-tab workbook + $OutputFile = Join-Path $OutputPath "Purview_Audit_MultiTab_$global:ScriptRunTimestamp.xlsx" + } +} elseif ($isCsv -and $initialCsvCombine) { + # CSV combined-activity naming; EntraUsers exported separately. + # Use the precomputed combined-audit leaf: when $ActivityTypes contains exactly + # one type (and -IncludeM365Usage is not expanding the set), the leaf is + # Purview_Audit_UsageActivity__.csv so the log file (derived from + # $OutputFile basename downstream) matches. Otherwise it is CombinedActivityTypes, + # and the post-export dynamic single-activity downgrade renames the file (and + # updates $OutputFile) when the actual data ends up containing a single type. + $OutputFile = Join-Path $OutputPath $_combinedAuditLeaf +} else { + $OutputFile = Join-Path $OutputPath "${filePrefix}_$global:ScriptRunTimestamp.$fileExtension" +} + +# -Deidentify data-consistency guard. Refuse to mix deidentified and raw data in an append +# target (corrupts user joins / distinct counts), and refuse a NON-rollup append into an +# already-deidentified file (the whole-file post-pass would re-hash existing rows). Runs for +# any -AppendFile / -AppendUserInfo, in both deid and non-deid runs. Local targets only; +# remote (SharePoint/Fabric) append URLs are validated downstream after scratch download. +if ($AppendFile -or $AppendUserInfo) { + $_deidResolve = { + param([string]$p) + if ([string]::IsNullOrWhiteSpace($p) -or $p -match '^https?://') { return $null } + if ([System.IO.Path]::IsPathRooted($p)) { return $p } + return (Join-Path $OutputPath $p) + } + $_deidGuardTargets = @() + if ($AppendFile) { + # Rollup appends target the user's rollup file ($AppendFile); non-rollup targets $OutputFile. + $_afTarget = if ($Rollup -or $RollupPlusRaw) { & $_deidResolve $AppendFile } else { $OutputFile } + if ($_afTarget) { $_deidGuardTargets += @{ Path = $_afTarget; Label = '-AppendFile' } } + } + if ($AppendUserInfo) { + $_auiTarget = & $_deidResolve $AppendUserInfo + if ($_auiTarget) { $_deidGuardTargets += @{ Path = $_auiTarget; Label = '-AppendUserInfo' } } + } + if ($_deidGuardTargets.Count -gt 0) { + try { + Assert-PaxDeidAppendConsistency -Targets $_deidGuardTargets -IsRollup ([bool]($Rollup -or $RollupPlusRaw)) + } catch { + Write-Host "" -ForegroundColor Red + Write-Host ("ERROR: {0}" -f $_.Exception.Message) -ForegroundColor Red + exit 1 + } + } +} + +# Ensure output directory exists after possible OutputPath override +if (-not (Test-Path $OutputPath)) { New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null } + +# When ExportWorkbook mode, set up intermediate CSV path (OutputFile stays as final .xlsx for log naming) +if ($ExportWorkbook -and -not $AppendFile) { + # CSV intermediate file uses same base name but .csv extension + $script:CsvOutputFile = Join-Path $OutputPath ([System.IO.Path]::GetFileNameWithoutExtension($OutputFile) + ".csv") +} else { + $script:CsvOutputFile = $OutputFile +} + +# ============================================================ +# CHECKPOINT SYSTEM: Initialize for new runs or set paths for resume +# ============================================================ +# Checkpoint is enabled for ALL auth modes (AppRegistration, WebLogin, DeviceCode) +# Enables resume after Ctrl+C, network interruptions, system restarts, or any failure +$script:CheckpointEnabled = (-not $RAWInputCSV) -and (-not $OnlyUserInfo) + +# NOTE: Use $ResumeSpecified here (set early via RemainingArgs parsing) rather than $script:IsResumeMode +# which is only set later during resume detection. This skips new checkpoint creation when -Resume is specified. +if ($ResumeSpecified) { + # Resume mode: checkpoint paths will be set later during resume detection + # Skip all checkpoint initialization here - it will be handled in the resume detection block + $script:FinalOutputPath = $null # Will be set during resume + $script:PartialOutputPath = $null # Will be set during resume + # Also skip log file setup - will be set after checkpoint is loaded + $script:DeferLogFileSetup = $true +} +elseif ($script:CheckpointEnabled) { + # New run with checkpoint enabled: Add _PARTIAL suffix + $script:FinalOutputPath = $OutputFile + $dir = Split-Path $OutputFile -Parent + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) + $ext = [System.IO.Path]::GetExtension($OutputFile) + $script:PartialOutputPath = Join-Path $dir "${baseName}_PARTIAL${ext}" + $OutputFile = $script:PartialOutputPath + # For ExportWorkbook mode, CsvOutputFile must use .csv extension (not .xlsx from PartialOutputPath) + if ($ExportWorkbook) { + $script:CsvOutputFile = Join-Path $dir "${baseName}_PARTIAL.csv" + } else { + $script:CsvOutputFile = $script:PartialOutputPath + } + + # Initialize checkpoint file for this run with ALL parameters for complete state restoration + $baseFileName = Split-Path $script:FinalOutputPath -Leaf + $allParams = @{ + # Date range + StartDate = $StartDate + EndDate = $EndDate + # Activity/Record filtering + ActivityTypes = $ActivityTypes + RecordTypes = $RecordTypes + ServiceTypes = $ServiceTypes + UserIds = $UserIds + GroupNames = $GroupNames + # Agent filtering + AgentId = $AgentId + AgentsOnly = $AgentsOnly.IsPresent + ExcludeAgents = $ExcludeAgents.IsPresent + # Prompt filtering + PromptFilter = $PromptFilter + # Schema/Explosion + ExplodeArrays = $ExplodeArrays.IsPresent + ExplodeDeep = $ExplodeDeep.IsPresent + FlatDepth = $FlatDepth + StreamingSchemaSample = $StreamingSchemaSample + StreamingChunkSize = $StreamingChunkSize + # M365/User info + IncludeM365Usage = $IncludeM365Usage.IsPresent + IncludeUserInfo = $IncludeUserInfo.IsPresent + IncludeCopilotInteraction = $IncludeCopilotInteraction.IsPresent + ExcludeCopilotInteraction = $ExcludeCopilotInteraction.IsPresent + IncludeAgent365Info = $IncludeAgent365Info.IsPresent + OnlyAgent365Info = $OnlyAgent365Info.IsPresent + # Partitioning + BlockHours = $BlockHours + PartitionHours = $PartitionHours + MaxPartitions = $MaxPartitions + # Output + ExportWorkbook = $ExportWorkbook.IsPresent + CombineOutput = $CombineOutput.IsPresent + Deidentify = $Deidentify.IsPresent + # FillerLabel: resolved hierarchy-filler mode (none|self|manager|fixed) + literal label. + FillerLabel = $script:HierarchyFillMode + FillerLabelText = $script:HierarchyFillLabel + # uploads to the original destinations — without these, Initialize-CheckpointForNewRun + # persists empty strings and the resume restore has nothing to rehydrate). + OutputPathUserInfo = $OutputPathUserInfo + OutputPathAgent365Info = $OutputPathAgent365Info + OutputPathLog = $OutputPathLog + AppendFile = $AppendFile + AppendUserInfo = $AppendUserInfo + AppendAgent365Info = $AppendAgent365Info + # Rollup mode (required so the resume restore re-arms -Rollup / -RollupPlusRaw + # correctly; without these the post-processor falls back to '-Rollup' semantics + # and deletes the raw EntraUsers CSV that -RollupPlusRaw is supposed to preserve). + Rollup = $Rollup.IsPresent + RollupPlusRaw = $RollupPlusRaw.IsPresent + Dashboard = $Dashboard + OutputPathDefenderUsage = $OutputPathDefenderUsage + AppendDefenderUsage = $AppendDefenderUsage + DisableAISIDDeltaCache = $DisableAISIDDeltaCache.IsPresent + # Auth (no secrets) + Auth = $Auth + TenantId = $TenantId + ClientId = $ClientId + # Other + ResultSize = $ResultSize + MaxConcurrency = $MaxConcurrency + MaxMemoryMB = $MaxMemoryMB + ExplosionThreads = $ExplosionThreads + UseEOM = $UseEOM.IsPresent + AutoCompleteness = $AutoCompleteness.IsPresent + IncludeTelemetry = $IncludeTelemetry.IsPresent + VerifyPartitionStability = $VerifyPartitionStability.IsPresent + # Hybrid directory enrichment source (path/URL only; no supplemental data VALUES are ever + # persisted). Restored on -Resume so the original supplemental input is reused. + UserInfoSupplement = $UserInfoSupplement + } + Initialize-CheckpointForNewRun -OutputPath $OutputPath -BaseOutputFileName $baseFileName -RunTimestamp $global:ScriptRunTimestamp -StartDate (script:Parse-DateSafe $StartDate) -EndDate (script:Parse-DateSafe $EndDate) -AllParameters $allParams | Out-Null +} +else { + # No checkpoint needed (AppRegistration mode or RAWInputCSV or OnlyUserInfo) + $script:FinalOutputPath = $OutputFile + $script:PartialOutputPath = $null +} + +# Update LogFile to match OutputFile base name (extension swapped to .log). +# When -OutputPathLog is bound to a Local path or Fabric Files/ URL, route the log +# there instead. Remote tiers (SharePoint/Fabric) for the audit destination still +# write the log to scratch and the existing run-finalization path uploads from there. +# Skip for resume mode - log file will be set after checkpoint is loaded +if (-not $script:DeferLogFileSetup) { + $logBaseName = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) + # Under non-rollup -AppendFile, $OutputFile is the customer's AppendFile target + # (or the _PARTIAL variant of it under checkpoint mode), so deriving the log + # basename from it would (a) inherit the target's ORIGINAL creation timestamp + # and (b) inherit the target's per-activity-type suffix (e.g. + # "_UsageActivity_CopilotInteraction") that the standard non-append path + # does NOT put in the log filename. Non-append runs name their log + # ${filePrefix}_.log (i.e. Purview_Audit_.log) — derived + # from the initial pre-splitter $OutputFile basename which itself starts as + # ${filePrefix}_.csv before the per-activity splitter renames its + # outputs to the long form. Match that canonical convention here so the + # append run's log file is named identically to the non-append run's log: + # the run log is a per-run artifact and is decoupled from the AppendFile + # target CSV's leaf identity. The _PARTIAL suffix convention (stripped at + # successful run-completion) is preserved. + if ($AppendFile -and -not ($Rollup -or $RollupPlusRaw)) { + $_hadPartialSuffix = ($logBaseName -match '_PARTIAL$') + $_currentRunTs = [string]$global:ScriptRunTimestamp + $_canonicalPfx = if ($filePrefix) { [string]$filePrefix } else { 'Purview_Audit' } + $_canonicalLogBase = if ($_currentRunTs) { "${_canonicalPfx}_${_currentRunTs}" } else { $_canonicalPfx } + $logBaseName = if ($_hadPartialSuffix) { "${_canonicalLogBase}_PARTIAL" } else { $_canonicalLogBase } + } + $logDir = Split-Path $OutputFile -Parent + try { + $dtL = script:Resolve-DataTypePaths -DataType 'Log' -DefaultBasename ("{0}.log" -f $logBaseName) + if ($dtL -and $dtL.IsBound -and $dtL.Tier -eq 'Local') { + $logDir = $dtL.EffectiveDir + if ($dtL.Basename) { $logBaseName = [System.IO.Path]::GetFileNameWithoutExtension($dtL.Basename) } + if (-not (Test-Path -LiteralPath $logDir -PathType Container)) { + New-Item -Path $logDir -ItemType Directory -Force | Out-Null + } + } + } catch {} + $finalLogFile = Join-Path $logDir ("{0}.log" -f $logBaseName) + + # Bootstrap-log migration: if the current + # $script:LogFile is a bootstrap temp-path file, move its contents to the + # resolved final location so the entire run (including pre-resolution + # entries) lands in one file. Falls back to copy+delete if Move-Item + # can't cross a volume boundary. + if ($script:LogFileIsBootstrap -and $script:LogFile -and (Test-Path -LiteralPath $script:LogFile)) { + try { + if (-not (Test-Path -LiteralPath $logDir -PathType Container)) { + New-Item -Path $logDir -ItemType Directory -Force | Out-Null + } + Move-Item -LiteralPath $script:LogFile -Destination $finalLogFile -Force -ErrorAction Stop + } catch { + try { + $bootstrapContent = Get-Content -LiteralPath $script:LogFile -Raw -ErrorAction Stop + Set-Content -LiteralPath $finalLogFile -Value $bootstrapContent -Encoding UTF8 -ErrorAction Stop + Remove-Item -LiteralPath $script:LogFile -Force -ErrorAction SilentlyContinue + } catch { + Microsoft.PowerShell.Utility\Write-Host ("WARNING: Could not migrate bootstrap log to final location ({0} -> {1}): {2}" -f $script:LogFile, $finalLogFile, $_.Exception.Message) -ForegroundColor Yellow + } + } + $script:LogFileIsBootstrap = $false + } + + $script:LogFile = $finalLogFile + $LogFile = $script:LogFile +} + +# Flush buffered logs now that log file is finalized (skip for resume - will flush after checkpoint load) +if ($script:LogBuffer -and $script:LogBuffer.Count -gt 0) { + foreach ($entry in $script:LogBuffer) { + try { Add-Content -Path $script:LogFile -Value $entry -Encoding UTF8 -ErrorAction SilentlyContinue } catch {} + } + $script:LogBuffer.Clear() +} + +# Note: $scriptMode already defined earlier for validation - reformat for display consistency +$scriptModeDisplay = if ($ExplodeDeep) { "Deep Column Explosion" } elseif ($ExplodeArrays -or $ForcedRawInputCsvExplosion) { if ($ForcedRawInputCsvExplosion -and -not $ExplodeArrays.IsPresent -and -not $ExplodeDeep.IsPresent) { "Array Explosion (RAWInput implied)" } else { "Array Explosion" } } else { "Standard (1:1)" } + +# Clean display names — strip _PARTIAL suffix so the startup banner always shows the expected final filename. +# Then route through Get-DisplayPath so remote -OutputPath* runs show the remote +# destination URL (where the artifact actually lives after upload), not the local scratch path. +$displayOutputFile = $OutputFile -replace '_PARTIAL(?=\.[^.]+$)', '' +$displayLogFile = $LogFile -replace '_PARTIAL(?=\.log$)', '' +$displayOutputFile = Get-DisplayPath -LocalPath $displayOutputFile +$displayLogFile = Get-DisplayPath -LocalPath $displayLogFile + +# Skip banner output to log file in resume mode (log file not set yet - will be set after checkpoint loads) +if (-not $script:DeferLogFileSetup) { +@" +=== Portable Audit eXporter (PAX) - Purview Audit Log Exporter === +Script Start Time (UTC): $((Get-Date).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ss')) UTC +Script Version: v$ScriptVersion +Mode: $scriptModeDisplay +Date Range: $(if ($RAWInputCSV) { if ([string]::IsNullOrWhiteSpace($StartDate) -and [string]::IsNullOrWhiteSpace($EndDate)) { 'Full CSV (no date filter)' } else { "$StartDate (inclusive) to $EndDate (exclusive) (filters)" } } else { "$StartDate (inclusive) to $EndDate (exclusive)" }) +Output File: $displayOutputFile +Log File: $displayLogFile +======================================================== + +"@ | Out-File -FilePath $LogFile -Encoding UTF8 -Append +} + + +# Fast-path: ensure M365 usage bundle is applied before output summary in raw/replay scenarios +if ($IncludeM365Usage -and -not ($PSBoundParameters.ContainsKey('ActivityTypes'))) { + $ActivityTypes = @($m365UsageActivityBundle + $copilotBaseActivityType) | Select-Object -Unique + # Activity types will be displayed in "Activity Types for This Run" section +} + +# Display active mode (Replay, EOM, or Graph API) +# Resume: suppress the parse-time mode/permissions/rollup/start/mode banner. +# On -Resume only the bootstrap log path, $Auth=WebLogin default, and a NEW +# $startTimeStamp are in scope at this point — the real values are restored +# AFTER this block by Read-Checkpoint. The resume restore block and the +# post-restore Parameter Snapshot print the authoritative auth context, file +# paths, and timestamps. Without this gate the early banner shows +# DELEGATED / WebLogin / bootstrap-log / new-timestamp on every resume run. +if (-not $ResumeSpecified) { +Write-LogHost "" +if ($RAWInputCSV) { + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " REPLAY MODE: Offline CSV (no service connections)" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " Source: $RAWInputCSV" -ForegroundColor White + Write-LogHost " Explosion: $scriptModeDisplay" -ForegroundColor White + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan +} +elseif ($UseEOM) { + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " QUERY MODE: Exchange Online Management" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " API Method: Search-UnifiedAuditLog cmdlet" -ForegroundColor White + Write-LogHost " Module: ExchangeOnlineManagement" -ForegroundColor White + Write-LogHost " Authentication: $Auth" -ForegroundColor White + Write-LogHost " Parallel Support: DISABLED (serial-only processing)" -ForegroundColor Yellow + Write-LogHost " Permissions: Exchange Online RBAC roles required" -ForegroundColor White + Write-LogHost " (View-Only Audit Logs, Compliance Management)" -ForegroundColor Gray + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan +} +else { + # Determine effective auth context(s) for this run: + # - audit phase context (app-only or delegated) + # - agent 365 phase context (always delegated when -IncludeAgent365Info / -OnlyAgent365Info) + # When -Auth AppRegistration is combined with Agent 365, BOTH contexts apply (sequentially). + # ManagedIdentity is treated as app-only (Connect-MgGraph -Identity uses application permissions + # from the user-assigned MI's service principal); it cannot satisfy Agent 365 (no interactive user). + $isAppOnlyAuth = ($Auth -eq 'AppRegistration' -or $Auth -eq 'ManagedIdentity') + $isManagedId = ($Auth -eq 'ManagedIdentity') + $auditContextLbl = if ($isManagedId) { 'APP-ONLY (managed identity / application permissions)' } + elseif ($isAppOnlyAuth) { 'APP-ONLY (application permissions)' } + else { 'DELEGATED (interactive user sign-in)' } + + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green + Write-LogHost " QUERY MODE: Microsoft Graph Security API (Default)" -ForegroundColor Green + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green + Write-LogHost " API Method: REST-based audit log queries" -ForegroundColor White + Write-LogHost " Module: Microsoft.Graph.Security" -ForegroundColor White + Write-LogHost " Authentication: $Auth (OAuth 2.0)" -ForegroundColor White + $parallelStatus = if ($PSVersionTable.PSVersion.Major -ge 7) { "AVAILABLE (PowerShell 7+)" } else { "LIMITED (PowerShell 5.1 detected)" } + Write-LogHost " Parallel Support: $parallelStatus" -ForegroundColor Green + + # ---- Auth Context summary ---- + Write-LogHost "" -ForegroundColor White + Write-LogHost " Auth Context: $auditContextLbl" -ForegroundColor White + if ($isAppOnlyAuth -and ($IncludeAgent365Info -or $OnlyAgent365Info)) { + Write-LogHost " Agent 365 phase reuses this SAME app-only context" -ForegroundColor Gray + Write-LogHost " (application app-role CopilotPackages.Read.All; no sign-in)." -ForegroundColor Gray + } + + # ---- Permissions list ---- + Write-LogHost "" -ForegroundColor White + Write-LogHost " Permissions Required for THIS run:" -ForegroundColor White + Write-LogHost " (Yellow = required this run; DarkGray = not needed this run)" -ForegroundColor Gray + + $tagLegendLines = New-Object System.Collections.Generic.List[string] + if ($isAppOnlyAuth) { + if ($isManagedId) { + $tagLegendLines.Add('[App-only] = application permission on managed identity service principal') + } else { + $tagLegendLines.Add('[App-only] = grant on app registration as Application permission') + } + } + if (-not $isAppOnlyAuth) { + $tagLegendLines.Add('[Delegated] = consented at interactive sign-in') + } + if (($IncludeAgent365Info -or $OnlyAgent365Info) -and (-not $isAppOnlyAuth)) { + $tagLegendLines.Add('[Role] = Entra directory role on signed-in user') + } + if ($script:RemoteOutputMode -eq 'Fabric') { + $tagLegendLines.Add('[Azure RBAC] = Azure role assignment on Fabric workspace (not a Graph scope)') + } + if ($tagLegendLines.Count -gt 0) { + Write-LogHost (" Tag legend: {0}" -f $tagLegendLines[0]) -ForegroundColor Gray + for ($legendIdx = 1; $legendIdx -lt $tagLegendLines.Count; $legendIdx++) { + Write-LogHost (" {0}" -f $tagLegendLines[$legendIdx]) -ForegroundColor Gray + } + } + + # Audit phase scopes (use the audit context tag) + $auditTag = if ($isAppOnlyAuth) { '[App-only] ' } else { '[Delegated]' } + if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Audit query (REQUIRED - this run reads audit logs):" -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery.Read.All (umbrella - all audit record types)" -f $auditTag) -ForegroundColor Yellow + } else { + $auditSkipReason = if ($OnlyAgent365Info) { '-OnlyAgent365Info' } else { '-OnlyUserInfo' } + Write-LogHost "" -ForegroundColor White + Write-LogHost (" Audit query (NOT NEEDED - {0} skips audit queries):" -f $auditSkipReason) -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery.Read.All (umbrella - all audit record types)" -f $auditTag) -ForegroundColor DarkGray + } + + # M365 usage bundle (audit context) + if ($IncludeM365Usage) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " M365 usage bundle (REQUIRED - because -IncludeM365Usage is set):" -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery-Exchange.Read.All" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} AuditLogsQuery-SharePoint.Read.All" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} AuditLogsQuery-OneDrive.Read.All" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " M365 usage bundle (only required with -IncludeM365Usage):" -ForegroundColor White + Write-LogHost (" {0} AuditLogsQuery-Exchange.Read.All" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} AuditLogsQuery-SharePoint.Read.All" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} AuditLogsQuery-OneDrive.Read.All" -f $auditTag) -ForegroundColor DarkGray + } + + # Entra directory + license enrichment (audit context) + if ($IncludeUserInfo -or $OnlyUserInfo) { + $userInfoTrigger = if ($OnlyUserInfo) { '-OnlyUserInfo' } else { '-IncludeUserInfo' } + Write-LogHost "" -ForegroundColor White + Write-LogHost (" Entra directory + license enrichment (REQUIRED - {0} is set):" -f $userInfoTrigger) -ForegroundColor White + Write-LogHost (" {0} User.Read.All (read /users)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost (" {0} Organization.Read.All (read /subscribedSkus)" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Entra directory + license enrichment (only required with -IncludeUserInfo or -OnlyUserInfo):" -ForegroundColor White + Write-LogHost (" {0} User.Read.All (read /users)" -f $auditTag) -ForegroundColor DarkGray + Write-LogHost (" {0} Organization.Read.All (read /subscribedSkus)" -f $auditTag) -ForegroundColor DarkGray + } + + # Group expansion (audit context) + if ($GroupNames -and $GroupNames.Count -gt 0) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Group expansion (REQUIRED - because -GroupNames is set):" -ForegroundColor White + Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/transitiveMembers)" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Group expansion (only required with -GroupNames):" -ForegroundColor White + Write-LogHost (" {0} GroupMember.Read.All (read /groups + /groups/{{id}}/transitiveMembers)" -f $auditTag) -ForegroundColor DarkGray + } + + # Remote output - SharePoint (audit context: same Graph token reused for SP drives API) + if ($script:RemoteOutputMode -eq 'SharePoint') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - SharePoint (REQUIRED - because -OutputPath resolves to a SharePoint URL):" -ForegroundColor White + Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive; create folder; upload; download)" -f $auditTag) -ForegroundColor Yellow + Write-LogHost " Destination user/identity ALSO needs at least Member access on the" -ForegroundColor Gray + Write-LogHost " target SharePoint site (granted via SharePoint, not Graph)." -ForegroundColor Gray + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - SharePoint (only required when an -OutputPath* value is a SharePoint URL):" -ForegroundColor White + Write-LogHost (" {0} Sites.ReadWrite.All (resolve site/drive; create folder; upload; download)" -f $auditTag) -ForegroundColor DarkGray + } + + # Remote output - Fabric / OneLake (separate token: storage.azure.com audience via Az.Accounts; + # Azure RBAC, not Graph scopes — this is the only section that uses the [Azure RBAC] tag) + if ($script:RemoteOutputMode -eq 'Fabric') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - Fabric/OneLake (REQUIRED - because -OutputPath resolves to a OneLake URL):" -ForegroundColor White + Write-LogHost " [Azure RBAC] Storage Blob Data Contributor on the Fabric workspace" -ForegroundColor Yellow + Write-LogHost " (assign at workspace scope to the audit identity)" -ForegroundColor Yellow + Write-LogHost " [Azure RBAC] Contributor on the Fabric workspace (Fabric portal)" -ForegroundColor Yellow + Write-LogHost " (Workspace settings -> Manage access; required by some" -ForegroundColor Yellow + Write-LogHost " tenants for OneLake DFS write access)" -ForegroundColor Yellow + Write-LogHost " NOTE: OneLake uses an INDEPENDENT token (audience https://storage.azure.com/)" -ForegroundColor Gray + Write-LogHost " acquired via Az.Accounts. Microsoft Graph scopes do NOT apply here." -ForegroundColor Gray + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Remote output - Fabric/OneLake (only required when an -OutputPath* value is a OneLake URL):" -ForegroundColor White + Write-LogHost " [Azure RBAC] Storage Blob Data Contributor on Fabric workspace" -ForegroundColor DarkGray + Write-LogHost " [Azure RBAC] Contributor on Fabric workspace (Fabric portal)" -ForegroundColor DarkGray + } + + # Managed identity advisory - shown only under -Auth ManagedIdentity + if ($isManagedId) { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Managed identity bootstrap (REQUIRED - because -Auth ManagedIdentity is set):" -ForegroundColor White + Write-LogHost " All scopes above must be ADMIN-CONSENTED on the managed identity's" -ForegroundColor Yellow + Write-LogHost " service principal as APPLICATION permissions (not delegated). One-time" -ForegroundColor Yellow + Write-LogHost " bootstrap via fabric_resources/.../Prereqs/Grant-PAXPermissions.ps1." -ForegroundColor Yellow + if ($env:AZURE_CLIENT_ID) { + Write-LogHost (" AZURE_CLIENT_ID is set: {0}" -f $env:AZURE_CLIENT_ID) -ForegroundColor Gray + } else { + Write-LogHost " AZURE_CLIENT_ID is NOT set; system-assigned MI will be used (if any)." -ForegroundColor Gray + } + } + + # Agent 365 enrichment - ALWAYS uses delegated scopes; ALSO requires Entra role. + # Suppressed from the permissions table while the Agent 365 switches are parked at + # the temporarily-disabled gate. Restore this block when the switches are re-enabled. + <# + if ($IncludeAgent365Info -or $OnlyAgent365Info) { + $agent365Trigger = if ($OnlyAgent365Info) { '-OnlyAgent365Info' } else { '-IncludeAgent365Info' } + Write-LogHost "" -ForegroundColor White + Write-LogHost (" Agent 365 enrichment (REQUIRED - {0} is set):" -f $agent365Trigger) -ForegroundColor White + if ($dualContextRun) { + Write-LogHost " NOTE: Performed in a SEPARATE delegated context (Phase 2)," -ForegroundColor Gray + Write-LogHost " prompted UP-FRONT immediately after Phase 1 connects." -ForegroundColor Gray + Write-LogHost " Your app registration is NOT used for these scopes." -ForegroundColor Gray + } + Write-LogHost " [Delegated] CopilotPackages.Read.All (read /copilot/admin/catalog/packages)" -ForegroundColor Yellow + Write-LogHost " [Delegated] Application.Read.All (resolve developer/owner via /applications)" -ForegroundColor Yellow + Write-LogHost " [Role] AI Administrator (preferred - least privilege)" -ForegroundColor Yellow + Write-LogHost " -- OR --" -ForegroundColor Gray + Write-LogHost " [Role] Global Administrator (alternative)" -ForegroundColor Yellow + Write-LogHost " The signed-in user MUST hold the Entra directory role above;" -ForegroundColor Gray + Write-LogHost " delegated Graph consent ALONE is not sufficient (returns 403)." -ForegroundColor Gray + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Agent 365 enrichment (only required with -IncludeAgent365Info or -OnlyAgent365Info):" -ForegroundColor White + Write-LogHost " [Delegated] CopilotPackages.Read.All (read /copilot/admin/catalog/packages)" -ForegroundColor DarkGray + Write-LogHost " [Delegated] Application.Read.All (resolve developer/owner via /applications)" -ForegroundColor DarkGray + Write-LogHost " [Role] AI Administrator OR Global Administrator (signed-in user)" -ForegroundColor DarkGray + } + #> + + # AISID dashboard (Defender advanced hunting) - reuses THIS Graph token via + # /security/runHuntingQuery. ThreatHunting.Read.All follows the same delegated-vs- + # application split as the audit scopes: delegated modes consent it at sign-in; + # app-only modes need it admin-consented as an APPLICATION permission. + if ($Dashboard -eq 'AISID') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " AISID dashboard (REQUIRED - because -Dashboard AISID is set):" -ForegroundColor White + Write-LogHost (" {0} ThreatHunting.Read.All (Defender advanced hunting via /security/runHuntingQuery)" -f $auditTag) -ForegroundColor Yellow + } else { + Write-LogHost "" -ForegroundColor White + Write-LogHost " AISID dashboard (only required with -Dashboard AISID):" -ForegroundColor White + Write-LogHost (" {0} ThreatHunting.Read.All (Defender advanced hunting via /security/runHuntingQuery)" -f $auditTag) -ForegroundColor DarkGray + } + + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Green +} +Write-LogHost "" + +# Per-data-type destination banner. Emitted whenever any -OutputPath* switch was supplied, +# or whenever the inferred Purview destination is remote (SP/Fabric). Shows resolved tier, +# full destination, and append surface for each data type in scope. Sits between the +# permissions banner and the rollup banner so users can see at a glance where each +# customer-visible artifact (CSV/XLSX/log) will land. +$anyDestBound = $false +foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { + if ($script:DestIsBound.ContainsKey($k) -and $script:DestIsBound[$k]) { $anyDestBound = $true; break } +} +# Bound -Append* values are destination bindings too — they tell the +# run where the canonical artifact lives. Treat them as such so the banner doesn't +# go silent on a pure -AppendFile/-AppendUserInfo/-AppendAgent365Info run with no +# -OutputPath* in scope. +if (-not $anyDestBound -and ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info -or $AppendDefenderUsage)) { + $anyDestBound = $true +} +if ($script:RemoteOutputMode -ne 'None' -or $anyDestBound) { + $scratchDirDisplay = if ($OutputPath.Length -le 70) { $OutputPath } else { $OutputPath.Substring(0, 67) + '...' } + $bannerHeader = if ($script:RemoteOutputMode -ne 'None') { "REMOTE OUTPUT MODE: $script:RemoteOutputMode" } else { 'OUTPUT DESTINATIONS' } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost (" {0}" -f $bannerHeader) -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + if ($script:RemoteOutputMode -ne 'None') { + # Top-line destination URL removed: per-data-type rows below carry the + # authoritative destination for each stream (which may differ across + # streams even within a single tier). Keep only auth + scratch dir here. + Write-LogHost (" Auth : {0}" -f $Auth) -ForegroundColor White + Write-LogHost " Scratch dir : $scratchDirDisplay" -ForegroundColor White + } + # Per-data-type rows (one line per data type in scope). + $dataTypeRows = @( + @{ Key = 'Purview' ; Label = 'Purview audit '; ScopeCondition = $true ; AppendVar = $AppendFile ; AppendName = 'AppendFile' }, + @{ Key = 'UserInfo' ; Label = 'EntraUsers CSV '; ScopeCondition = ($IncludeUserInfo -or $OnlyUserInfo) ; AppendVar = $AppendUserInfo ; AppendName = 'AppendUserInfo' }, + @{ Key = 'Agent365Info' ; Label = 'Agent 365 CSV '; ScopeCondition = ($IncludeAgent365Info -or $OnlyAgent365Info) ; AppendVar = $AppendAgent365Info ; AppendName = 'AppendAgent365Info' }, + @{ Key = 'DefenderUsage'; Label = 'AISID CSVs '; ScopeCondition = ($Dashboard -eq 'AISID') ; AppendVar = $AppendDefenderUsage ; AppendName = 'AppendDefenderUsage' }, + @{ Key = 'Log' ; Label = 'Run log '; ScopeCondition = $true ; AppendVar = $null ; AppendName = $null } + ) + # Always emit the per-data-type rows when the banner shows at all + # (a run with only -OutputPath — no per-stream override and no Append* — must + # still show what each stream resolves to). + Write-LogHost "" -ForegroundColor White + Write-LogHost " Per-data-type destinations:" -ForegroundColor White + foreach ($row in $dataTypeRows) { + if (-not $row.ScopeCondition) { continue } + $tier = if ($script:DestTier.ContainsKey($row.Key)) { $script:DestTier[$row.Key] } + elseif ($row.Key -eq 'Purview' -and $script:DestTier.ContainsKey('Purview')) { $script:DestTier['Purview'] } + elseif ($row.Key -eq 'Log' -and $script:DestTier.ContainsKey('Purview')) { + # Log inherits the Purview tier when -OutputPathLog wasn't bound — the + # log lives in the Purview scratch dir (Local) or is uploaded alongside + # the audit CSV on remote runs. + $script:DestTier['Purview'] + } + else { '(default)' } + # Path resolution: prefer the bound Append target (it IS the destination + # when -Append* is set), then a resolved FILE URL per row, then sensible + # defaults. + # + # Every row renders as / via Get-DisplayPath (or a + # pattern leaf in per-activity-split mode where the leaf varies by + # ActivityType), so adjacent rows in the same banner stay internally + # consistent (file URL on every row, not a mix of folder + file URLs) + # and give customers a verifiable destination per stream. The Run log + # row uses the same Get-DisplayPath transform via $displayLogFile so the + # banner and the 'Log File:' preamble agree on the final destination. + $path = if ($row.AppendName -and $row.AppendVar) { $row.AppendVar } + elseif ($row.Key -eq 'Purview') { + if ($OutputFile) { + # Single-CSV cases: -CombineOutput on, OR a single-activity-type + # run (where the file leaf is fully determined). In both cases + # $OutputFile is the authoritative source for the final leaf. + $_isSingleActivity = ($ActivityTypes -and @($ActivityTypes).Count -eq 1) + if ($CombineOutput -or $_isSingleActivity) { + $_pvFinal = $OutputFile -replace '_PARTIAL(?=\.[^.]+$)', '' + Get-DisplayPath -LocalPath $_pvFinal + } + else { + # Per-activity split: leaf varies by ActivityType — show a + # pattern URL so customers see the naming shape and the + # folder, just like the existing 'Output Files: ...' line. + $_outDir = Split-Path $OutputFile -Parent + $_dispDir = (Get-DisplayPath -LocalPath $_outDir -Directory).TrimEnd('/','\') + $_ts = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) -replace '.*_(\d{8}_\d{6}).*', '$1' + "${_dispDir}/Purview_Audit_UsageActivity__${_ts}.csv" + } + } + elseif ($script:DestRaw.ContainsKey('Purview')) { $script:DestRaw['Purview'] } + else { '(inherits -OutputPath)' } + } + elseif ($row.Key -eq 'UserInfo') { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound; + # otherwise inherit -OutputPath. Without consulting Resolve-DataTypePaths the + # banner unconditionally joined the EntraUsers leaf onto $OutputFile's parent + # (the Purview -OutputPath dir), masking any -OutputPathUserInfo override that + # the runtime writer correctly routed to. + $_uiDt = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -eq 'Local') { + Get-DisplayPath -LocalPath (Join-Path $_uiDt.EffectiveDir $_uiDt.Basename) + } + elseif ($_uiDt -and $_uiDt.IsBound) { + # Remote per-stream override (Fabric / SharePoint URL) — surface the resolved file URL. + ($_uiDt.EffectiveDir.TrimEnd('/') + '/' + $_uiDt.Basename) + } + elseif ($OutputFile -and $script:GetEffectiveEntraBasename) { + $_entraLeaf = & $script:GetEffectiveEntraBasename + $_entraLocal = Join-Path (Split-Path $OutputFile -Parent) $_entraLeaf + Get-DisplayPath -LocalPath $_entraLocal + } + elseif ($script:DestRaw.ContainsKey('UserInfo')) { $script:DestRaw['UserInfo'] } + else { '(inherits -OutputPath)' } + } + elseif ($row.Key -eq 'Agent365Info') { + # Symmetric fix to the UserInfo branch above: honor -OutputPathAgent365Info + # (or an Append-side promotion) before falling back to the Purview -OutputPath dir. + $_a365Dt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + if ($_a365Dt -and $_a365Dt.IsBound -and $_a365Dt.Tier -eq 'Local') { + Get-DisplayPath -LocalPath (Join-Path $_a365Dt.EffectiveDir $_a365Dt.Basename) + } + elseif ($_a365Dt -and $_a365Dt.IsBound) { + ($_a365Dt.EffectiveDir.TrimEnd('/') + '/' + $_a365Dt.Basename) + } + elseif ($OutputFile -and $script:GetEffectiveAgent365Basename) { + $_a365Leaf = & $script:GetEffectiveAgent365Basename + $_a365Local = Join-Path (Split-Path $OutputFile -Parent) $_a365Leaf + Get-DisplayPath -LocalPath $_a365Local + } + elseif ($script:DestRaw.ContainsKey('Agent365Info')) { $script:DestRaw['Agent365Info'] } + else { '(inherits -OutputPath)' } + } + elseif ($row.Key -eq 'DefenderUsage') { + # AISID dashboard: twelve fixed-name CSVs land together in ONE folder; show the + # resolved folder (honoring -OutputPathDefenderUsage / -AppendDefenderUsage, else + # co-located with -OutputPath) with a count hint rather than a single leaf. + $_duDtB = script:Resolve-DataTypePaths -DataType 'DefenderUsage' + if ($_duDtB -and $_duDtB.IsBound -and $_duDtB.Tier -eq 'Local') { + (Get-DisplayPath -LocalPath $_duDtB.EffectiveDir -Directory).TrimEnd('/','\') + '/ (12 dashboard CSVs)' + } + elseif ($_duDtB -and $_duDtB.IsBound) { + $_duDtB.EffectiveDir.TrimEnd('/') + '/ (12 dashboard CSVs)' + } + elseif ($OutputFile) { + (Get-DisplayPath -LocalPath (Split-Path $OutputFile -Parent) -Directory).TrimEnd('/','\') + '/ (12 dashboard CSVs)' + } + else { '(inherits -OutputPath) (12 dashboard CSVs)' } + } + elseif ($row.Key -eq 'Log') { + # Surface the resolved log destination URL (or final local path), + # NOT the local _PARTIAL scratch path. $script:LogFile is the + # _PARTIAL working copy throughout the run; strip the suffix and + # pass through Get-DisplayPath so remote-mode banners show the + # SharePoint / OneLake URL the log will land at — matching the + # 'Log File:' preamble line which uses the same transform via + # $displayLogFile. ($script:LogFile is not yet resolved to its + # final path at this point; only the local _PARTIAL copy exists + # until end-of-run upload + rename.) + if ($script:LogFile) { + $_logFinal = $script:LogFile -replace '_PARTIAL(?=\.log$)', '' + Get-DisplayPath -LocalPath $_logFinal + } + elseif ($script:RemoteOutputMode -ne 'None') { "Files/pax_run_logs/$($global:ScriptRunTimestamp)/" } + else { $OutputPath } + } + else { '(inherits -OutputPath)' } + # Never truncate. Customers need the full URL for verification and copy-paste. + $pathDisplay = $path + $appendMarker = '' + if ($row.AppendName -and $row.AppendVar) { + # Single consistent marker across all tiers / both first-time and + # subsequent appends. Init-vs-not distinction was inconsistent across + # tiers (Local could Test-Path, Remote could not) and produced mixed + # '[append-init:...]' / '[append:...]' tags within a single banner. + $appendMarker = " [append:$($row.AppendName)]" + } + Write-LogHost (" {0} [{1,-10}] {2}{3}" -f $row.Label, $tier, $pathDisplay, $appendMarker) -ForegroundColor White + } + # Per-tier truth in the Behavior summary. The Log stream in particular may + # stay local even on a remote-mode run (only the Purview/UserInfo/Agent365 + # streams have remote bindings here). + if ($script:RemoteOutputMode -ne 'None') { + Write-LogHost "" -ForegroundColor White + Write-LogHost " Behavior : Each stream is written to the tier shown above. Remote streams" -ForegroundColor White + Write-LogHost " are staged in scratch and uploaded at run completion." -ForegroundColor White + } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost "" +} + +# One-time legend explaining the Append/Merge tally vocabulary used by the +# various 'EntraUsers append-merge complete:' / 'Rollup: -AppendUserInfo +# merge:' / 'Rollup: -AppendFile merge:' lines emitted later in the run. +# The label 'Departed' in particular can mislead a casual reader into thinking +# rows were REMOVED from the file — they are not; 'Departed' rows are carried +# forward into the union with In_Latest_Append=FALSE. Gated on any -Append* +# switch being bound so the legend only appears when at least one merge tally +# will actually be emitted. +if ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info -or $AppendDefenderUsage) { + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " APPEND/MERGE TALLY LEGEND" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " Each '-Append*' run emits a merge tally with four counts:" -ForegroundColor White + Write-LogHost " Retained = rows present in BOTH the target file and this run's output (overlap)" -ForegroundColor White + Write-LogHost " New = rows added by THIS run only (current-only; not in target before)" -ForegroundColor White + Write-LogHost " Departed = rows present in TARGET ONLY (not seen this run); KEPT in the union" -ForegroundColor White + Write-LogHost " with In_Latest_Append=FALSE — NOT removed from the file" -ForegroundColor White + Write-LogHost " Union = total rows in the merged output (= Retained + New + Departed)" -ForegroundColor White + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost "" +} + +# Rollup post-processor banner. Emitted only when -Rollup or -RollupPlusRaw is in effect +# (either passed on the CLI or restored from checkpoint on resume). Sits immediately after +# the permissions banner so users see at a glance that an embedded Python post-processor +# will run at end-of-run, what mode, and what raw-CSV retention behavior to expect. +if ($Rollup -or $RollupPlusRaw) { + $rollupSwitchBannerName = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + $rollupRetention = if ($Rollup) { + switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { 'Raw Purview CSV and Entra users CSV will be DELETED on processor success (only rolled-up output kept).' } + 'M365Bundle' { + if ($IncludeUserInfo) { + 'Raw Purview CSV will be DELETED on processor success; Entra users CSV will be KEPT (not consumed by this processor).' + } else { + 'Raw Purview CSV will be DELETED on processor success (only rolled-up output kept).' + } + } + default { 'Raw CSV(s) will be DELETED on processor success (only rolled-up output kept).' } + } + } + else { + switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { 'Raw Purview CSV and Entra users CSV will be KEPT alongside the rolled-up output.' } + 'M365Bundle' { + if ($IncludeUserInfo) { + 'Raw Purview CSV and Entra users CSV will be KEPT alongside the rolled-up output.' + } else { + 'Raw Purview CSV will be KEPT alongside the rolled-up output.' + } + } + default { 'Raw CSV(s) will be KEPT alongside the rolled-up output.' } + } + } + $processorLabel = switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { "Purview_CopilotInteraction_Processor v$($Script:EMBEDDED_PROCESSOR_COPILOT_VERSION) ($(if ($script:RollupDashboard -eq 'AIBV') { 'ValueLens profile' } else { '--profile aio' }); inputs: Purview CSV + Entra users CSV)" } + 'M365Bundle' { "Purview_M365_Usage_Bundle_Explosion_Processor v$($Script:EMBEDDED_PROCESSOR_M365_VERSION) (input: Purview CSV)" } + default { 'unresolved' } + } + $rollupTargetDashboard = switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { + if ($script:RollupDashboard -eq 'AIBV') { 'ValueLens (Analytics-Hub)' } + else { 'AI-in-One (Analytics-Hub)' } + } + 'M365Bundle' { 'M365 Usage Analytics (Analytics-Hub)' } + default { 'unresolved' } + } + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " ROLLUP POST-PROCESSOR ENABLED ($rollupSwitchBannerName)" -ForegroundColor Cyan + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost " Purpose : Produces input files for the Microsoft Copilot Growth ROI Advisory Team's" -ForegroundColor White + Write-LogHost " Power BI templates at https://github.com/microsoft/Analytics-Hub." -ForegroundColor White + Write-LogHost " Output is NOT intended for any other downstream use." -ForegroundColor White + Write-LogHost " Target : $rollupTargetDashboard" -ForegroundColor White + Write-LogHost " Processor : $processorLabel" -ForegroundColor White + Write-LogHost " Retention : $rollupRetention" -ForegroundColor White + if ($IncludeAgent365Info) { + Write-LogHost " Companion : Agent365_.csv (point-in-time snapshot; ALWAYS retained — consumed by the same Analytics-Hub dashboards alongside the rollup output)." -ForegroundColor White + } + Write-LogHost " Runtime : Python $($Script:ROLLUP_PYTHON_MIN_MAJOR).$($Script:ROLLUP_PYTHON_MIN_MINOR)+ required (auto-installed if missing); 'orjson' auto-installed for speed (optional)." -ForegroundColor Gray + # Destination-aware Output line. The per-data-type destination banner above + # already shows the resolved tier/path for every stream, so this line just + # summarizes where the rollup output lands relative to the audit CSV. + # In M365Bundle mode the processor emits four files (Rollup + UserStats + + # SessionCohort + SessionStats sidecars) sharing a common stem, all written + # to the same destination; CopilotInteraction emits a single Interactions Fact CSV. + $rollupOutputLabel = switch ($script:RollupProcessorMode) { + 'CopilotInteraction' { 'Rollup Fact CSV' } + 'M365Bundle' { 'Rollup CSV bundle (Rollup + UserStats + SessionCohort + SessionStats sidecars sharing the same stem)' } + default { 'Rollup CSV' } + } + $rollupOutputSummary = + if ($AppendFile) { + "$rollupOutputLabel destination follows the Append target shown in the Per-data-type destinations banner above." + } elseif ($script:RemoteOutputMode -ne 'None') { + "$rollupOutputLabel is uploaded to the $script:RemoteOutputMode destination resolved for the Purview audit stream (see banner above)." + } else { + "$rollupOutputLabel is written to the local Purview audit destination resolved above." + } + Write-LogHost (" Output : {0}" -f $rollupOutputSummary) -ForegroundColor Gray + Write-LogHost "═══════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-LogHost "" +} + +$startTimeStamp = try { $script:metrics.StartTime.ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ss') } catch { (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd HH:mm:ss') } +Write-LogHost ("Script execution started at $startTimeStamp UTC") -ForegroundColor White + +# For OnlyUserInfo mode, show simplified header (no Mode/DateRange/Purview output info) +if (-not $OnlyUserInfo -and -not $OnlyAgent365Info) { + Write-LogHost "Mode: $scriptMode" -ForegroundColor White + ${rangeText} = if ($RAWInputCSV) { if ([string]::IsNullOrWhiteSpace($StartDate) -and [string]::IsNullOrWhiteSpace($EndDate)) { 'Full CSV (no date filter)' } else { "$StartDate (inclusive) to $EndDate (exclusive) (filters)" } } else { "$StartDate (inclusive) to $EndDate (exclusive)" } + Write-LogHost "Date Range: $rangeText" -ForegroundColor White +} elseif ($OnlyAgent365Info) { + Write-LogHost "Mode: OnlyAgent365Info (Microsoft Agent 365 catalog export only - no audit logs)" -ForegroundColor Cyan +} else { + Write-LogHost "Mode: OnlyUserInfo (Entra user and MAC licensing export only)" -ForegroundColor Cyan +} +} # end: if (-not $ResumeSpecified) — parse-time mode/permissions/rollup/start banner + +# --- Early build of $finalActivityTypes for user warning checks (before auth) --- +# This preview build is used ONLY for multi-output warning and PAYG billing warning +# The full/authoritative build happens later in the pipeline after authentication +$finalActivityTypes = @() +if ($PSBoundParameters.ContainsKey('ActivityTypes') -and $ActivityTypes) { + $finalActivityTypes += $ActivityTypes +} +if ($IncludeM365Usage) { + $finalActivityTypes += $m365UsageActivityBundle +} +# Add CopilotInteraction as default if no custom types and not excluded +$userProvidedCustomTypes = $PSBoundParameters.ContainsKey('ActivityTypes') +if (-not $ExcludeCopilotInteraction -and -not $userProvidedCustomTypes) { + if (-not ($finalActivityTypes -contains 'CopilotInteraction')) { + $finalActivityTypes += 'CopilotInteraction' + } +} +# Remove CopilotInteraction if explicitly excluded +if ($ExcludeCopilotInteraction) { + $finalActivityTypes = $finalActivityTypes | Where-Object { $_ -ne 'CopilotInteraction' } +} +$finalActivityTypes = $finalActivityTypes | Select-Object -Unique + +# --- Multi-Output Warning: Prompt when many files/tabs expected without -CombineOutput --- +$activityTypeCount = $finalActivityTypes.Count +$isMultiOutputScenario = ($activityTypeCount -gt 10) -and (-not $CombineOutput) +$outputType = if ($ExportWorkbook) { "tabs" } else { "CSV files" } + +if ($isMultiOutputScenario -and -not $Force) { + Write-LogHost "" + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-Host "WARNING: Multiple Output $outputType Detected" -ForegroundColor Yellow + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "You have $activityTypeCount activity types selected." -ForegroundColor Cyan + Write-LogHost "Without -CombineOutput, this will create $activityTypeCount separate $outputType." -ForegroundColor Cyan + Write-LogHost "" + Write-LogHost "Recommendation:" -ForegroundColor Green + Write-LogHost " • Add -CombineOutput to merge all activity types into a single $(if ($ExportWorkbook) { 'tab' } else { 'CSV file' })" -ForegroundColor Green + Write-LogHost "" + Write-LogHost "Do you want to continue with $activityTypeCount separate $outputType?" -ForegroundColor White + Write-LogHost "" + Write-LogHost " [Y] YES - Continue with separate $outputType (I understand there will be many $outputType)" -ForegroundColor Green + Write-LogHost " [C] COMBINE - Enable -CombineOutput and continue (single merged $(if ($ExportWorkbook) { 'tab' } else { 'CSV' }))" -ForegroundColor Cyan + Write-LogHost " [E] EXIT - Cancel script execution" -ForegroundColor Red + Write-LogHost "" + + # Strict noninteractive guard: on a container/ACA Job/CI + # runner there is nobody to choose Y/C/E. Default to [Y] CONTINUE so scheduled runs + # do not hang on stdin. Operators who want the [C] COMBINE behavior should pass + # -CombineOutput explicitly; the script's -Force path treats this prompt as Y anyway. + if (script:Test-IsNonInteractive) { + Write-LogHost " Noninteractive host detected: auto-selecting [Y] - continuing with $activityTypeCount separate $outputType." -ForegroundColor DarkGray + Write-LogHost " Hint: pass -CombineOutput to merge into a single output, or -Force to suppress this banner." -ForegroundColor DarkGray + $multiOutput_choice = 'Y' + } else { + Send-PromptNotification + $multiOutput_choice = Read-Host "Enter your choice (Y/C/E)" + } + + if ($multiOutput_choice -eq 'Y' -or $multiOutput_choice -eq 'y') { + Write-LogHost "" + Write-LogHost "Continuing with $activityTypeCount separate output files..." -ForegroundColor Green + Write-LogHost "" + } + elseif ($multiOutput_choice -eq 'C' -or $multiOutput_choice -eq 'c') { + Write-LogHost "" + Write-LogHost "ENABLED: -CombineOutput mode" -ForegroundColor Green + Write-LogHost " All $activityTypeCount activity types will be merged into a single CSV file." -ForegroundColor Cyan + Write-LogHost "" + $CombineOutput = $true + } + else { + Write-LogHost "" + Write-LogHost "User choice: EXIT - Script execution cancelled" -ForegroundColor Red + Write-LogHost "" + exit 0 + } +} +elseif ($isMultiOutputScenario -and $Force) { + Write-LogHost "Force mode: Skipping multi-output warning ($activityTypeCount activity types, separate $outputType)" -ForegroundColor DarkGray + Write-LogHost " → Continuing with separate $outputType (use -CombineOutput to merge if desired)" -ForegroundColor DarkGray +} +# --- End Multi-Output Warning --- + +# --- DSPM for AI: Billing Information Warning --- +if (($finalActivityTypes -contains 'AIAppInteraction') -or ($finalActivityTypes -contains 'ConnectedAIAppInteraction') -or ($finalActivityTypes -contains 'AIInteraction')) { + if ($IncludeM365Usage -and -not ($finalActivityTypes -contains 'AIAppInteraction')) { + # Bundle-attributed DSPM signal: -IncludeM365Usage carries ConnectedAIAppInteraction + # as part of the curated activity bundle. The DSPM informational prompt is redundant + # because AIAppInteraction (the PAYG path) is not requested; silently bypass. + } + elseif (-not $Force) { + Write-LogHost "" + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-Host "INFORMATION: DSPM for AI Audit Logging - Billing Details" -ForegroundColor Cyan + Write-LogHost "============================================================================================================" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "DSPM Activity Types:" -ForegroundColor Cyan + Write-LogHost " • AIInteraction - FREE (Microsoft platforms: Copilot Studio, Azure AI Studio)" -ForegroundColor Green + Write-LogHost " • ConnectedAIAppInteraction - MIXED (FREE for Microsoft apps, PAYG for third-party)" -ForegroundColor Yellow + if ($finalActivityTypes -contains 'AIAppInteraction') { + Write-LogHost " • AIAppInteraction - PAYG BILLING REQUIRED (third-party AI like ChatGPT)" -ForegroundColor DarkYellow + } + Write-LogHost "" + # Check if AIAppInteraction is included - offer options + if ($finalActivityTypes -contains 'AIAppInteraction') { + Write-LogHost "[!] IMPORTANT: AIAppInteraction REQUIRES Microsoft Purview PAYG billing" -ForegroundColor Yellow + Write-LogHost "" + Write-LogHost "PAYG Requirements:" -ForegroundColor Cyan + Write-LogHost " • Azure subscription linked to M365 tenant" -ForegroundColor Cyan + Write-LogHost " • Microsoft Purview PAYG billing enabled in Compliance portal" -ForegroundColor Cyan + Write-LogHost "" + Write-LogHost "Do you have PAYG billing configured in your tenant?" -ForegroundColor White + Write-LogHost "" + Write-LogHost " [Y] YES - I have PAYG billing, continue with all DSPM types" -ForegroundColor Green + Write-LogHost " [N] NO - I don't have PAYG billing, remove AIAppInteraction and continue" -ForegroundColor Yellow + Write-LogHost " (Third-party AI records will NOT be included)" -ForegroundColor DarkGray + Write-LogHost " [E] EXIT - Cancel script execution" -ForegroundColor Red + Write-LogHost "" + + # Strict noninteractive guard: cannot prompt for + # PAYG confirmation on a container/CI runner. Fail-fast with explicit recovery + # guidance rather than silently including or removing AIAppInteraction. + if (script:Test-IsNonInteractive) { + Write-LogHost "ERROR: Cannot prompt for PAYG billing confirmation on a noninteractive host." -ForegroundColor Red + Write-LogHost " Recovery: re-run WITHOUT -ActivityTypes AIAppInteraction if PAYG is not configured," -ForegroundColor Yellow + Write-LogHost " OR pass -Force to acknowledge PAYG is configured and continue with AIAppInteraction." -ForegroundColor Yellow + exit 1 + } + + Send-PromptNotification + $payg_choice = Read-Host "Enter your choice (Y/N/E)" + + if ($payg_choice -eq 'Y' -or $payg_choice -eq 'y') { + Write-LogHost "" + Write-LogHost "Continuing with all DSPM types (AIInteraction, ConnectedAIAppInteraction, AIAppInteraction)..." -ForegroundColor Green + Write-LogHost "" + } + elseif ($payg_choice -eq 'N' -or $payg_choice -eq 'n') { + Write-LogHost "" + Write-LogHost "REMOVED: AIAppInteraction (third-party AI records will NOT be captured)" -ForegroundColor Yellow + Write-LogHost "Continuing with: AIInteraction, ConnectedAIAppInteraction (Microsoft platforms only)" -ForegroundColor Green + Write-LogHost "" + Write-LogHost "Note: Without PAYG billing, only Microsoft-hosted AI activity will be captured." -ForegroundColor Yellow + Write-LogHost " Third-party AI apps (ChatGPT, etc.) require PAYG billing." -ForegroundColor Yellow + Write-LogHost "" + + # Set flag to remove AIAppInteraction during later rebuild + $script:RemoveAIAppInteraction = $true + $finalActivityTypes = $finalActivityTypes | Where-Object { $_ -ne 'AIAppInteraction' } + } + else { + Write-LogHost "" + Write-LogHost "User choice: EXIT - Script execution cancelled" -ForegroundColor Red + Write-LogHost "" + exit 0 + } + } + else { + # No AIAppInteraction - simple Y/N prompt + Write-LogHost "Note: AIAppInteraction (PAYG-only third-party AI) is NOT included." -ForegroundColor DarkGray + Write-LogHost " Only Microsoft-hosted AI activity will be captured (AIInteraction, ConnectedAIAppInteraction)." -ForegroundColor DarkGray + Write-LogHost "" + + # Strict noninteractive guard: default to [Y] + # CONTINUE because the surrounding banner is informational-only (no PAYG + # entitlements are required for the activity types that remain in scope). + if (script:Test-IsNonInteractive) { + Write-LogHost " Noninteractive host detected: auto-selecting [Y] - continuing with DSPM for AI export." -ForegroundColor DarkGray + Write-LogHost "" + $payg_choice = 'Y' + } else { + Send-PromptNotification + $payg_choice = Read-Host "Continue with DSPM for AI export? (Y/N)" + } + + if ($payg_choice -eq 'Y' -or $payg_choice -eq 'y') { + Write-LogHost "" + Write-LogHost "Continuing with DSPM for AI export..." -ForegroundColor Green + Write-LogHost "" + } + else { + Write-LogHost "" + Write-LogHost "User choice: ABORT - DSPM for AI export declined" -ForegroundColor Red + Write-LogHost "Script execution cancelled by user." -ForegroundColor Yellow + Write-LogHost "" + exit 0 + } + } + } + else { + Write-LogHost "Force mode enabled: Skipping DSPM for AI billing information prompt" -ForegroundColor DarkGray + Write-LogHost "User choice: CONTINUE (Force mode - automatic acceptance)" -ForegroundColor Gray + } +} +# --- End PAYG Billing Warning --- + +# Resume: suppress the parse-time Output Directory / Output Files / Log File / +# Authentication / Append/Destination banner. On -Resume the destination state +# ($OutputFile, $script:RemoteOutputMode, $displayLogFile, $Auth, $AppendFile…) +# is restored AFTER this point by Read-Checkpoint and re-resolved by the +# post-restore destination block. The post-restore Parameter Snapshot prints +# the authoritative values. Without this gate the early banner shows the +# bootstrap log path, a fresh non-restored timestamp in the Output Files line, +# and stale Auth=WebLogin on every resume run. +if (-not $ResumeSpecified) { +# Output file/directory display based on export mode +# Note: If activity type switches are used, detailed filenames will be shown after activity types are finalized +# For OnlyUserInfo / OnlyAgent365Info modes, show only the relevant standalone files +if ($OnlyUserInfo) { + $outputDir = if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + if ($ExportWorkbook) { + $entraOutputFile = Join-Path $outputDir "EntraUsers_MAClicensing_${global:ScriptRunTimestamp}.xlsx" + Write-LogHost "Output File: $(Get-DisplayPath -LocalPath $entraOutputFile) (Entra users workbook)" -ForegroundColor White + } else { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound; otherwise inherit $outputDir. + $_uiDt6 = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + $entraOutputFile = if ($_uiDt6 -and $_uiDt6.IsBound -and $_uiDt6.Tier -eq 'Local') { Join-Path $_uiDt6.EffectiveDir $_uiDt6.Basename } + elseif ($_uiDt6 -and $_uiDt6.IsBound) { $_uiDt6.EffectiveDir.TrimEnd('/') + '/' + $_uiDt6.Basename } + else { Join-Path $outputDir (& $script:GetEffectiveEntraBasename) } + if ($_uiDt6 -and $_uiDt6.IsBound -and $_uiDt6.Tier -ne 'Local') { Write-LogHost "Output File: $entraOutputFile" -ForegroundColor White } + else { Write-LogHost "Output File: $(Get-DisplayPath -LocalPath $entraOutputFile)" -ForegroundColor White } + } + if ($IncludeAgent365Info) { + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; otherwise inherit $outputDir. + $_aiDt6 = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt6 -and $_aiDt6.IsBound -and $_aiDt6.Tier -eq 'Local') { Join-Path $_aiDt6.EffectiveDir $_aiDt6.Basename } + elseif ($_aiDt6 -and $_aiDt6.IsBound) { $_aiDt6.EffectiveDir.TrimEnd('/') + '/' + $_aiDt6.Basename } + else { Join-Path $outputDir (& $script:GetEffectiveAgent365Basename) } + if ($_aiDt6 -and $_aiDt6.IsBound -and $_aiDt6.Tier -ne 'Local') { Write-LogHost " Agent 365 File: $agent365PreviewFile" -ForegroundColor Gray } + else { Write-LogHost " Agent 365 File: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray } + } +} elseif ($OnlyAgent365Info) { + # OnlyAgent365Info mode: only the Agents365 catalog CSV is produced + $outputDir = if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; otherwise inherit $outputDir. + $_aiDt7 = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt7 -and $_aiDt7.IsBound -and $_aiDt7.Tier -eq 'Local') { Join-Path $_aiDt7.EffectiveDir $_aiDt7.Basename } + elseif ($_aiDt7 -and $_aiDt7.IsBound) { $_aiDt7.EffectiveDir.TrimEnd('/') + '/' + $_aiDt7.Basename } + else { Join-Path $outputDir (& $script:GetEffectiveAgent365Basename) } + if ($_aiDt7 -and $_aiDt7.IsBound -and $_aiDt7.Tier -ne 'Local') { Write-LogHost "Output File: $agent365PreviewFile (Microsoft Agent 365 catalog)" -ForegroundColor White } + else { Write-LogHost "Output File: $(Get-DisplayPath -LocalPath $agent365PreviewFile) (Microsoft Agent 365 catalog)" -ForegroundColor White } +} elseif ($AppendFile) { + # AppendFile mode: the destination URL is already surfaced by the parameter + # snapshot's AppendFile row. Re-emitting an 'Output File: ...' + 'Mode: + # Appending to existing file' pair here duplicates that row (and in remote + # tiers it also leaks the scratch-shadow URL via $displayOutputFile). Only + # emit Agent 365 sibling info, which the snapshot does not cover. + if ($IncludeAgent365Info) { + # Agent 365 always writes a separate CSV alongside the appended file (or its own tab if Excel). + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; otherwise inherit the LOCAL OutputFile parent. + $agent365TargetDir = Split-Path $OutputFile -Parent + if ($ExportWorkbook) { + Write-LogHost " Agent 365 Tab: Agents365 (appended to workbook)" -ForegroundColor Gray + } else { + $_aiDt8 = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt8 -and $_aiDt8.IsBound -and $_aiDt8.Tier -eq 'Local') { Join-Path $_aiDt8.EffectiveDir $_aiDt8.Basename } + elseif ($_aiDt8 -and $_aiDt8.IsBound) { $_aiDt8.EffectiveDir.TrimEnd('/') + '/' + $_aiDt8.Basename } + else { Join-Path $agent365TargetDir (& $script:GetEffectiveAgent365Basename) } + if ($_aiDt8 -and $_aiDt8.IsBound -and $_aiDt8.Tier -ne 'Local') { Write-LogHost " Agent 365 File: $agent365PreviewFile" -ForegroundColor Gray } + else { Write-LogHost " Agent 365 File: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray } + } + } +} elseif ($ExcludeCopilotInteraction) { + # Activity type switches present - defer detailed filename listing until after activity types are finalized + $outputDir = if ($ExportWorkbook) { + if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + } else { + Split-Path $OutputFile -Parent + } + Write-LogHost "Output Directory: $(Get-DisplayPath -LocalPath $outputDir -Directory)\" -ForegroundColor White + Write-LogHost " (Detailed filenames will be shown after activity types are finalized)" -ForegroundColor Gray +} elseif ($ExportWorkbook) { + # Excel mode: always one .xlsx file (combined tab or multiple tabs) + $outputDir = if ($OutputPath) { $OutputPath } else { "C:\Temp\" } + $displayDir = (Get-DisplayPath -LocalPath $outputDir -Directory).TrimEnd('/','\') + '/' + if ($CombineOutput) { + # New naming: Purview_Audit_CombinedUsageActivity[_EntraUsers]_timestamp.xlsx + $baseName = "Purview_Audit_CombinedUsageActivity" + if ($IncludeUserInfo -and -not $UseEOM) { $baseName += "_EntraUsers" } + Write-LogHost "Output File: ${displayDir}${baseName}_.xlsx (single-tab workbook)" -ForegroundColor White + } else { + Write-LogHost "Output File: ${displayDir}Purview_Audit_MultiTab_.xlsx (multi-tab workbook)" -ForegroundColor White + } + if ($IncludeAgent365Info) { Write-LogHost " Agent 365 Tab: Agents365 (appended to workbook after audit + EntraUsers)" -ForegroundColor Gray } +} else { + # CSV mode: combined file or separate files per activity type + if ($CombineOutput) { + # Single combined CSV file. Suppress the "(combined - all activity types)" + # qualifier when the run is gated to exactly one activity type — the file + # name itself already carries the single activity-type name in that case. + $_isSingleActivityRun = ($ActivityTypes -and @($ActivityTypes).Count -eq 1 -and -not $IncludeM365Usage) + $_combinedQualifier = if ($_isSingleActivityRun) { '' } else { ' (combined - all activity types)' } + Write-LogHost "Output File: ${displayOutputFile}${_combinedQualifier}" -ForegroundColor White + if ($IncludeUserInfo -and -not $UseEOM) { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound, so the + # preview line matches the runtime writer's actual destination instead of unconditionally + # joining the EntraUsers leaf onto $OutputFile's parent (the Purview -OutputPath dir). + $_uiDt = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + $entraFile = if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -eq 'Local') { + Join-Path $_uiDt.EffectiveDir $_uiDt.Basename + } elseif ($_uiDt -and $_uiDt.IsBound) { + $_uiDt.EffectiveDir.TrimEnd('/') + '/' + $_uiDt.Basename + } else { + Join-Path (Split-Path $OutputFile -Parent) (& $script:GetEffectiveEntraBasename) + } + if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -ne 'Local') { + Write-LogHost " Entra Users File: $entraFile" -ForegroundColor Gray + } else { + Write-LogHost " Entra Users File: $(Get-DisplayPath -LocalPath $entraFile)" -ForegroundColor Gray + } + } + if ($IncludeAgent365Info) { + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; same + # rationale as the EntraUsers branch above. + $_aiDt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -eq 'Local') { + Join-Path $_aiDt.EffectiveDir $_aiDt.Basename + } elseif ($_aiDt -and $_aiDt.IsBound) { + $_aiDt.EffectiveDir.TrimEnd('/') + '/' + $_aiDt.Basename + } else { + Join-Path (Split-Path $OutputFile -Parent) (& $script:GetEffectiveAgent365Basename) + } + if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -ne 'Local') { + Write-LogHost " Agent 365 File: $agent365PreviewFile" -ForegroundColor Gray + } else { + Write-LogHost " Agent 365 File: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray + } + } + } else { + # Separate CSV files per activity type + $outputDir = Split-Path $OutputFile -Parent + $displayDir = (Get-DisplayPath -LocalPath $outputDir -Directory).TrimEnd('/','\') + $timestamp = [System.IO.Path]::GetFileNameWithoutExtension($OutputFile) -replace '.*_(\d{8}_\d{6}).*', '$1' + Write-LogHost "Output Directory: $displayDir\" -ForegroundColor White + Write-LogHost "Output Files: ${displayDir}\Purview_Audit_UsageActivity__${timestamp}.csv" -ForegroundColor Gray + if ($IncludeUserInfo -and -not $UseEOM) { + # Honor -OutputPathUserInfo (or an Append-side promotion of DestRaw) when bound, so the + # separate-files preview matches the runtime writer's actual destination instead of + # unconditionally joining the EntraUsers leaf onto $OutputFile's parent. + $_uiDt = script:Resolve-DataTypePaths -DataType 'UserInfo' -DefaultBasename (& $script:GetEffectiveEntraBasename) + $entraFile = if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -eq 'Local') { + Join-Path $_uiDt.EffectiveDir $_uiDt.Basename + } elseif ($_uiDt -and $_uiDt.IsBound) { + $_uiDt.EffectiveDir.TrimEnd('/') + '/' + $_uiDt.Basename + } else { + Join-Path $outputDir (& $script:GetEffectiveEntraBasename) + } + if ($_uiDt -and $_uiDt.IsBound -and $_uiDt.Tier -ne 'Local') { + Write-LogHost " Entra Users: $entraFile" -ForegroundColor Gray + } else { + Write-LogHost " Entra Users: $(Get-DisplayPath -LocalPath $entraFile)" -ForegroundColor Gray + } + } + if ($IncludeAgent365Info) { + # Honor -OutputPathAgent365Info (or an Append-side promotion of DestRaw) when bound; same + # rationale as the EntraUsers branch above. + $_aiDt = script:Resolve-DataTypePaths -DataType 'Agent365Info' -DefaultBasename (& $script:GetEffectiveAgent365Basename) + $agent365PreviewFile = if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -eq 'Local') { + Join-Path $_aiDt.EffectiveDir $_aiDt.Basename + } elseif ($_aiDt -and $_aiDt.IsBound) { + $_aiDt.EffectiveDir.TrimEnd('/') + '/' + $_aiDt.Basename + } else { + Join-Path $outputDir (& $script:GetEffectiveAgent365Basename) + } + if ($_aiDt -and $_aiDt.IsBound -and $_aiDt.Tier -ne 'Local') { + Write-LogHost " Agent 365: $agent365PreviewFile" -ForegroundColor Gray + } else { + Write-LogHost " Agent 365: $(Get-DisplayPath -LocalPath $agent365PreviewFile)" -ForegroundColor Gray + } + } + } +} + +Write-LogHost "Log File: $displayLogFile" -ForegroundColor White +if (-not $RAWInputCSV) { + Write-LogHost "Authentication: $Auth" -ForegroundColor White +} + +# Append-target / Fabric Delta destination summary. Only emitted when at least one +# is in effect, so non-append local runs see no extra banner noise. +if ($AppendFile -or $AppendUserInfo -or $AppendAgent365Info -or + ($script:RemoteOutputMode -eq 'Fabric' -and $script:RemoteOutputUrl -match '/Tables/?$')) { + Write-LogHost "Append / Destination:" -ForegroundColor Yellow + if ($AppendFile) { Write-LogHost (" -AppendFile : {0}" -f $AppendFile) -ForegroundColor Gray } + if ($AppendUserInfo) { Write-LogHost (" -AppendUserInfo : {0}" -f $AppendUserInfo) -ForegroundColor Gray } + if ($AppendAgent365Info) { Write-LogHost (" -AppendAgent365Info : {0}" -f $AppendAgent365Info) -ForegroundColor Gray } + if ($script:RemoteOutputMode -eq 'Fabric' -and $script:RemoteOutputUrl -match '/Tables/?$') { + Write-LogHost (" Fabric destination : {0} (CSV outputs land as Delta tables)" -f $script:RemoteOutputUrl) -ForegroundColor Gray + } +} +} # end: if (-not $ResumeSpecified) — parse-time output/log/auth/append banner + +if ($AgentId -or $AgentsOnly -or $ExcludeAgents -or $PromptFilter -or $UserIds -or $GroupNames) { + Write-LogHost "Filters:" -ForegroundColor Yellow + if ($AgentsOnly) { Write-LogHost " AgentsOnly: Only records with AgentId present" -ForegroundColor Gray } + if ($AgentId) { + $agentDisplay = if ($AgentId.Count -eq 1) { + "Specific AgentId: $($AgentId[0])" + } + elseif ($AgentId.Count -le 3) { + "Specific AgentIds ($($AgentId.Count)): " + ($AgentId -join '; ') + } + else { + "Specific AgentIds ($($AgentId.Count) total):" + } + Write-LogHost " $agentDisplay" -ForegroundColor Gray + if ($AgentId.Count -gt 3) { + for ($i = 0; $i -lt [Math]::Min(3, $AgentId.Count); $i++) { + $displayId = if ($AgentId[$i].Length -gt 80) { $AgentId[$i].Substring(0, 77) + '...' } else { $AgentId[$i] } + Write-LogHost " [$($i+1)] $displayId" -ForegroundColor DarkGray + } + if ($AgentId.Count -gt 3) { + Write-LogHost " ... and $($AgentId.Count - 3) more" -ForegroundColor DarkGray + } + } + } + if ($ExcludeAgents) { Write-LogHost " ExcludeAgents: Only records without AgentId" -ForegroundColor Gray } + if ($PromptFilter) { + $promptLabel = switch ($PromptFilter) { + 'Prompt' { 'Only prompts (Message_isPrompt = True)' } + 'Response' { 'Only responses (Message_isPrompt = False)' } + 'Both' { 'Both prompts and responses (Message_isPrompt = True or False)' } + 'Null' { 'Only records with no Message_isPrompt values (Null/Empty)' } + } + Write-LogHost " PromptFilter: $promptLabel" -ForegroundColor Gray + } + if ($UserIds -or $GroupNames) { + if ($UserIds) { + if ($UserIds.Count -eq 1) { Write-LogHost " UserIds: 1 user" -ForegroundColor Gray } else { Write-LogHost " UserIds: $($UserIds.Count) users" -ForegroundColor Gray } + } + if ($GroupNames) { + if ($GroupNames.Count -eq 1) { Write-LogHost " GroupNames: 1 group" -ForegroundColor Gray } else { Write-LogHost " GroupNames: $($GroupNames.Count) groups" -ForegroundColor Gray } + } + } +} + +Write-LogHost "=============================================" -ForegroundColor Cyan +Write-LogHost "" + +# Now perform AppendFile validation if needed (after banner display). +# +# Test-AppendFileCompatibility's sole purpose is to detect a mismatch between +# the existing file's explosion schema (column shape produced by -ExplodeArrays +# or -ExplodeDeep) and the current run's explosion params. In a Standard → +# Standard append both modes are 'Standard (no explosion)' by construction +# and a mismatch is impossible — the only outcomes are 'compatible' (no-op) +# or a spurious failure when the existing-file probe (Get-Content -First 1) +# can't read the file (e.g. remote AppendFile URL not yet downloaded to +# scratch, file briefly locked by another process). Gate the validator on +# explosion actually being in play OR a replay (-RAWInputCSV) run so the +# only people who see this check are the ones it was designed for. +if ($AppendFile -and ($ExplodeArrays -or $ExplodeDeep -or $RAWInputCSV)) { + + $validation = Test-AppendFileCompatibility ` + -FilePath $OutputFile ` + -IsExcel $ExportWorkbook ` + -ExplodeArrays:$ExplodeArrays ` + -ExplodeDeep:$ExplodeDeep + + if (-not $validation.Compatible) { + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: Explosion Parameter Mismatch - Cannot Append" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host "The existing file was created with different explosion parameters than" -ForegroundColor Yellow + Write-Host "the current command. Appending would create incompatible data structures." -ForegroundColor Yellow + Write-Host "" + Write-Host "Existing file: $OutputFile" -ForegroundColor White + Write-Host " Columns: $($validation.ExistingCount)" -ForegroundColor Gray + Write-Host " Mode: $($validation.ExistingMode.DisplayName)" -ForegroundColor Gray + Write-Host "" + Write-Host "Current command:" -ForegroundColor White + Write-Host " Mode: $($validation.CurrentMode.DisplayName)" -ForegroundColor Gray + Write-Host "" + Write-Host "Root Cause:" -ForegroundColor Cyan + Write-Host " Explosion parameters must match between original file and append operation." -ForegroundColor Yellow + Write-Host "" + Write-Host "Resolution Options:" -ForegroundColor Cyan + Write-Host " 1. Match the original file's parameters:" -ForegroundColor White + Write-Host " Use: $($validation.ExistingMode.DisplayName)" -ForegroundColor Gray + Write-Host " 2. Create new output file instead:" -ForegroundColor White + Write-Host " Remove -AppendFile parameter" -ForegroundColor Gray + + if ($ExportWorkbook) { + Write-Host "" + Write-Host "Note for Excel mode:" -ForegroundColor DarkGray + Write-Host " If parameters matched, mismatched columns would create timestamped" -ForegroundColor DarkGray + Write-Host " duplicate tabs instead of appending (no data loss)." -ForegroundColor DarkGray + } + else { + Write-Host "" + Write-Host "CRITICAL for CSV mode:" -ForegroundColor Yellow + Write-Host " CSV append with mismatched explosion parameters creates CORRUPTED files!" -ForegroundColor Yellow + Write-Host " This validation prevents data corruption by failing early." -ForegroundColor Yellow + } + + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════" -ForegroundColor Red + exit 1 + } + + Write-LogHost " Explosion parameters compatible ($($validation.CurrentMode.DisplayName)) - safe to append" -ForegroundColor Green +} + +if ($ExplodeDeep -and $ExplodeArrays) { Write-LogHost "Note: -ExplodeDeep takes precedence over -ExplodeArrays (arrays will still explode, plus deep flatten)." -ForegroundColor DarkYellow } +if ($ForcedRawInputCsvExplosion -and -not $ExplodeDeep -and -not $ExplodeArrays.IsPresent) { Write-LogHost "RAWInputCSV provided -> forcing Purview array explosion (non-exploded mode disabled)." -ForegroundColor Yellow } +if ($script:memoryFlushEnabled) { + $memSource = if ($MaxMemoryMB -eq -1) { "auto-detected" } else { "user-specified" } + Write-LogHost "Memory management: $($script:ResolvedMaxMemoryMB)MB limit ($memSource) - will flush to disk when exceeded" -ForegroundColor Cyan +} elseif ($script:ResolvedMaxMemoryMB -gt 0 -and ($ExplodeDeep -or $ExplodeArrays -or $ForcedRawInputCsvExplosion)) { + Write-LogHost "Note: Memory limit ($($script:ResolvedMaxMemoryMB)MB) ignored because explosion mode is active" -ForegroundColor DarkYellow +} + +if ($RAWInputCSV) { + # Replay mode (offline): no live EOM/Graph query; show only relevant parameters. + # The output-destinations cluster at the top mirrors the Graph-mode snapshot so + # operators see all file paths in one block. Replay forbids remote destination + # tiers in parameter validation, so the resolveAuxDisplayPath helper effectively + # always falls through to local Get-DisplayPath here — but the structure is kept + # identical for visual consistency between modes. + $resolveAuxDisplayPath = { + param([string]$DataTypeKey, [string]$FileName) + if ($script:DestTier -and $script:DestTier.ContainsKey($DataTypeKey) -and $script:DestTier[$DataTypeKey] -and $script:DestTier[$DataTypeKey] -ne 'Local') { + $folder = if ($script:DestParentUrl -and $script:DestParentUrl.ContainsKey($DataTypeKey) -and $script:DestParentUrl[$DataTypeKey]) { $script:DestParentUrl[$DataTypeKey] } else { $script:DestRaw[$DataTypeKey] } + return ($folder.TrimEnd('/','\') + '/' + $FileName) + } + # Local tier: honor a per-stream -OutputPath override (or an Append-side promotion of + # DestRaw) so the snapshot reflects the actual on-disk destination, not the inherited -OutputPath + # dir. Without this, EntraUsersOutput / Agents365Output displayed the -OutputPath parent even + # when the runtime writer was routing the stream to its own -OutputPath directory. + $auxDt = script:Resolve-DataTypePaths -DataType $DataTypeKey -DefaultBasename $FileName + if ($auxDt -and $auxDt.IsBound -and $auxDt.Tier -eq 'Local') { + return (Get-DisplayPath -LocalPath (Join-Path $auxDt.EffectiveDir $FileName)) + } + $localSibling = Join-Path (Split-Path $OutputFile -Parent) $FileName + return (Get-DisplayPath -LocalPath $localSibling) + } + + $paramSnapshot = [ordered]@{ + Mode = $scriptMode + RAWInputCSV = $RAWInputCSV + 'StartDate (inclusive)' = $StartDate + 'EndDate (exclusive)' = $EndDate + } + + # ── Output destinations cluster ────────────────────────────────────── + # Append-vs-Output suppression: when an -Append* flag is bound the + # resolved destination of that stream IS the Append target — emitting both + # OutputFile/EntraUsersOutput/Agents365Output AND the matching Append* line + # produces two identical URLs that read like a bug. Suppress the resolved- + # destination line in append mode; the Append* line below is the single + # source of truth for that stream. + # Purview stream + if ($PSBoundParameters.ContainsKey('AppendFile') -and $AppendFile) { + $paramSnapshot['AppendFile'] = $AppendFile + } else { + $paramSnapshot['OutputFile'] = $displayOutputFile + } + + # UserInfo stream (Entra users licensing export). Replay mode currently + # does not regenerate EntraUsers, but if -IncludeUserInfo is in scope the + # path is surfaced for parity with the standard snapshot. + if ($PSBoundParameters.ContainsKey('AppendUserInfo') -and $AppendUserInfo) { + $paramSnapshot['AppendUserInfo'] = $AppendUserInfo + } elseif (($IncludeUserInfo -or $OnlyUserInfo) -and -not $UseEOM) { + if ($ExportWorkbook) { + $paramSnapshot['EntraUsersOutput'] = 'Workbook Tab: EntraUsers' + } else { + $paramSnapshot['EntraUsersOutput'] = & $resolveAuxDisplayPath 'UserInfo' (& $script:GetEffectiveEntraBasename) + } + } + + # Agent365Info stream + if ($PSBoundParameters.ContainsKey('AppendAgent365Info') -and $AppendAgent365Info) { + $paramSnapshot['AppendAgent365Info'] = $AppendAgent365Info + } elseif ($IncludeAgent365Info -or $OnlyAgent365Info) { + if ($ExportWorkbook) { + $paramSnapshot['Agents365Output'] = 'Workbook Tab: Agents365' + } else { + $paramSnapshot['Agents365Output'] = & $resolveAuxDisplayPath 'Agent365Info' (& $script:GetEffectiveAgent365Basename) + } + } + + # Log stream + $paramSnapshot['LogFile'] = $displayLogFile + # ── End output destinations cluster ────────────────────────────────── + + $paramSnapshot['ActivityTypes'] = ($ActivityTypes -join ';') + $paramSnapshot['ExcludeCopilotInteraction'] = $ExcludeCopilotInteraction.IsPresent + $paramSnapshot['ExplodeArrays'] = $ForcedRawInputCsvExplosion + $paramSnapshot['ExplodeDeep'] = $ExplodeDeep.IsPresent + if ($ExplodeDeep -or $PSBoundParameters.ContainsKey('FlatDepth')) { $paramSnapshot['FlatDepth'] = $FlatDepth } + $paramSnapshot['UseEOM'] = $UseEOM.IsPresent + $paramSnapshot['MaxMemoryMB'] = $(if ($script:ResolvedMaxMemoryMB -eq 0) { 'Off' } else { "$($script:ResolvedMaxMemoryMB)MB" + $(if ($MaxMemoryMB -eq -1) { ' (auto)' } else { '' }) }) + $paramSnapshot['StatusIntervalSeconds'] = $StatusIntervalSeconds + $paramSnapshot['ExportWorkbook'] = $ExportWorkbook.IsPresent + $paramSnapshot['CombineOutput'] = $CombineOutput.IsPresent + $paramSnapshot['Deidentify'] = $Deidentify.IsPresent + $paramSnapshot['FillerLabel'] = $script:HierarchyFillMode + $paramSnapshot['FillerLabelText'] = $script:HierarchyFillLabel + $paramSnapshot['Force'] = $Force.IsPresent + $paramSnapshot['SkipDiagnostics'] = $SkipDiagnostics.IsPresent + $paramSnapshot['SkipVersionCheck'] = $SkipVersionCheck.IsPresent + $paramSnapshot['EmitMetricsJson'] = $EmitMetricsJson.IsPresent + $paramSnapshot['MetricsPath'] = $(if ($MetricsPath) { $MetricsPath } else { '' }) + $paramSnapshot['StreamingSchemaSample'] = $StreamingSchemaSample + $paramSnapshot['StreamingChunkSize'] = $StreamingChunkSize + $paramSnapshot['PSVersion'] = $PSVersionTable.PSVersion.ToString() + $paramSnapshot['PSEdition'] = $PSVersionTable.PSEdition + $paramSnapshot['HostName'] = $Host.Name + $paramSnapshot['HostVersion'] = $(try { $Host.Version.ToString() } catch { '' }) + + $copilotIncluded = $IncludeCopilotInteraction.IsPresent -or ($ActivityTypes -contains $copilotBaseActivityType) + $paramSnapshot['IncludeCopilotInteraction'] = $copilotIncluded +} +else { + # Smart parameter snapshot: Show only applicable parameters for the chosen query mode. + # + # Top of the snapshot is a single, contiguous "Output destinations" cluster so + # operators can audit every file the run will produce at a glance. Each stream + # (Purview / UserInfo / Agent365Info / Log) gets its lines kept together: + # 1. The user-supplied -OutputPath* (if bound) — shown verbatim (DestRaw) + # 2. The resolved artifact path — shown via Get-DisplayPath / DestParentUrl + # so SharePoint/Fabric tiers display as remote URLs, not local scratch + # 3. The companion -Append* (if bound) — shown verbatim + # Auxiliary helper resolves per-data-type display path for sibling artifacts + # (EntraUsers, Agent365): if the data type's tier is non-Local, use that + # tier's folder URL; otherwise fall through Get-DisplayPath (which maps onto + # the Purview remote URL when the Purview output is remote, or returns the + # local path verbatim when fully local). + $resolveAuxDisplayPath = { + param([string]$DataTypeKey, [string]$FileName) + if ($script:DestTier -and $script:DestTier.ContainsKey($DataTypeKey) -and $script:DestTier[$DataTypeKey] -and $script:DestTier[$DataTypeKey] -ne 'Local') { + $folder = if ($script:DestParentUrl -and $script:DestParentUrl.ContainsKey($DataTypeKey) -and $script:DestParentUrl[$DataTypeKey]) { $script:DestParentUrl[$DataTypeKey] } else { $script:DestRaw[$DataTypeKey] } + return ($folder.TrimEnd('/','\') + '/' + $FileName) + } + # Local tier: honor a per-stream -OutputPath override (or an Append-side promotion of + # DestRaw) so the snapshot reflects the actual on-disk destination, not the inherited -OutputPath + # dir. Without this, EntraUsersOutput / Agents365Output displayed the -OutputPath parent even + # when the runtime writer was routing the stream to its own -OutputPath directory. + $auxDt = script:Resolve-DataTypePaths -DataType $DataTypeKey -DefaultBasename $FileName + if ($auxDt -and $auxDt.IsBound -and $auxDt.Tier -eq 'Local') { + return (Get-DisplayPath -LocalPath (Join-Path $auxDt.EffectiveDir $FileName)) + } + $localSibling = Join-Path (Split-Path $OutputFile -Parent) $FileName + return (Get-DisplayPath -LocalPath $localSibling) + } + + $paramSnapshot = [ordered]@{ + 'StartDate (inclusive)' = $StartDate + 'EndDate (exclusive)' = $EndDate + } + + # ── Output destinations cluster ────────────────────────────────────── + # Append-vs-Output suppression: see RAWInputCSV branch for rationale. + # Purview stream + if ($PSBoundParameters.ContainsKey('AppendFile') -and $AppendFile) { + $paramSnapshot['AppendFile'] = $AppendFile + } else { + $paramSnapshot['OutputFile'] = $displayOutputFile + } + + # UserInfo stream (Entra users licensing export) + if ($PSBoundParameters.ContainsKey('AppendUserInfo') -and $AppendUserInfo) { + $paramSnapshot['AppendUserInfo'] = $AppendUserInfo + } elseif (($IncludeUserInfo -or $OnlyUserInfo) -and -not $UseEOM) { + if ($ExportWorkbook) { + $paramSnapshot['EntraUsersOutput'] = 'Workbook Tab: EntraUsers' + } else { + $paramSnapshot['EntraUsersOutput'] = & $resolveAuxDisplayPath 'UserInfo' (& $script:GetEffectiveEntraBasename) + } + } + + # Agent365Info stream + if ($PSBoundParameters.ContainsKey('AppendAgent365Info') -and $AppendAgent365Info) { + $paramSnapshot['AppendAgent365Info'] = $AppendAgent365Info + } elseif ($IncludeAgent365Info -or $OnlyAgent365Info) { + if ($ExportWorkbook) { + $paramSnapshot['Agents365Output'] = 'Workbook Tab: Agents365' + } else { + $paramSnapshot['Agents365Output'] = & $resolveAuxDisplayPath 'Agent365Info' (& $script:GetEffectiveAgent365Basename) + } + } + + # Log stream + $paramSnapshot['LogFile'] = $displayLogFile + # ── End output destinations cluster ────────────────────────────────── + + # Authentication (both modes, but different usage) + if ($UseEOM) { + # EOM mode: Auth parameter controls connection + $paramSnapshot['Auth'] = $Auth + } else { + # Graph API mode: Auth parameter controls connection + $paramSnapshot['Auth'] = $Auth + # Capture AppRegistration context (without exposing secrets) + $paramSnapshot['TenantId'] = $(if ($TenantId) { $TenantId } elseif ($env:GRAPH_TENANT_ID) { '[GRAPH_TENANT_ID]' } else { '' }) + $paramSnapshot['ClientId'] = $(if ($ClientId) { $ClientId } elseif ($env:GRAPH_CLIENT_ID) { '[GRAPH_CLIENT_ID]' } else { '' }) + if ($PSBoundParameters.ContainsKey('ClientSecret') -or $env:GRAPH_CLIENT_SECRET) { + $paramSnapshot['ClientSecret'] = '[securestring provided]' + } + if ($PSBoundParameters.ContainsKey('ClientCertificateThumbprint')) { + $paramSnapshot['ClientCertificateThumbprint'] = $ClientCertificateThumbprint + } + if ($PSBoundParameters.ContainsKey('ClientCertificateStoreLocation')) { + $paramSnapshot['ClientCertificateStoreLocation'] = $ClientCertificateStoreLocation + } + if ($PSBoundParameters.ContainsKey('ClientCertificatePath')) { + $paramSnapshot['ClientCertificatePath'] = $ClientCertificatePath + } + if ($PSBoundParameters.ContainsKey('ClientCertificatePassword')) { + $paramSnapshot['ClientCertificatePassword'] = '[securestring provided]' + } + } + + # Query parameters specific to each mode + if ($UseEOM) { + # EOM-only: Search-UnifiedAuditLog parameters + $paramSnapshot['BlockHours'] = $BlockHours + $paramSnapshot['ResultSize'] = $ResultSize + $paramSnapshot['PacingMs'] = $PacingMs + $paramSnapshot['DisableAdaptive'] = $DisableAdaptive.IsPresent + $paramSnapshot['MemoryPressureMB'] = $MemoryPressureMB + $paramSnapshot['BackoffBaseSeconds'] = $BackoffBaseSeconds + $paramSnapshot['BackoffMaxSeconds'] = $BackoffMaxSeconds + $paramSnapshot['CircuitBreakerThreshold'] = $CircuitBreakerThreshold + $paramSnapshot['CircuitBreakerCooldownSeconds'] = $CircuitBreakerCooldownSeconds + $paramSnapshot['ProgressSmoothingAlpha'] = $ProgressSmoothingAlpha + # Adaptive-pacing controls — surfaced whenever -UseEOM is in play so the + # effective tuning that drove the run is auditable. These do nothing under + # -DisableAdaptive but are still informative for post-run analysis. + $paramSnapshot['HighLatencyMs'] = $HighLatencyMs + $paramSnapshot['LowLatencyMs'] = $LowLatencyMs + $paramSnapshot['LowLatencyConsecutive'] = $LowLatencyConsecutive + $paramSnapshot['ThroughputDropPct'] = $ThroughputDropPct + $paramSnapshot['ThroughputSmoothingAlpha'] = $ThroughputSmoothingAlpha + $paramSnapshot['AdaptiveConcurrencyCeiling'] = $AdaptiveConcurrencyCeiling + $paramSnapshot['AutoCompleteness'] = $AutoCompleteness.IsPresent + } else { + # Graph-only: parallel/partition orchestration + $paramSnapshot['ParallelMode'] = $ParallelMode + $paramSnapshot['MaxParallelGroups'] = $MaxParallelGroups + $paramSnapshot['EnableParallel'] = $EnableParallel.IsPresent + $paramSnapshot['PartitionHours'] = if ($PartitionHours -gt 0) { $PartitionHours } else { 'auto' } + $paramSnapshot['MaxPartitions'] = $MaxPartitions + $paramSnapshot['MaxNetworkOutageMinutes'] = $MaxNetworkOutageMinutes + $paramSnapshot['IncludeUserInfo'] = $IncludeUserInfo.IsPresent + $paramSnapshot['OnlyUserInfo'] = $OnlyUserInfo.IsPresent + $paramSnapshot['IncludeTelemetry'] = $IncludeTelemetry.IsPresent + } + + # AISID (Defender / AI Solutions Intelligence Dashboard) parameters — surfaced only for + # -Dashboard AISID runs so operators can audit the resolved AISID output folder plus the + # append/delta-cache switches that drove the run. The resolved folder comes from the same + # Resolve-DataTypePaths 'DefenderUsage' lookup used by the destinations banner and the + # metrics JSON, so every audit surface agrees on one value. + if ($Dashboard -eq 'AISID') { + $paramSnapshot['DefenderUsageOutput'] = (script:Get-AISIDResolvedOutputDir) + $paramSnapshot['AppendDefenderUsage'] = $(if ($AppendDefenderUsage) { $AppendDefenderUsage } else { '' }) + $paramSnapshot['DisableAISIDDeltaCache'] = $DisableAISIDDeltaCache.IsPresent + } + + # Crossover (both modes) + $paramSnapshot['MaxConcurrency'] = $MaxConcurrency + $paramSnapshot['MaxMemoryMB'] = $(if ($script:ResolvedMaxMemoryMB -eq 0) { 'Off' } else { "$($script:ResolvedMaxMemoryMB)MB" + $(if ($MaxMemoryMB -eq -1) { ' (auto)' } else { '' }) }) + $paramSnapshot['StatusIntervalSeconds'] = $StatusIntervalSeconds + + # Common toggles and output options + $paramSnapshot['UseEOM'] = $UseEOM.IsPresent + $paramSnapshot['ExportWorkbook'] = $ExportWorkbook.IsPresent + $paramSnapshot['CombineOutput'] = $CombineOutput.IsPresent + $paramSnapshot['Deidentify'] = $Deidentify.IsPresent + $paramSnapshot['FillerLabel'] = $script:HierarchyFillMode + $paramSnapshot['FillerLabelText'] = $script:HierarchyFillLabel + # AppendFile / AppendUserInfo / AppendAgent365Info live in the output-destinations + # cluster at the top of the snapshot, kept next to their corresponding -OutputPath* + # and resolved artifact paths so operators see the full file-routing picture in one + # block. They are intentionally not repeated here. + $paramSnapshot['Force'] = $Force.IsPresent + $paramSnapshot['SkipDiagnostics'] = $SkipDiagnostics.IsPresent + $paramSnapshot['SkipVersionCheck'] = $SkipVersionCheck.IsPresent + $paramSnapshot['Rollup'] = $Rollup.IsPresent + $paramSnapshot['RollupPlusRaw'] = $RollupPlusRaw.IsPresent + $paramSnapshot['Dashboard'] = $Dashboard + $paramSnapshot['EmitMetricsJson'] = $EmitMetricsJson.IsPresent + $paramSnapshot['MetricsPath'] = $(if ($MetricsPath) { $MetricsPath } else { '' }) + $paramSnapshot['StreamingSchemaSample'] = $StreamingSchemaSample + $paramSnapshot['StreamingChunkSize'] = $StreamingChunkSize + + # Common parameters (work in both modes) + $paramSnapshot['ActivityTypes'] = ($ActivityTypes -join ';') + $paramSnapshot['RecordTypes'] = $(if ($RecordTypes) { ($RecordTypes -join ';') } else { '' }) + $paramSnapshot['ServiceTypes'] = $(if ($ServiceTypes) { ($ServiceTypes -join ';') } else { '' }) + $copilotIncluded = $IncludeCopilotInteraction.IsPresent -or ($ActivityTypes -contains $copilotBaseActivityType) + $paramSnapshot['IncludeCopilotInteraction'] = $copilotIncluded + $paramSnapshot['IncludeM365Usage'] = $IncludeM365Usage.IsPresent + $paramSnapshot['IncludeAgent365Info'] = $IncludeAgent365Info.IsPresent + $paramSnapshot['OnlyAgent365Info'] = $OnlyAgent365Info.IsPresent + $paramSnapshot['ExcludeCopilotInteraction'] = $ExcludeCopilotInteraction.IsPresent + + # Post-processing filters (work in both modes - applied during/after explosion) + $paramSnapshot['AgentsOnly'] = $AgentsOnly.IsPresent + $paramSnapshot['AgentId'] = $(if ($AgentId) { ($AgentId -join ';') } else { '' }) + $paramSnapshot['ExcludeAgents'] = $ExcludeAgents.IsPresent + $paramSnapshot['UserId'] = $(if ($UserIds) { ($UserIds -join ';') } else { '' }) + + # GroupNames only works in live mode (requires auth for expansion) + if (-not $UseEOM -and $GroupNames) { + $paramSnapshot['GroupName'] = ($GroupNames -join ';') + } + + $paramSnapshot['PromptFilter'] = $(if ($PromptFilter) { $PromptFilter } else { '' }) + + # Output format parameters (work in both modes) + $paramSnapshot['ExplodeArrays'] = ($ExplodeArrays.IsPresent -or $ForcedRawInputCsvExplosion -or $ExplodeDeep.IsPresent) + $paramSnapshot['ExplodeDeep'] = $ExplodeDeep.IsPresent + # FlatDepth governs how many nested levels -ExplodeDeep walks. Surface it whenever + # ExplodeDeep is in play, OR whenever the user explicitly set it on the CLI, so + # operators can verify depth was honored. + if ($ExplodeDeep -or $PSBoundParameters.ContainsKey('FlatDepth')) { $paramSnapshot['FlatDepth'] = $FlatDepth } + $paramSnapshot['ExplosionThreads'] = $(if ($ExplosionThreads -eq 0) { 'auto' } else { $ExplosionThreads }) + + # Environment info + $paramSnapshot['PSVersion'] = $PSVersionTable.PSVersion.ToString() + $paramSnapshot['PSEdition'] = $PSVersionTable.PSEdition + $paramSnapshot['HostName'] = $Host.Name + $paramSnapshot['HostVersion'] = $(try { $Host.Version.ToString() } catch { '' }) + + # EntraUsersOutput / Agents365Output are emitted up in the output-destinations + # cluster at the top of the snapshot so all per-stream file paths stay together. + # The earlier injection-via-rebuild logic was removed when that cluster was added. +} +# Parameter Snapshot will be displayed after DSPM processing (see line ~4150) + +# Predeclare script-scope collections to satisfy StrictMode before first access +if (-not (Get-Variable -Name DeepExtraColumns -Scope Script -ErrorAction SilentlyContinue)) { $script:DeepExtraColumns = $null } + +<# +===================================================================== + Operational Logic +===================================================================== +#> + +function Find-AllArrays { + param( + $Data, + [string]$Path = '', + [int]$Depth = 0, + [hashtable]$Arrays + ) + if ($null -eq $Data) { return @{} } + if (-not $Arrays) { $Arrays = @{} } + if ($Depth -gt 6) { return $Arrays } + if ($null -eq $Data) { return $Arrays } + + $isArray = ($Data -is [System.Collections.IEnumerable] -and -not ($Data -is [string]) -and (($Data -is [System.Collections.IList]) -or $Data.GetType().IsArray)) + if ($isArray) { + $key = if ($Path) { $Path } else { 'root' } + if (-not $Arrays.ContainsKey($key)) { + $Arrays[$key] = [pscustomobject]@{ Path = $Path; Data = $Data; Count = ($Data | Measure-Object).Count } + } + } + + $props = $null + if ($Data -is [System.Management.Automation.PSObject]) { $props = $Data.PSObject.Properties } + elseif ($Data -is [System.Collections.IDictionary]) { $props = $Data.GetEnumerator() } + + if ($props) { + foreach ($p in $props) { + $name = if ($p -is [System.Collections.DictionaryEntry]) { $p.Key } else { $p.Name } + $val = if ($p -is [System.Collections.DictionaryEntry]) { $p.Value } else { $p.Value } + $childPath = if ($Path) { "$Path.$name" } else { $name } + Find-AllArrays -Data $val -Path $childPath -Depth ($Depth + 1) -Arrays $Arrays | Out-Null + } + } + # Note: Do NOT recurse into array elements - arrays are treated as terminal values + # that will be converted to JSON strings for predictable column names + return $Arrays +} + +function Test-ScalarValue { param($v) ($null -eq $v -or $v -is [string] -or $v -is [char] -or $v -is [bool] -or $v -is [int] -or $v -is [long] -or $v -is [double] -or $v -is [decimal] -or $v -is [float] -or $v -is [datetime] -or $v -is [guid]) } + +function Import-CsvToDataTable { + <# + .SYNOPSIS + Imports a CSV file directly into a System.Data.DataTable using fast .NET StreamReader. + + .DESCRIPTION + This is 10-50x faster than Import-Csv | ConvertTo-DataTable for large files because it: + 1. Uses .NET StreamReader instead of PowerShell's Import-Csv + 2. Avoids creating intermediate PSObjects + 3. Parses CSV directly into DataTable rows + + .PARAMETER Path + The path to the CSV file to import. + + .OUTPUTS + System.Data.DataTable + #> + param( + [Parameter(Mandatory = $true)] + [string]$Path + ) + + $dataTable = New-Object System.Data.DataTable + $reader = $null + + try { + $reader = New-Object System.IO.StreamReader($Path, [System.Text.Encoding]::UTF8) + $lineNum = 0 + $columns = @() + + while ($null -ne ($line = $reader.ReadLine())) { + # Parse CSV line (handles quoted fields with commas) + $fields = [System.Collections.Generic.List[string]]::new() + $field = [System.Text.StringBuilder]::new() + $inQuotes = $false + + for ($i = 0; $i -lt $line.Length; $i++) { + $c = $line[$i] + if ($c -eq '"') { + if ($inQuotes -and $i + 1 -lt $line.Length -and $line[$i + 1] -eq '"') { + [void]$field.Append('"') + $i++ + } else { + $inQuotes = -not $inQuotes + } + } elseif ($c -eq ',' -and -not $inQuotes) { + [void]$fields.Add($field.ToString()) + [void]$field.Clear() + } else { + [void]$field.Append($c) + } + } + [void]$fields.Add($field.ToString()) + + if ($lineNum -eq 0) { + # Header row - create columns + $columns = $fields.ToArray() + foreach ($col in $columns) { + [void]$dataTable.Columns.Add($col, [string]) + } + } else { + # Data row + $row = $dataTable.NewRow() + for ($j = 0; $j -lt [Math]::Min($columns.Count, $fields.Count); $j++) { + $val = $fields[$j] + $row[$j] = if ([string]::IsNullOrEmpty($val)) { [DBNull]::Value } else { $val } + } + [void]$dataTable.Rows.Add($row) + } + $lineNum++ + } + } + finally { + if ($reader) { $reader.Dispose() } + } + + return ,$dataTable +} + +function ConvertTo-DataTable { + <# + .SYNOPSIS + Converts an array of PSObjects to a System.Data.DataTable for high-performance Excel export. + + .DESCRIPTION + Export-Excel with piped PSObjects processes cells one-by-one (~400 cells/sec), which is extremely + slow for large datasets. Send-SQLDataToExcel with DataTable uses bulk insert and is 100-1000x faster. + This function converts PSObject arrays to DataTable format for use with Send-SQLDataToExcel. + + .PARAMETER InputObject + The array of PSObjects to convert to a DataTable. + + .OUTPUTS + System.Data.DataTable + #> + param( + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] + [object[]]$InputObject + ) + + begin { + $dataTable = New-Object System.Data.DataTable + $isFirstRow = $true + $columns = @() + } + + process { + foreach ($obj in $InputObject) { + if ($isFirstRow) { + $columns = @($obj.PSObject.Properties.Name) + foreach ($colName in $columns) { + [void]$dataTable.Columns.Add($colName, [string]) + } + $isFirstRow = $false + } + + $row = $dataTable.NewRow() + foreach ($colName in $columns) { + $val = $obj.$colName + $row[$colName] = if ($null -eq $val) { [DBNull]::Value } else { [string]$val } + } + [void]$dataTable.Rows.Add($row) + } + } + + end { + return ,$dataTable + } +} + +function Export-DataTableToExcel { + <# + .SYNOPSIS + High-performance Excel export using DataTable bulk insert method. + + .DESCRIPTION + Wrapper function that converts PSObjects to DataTable and exports using Send-SQLDataToExcel. + This is 100-1000x faster than piping to Export-Excel for large datasets. + + .PARAMETER Data + The array of PSObjects to export. + + .PARAMETER Path + The path to the Excel file. + + .PARAMETER WorksheetName + The name of the worksheet/tab. + #> + param( + [Parameter(Mandatory = $true)] + [object[]]$Data, + + [Parameter(Mandatory = $true)] + [string]$Path, + + [Parameter(Mandatory = $true)] + [string]$WorksheetName + ) + + $dataTable = $Data | ConvertTo-DataTable + Send-SQLDataToExcel -DataTable $dataTable -Path $Path -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' +} + +function ConvertFrom-PaxCsvRecordText { + <# + .SYNOPSIS + Parses one complete RFC-4180 CSV record into an array of field strings. + .DESCRIPTION + Handles quoted fields, escaped double-quotes ("") and embedded delimiters. + The record text may contain embedded newlines inside quoted fields (see + Read-PaxCsvRecordText). Bounded to a single record. + #> + param([Parameter(Mandatory = $true)][string]$Text) + $fields = [System.Collections.Generic.List[string]]::new() + $sb = [System.Text.StringBuilder]::new() + $inQuotes = $false + for ($i = 0; $i -lt $Text.Length; $i++) { + $c = $Text[$i] + if ($c -eq '"') { + if ($inQuotes -and $i + 1 -lt $Text.Length -and $Text[$i + 1] -eq '"') { [void]$sb.Append('"'); $i++ } + else { $inQuotes = -not $inQuotes } + } elseif ($c -eq ',' -and -not $inQuotes) { + [void]$fields.Add($sb.ToString()); [void]$sb.Clear() + } else { + [void]$sb.Append($c) + } + } + [void]$fields.Add($sb.ToString()) + return ,$fields.ToArray() +} + +function Read-PaxCsvRecordText { + <# + .SYNOPSIS + Reads one complete CSV record from a StreamReader. + .DESCRIPTION + Joins continuation physical lines when a quoted field contains an embedded + newline (RFC-4180), returning the complete record text. Returns $null at end + of stream. Only one record is ever held in memory. + #> + param([Parameter(Mandatory = $true)][System.IO.StreamReader]$Reader) + $line = $Reader.ReadLine() + if ($null -eq $line) { return $null } + $buffer = [System.Text.StringBuilder]::new() + [void]$buffer.Append($line) + $quoteCount = 0 + foreach ($ch in $line.ToCharArray()) { if ($ch -eq '"') { $quoteCount++ } } + while (($quoteCount % 2) -ne 0) { + $next = $Reader.ReadLine() + if ($null -eq $next) { break } + [void]$buffer.Append("`n") + [void]$buffer.Append($next) + foreach ($ch in $next.ToCharArray()) { if ($ch -eq '"') { $quoteCount++ } } + } + return $buffer.ToString() +} + +function Get-PaxCsvHeader { + <# + .SYNOPSIS + Reads only the header record of a CSV as a string[] of column names. + Bounded — reads a single record. + #> + param([Parameter(Mandatory = $true)][string]$CsvPath) + $reader = $null + try { + $reader = New-Object System.IO.StreamReader($CsvPath, [System.Text.Encoding]::UTF8) + $h = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $h) { return @() } + return ,(ConvertFrom-PaxCsvRecordText -Text $h) + } + finally { + if ($reader) { $reader.Dispose() } + } +} + +function Write-PaxCsvToWorksheet { + <# + .SYNOPSIS + Streams a CSV file into an Excel worksheet in bounded batches. + .DESCRIPTION + Reads the CSV record-by-record and writes fixed-size batches of rows via + Send-SQLDataToExcel (the first batch with -Force to create the sheet + bold + frozen header, subsequent batches with -Append). At most one BatchSize slab + of rows is ever materialized, so peak memory is independent of file size. + All cells are written as text (-NoNumberConversion '*'), identical to the + prior whole-file DataTable path. AutoSize is applied on the first (header + + bounded sample) write only. Returns the number of DATA rows written. + .PARAMETER Append + Append rows to an existing worksheet (no new sheet / header is created). + #> + param( + [Parameter(Mandatory = $true)][string]$CsvPath, + [Parameter(Mandatory = $true)][string]$WorkbookPath, + [Parameter(Mandatory = $true)][string]$WorksheetName, + [int]$BatchSize = 5000, + [switch]$Append + ) + $reader = $null + $rowsWritten = 0 + $batch = $null + try { + $reader = New-Object System.IO.StreamReader($CsvPath, [System.Text.Encoding]::UTF8) + $headerText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $headerText) { throw "Workbook source CSV has no header row: $CsvPath" } + $columns = ConvertFrom-PaxCsvRecordText -Text $headerText + $colCount = $columns.Count + $appendMode = [bool]$Append + $firstWrite = $true + + $batch = New-Object System.Data.DataTable + foreach ($c in $columns) { [void]$batch.Columns.Add($c, [string]) } + + while ($true) { + $recText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $recText) { break } + $fields = ConvertFrom-PaxCsvRecordText -Text $recText + $nr = $batch.NewRow() + $lim = [Math]::Min($colCount, $fields.Count) + for ($j = 0; $j -lt $lim; $j++) { + $v = $fields[$j] + $nr[$j] = if ([string]::IsNullOrEmpty($v)) { [DBNull]::Value } else { $v } + } + [void]$batch.Rows.Add($nr) + $rowsWritten++ + + if ($batch.Rows.Count -ge $BatchSize) { + if ($firstWrite -and -not $appendMode) { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + } else { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Append -FreezeTopRow -BoldTopRow -NoNumberConversion '*' + } + $firstWrite = $false + $batch.Rows.Clear() + } + } + + if ($batch.Rows.Count -gt 0) { + if ($firstWrite -and -not $appendMode) { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + } else { + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Append -FreezeTopRow -BoldTopRow -NoNumberConversion '*' + } + } elseif ($firstWrite -and -not $appendMode) { + # Header-only source: create the sheet with just the (bold, frozen) header. + Send-SQLDataToExcel -DataTable $batch -Path $WorkbookPath -WorkSheetName $WorksheetName -Force -FreezeTopRow -BoldTopRow -AutoSize -NoNumberConversion '*' + } + } + finally { + if ($batch) { $batch.Dispose() } + if ($reader) { $reader.Dispose() } + } + return $rowsWritten +} + +function Split-PaxCsvByColumn { + <# + .SYNOPSIS + Partitions a CSV into one temp CSV per distinct value of a column. + .DESCRIPTION + Streams the source CSV once, writing each record verbatim into a per-group + temp CSV (each carrying the original header). First-appearance group order is + preserved. Peak memory is bounded to one open writer per distinct group value + (the number of distinct activity types, not the row count). Returns an ordered + array of [pscustomobject]@{ Value; Path; RowCount }. + #> + param( + [Parameter(Mandatory = $true)][string]$CsvPath, + [Parameter(Mandatory = $true)][string]$ColumnName, + [Parameter(Mandatory = $true)][string]$WorkDir + ) + $reader = $null + $writers = @{} + $paths = @{} + $counts = @{} + $order = [System.Collections.Generic.List[string]]::new() + try { + $reader = New-Object System.IO.StreamReader($CsvPath, [System.Text.Encoding]::UTF8) + $headerText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $headerText) { return ,@() } + $columns = ConvertFrom-PaxCsvRecordText -Text $headerText + $colIdx = [Array]::IndexOf($columns, $ColumnName) + if ($colIdx -lt 0) { throw "Column '$ColumnName' not found in CSV header: $CsvPath" } + $idx = 0 + while ($true) { + $recText = Read-PaxCsvRecordText -Reader $reader + if ($null -eq $recText) { break } + $fields = ConvertFrom-PaxCsvRecordText -Text $recText + $val = if ($colIdx -lt $fields.Count) { $fields[$colIdx] } else { '' } + if (-not $writers.ContainsKey($val)) { + $idx++ + $safe = ($val -replace '[^A-Za-z0-9_.-]', '_') + if ([string]::IsNullOrEmpty($safe)) { $safe = 'blank' } + $fp = Join-Path $WorkDir ("part_{0:D4}_{1}.csv" -f $idx, $safe) + $w = New-Object System.IO.StreamWriter($fp, $false, (New-Object System.Text.UTF8Encoding($false))) + $w.WriteLine($headerText) + $writers[$val] = $w + $paths[$val] = $fp + $counts[$val] = 0 + [void]$order.Add($val) + } + $writers[$val].WriteLine($recText) + $counts[$val] = $counts[$val] + 1 + } + } + finally { + foreach ($w in $writers.Values) { try { $w.Flush(); $w.Dispose() } catch { } } + if ($reader) { $reader.Dispose() } + } + $result = foreach ($v in $order) { [pscustomobject]@{ Value = $v; Path = $paths[$v]; RowCount = $counts[$v] } } + return ,@($result) +} + +function ConvertTo-UniqueString { + param([object]$items, [char]$Sep = ';') + if ($null -eq $items) { return $null } + $set = New-Object System.Collections.Generic.HashSet[string] + foreach ($v in $items) { if ($null -ne $v -and $v -ne '') { [void]$set.Add([string]$v) } } + ([string]::Join($Sep, $set)) +} + +function ConvertTo-FlatColumns { + param([object]$Node, [string]$Prefix = '', [int]$MaxDepth = 60) + $cols = @{} + function Recurse([object]$n, [string]$p, [int]$d) { + if ($d -gt $MaxDepth) { return } + if ($null -eq $n) { if ($p) { $cols[$p.TrimEnd('.')] = $null }; return } + if (Test-ScalarValue $n) { if ($p) { $cols[$p.TrimEnd('.')] = $n }; return } + if ($n -is [System.Collections.IEnumerable] -and -not ($n -is [string]) -and -not ($n -is [System.Collections.IDictionary])) { + # Smart array handling: single-element arrays recurse without index, multi-element become JSON + $arr = @($n) + if ($arr.Count -eq 1) { + # Single element: recurse into it without adding index to path (clean column names) + Recurse -n $arr[0] -p $p -d ($d + 1) + } elseif ($arr.Count -gt 1) { + # Multiple elements: serialize to JSON (row explosion handles important arrays separately) + if ($p) { + try { $cols[$p.TrimEnd('.')] = ($n | ConvertTo-Json -Depth 10 -Compress -ErrorAction SilentlyContinue) } + catch { $cols[$p.TrimEnd('.')] = '' } + } + } else { + # Empty array + if ($p) { $cols[$p.TrimEnd('.')] = '' } + } + return + } + $props = $null; try { $props = $n.PSObject.Properties } catch {} + if ($props) { + foreach ($prop in $props) { $name = [string]$prop.Name; $child = $prop.Value; $cp = if ($p) { $p + $name + '.' } else { $name + '.' }; Recurse -n $child -p $cp -d ($d + 1) } + } + } + Recurse -n $Node -p $Prefix -d 0 + return $cols +} + +function To-RecordArray { + param($records) + $result = @() + if ($null -eq $records) { return $result } + $isEnumerable = ($records -is [System.Collections.IEnumerable]) + $isScalarish = ($records -is [string] -or $records -is [System.Management.Automation.PSObject] -or $records -is [System.Management.Automation.PSCustomObject]) + if ($isEnumerable -and -not $isScalarish) { + foreach ($r in $records) { $result += ,$r } + } + else { + $result += ,$records + } + return $result +} + +try { + $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path + $schemaHelper = Join-Path $scriptDir "..\..\scripts\lib\M365UsageSchema.ps1" + if (-not (Test-Path $schemaHelper)) { $schemaHelper = Join-Path $scriptDir "M365UsageSchema.ps1" } + if (Test-Path $schemaHelper) { . $schemaHelper } +} catch {} + +function Invoke-ReplayInlineExport { + param( + [Parameter(Mandatory)] [string]$SpoolPath + ) + Write-LogHost "Replay inline export starting..." -ForegroundColor Magenta + try { Write-PaxMemoryObservation -Stage 'ReplayExport' -Force } catch {} + $exportTemp = Join-Path ([System.IO.Path]::GetTempPath()) ("pax_export_" + [guid]::NewGuid().ToString() + ".tmp") + # Fixed 153-column M code schema (matches live explosion output exactly) + $columnOrder = $PurviewExplodedHeader + $total = 0 + $idx = 0 + $errCount = 0 + $spoolReader = $null + try { + Open-CsvWriter -Path $exportTemp -Columns $columnOrder + # Stream the re-iterable JSON-lines spool; rebuild the normalized replay record + # per line (identical to the prior in-memory build). The complete replay set is + # never held in memory. + $spoolReader = [System.IO.StreamReader]::new($SpoolPath, ([System.Text.UTF8Encoding]::new($false))) + while ($true) { + $spoolLine = $spoolReader.ReadLine() + if ($null -eq $spoolLine) { break } + if ([string]::IsNullOrWhiteSpace($spoolLine)) { continue } + $idx++ + if ($idx % 5000 -eq 0) { Write-LogHost ("Replay inline progress: {0} records" -f $idx) -ForegroundColor DarkGray; try { Write-PaxMemoryObservation -Stage 'ReplayExport' -RowsProcessed ([long]$idx) } catch {} } + try { + $row = $spoolLine | ConvertFrom-Json + $creation = if ($row.CreationDate) { $d = script:Parse-DateSafe $row.CreationDate; if ($d) { $d } else { Get-Date } } else { Get-Date } + $identity = if ($row.Id) { $row.Id } elseif ($row.RecordId) { $row.RecordId } else { [guid]::NewGuid().ToString() } + $log = [pscustomobject]@{ + RecordType = $(try { [int]$row.RecordType } catch { 0 }) + CreationDate = $creation + UserIds = @(if ($row.UserIds) { $row.UserIds } elseif ($row.UserId) { $row.UserId } else { $null }) + Operations = if ($row.Operations) { $row.Operations } elseif ($row.Operation) { $row.Operation } else { $null } + ResultStatus = $(try { $row.ResultStatus } catch { '' }) + ResultCount = 0 + Identity = $identity + IsValid = $true + ObjectState = '' + AuditData = $row.AuditData + Operation = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } + UserId = if ($row.UserId) { $row.UserId } elseif ($row.UserIds) { $row.UserIds } else { $null } + } + $records = Convert-ToPurviewExplodedRecords -Record $log -Deep:$ExplodeDeep -PromptFilterValue $PromptFilter + $recordsArr = To-RecordArray $records + if ($recordsArr.Count -gt 0) { + $total += $recordsArr.Count + $emitSet = $recordsArr | ForEach-Object { $_ | Select-Object -Property $columnOrder } + $rowsOut = @($emitSet) + if ($rowsOut.Count -gt 0) { Write-CsvRows -Rows $rowsOut -Columns $columnOrder } + } + } catch { + $errCount++ + } + } + } + finally { + if ($spoolReader) { $spoolReader.Dispose() } + try { Close-CsvWriter } catch {} + } + # Fail-closed publish: terminate on failure so the caller preserves the spool. + try { + Move-Item -Force -Path $exportTemp -Destination $OutputFile -ErrorAction Stop + } + catch { + if (Test-Path -LiteralPath $exportTemp) { Remove-Item -LiteralPath $exportTemp -Force -ErrorAction SilentlyContinue } + throw + } + try { $script:metrics.TotalStructuredRows = $total } catch {} + Write-LogHost ("Replay inline export complete: {0} rows" -f $total) -ForegroundColor Green + + # Explosion summary for replay mode + Write-LogHost "" + Write-LogHost "=== REPLAY EXPLOSION SUMMARY ===" -ForegroundColor Cyan + Write-LogHost (" Input records: {0:N0}" -f $idx) -ForegroundColor White + Write-LogHost (" Output rows: {0:N0}" -f $total) -ForegroundColor White + if ($total -gt $idx) { + $explosionRatio = [Math]::Round($total / $idx, 2) + Write-LogHost (" Expansion: {0}x ({1:N0} additional rows from array explosion)" -f $explosionRatio, ($total - $idx)) -ForegroundColor Green + } elseif ($total -eq $idx) { + Write-LogHost " Expansion: 1:1 (no arrays exploded)" -ForegroundColor Yellow + } else { + Write-LogHost (" Reduction: {0:N0} records filtered out" -f ($idx - $total)) -ForegroundColor DarkYellow + } + if ($errCount -gt 0) { + Write-LogHost (" Errors: {0:N0} record(s) failed to process" -f $errCount) -ForegroundColor Red + } + # Replay summary label tracks append mode. + if ($AppendFile) { + Write-LogHost (" Appended to: {0}" -f $AppendFile) -ForegroundColor Gray + } else { + Write-LogHost (" Output file: {0}" -f (Get-DisplayPath -LocalPath $OutputFile)) -ForegroundColor Gray + } + Write-LogHost "" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Explosion-mode bounded input spool (Blocker 1) +# Streams the (deduped, trimmed) normalized records to a re-iterable JSON-lines +# disk spool so the explosion pass never holds the complete normalized record set +# — nor the exploded rows — in memory. Mirrors the accepted RAW replay spool +# pattern (Invoke-ReplayInlineExport): the same normalized record shape is written +# per line and reconstructed on read, and each pass opens a fresh bounded reader. +# ───────────────────────────────────────────────────────────────────────────── +function Write-PaxNormalizedSpool { + param( + [Parameter(Mandatory)] $Records, + [Parameter(Mandatory)] [string]$SpoolPath + ) + $enc = [System.Text.UTF8Encoding]::new($false) + $sw = [System.IO.StreamWriter]::new($SpoolPath, $false, $enc) + $count = 0 + try { + foreach ($log in $Records) { + if ($null -eq $log) { continue } + $creationOut = $null + try { + if ($log.PSObject.Properties['CreationDate'] -and $log.CreationDate) { + $creationOut = ([datetime]$log.CreationDate).ToString('o', [System.Globalization.CultureInfo]::InvariantCulture) + } + } catch { try { $creationOut = [string]$log.CreationDate } catch { $creationOut = $null } } + $obj = [ordered]@{ + Id = $(try { if ($log.PSObject.Properties['Identity'] -and $log.Identity) { $log.Identity } elseif ($log.PSObject.Properties['RecordId'] -and $log.RecordId) { $log.RecordId } elseif ($log.PSObject.Properties['Id']) { $log.Id } else { $null } } catch { $null }) + RecordId = $(try { if ($log.PSObject.Properties['RecordId']) { $log.RecordId } elseif ($log.PSObject.Properties['Identity']) { $log.Identity } else { $null } } catch { $null }) + CreationDate = $creationOut + RecordType = $(try { if ($log.PSObject.Properties['RecordType']) { $log.RecordType } else { $null } } catch { $null }) + Operations = $(try { if ($log.PSObject.Properties['Operations']) { $log.Operations } else { $null } } catch { $null }) + Operation = $(try { if ($log.PSObject.Properties['Operation']) { $log.Operation } else { $null } } catch { $null }) + UserIds = $(try { if ($log.PSObject.Properties['UserIds']) { $log.UserIds } else { $null } } catch { $null }) + UserId = $(try { if ($log.PSObject.Properties['UserId']) { $log.UserId } else { $null } } catch { $null }) + ResultStatus = $(try { if ($log.PSObject.Properties['ResultStatus']) { $log.ResultStatus } else { $null } } catch { $null }) + AuditData = $(try { if ($log.PSObject.Properties['AuditData']) { $log.AuditData } else { $null } } catch { $null }) + } + $sw.WriteLine(($obj | ConvertTo-Json -Compress -Depth 20)) + $count++ + } + } + finally { + $sw.Flush(); $sw.Dispose() + } + return $count +} + +function ConvertFrom-PaxNormalizedSpoolLine { + param([Parameter(Mandatory)] [AllowEmptyString()] [string]$Line) + if ([string]::IsNullOrWhiteSpace($Line)) { return $null } + $row = $Line | ConvertFrom-Json + $creation = if ($row.CreationDate) { $d = script:Parse-DateSafe $row.CreationDate; if ($d) { $d } else { Get-Date } } else { Get-Date } + $identity = if ($row.Id) { $row.Id } elseif ($row.RecordId) { $row.RecordId } else { [guid]::NewGuid().ToString() } + return [pscustomobject]@{ + RecordType = $(try { [int]$row.RecordType } catch { 0 }) + CreationDate = $creation + UserIds = @(if ($row.UserIds) { $row.UserIds } elseif ($row.UserId) { $row.UserId } else { $null }) + Operations = if ($row.Operations) { $row.Operations } elseif ($row.Operation) { $row.Operation } else { $null } + ResultStatus = $(try { $row.ResultStatus } catch { '' }) + ResultCount = 0 + Identity = $identity + IsValid = $true + ObjectState = '' + AuditData = $row.AuditData + Operation = if ($row.Operation) { $row.Operation } elseif ($row.Operations) { $row.Operations } else { $null } + UserId = if ($row.UserId) { $row.UserId } elseif ($row.UserIds) { $row.UserIds } else { $null } + RecordId = if ($row.RecordId) { $row.RecordId } elseif ($row.Id) { $row.Id } else { $identity } + } +} + +function Get-SafeProperty { param($obj, [string]$name) try { if ($null -ne $obj -and $obj.PSObject.Properties[$name]) { return $obj.($name) } } catch {}; return $null } + +# --- Purview Exploded Schema (153 columns — matches M code #"Changed Type" step exactly) --- +$PurviewExplodedHeader = @( + 'RecordId', 'CreationDate', 'RecordType', 'Operation', 'UserId', + 'AssociatedAdminUnits', 'AssociatedAdminUnitsNames', + '@odata.type', 'CreationTime', 'Id', 'OrganizationId', + 'ResultStatus', 'UserKey', 'UserType', 'Version', 'Workload', + 'ClientIP', 'ObjectId', 'AzureActiveDirectoryEventType', + 'ActorContextId', 'ActorIpAddress', 'InterSystemsId', 'IntraSystemId', + 'SupportTicketId', 'TargetContextId', 'ApplicationId', + 'DeviceProperties.OS', 'DeviceProperties.BrowserType', + 'ErrorNumber', + 'SiteUrl', 'SourceRelativeUrl', 'SourceFileName', 'SourceFileExtension', + 'ListId', 'ListItemUniqueId', 'WebId', 'ApplicationDisplayName', 'EventSource', + 'ItemType', 'SiteSensitivityLabelId', 'GeoLocation', 'IsManagedDevice', + 'DeviceDisplayName', 'ListBaseType', 'ListServerTemplate', + 'AuthenticationType', 'Site', 'DoNotDistributeEvent', 'HighPriorityMediaProcessing', + 'BrowserName', 'BrowserVersion', 'CorrelationId', 'Platform', 'UserAgent', + 'ActorInfoString', 'AppId', 'AuthType', 'ClientAppId', 'ClientIPAddress', + 'ClientInfoString', 'ExternalAccess', 'InternalLogonType', 'LogonType', + 'LogonUserSid', 'MailboxGuid', 'MailboxOwnerSid', 'MailboxOwnerUPN', + 'OrganizationName', 'OriginatingServer', 'SessionId', + 'TokenObjectId', 'TokenTenantId', 'TokenType', 'SaveToSentItems', + 'OperationCount', 'FileSizeBytes', + 'MeetingId', 'MeetingType', 'EventSignature', 'EventData', + 'Permission', 'SensitivityLabelId', 'SharingLinkScope', + 'TargetUserOrGroupType', 'TargetUserOrGroupName', + 'MeetingURL', 'ChatId', 'MessageId', 'MessageSizeInBytes', 'MessageType', + 'FormId', 'FormName', 'VideoId', 'VideoName', 'ChannelId', 'ViewDuration', + 'ClientRegion', 'CopilotLogVersion', 'TargetId', + 'TeamName', 'TeamGuid', 'ResponseId', 'IsAnonymous', 'DeviceType', + 'ChannelName', 'ChannelGuid', 'ChannelType', 'AppName', 'EnvironmentName', + 'PlanId', 'PlanName', 'TaskId', 'TaskName', 'PercentComplete', + 'CrossMailboxOperation', + 'RecordTypeNum', 'ResultStatus_Audit', + 'ModelId', 'ModelProvider', 'ModelFamily', + 'TokensTotal', 'TokensInput', 'TokensOutput', 'DurationMs', 'OutcomeStatus', + 'ConversationId', 'TurnNumber', 'RetryCount', 'ClientVersion', 'ClientPlatform', + 'AgentId', 'AgentName', 'AgentVersion', 'AgentCategory', 'ApplicationName', + 'AppHost', 'ThreadId', + 'Context_Id', 'Context_Type', + 'Message_Id', 'Message_isPrompt', + 'AccessedResource_Action', 'AccessedResource_PolicyDetails', 'AccessedResource_SiteUrl', + 'AISystemPlugin_Id', 'AISystemPlugin_Name', + 'ModelTransparencyDetails_ModelName', 'MessageIds', + 'AccessedResource_Name', 'AccessedResource_SensitivityLabel', + 'AccessedResource_ResourceType', 'SensitivityLabel', 'Context_Item' +) + +# --- M365 Usage Base Header --- +$M365UsageBaseHeader = @( + 'RecordId','CreationDate','RecordType','Operation','UserId','AuditData','AssociatedAdminUnits','AssociatedAdminUnitsNames','CreationTime','Id','OrganizationId','ResultStatus','UserKey','UserType','Version','Workload','ClientIP','ObjectId','AzureActiveDirectoryEventType','ExtendedProperties','ExtendedProperties.ResultStatusDetail','ExtendedProperties.Name','ExtendedProperties.Value','ExtendedProperties.UserAgent','ExtendedProperties.RequestType','ModifiedProperties','Actor','Actor.ID','Actor.Type','ActorContextId','ActorIpAddress','InterSystemsId','IntraSystemId','SupportTicketId','Target','Target.ID','Target.Type','TargetContextId','ApplicationId','DeviceProperties','DeviceProperties.OS','DeviceProperties.Name','DeviceProperties.Value','DeviceProperties.BrowserType','DeviceProperties.SessionId','ErrorNumber','ExtendedProperties.KeepMeSignedIn','DeviceProperties.Id','DeviceProperties.DisplayName','DeviceProperties.TrustType','ExtendedProperties.UserAuthenticationMethod','DeviceProperties.IsCompliant','DeviceProperties.IsCompliantAndManaged', + # SharePoint / OneDrive + 'SiteUrl','SourceRelativeUrl','SourceFileName','SourceFileExtension','ListId','ListItemUniqueId','WebId','ApplicationDisplayName','EventSource','ItemType','SiteSensitivityLabelId','GeoLocation','IsManagedDevice','DeviceDisplayName','ListBaseType','ListServerTemplate','AuthenticationType','Site','DoNotDistributeEvent','HighPriorityMediaProcessing', + # App Access Context + 'AppAccessContext.ClientAppId','AppAccessContext.ClientAppName','AppAccessContext.CorrelationId','AppAccessContext.AADSessionId','AppAccessContext.UniqueTokenId','AppAccessContext.AuthTime','AppAccessContext.TokenIssuedAtTime','AppAccessContext.UserObjectId','AppAccessContext.DeviceId' +) + +# --- Unified Replay Header (auto-detects all activity types) --- +# Scans input CSV to detect columns from any record type, merges with PurviewExplodedHeader for Copilot +# Skips CopilotEventData.* paths since explosion produces flat column names +function Get-UnifiedReplayHeader { + param( + [Parameter(Mandatory)][string]$RawCsvPath, + [int]$Sample = 500 + ) + # Base columns common to all activity types + $base = @('RecordId','CreationDate','RecordType','Operation','UserId','AuditData','AssociatedAdminUnits','AssociatedAdminUnitsNames','CreationTime','Id','OrganizationId','ResultStatus','UserKey','UserType','Version','Workload','ClientIP','ObjectId','AzureActiveDirectoryEventType','ExtendedProperties','ExtendedProperties.ResultStatusDetail','ExtendedProperties.Name','ExtendedProperties.Value','ExtendedProperties.UserAgent','ExtendedProperties.RequestType','ModifiedProperties','Actor','Actor.ID','Actor.Type','ActorContextId','ActorIpAddress','InterSystemsId','IntraSystemId','SupportTicketId','Target','Target.ID','Target.Type','TargetContextId','ApplicationId','DeviceProperties','DeviceProperties.OS','DeviceProperties.Name','DeviceProperties.Value','DeviceProperties.BrowserType','DeviceProperties.SessionId','ErrorNumber','ExtendedProperties.KeepMeSignedIn','DeviceProperties.Id','DeviceProperties.DisplayName','DeviceProperties.TrustType','ExtendedProperties.UserAuthenticationMethod','DeviceProperties.IsCompliant','DeviceProperties.IsCompliantAndManaged') + $aug = @( + 'SiteUrl','SourceRelativeUrl','SourceFileName','SourceFileExtension','ListId','ListItemUniqueId','WebId','ApplicationDisplayName','EventSource','ItemType','SiteSensitivityLabelId','GeoLocation','IsManagedDevice','DeviceDisplayName','ListBaseType','ListServerTemplate','AuthenticationType','Site','DoNotDistributeEvent','HighPriorityMediaProcessing', + 'AppAccessContext.ClientAppId','AppAccessContext.ClientAppName','AppAccessContext.CorrelationId','AppAccessContext.AADSessionId','AppAccessContext.UniqueTokenId','AppAccessContext.AuthTime','AppAccessContext.TokenIssuedAtTime','AppAccessContext.UserObjectId','AppAccessContext.DeviceId','AppAccessContext.@odata.type','AppAccessContext.APIId','AppAccessContext.IssuedAtTime' + ) + $detected = New-Object System.Collections.Generic.List[string] + $hasCopilot = $false + + # Recursively detect column paths from JSON, skipping CopilotEventData (handled by explosion with flat names) + function Add-Paths([object]$node, [string]$prefix, [System.Collections.Generic.List[string]]$collector) { + if ($null -eq $node) { return } + if (Test-ScalarValue $node) { if ($prefix) { $collector.Add($prefix) | Out-Null }; return } + if ($node -is [System.Collections.IEnumerable] -and $node -isnot [string]) { + foreach ($item in $node) { Add-Paths $item $prefix $collector } + return + } + if ($node.PSObject -and $node.PSObject.Properties) { + foreach ($prop in $node.PSObject.Properties) { + $pn = $prop.Name; $pv = $prop.Value + $path = if ($prefix) { "$prefix.$pn" } else { $pn } + # SKIP CopilotEventData - explosion handles these with flat column names + if ($pn -eq 'CopilotEventData') { continue } + # Special handling for Name/Value arrays (pivot into columns) + if ($pn -eq 'ExtendedProperties' -and $pv -is [System.Collections.IEnumerable]) { + foreach ($item in $pv) { try { if ($item.Name) { $collector.Add("ExtendedProperties.$($item.Name)") | Out-Null } } catch {} } + continue + } + if ($pn -eq 'DeviceProperties' -and $pv -is [System.Collections.IEnumerable]) { + foreach ($item in $pv) { try { if ($item.Name) { $collector.Add("DeviceProperties.$($item.Name)") | Out-Null } } catch {} } + continue + } + Add-Paths $pv $path $collector + } + } + } + if ($RawCsvPath -and (Test-Path $RawCsvPath)) { + try { + $rows = Import-Csv $RawCsvPath | Select-Object -First $Sample + foreach ($r in $rows) { + try { + $audit = $r.AuditData | ConvertFrom-Json -ErrorAction Stop + if ($audit) { + # Detect if any Copilot records exist + if ($audit.CopilotEventData) { $hasCopilot = $true } + Add-Paths $audit '' $detected + } + } catch {} + } + } catch {} + } + # Build unified header: base + augmented + detected (non-Copilot) + PurviewExplodedHeader (flat Copilot columns) + $header = New-Object System.Collections.Generic.List[string] + foreach ($c in $base) { if (-not $header.Contains($c)) { $header.Add($c) } } + foreach ($c in $aug) { if (-not $header.Contains($c)) { $header.Add($c) } } + foreach ($c in $detected) { if (-not $header.Contains($c)) { $header.Add($c) } } + # Always include flat Copilot columns from PurviewExplodedHeader (supports all activity types) + foreach ($c in $PurviewExplodedHeader) { if (-not $header.Contains($c)) { $header.Add($c) } } + try { + if ($RawCsvPath) { + $hdrPath = Join-Path (Split-Path $RawCsvPath -Parent) 'UnifiedReplayHeader.txt' + $header | Set-Content -Path $hdrPath -Encoding utf8 + } + } catch {} + return $header +} + +# --- M365 Usage Wide Header (used for replay of previously exported raw CSVs) --- +function Get-M365UsageWideHeader { + param( + [string]$RawCsvPath, + [int]$Sample = 500 + ) + # Delegate to unified header function + return Get-UnifiedReplayHeader -RawCsvPath $RawCsvPath -Sample $Sample +} + +# --- Entra Users Schema (47 columns) --- +# 30 core + 5 manager + 2 license columns + 10 Power BI template compatibility columns +$EntraUsersHeader = @( + 'userPrincipalName','displayName','id','mail','givenName','surname','jobTitle','department','employeeType','employeeId','employeeHireDate', + 'officeLocation','city','state','country','postalCode','companyName','employeeOrgData_division','employeeOrgData_costCenter', + 'accountEnabled','userType','createdDateTime','usageLocation','preferredLanguage','onPremisesSyncEnabled','onPremisesImmutableId','externalUserState', + 'proxyAddresses_Primary','proxyAddresses_Count','proxyAddresses_All', + 'manager_id','manager_displayName','manager_userPrincipalName','manager_mail','manager_jobTitle', + 'assignedLicenses','hasLicense', + # Power BI template compatibility columns (alias mappings) + 'ManagerID','BusinessAreaLabel','CountryofEmployment','CompanyCodeLabel','CostCentreLabel','UserName', + # Power BI template compatibility columns (null placeholders for Viva Insights fields) + 'EffectiveDate','FunctionType','BusinessAreaCode','OrgLevel_3Label' +) + +function Test-EntraUsersSchema { + param( + [Parameter(Mandatory=$true)][array]$Users, + [switch]$Quiet + ) + if (-not $Users -or $Users.Count -eq 0) { return } + $expected = $EntraUsersHeader + $actual = $Users[0].PSObject.Properties.Name + $missing = @(); foreach ($c in $expected) { if ($c -notin $actual) { $missing += $c } } + $extra = @(); foreach ($c in $actual) { if ($c -notin $expected) { $extra += $c } } + if ($missing.Count -gt 0 -or $extra.Count -gt 0) { + Write-LogHost ("WARNING: EntraUsers schema mismatch. Missing: {0}; Extra: {1}" -f ($missing -join ', '), ($extra -join ', ')) -ForegroundColor Yellow + } elseif (-not $Quiet) { + Write-LogHost "Validated EntraUsers schema ($($expected.Count) columns)." -ForegroundColor DarkGray + } +} + +# ═══════════════════════════════════════════════════════════════════════════════ +# HYBRID DIRECTORY SUPPLEMENT (-UserInfoSupplement) — shared input transport + parse/validate + merge +# The three functions below implement the customer-facing hybrid enrichment mode. They are script-scoped +# and pure (no global side effects beyond their return values) so the focused test suite can AST-extract +# and exercise the SHIPPED code paths directly. +# ═══════════════════════════════════════════════════════════════════════════════ + +function script:Get-PaxReservedDirectoryColumns { + # Authoritative reserved-column set that a supplemental column must never collide with + # (case-insensitive). Derived from the live Entra users contract ($EntraUsersHeader) UNION the + # PAX/Rollup-owned identity, licensing, append-provenance, and hierarchy columns produced downstream + # by the Users dimension. Deriving from $EntraUsersHeader keeps the Entra half in sync with the schema. + $reserved = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($c in $EntraUsersHeader) { [void]$reserved.Add([string]$c) } + foreach ($c in @('PersonId','UserKey','PersonId_Normalized','License Status','TotalEmployees', + 'Date_Added','Latest_Append_Date','In_Latest_Append','Manager_UserKey','OrgLevel','HierarchyPath', + 'TopOfChain_UserKey','IsManager','DirectReports','TotalReports')) { [void]$reserved.Add($c) } + for ($lvl = 0; $lvl -le 14; $lvl++) { [void]$reserved.Add("Level${lvl}_UserKey"); [void]$reserved.Add("Level${lvl}_Name") } + # Unary comma prevents PowerShell from unrolling the HashSet into an array on return (which would + # silently make .Contains() a case-SENSITIVE array match). Callers receive the case-insensitive set. + return ,$reserved +} + +function script:Resolve-DirectoryCsvInput { + # Shared, hardened classifier + stager for a customer-provided directory CSV INPUT (used by BOTH + # -UserInfoFile and -UserInfoSupplement). Classifies the input value's OWN tier independent of the + # output tier, requires an actual .csv file leaf (folder URLs, Fabric roots, and Fabric Tables URLs are + # rejected; a Fabric input must live under /Files/.../*.csv), and for remote tiers stages the file to + # run-specific local scratch via the matching Get-RemoteFile-* using the input URL's own parent + leaf. + # Staged inputs are never registered for upload. Returns @{ Tier; LocalPath; Leaf; Parent; Staged }. + # Throws (or Get-PathTier hard-stops) on any classification/fetch failure — callers do NOT fall back. + param( + [Parameter(Mandatory)][string]$Value, + [Parameter(Mandatory)][string]$SwitchName, + [Parameter(Mandatory)][string]$ScratchDir, + [Parameter(Mandatory)][string]$ScratchPrefix + ) + $raw = $Value.Trim() + $tier = script:Get-PathTier -Value $raw -SwitchName $SwitchName -AllowFabricFilesOnly + $leaf = ($raw -split '[\\/]' | Where-Object { $_ }) | Select-Object -Last 1 + if ([string]::IsNullOrWhiteSpace($leaf) -or ($leaf -notmatch '(?i)\.csv$')) { + Write-Host ("ERROR: -{0} must reference a .csv file (not a folder). Provided: {1}" -f $SwitchName, $raw) -ForegroundColor Red + throw ("-{0} must reference a .csv file: {1}" -f $SwitchName, $raw) + } + if ($tier -eq 'Local') { + return @{ Tier = 'Local'; LocalPath = $raw; Leaf = $leaf; Parent = (Split-Path -Path $raw -Parent); Staged = $false } + } + $parent = $raw.Substring(0, $raw.Length - $leaf.Length).TrimEnd('/', '\') + $localPath = Join-Path $ScratchDir ('{0}{1}' -f $ScratchPrefix, $leaf) + if ($tier -eq 'SharePoint') { + Get-RemoteFile-SharePoint -RelativeName $leaf -DestinationPath $localPath -ParentOverride $parent -ErrorAction Stop + } + else { + Get-RemoteFile-OneLake -RelativeName $leaf -DestinationPath $localPath -ParentOverride $parent -ErrorAction Stop + } + return @{ Tier = $tier; LocalPath = $localPath; Leaf = $leaf; Parent = $parent; Staged = $true } +} + +function script:Import-PaxSupplementCsv { + # Parse + validate a supplemental directory CSV for -UserInfoSupplement. Structured CSV parsing only + # (Import-Csv), never string splitting. Enforces: readable CSV header; >=1 data row; exactly one + # 'UserPrincipalName' header (case-insensitive; NO aliases); no case-insensitive duplicate headers; + # non-blank UPN on every row; unique normalized UPNs; and no supplemental (passthrough) column colliding + # with a reserved Entra/PAX-owned column. The join key is removed from the passthrough set (it is never + # copied into output). Returns @{ Rows; UpnColumn; PassthroughColumns; Lookup } where Lookup maps a + # normalized UPN (Trim().ToLowerInvariant()) to the ORIGINAL supplemental row. Throws a clear, + # count/list-bearing message on any violation. + param([Parameter(Mandatory)][string]$Path) + $rows = @(Import-Csv -LiteralPath $Path -ErrorAction Stop) + if ($rows.Count -eq 0) { + $firstLine = Get-Content -LiteralPath $Path -TotalCount 1 -ErrorAction SilentlyContinue + if ([string]::IsNullOrWhiteSpace($firstLine)) { throw "-UserInfoSupplement has no header row (not a CSV): $Path" } + throw "-UserInfoSupplement contains a header but zero data rows: $Path" + } + $headers = @($rows[0].PSObject.Properties.Name) + $seen = @{}; $dupHeaders = @() + foreach ($h in $headers) { $k = ([string]$h).Trim().ToLowerInvariant(); if ($seen.ContainsKey($k)) { $dupHeaders += $h } else { $seen[$k] = $h } } + if ($dupHeaders.Count -gt 0) { throw ("-UserInfoSupplement has duplicate column header(s) (case-insensitive): {0}" -f (($dupHeaders | Select-Object -Unique) -join ', ')) } + $upnCols = @($headers | Where-Object { ([string]$_).Trim().ToLowerInvariant() -eq 'userprincipalname' }) + if ($upnCols.Count -eq 0) { throw ("-UserInfoSupplement is missing the required 'UserPrincipalName' column. Found: {0}" -f ($headers -join ', ')) } + if ($upnCols.Count -gt 1) { throw ("-UserInfoSupplement has more than one 'UserPrincipalName' column; exactly one is required.") } + $upnCol = $upnCols[0] + $passthrough = @($headers | Where-Object { $_ -ne $upnCol }) + $reserved = script:Get-PaxReservedDirectoryColumns + $collisions = @($passthrough | Where-Object { $reserved.Contains([string]$_) }) + if ($collisions.Count -gt 0) { throw ("-UserInfoSupplement column(s) collide with reserved Entra/PAX-owned columns (supplemental columns are additive-only and cannot override PAX data): {0}" -f ($collisions -join ', ')) } + $lookup = [System.Collections.Generic.Dictionary[string,object]]::new([System.StringComparer]::OrdinalIgnoreCase) + $blankCount = 0; $dupUpns = [System.Collections.Generic.List[string]]::new() + foreach ($row in $rows) { + $rawUpn = [string]$row.$upnCol + if ([string]::IsNullOrWhiteSpace($rawUpn)) { $blankCount++; continue } + $norm = $rawUpn.Trim().ToLowerInvariant() + if ($lookup.ContainsKey($norm)) { $dupUpns.Add($rawUpn.Trim()) } else { $lookup[$norm] = $row } + } + if ($blankCount -gt 0) { throw ("-UserInfoSupplement has {0} row(s) with a blank UserPrincipalName; every supplemental row must carry a UPN." -f $blankCount) } + if ($dupUpns.Count -gt 0) { throw ("-UserInfoSupplement has duplicate UserPrincipalName value(s) (normalized, case-insensitive): {0}" -f (($dupUpns | Select-Object -Unique) -join ', ')) } + return @{ Rows = $rows; UpnColumn = $upnCol; PassthroughColumns = $passthrough; Lookup = $lookup } +} + +function script:Merge-PaxEntraSupplement { + # Entra-LEFT join: append the supplemental passthrough columns to each Entra row, preserving Entra row + # order and supplemental column order. A matching Entra user receives the exact supplemental value; a + # non-matching Entra user receives '' for every supplemental column. Supplemental-only users are NEVER + # added (Entra stays authoritative); the supplemental UPN is NEVER copied into output. Entra rows are + # enriched in place (each supplemental column appended AFTER all existing properties). Returns + # @{ Rows; Matched; UnmatchedSupplemental; EntraWithout; ColumnsAdded }. + param( + [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$EntraRows, + [Parameter(Mandatory)][string[]]$PassthroughColumns, + [Parameter(Mandatory)]$Lookup, + [Parameter(Mandatory)][string]$UpnColumn + ) + $matchedKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + $entraWithout = 0 + foreach ($er in $EntraRows) { + $eupn = [string]$er.userPrincipalName + $norm = if ($eupn) { $eupn.Trim().ToLowerInvariant() } else { '' } + $supRow = $null + if ($norm -and $Lookup.ContainsKey($norm)) { $supRow = $Lookup[$norm]; [void]$matchedKeys.Add($norm) } else { $entraWithout++ } + foreach ($pc in $PassthroughColumns) { + $val = if ($supRow) { [string]$supRow.$pc } else { '' } + $er.PSObject.Properties.Remove($pc) + $er | Add-Member -NotePropertyName $pc -NotePropertyValue $val -Force + } + } + $unmatched = [System.Collections.Generic.List[string]]::new() + foreach ($k in $Lookup.Keys) { if (-not $matchedKeys.Contains($k)) { $unmatched.Add([string]$Lookup[$k].$UpnColumn) } } + return @{ Rows = $EntraRows; Matched = $matchedKeys.Count; UnmatchedSupplemental = @($unmatched); EntraWithout = $entraWithout; ColumnsAdded = @($PassthroughColumns) } +} + +# ═══════════════════════════════════════════════════════════════════════════════ +# FAST ROW CREATION HELPER + +# Converts a hashtable to PSCustomObject in a single operation (avoids Add-Member overhead) +# Used by explosion logic to build rows efficiently via hashtable accumulation +# ═══════════════════════════════════════════════════════════════════════════════ +function New-FastRow { + <# + .SYNOPSIS + Creates a PSCustomObject from a hashtable in a single operation. + .DESCRIPTION + Builds a row by accumulating properties in a hashtable first, then converting + to PSCustomObject once. This is significantly faster than repeated Add-Member calls. + .PARAMETER Properties + A hashtable containing property names and values for the new object. + .EXAMPLE + $props = @{ Name = 'Test'; Value = 123 } + $row = New-FastRow -Properties $props + #> + [CmdletBinding()] + param([Parameter(Mandatory)][hashtable]$Properties) + return [PSCustomObject]$Properties +} + +$existingDeep = Get-Variable -Name DeepExtraColumns -Scope Script -ErrorAction SilentlyContinue +if (-not $existingDeep -or -not $script:DeepExtraColumns) { $script:DeepExtraColumns = New-Object System.Collections.Generic.List[string] } + +function Convert-ToPurviewExplodedRecords { + [CmdletBinding()] + param( + [Parameter(Mandatory)] $Record, + [switch]$Deep, + [switch]$PartialExplode, # NEW: Prompt-specific explosion only (preserves AuditData) + [string]$PromptFilterValue, + [switch]$SkipMetrics # Used by parallel replay to defer metrics aggregation to parent thread + ) + try { + $auditData = if ($Record.PSObject.Properties['_ParsedAuditData']) { $Record._ParsedAuditData } else { try { $Record.AuditData | ConvertFrom-Json -ErrorAction Stop } catch { $null } } + if (-not $auditData) { + if (-not $SkipMetrics) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringMissingAuditData++ + } + return @() + } + try { Profile-AuditData $auditData } catch {} + # Ensure helper is available when re-entering session (PS alias scoping) + if (-not (Get-Command Find-AllArrays -ErrorAction SilentlyContinue)) { Set-Alias -Name Find-AllArrays -Value Find-AllArrays -ErrorAction SilentlyContinue | Out-Null } + + $ced = Get-SafeProperty $auditData 'CopilotEventData' + if (-not $ced) { + # ── M code-aligned non-Copilot path: fixed 153-column extraction (no dynamic discovery) ── + # Produces exactly 1 row per record with all 153 M code columns populated from AuditData. + # No array explosion for non-Copilot records (matches M code behaviour). + # DeviceProperties NV-pivot: only .OS and .BrowserType (matches M code GetNVProp). + $recordId = if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id } + $creationDate = script:Format-DatePurviewFast $Record.CreationDate + $creationTime = try { script:Format-DatePurviewFast $auditData.CreationTime } catch { '' } + $opValue = try { $auditData.Operation } catch { if ($Record.Operation) { $Record.Operation } else { $Record.Operations } } + $uidValue = try { $auditData.UserId } catch { if ($Record.UserId) { $Record.UserId } elseif ($Record.UserIds) { $Record.UserIds } else { '' } } + $recordType = $Record.RecordType + $resultStatus = Get-SafeProperty $auditData 'ResultStatus' + $recordTypeNum = try { [int]$recordType } catch { $recordType } + $applicationId = Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'ApplicationId'), (Get-SafeProperty $auditData 'AppId'), (Get-SafeProperty $auditData 'ClientAppId')) + # DeviceProperties NV-pivot: only .OS and .BrowserType (matches M code GetNVProp) + $devProps = Get-SafeProperty $auditData 'DeviceProperties' + $dpOS = ''; $dpBrowser = '' + if ($devProps -and ($devProps -is [System.Collections.IEnumerable])) { + foreach ($dp in $devProps) { + try { + if ($dp.Name -eq 'OS') { $dpOS = $dp.Value } + elseif ($dp.Name -eq 'BrowserType') { $dpBrowser = $dp.Value } + } catch {} + } + } + # AgentCategory + $agentIdVal = Get-SafeProperty $auditData 'AgentId' + $agentCat = '' + if ($agentIdVal) { + if ($agentIdVal -like "CopilotStudio.Declarative.*") { $agentCat = "Declarative Agent" } + elseif ($agentIdVal -like "CopilotStudio.CustomEngine.*") { $agentCat = "Custom Engine Agent" } + elseif ($agentIdVal -like "P_*") { $agentCat = "Declarative Agent (Purview)" } + else { $agentCat = "Other Agent" } + } + $rowObj = [PSCustomObject][ordered]@{ + RecordId = $recordId + CreationDate = $creationDate + RecordType = $recordType + Operation = $opValue + UserId = $uidValue + AssociatedAdminUnits = $(try { if ($Record.AssociatedAdminUnits) { $Record.AssociatedAdminUnits } elseif ($auditData.AssociatedAdminUnits) { $auditData.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($Record.AssociatedAdminUnitsNames) { $Record.AssociatedAdminUnitsNames } elseif ($auditData.AssociatedAdminUnitsNames) { $auditData.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + '@odata.type' = (Get-SafeProperty $auditData '@odata.type') + CreationTime = $creationTime + Id = (Get-SafeProperty $auditData 'Id') + OrganizationId = (Get-SafeProperty $auditData 'OrganizationId') + ResultStatus = $resultStatus + UserKey = (Get-SafeProperty $auditData 'UserKey') + UserType = (Get-SafeProperty $auditData 'UserType') + Version = (Get-SafeProperty $auditData 'Version') + Workload = (Get-SafeProperty $auditData 'Workload') + ClientIP = (Get-SafeProperty $auditData 'ClientIP') + ObjectId = (Get-SafeProperty $auditData 'ObjectId') + AzureActiveDirectoryEventType = (Get-SafeProperty $auditData 'AzureActiveDirectoryEventType') + ActorContextId = (Get-SafeProperty $auditData 'ActorContextId') + ActorIpAddress = (Get-SafeProperty $auditData 'ActorIpAddress') + InterSystemsId = (Get-SafeProperty $auditData 'InterSystemsId') + IntraSystemId = (Get-SafeProperty $auditData 'IntraSystemId') + SupportTicketId = (Get-SafeProperty $auditData 'SupportTicketId') + TargetContextId = (Get-SafeProperty $auditData 'TargetContextId') + ApplicationId = $applicationId + 'DeviceProperties.OS' = $dpOS + 'DeviceProperties.BrowserType' = $dpBrowser + ErrorNumber = (Get-SafeProperty $auditData 'ErrorNumber') + SiteUrl = (Get-SafeProperty $auditData 'SiteUrl') + SourceRelativeUrl = (Get-SafeProperty $auditData 'SourceRelativeUrl') + SourceFileName = (Get-SafeProperty $auditData 'SourceFileName') + SourceFileExtension = (Get-SafeProperty $auditData 'SourceFileExtension') + ListId = (Get-SafeProperty $auditData 'ListId') + ListItemUniqueId = (Get-SafeProperty $auditData 'ListItemUniqueId') + WebId = (Get-SafeProperty $auditData 'WebId') + ApplicationDisplayName = (Get-SafeProperty $auditData 'ApplicationDisplayName') + EventSource = (Get-SafeProperty $auditData 'EventSource') + ItemType = (Get-SafeProperty $auditData 'ItemType') + SiteSensitivityLabelId = (Get-SafeProperty $auditData 'SiteSensitivityLabelId') + GeoLocation = (Get-SafeProperty $auditData 'GeoLocation') + IsManagedDevice = (Get-SafeProperty $auditData 'IsManagedDevice') + DeviceDisplayName = (Get-SafeProperty $auditData 'DeviceDisplayName') + ListBaseType = (Get-SafeProperty $auditData 'ListBaseType') + ListServerTemplate = (Get-SafeProperty $auditData 'ListServerTemplate') + AuthenticationType = (Get-SafeProperty $auditData 'AuthenticationType') + Site = (Get-SafeProperty $auditData 'Site') + DoNotDistributeEvent = (Get-SafeProperty $auditData 'DoNotDistributeEvent') + HighPriorityMediaProcessing = (Get-SafeProperty $auditData 'HighPriorityMediaProcessing') + BrowserName = (Get-SafeProperty $auditData 'BrowserName') + BrowserVersion = (Get-SafeProperty $auditData 'BrowserVersion') + CorrelationId = (Get-SafeProperty $auditData 'CorrelationId') + Platform = (Get-SafeProperty $auditData 'Platform') + UserAgent = (Get-SafeProperty $auditData 'UserAgent') + ActorInfoString = (Get-SafeProperty $auditData 'ActorInfoString') + AppId = (Get-SafeProperty $auditData 'AppId') + AuthType = (Get-SafeProperty $auditData 'AuthType') + ClientAppId = (Get-SafeProperty $auditData 'ClientAppId') + ClientIPAddress = (Get-SafeProperty $auditData 'ClientIPAddress') + ClientInfoString = (Get-SafeProperty $auditData 'ClientInfoString') + ExternalAccess = (Get-SafeProperty $auditData 'ExternalAccess') + InternalLogonType = (Get-SafeProperty $auditData 'InternalLogonType') + LogonType = (Get-SafeProperty $auditData 'LogonType') + LogonUserSid = (Get-SafeProperty $auditData 'LogonUserSid') + MailboxGuid = (Get-SafeProperty $auditData 'MailboxGuid') + MailboxOwnerSid = (Get-SafeProperty $auditData 'MailboxOwnerSid') + MailboxOwnerUPN = (Get-SafeProperty $auditData 'MailboxOwnerUPN') + OrganizationName = (Get-SafeProperty $auditData 'OrganizationName') + OriginatingServer = (Get-SafeProperty $auditData 'OriginatingServer') + SessionId = (Get-SafeProperty $auditData 'SessionId') + TokenObjectId = (Get-SafeProperty $auditData 'TokenObjectId') + TokenTenantId = (Get-SafeProperty $auditData 'TokenTenantId') + TokenType = (Get-SafeProperty $auditData 'TokenType') + SaveToSentItems = (Get-SafeProperty $auditData 'SaveToSentItems') + OperationCount = (Get-SafeProperty $auditData 'OperationCount') + FileSizeBytes = (Get-SafeProperty $auditData 'FileSizeBytes') + MeetingId = (Get-SafeProperty $auditData 'MeetingId') + MeetingType = (Get-SafeProperty $auditData 'MeetingType') + EventSignature = (Get-SafeProperty $auditData 'EventSignature') + EventData = (Get-SafeProperty $auditData 'EventData') + Permission = (Get-SafeProperty $auditData 'Permission') + SensitivityLabelId = (Get-SafeProperty $auditData 'SensitivityLabelId') + SharingLinkScope = (Get-SafeProperty $auditData 'SharingLinkScope') + TargetUserOrGroupType = (Get-SafeProperty $auditData 'TargetUserOrGroupType') + TargetUserOrGroupName = (Get-SafeProperty $auditData 'TargetUserOrGroupName') + MeetingURL = (Get-SafeProperty $auditData 'MeetingURL') + ChatId = (Get-SafeProperty $auditData 'ChatId') + MessageId = (Get-SafeProperty $auditData 'MessageId') + MessageSizeInBytes = (Get-SafeProperty $auditData 'MessageSizeInBytes') + MessageType = (Get-SafeProperty $auditData 'MessageType') + FormId = (Get-SafeProperty $auditData 'FormId') + FormName = (Get-SafeProperty $auditData 'FormName') + VideoId = (Get-SafeProperty $auditData 'VideoId') + VideoName = (Get-SafeProperty $auditData 'VideoName') + ChannelId = (Get-SafeProperty $auditData 'ChannelId') + ViewDuration = (Get-SafeProperty $auditData 'ViewDuration') + ClientRegion = (Get-SafeProperty $auditData 'ClientRegion') + CopilotLogVersion = (Get-SafeProperty $auditData 'CopilotLogVersion') + TargetId = (Get-SafeProperty $auditData 'TargetId') + TeamName = (Get-SafeProperty $auditData 'TeamName') + TeamGuid = (Get-SafeProperty $auditData 'TeamGuid') + ResponseId = (Get-SafeProperty $auditData 'ResponseId') + IsAnonymous = (Get-SafeProperty $auditData 'IsAnonymous') + DeviceType = (Get-SafeProperty $auditData 'DeviceType') + ChannelName = (Get-SafeProperty $auditData 'ChannelName') + ChannelGuid = (Get-SafeProperty $auditData 'ChannelGuid') + ChannelType = (Get-SafeProperty $auditData 'ChannelType') + AppName = (Get-SafeProperty $auditData 'AppName') + EnvironmentName = (Get-SafeProperty $auditData 'EnvironmentName') + PlanId = (Get-SafeProperty $auditData 'PlanId') + PlanName = (Get-SafeProperty $auditData 'PlanName') + TaskId = (Get-SafeProperty $auditData 'TaskId') + TaskName = (Get-SafeProperty $auditData 'TaskName') + PercentComplete = (Get-SafeProperty $auditData 'PercentComplete') + CrossMailboxOperation = (Get-SafeProperty $auditData 'CrossMailboxOperation') + RecordTypeNum = $recordTypeNum + ResultStatus_Audit = $resultStatus + ModelId = (Get-SafeProperty $auditData 'ModelId') + ModelProvider = (Get-SafeProperty $auditData 'ModelProvider') + ModelFamily = (Get-SafeProperty $auditData 'ModelFamily') + TokensTotal = (Get-SafeProperty $auditData 'TokensTotal') + TokensInput = (Get-SafeProperty $auditData 'TokensInput') + TokensOutput = (Get-SafeProperty $auditData 'TokensOutput') + DurationMs = (Get-SafeProperty $auditData 'DurationMs') + OutcomeStatus = (Get-SafeProperty $auditData 'OutcomeStatus') + ConversationId = (Get-SafeProperty $auditData 'ConversationId') + TurnNumber = (Get-SafeProperty $auditData 'TurnNumber') + RetryCount = (Get-SafeProperty $auditData 'RetryCount') + ClientVersion = (Get-SafeProperty $auditData 'ClientVersion') + ClientPlatform = (Get-SafeProperty $auditData 'ClientPlatform') + AgentId = $agentIdVal + AgentName = (Get-SafeProperty $auditData 'AgentName') + AgentVersion = (Get-SafeProperty $auditData 'AgentVersion') + AgentCategory = $agentCat + ApplicationName = (Get-SafeProperty $auditData 'ApplicationName') + SensitivityLabel = (Get-SafeProperty $auditData 'SensitivityLabel') + # CED sub-fields — empty for non-Copilot records + AppHost = '' + ThreadId = '' + Context_Id = '' + Context_Type = '' + Message_Id = '' + Message_isPrompt = '' + AccessedResource_Action = '' + AccessedResource_PolicyDetails = '' + AccessedResource_SiteUrl = '' + AISystemPlugin_Id = '' + AISystemPlugin_Name = '' + ModelTransparencyDetails_ModelName = '' + MessageIds = '' + AccessedResource_Name = '' + AccessedResource_SensitivityLabel = '' + AccessedResource_ResourceType = '' + Context_Item = '' + } + if ($Deep) { + # Deep flatten entire AuditData for each row (no raw JSON) + $flatAudit = ConvertTo-FlatColumns -Node $auditData -Prefix '' -MaxDepth $FlatDepthDeep + foreach ($k in $flatAudit.Keys) { if (-not $rowObj.PSObject.Properties[$k]) { Add-Member -InputObject $rowObj -NotePropertyName $k -NotePropertyValue $flatAudit[$k] -Force } } + } + return @($rowObj) + } + $messages = script:GetArrayFast $ced 'Messages' + if ($PromptFilterValue) { + $filteredMessages = New-Object System.Collections.Generic.List[object] + if ($PromptFilterValue -eq 'Null') { + foreach ($msg in $messages) { if ($null -eq $msg.isPrompt) { $filteredMessages.Add($msg) } } + } + elseif ($PromptFilterValue -eq 'Both') { + foreach ($msg in $messages) { if ($null -ne $msg.isPrompt) { $filteredMessages.Add($msg) } } + } + else { + $targetValue = ($PromptFilterValue -eq 'Prompt') + foreach ($msg in $messages) { try { if ($msg.isPrompt -eq $targetValue) { $filteredMessages.Add($msg) } } catch {} } + } + $messages = $filteredMessages + if ($messages.Count -eq 0) { + if (-not $SkipMetrics) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringPromptFiltered++ + } + return @() + } + } + $contexts = script:GetArrayFast $ced 'Contexts' + $resources = script:GetArrayFast $ced 'AccessedResources' + $pluginsRaw = script:GetArrayFast $ced 'AISystemPlugin' + $modelDetRaw = script:GetArrayFast $ced 'ModelTransparencyDetails' + $messageIds = script:GetArrayFast $ced 'MessageIds' + + # DSPM for AI: Extract SensitivityLabels array + $sensitivityLabels = script:GetArrayFast $ced 'SensitivityLabels' + + # DSPM for AI: Determine activity type for conditional 2-level explosion + $activityType = try { $auditData.Operation } catch { $null } + + # DSPM for AI: Extract 2nd-level arrays (for full explosion mode) + $plugins = $null + $recordingSessions = $null + $contextItems = $null + + if (-not $PartialExplode) { + # Full explosion mode: Extract 2nd-level arrays for row count calculation + if ($activityType -eq 'ConnectedAIAppInteraction' -and $appIdentityRaw) { + $plugins = script:GetArrayFast $appIdentityRaw 'Plugins' + } + if ($activityType -eq 'CopilotInteraction' -and $contexts.Count -gt 0) { + # Find max Items[] count across all Contexts + $maxItemsCount = 0 + foreach ($ctx in $contexts) { + if ($ctx) { + $items = script:GetArrayFast $ctx 'Items' + if ($items -and $items.Count -gt $maxItemsCount) { + $maxItemsCount = $items.Count + } + } + } + if ($maxItemsCount -gt 0) { + $contextItems = $maxItemsCount # Store count for row calculation + } + } + } + + if ($PromptFilterValue) { $rowCount = [Math]::Max(1, $messages.Count) } else { + # DSPM for AI: Include all arrays in row count calculation (including AISystemPlugin and ModelTransparencyDetails) + $arrayCounts = @(1, $messages.Count, $contexts.Count, $resources.Count, $sensitivityLabels.Count, $pluginsRaw.Count, $modelDetRaw.Count) + + # Full explosion: include 2nd-level arrays in row count + if (-not $PartialExplode) { + if ($plugins) { $arrayCounts += $plugins.Count } + if ($recordingSessions) { $arrayCounts += $recordingSessions.Count } + if ($contextItems) { $arrayCounts += $contextItems } + } + + $rowCount = ($arrayCounts | Measure-Object -Maximum).Maximum + } + # Removed $plugin0 and $model0 - now using indexed access in row loop for full explosion + $creationDate = script:Format-DatePurviewFast $Record.CreationDate + $creationTime = try { script:Format-DatePurviewFast $auditData.CreationTime } catch { '' } + $appIdentityRaw = (Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'AppIdentity'), (Get-SafeProperty $ced 'AppIdentity'))) + $applicationId = Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'ApplicationId'), (Get-SafeProperty $auditData 'AppId'), (Get-SafeProperty $auditData 'ClientAppId')) + # DeviceProperties NV-pivot: only .OS and .BrowserType (matches M code GetNVProp) + $devProps = Get-SafeProperty $auditData 'DeviceProperties' + $dpOS = ''; $dpBrowser = '' + if ($devProps -and ($devProps -is [System.Collections.IEnumerable])) { + foreach ($dp in $devProps) { + try { + if ($dp.Name -eq 'OS') { $dpOS = $dp.Value } + elseif ($dp.Name -eq 'BrowserType') { $dpBrowser = $dp.Value } + } catch {} + } + } + $appHost = (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AppHost'), (Get-SafeProperty $auditData 'AppHost'), (Get-SafeProperty $auditData 'Workload'))) + $clientRegion = (Get-SafeProperty $auditData 'ClientRegion') + $agentId = (Get-SafeProperty $auditData 'AgentId') + $agentName = (Get-SafeProperty $auditData 'AgentName') + $agentVersion = (Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'AgentVersion'), (Get-SafeProperty $ced 'AgentVersion'), (Get-SafeProperty $ced 'Version'))) + + # Agent categorization based on AgentId pattern + $agentCategory = "" + if ($agentId) { + if ($agentId -like "CopilotStudio.Declarative.*") { + $agentCategory = "Declarative Agent" + } elseif ($agentId -like "CopilotStudio.CustomEngine.*") { + $agentCategory = "Custom Engine Agent" + } elseif ($agentId -like "P_*") { + $agentCategory = "Declarative Agent (Purview)" + } elseif ($agentId) { + $agentCategory = "Other Agent" + } + } + + # With -IncludeUserInfo, license data now appears only in EntraUsers output + # (or in combined mode via left join) + + $appName = (Select-FirstNonNull -Values @((Get-SafeProperty $auditData 'ApplicationName'), (Get-SafeProperty $ced 'HostAppName'), (Get-SafeProperty $ced 'ClientAppName'))) + $threadId = (Get-SafeProperty $ced 'ThreadId') + $auditUserKey = try { $auditData.UserKey } catch { $null } + # $modelName moved to row loop for indexed access + $clientIP = (Get-SafeProperty $auditData 'ClientIP') + $organizationId = (Get-SafeProperty $auditData 'OrganizationId') + $version = (Get-SafeProperty $auditData 'Version') + $userType = (Get-SafeProperty $auditData 'UserType') + $copilotLogVersion = (Get-SafeProperty $auditData 'CopilotLogVersion') + $workload = (Get-SafeProperty $auditData 'Workload') + + # Extract fields to match ExplodeArrays output for Power BI compatibility + $auditDataId = try { $auditData.Id } catch { $null } + $recordTypeNum = try { $auditData.RecordType } catch { $null } + $resultStatusAudit = try { $auditData.ResultStatus } catch { $null } + $appId = try { $auditData.AppId } catch { $null } + $clientAppId = try { $auditData.ClientAppId } catch { $null } + $correlationId = try { $auditData.CorrelationId } catch { $null } + + # Model and token fields (same as ExplodeArrays) + $modelId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelId'), (Get-SafeProperty $ced 'ModelID'), (Get-SafeProperty $auditData 'ModelId')) + $modelProvider = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelProvider'), (Get-SafeProperty $ced 'Provider'), (Get-SafeProperty $ced 'ModelVendor')) + $modelFamily = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelFamily'), (Get-SafeProperty $ced 'ModelType')) + $usageNode = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'Usage'), (Get-SafeProperty $ced 'TokenUsage'), (Get-SafeProperty $ced 'Tokens'), (Get-SafeProperty $auditData 'Usage')) + $tokensTotal = $null; $tokensInput = $null; $tokensOutput = $null + if ($usageNode) { + function Local:Get-Num([object]$v) { if ($null -eq $v) { return $null }; try { if ($v -is [string] -and [string]::IsNullOrWhiteSpace($v)) { return $null }; return [double]$v } catch { return $null } } + $tokensTotal = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Total'), (Get-SafeProperty $usageNode 'TotalTokens'), (Get-SafeProperty $usageNode 'TokensTotal'))) + $tokensInput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Input'), (Get-SafeProperty $usageNode 'Prompt'), (Get-SafeProperty $usageNode 'InputTokens'), (Get-SafeProperty $usageNode 'TokensInput'))) + $tokensOutput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Output'), (Get-SafeProperty $usageNode 'Completion'), (Get-SafeProperty $usageNode 'OutputTokens'), (Get-SafeProperty $usageNode 'TokensOutput'))) + } + if (-not $tokensTotal -and ($tokensInput -or $tokensOutput)) { try { $tokensTotal = ($tokensInput + $tokensOutput) } catch {} } + + # Duration, outcome, conversation fields (same as ExplodeArrays) + function Local:Get-NumSafe([object]$v) { if ($null -eq $v) { return $null }; try { if ($v -is [string] -and [string]::IsNullOrWhiteSpace($v)) { return $null }; return [double]$v } catch { return $null } } + $durationMs = Local:Get-NumSafe (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'DurationMs'), (Get-SafeProperty $ced 'ElapsedMs'), (Get-SafeProperty $ced 'ProcessingTimeMs'), (Get-SafeProperty $ced 'LatencyMs'))) + $outcomeStatus = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'OutcomeStatus'), (Get-SafeProperty $ced 'Outcome'), (Get-SafeProperty $ced 'Result'), (Get-SafeProperty $ced 'Status')) + if ($outcomeStatus -is [bool]) { $outcomeStatus = if ($outcomeStatus) { 'Success' } else { 'Failure' } } + $conversationId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ConversationId'), (Get-SafeProperty $ced 'ConversationID'), (Get-SafeProperty $ced 'SessionId')) + $turnNumber = Local:Get-NumSafe (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'TurnNumber'), (Get-SafeProperty $ced 'TurnIndex'), (Get-SafeProperty $ced 'MessageIndex'))) + $retryCount = Local:Get-NumSafe (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'RetryCount'), (Get-SafeProperty $ced 'Retries'))) + $clientVersion = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientVersion'), (Get-SafeProperty $ced 'Version'), (Get-SafeProperty $ced 'Build')) + $clientPlatform = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientPlatform'), (Get-SafeProperty $ced 'Platform'), (Get-SafeProperty $ced 'OS')) + + $baseSet = New-Object System.Collections.Generic.HashSet[string]; foreach ($c in $PurviewExplodedHeader) { $null = $baseSet.Add($c) } + $rows = New-Object System.Collections.Generic.List[object] + for ($i = 0; $i -lt $rowCount; $i++) { + $rowObj = [PSCustomObject][ordered]@{ + RecordId = $(if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id }) + CreationDate = $creationDate + RecordType = $Record.RecordType + Operation = $auditData.Operation + UserId = $auditData.UserId + AssociatedAdminUnits = $(try { if ($Record.AssociatedAdminUnits) { $Record.AssociatedAdminUnits } elseif ($auditData.AssociatedAdminUnits) { $auditData.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($Record.AssociatedAdminUnitsNames) { $Record.AssociatedAdminUnitsNames } elseif ($auditData.AssociatedAdminUnitsNames) { $auditData.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + '@odata.type' = (Get-SafeProperty $auditData '@odata.type') + CreationTime = $creationTime + Id = $auditDataId + OrganizationId = $organizationId + ResultStatus = $resultStatusAudit + UserKey = $auditUserKey + UserType = $userType + Version = $version + Workload = $workload + ClientIP = $clientIP + ObjectId = (Get-SafeProperty $auditData 'ObjectId') + AzureActiveDirectoryEventType = (Get-SafeProperty $auditData 'AzureActiveDirectoryEventType') + ActorContextId = (Get-SafeProperty $auditData 'ActorContextId') + ActorIpAddress = (Get-SafeProperty $auditData 'ActorIpAddress') + InterSystemsId = (Get-SafeProperty $auditData 'InterSystemsId') + IntraSystemId = (Get-SafeProperty $auditData 'IntraSystemId') + SupportTicketId = (Get-SafeProperty $auditData 'SupportTicketId') + TargetContextId = (Get-SafeProperty $auditData 'TargetContextId') + ApplicationId = $applicationId + 'DeviceProperties.OS' = $dpOS + 'DeviceProperties.BrowserType' = $dpBrowser + ErrorNumber = (Get-SafeProperty $auditData 'ErrorNumber') + SiteUrl = (Get-SafeProperty $auditData 'SiteUrl') + SourceRelativeUrl = (Get-SafeProperty $auditData 'SourceRelativeUrl') + SourceFileName = (Get-SafeProperty $auditData 'SourceFileName') + SourceFileExtension = (Get-SafeProperty $auditData 'SourceFileExtension') + ListId = (Get-SafeProperty $auditData 'ListId') + ListItemUniqueId = (Get-SafeProperty $auditData 'ListItemUniqueId') + WebId = (Get-SafeProperty $auditData 'WebId') + ApplicationDisplayName = (Get-SafeProperty $auditData 'ApplicationDisplayName') + EventSource = (Get-SafeProperty $auditData 'EventSource') + ItemType = (Get-SafeProperty $auditData 'ItemType') + SiteSensitivityLabelId = (Get-SafeProperty $auditData 'SiteSensitivityLabelId') + GeoLocation = (Get-SafeProperty $auditData 'GeoLocation') + IsManagedDevice = (Get-SafeProperty $auditData 'IsManagedDevice') + DeviceDisplayName = (Get-SafeProperty $auditData 'DeviceDisplayName') + ListBaseType = (Get-SafeProperty $auditData 'ListBaseType') + ListServerTemplate = (Get-SafeProperty $auditData 'ListServerTemplate') + AuthenticationType = (Get-SafeProperty $auditData 'AuthenticationType') + Site = (Get-SafeProperty $auditData 'Site') + DoNotDistributeEvent = (Get-SafeProperty $auditData 'DoNotDistributeEvent') + HighPriorityMediaProcessing = (Get-SafeProperty $auditData 'HighPriorityMediaProcessing') + BrowserName = (Get-SafeProperty $auditData 'BrowserName') + BrowserVersion = (Get-SafeProperty $auditData 'BrowserVersion') + CorrelationId = $correlationId + Platform = (Get-SafeProperty $auditData 'Platform') + UserAgent = (Get-SafeProperty $auditData 'UserAgent') + ActorInfoString = (Get-SafeProperty $auditData 'ActorInfoString') + AppId = $appId + AuthType = (Get-SafeProperty $auditData 'AuthType') + ClientAppId = $clientAppId + ClientIPAddress = (Get-SafeProperty $auditData 'ClientIPAddress') + ClientInfoString = (Get-SafeProperty $auditData 'ClientInfoString') + ExternalAccess = (Get-SafeProperty $auditData 'ExternalAccess') + InternalLogonType = (Get-SafeProperty $auditData 'InternalLogonType') + LogonType = (Get-SafeProperty $auditData 'LogonType') + LogonUserSid = (Get-SafeProperty $auditData 'LogonUserSid') + MailboxGuid = (Get-SafeProperty $auditData 'MailboxGuid') + MailboxOwnerSid = (Get-SafeProperty $auditData 'MailboxOwnerSid') + MailboxOwnerUPN = (Get-SafeProperty $auditData 'MailboxOwnerUPN') + OrganizationName = (Get-SafeProperty $auditData 'OrganizationName') + OriginatingServer = (Get-SafeProperty $auditData 'OriginatingServer') + SessionId = (Get-SafeProperty $auditData 'SessionId') + TokenObjectId = (Get-SafeProperty $auditData 'TokenObjectId') + TokenTenantId = (Get-SafeProperty $auditData 'TokenTenantId') + TokenType = (Get-SafeProperty $auditData 'TokenType') + SaveToSentItems = (Get-SafeProperty $auditData 'SaveToSentItems') + OperationCount = (Get-SafeProperty $auditData 'OperationCount') + FileSizeBytes = (Get-SafeProperty $auditData 'FileSizeBytes') + MeetingId = (Get-SafeProperty $auditData 'MeetingId') + MeetingType = (Get-SafeProperty $auditData 'MeetingType') + EventSignature = (Get-SafeProperty $auditData 'EventSignature') + EventData = (Get-SafeProperty $auditData 'EventData') + Permission = (Get-SafeProperty $auditData 'Permission') + SensitivityLabelId = (Get-SafeProperty $auditData 'SensitivityLabelId') + SharingLinkScope = (Get-SafeProperty $auditData 'SharingLinkScope') + TargetUserOrGroupType = (Get-SafeProperty $auditData 'TargetUserOrGroupType') + TargetUserOrGroupName = (Get-SafeProperty $auditData 'TargetUserOrGroupName') + MeetingURL = (Get-SafeProperty $auditData 'MeetingURL') + ChatId = (Get-SafeProperty $auditData 'ChatId') + MessageId = (Get-SafeProperty $auditData 'MessageId') + MessageSizeInBytes = (Get-SafeProperty $auditData 'MessageSizeInBytes') + MessageType = (Get-SafeProperty $auditData 'MessageType') + FormId = (Get-SafeProperty $auditData 'FormId') + FormName = (Get-SafeProperty $auditData 'FormName') + VideoId = (Get-SafeProperty $auditData 'VideoId') + VideoName = (Get-SafeProperty $auditData 'VideoName') + ChannelId = (Get-SafeProperty $auditData 'ChannelId') + ViewDuration = (Get-SafeProperty $auditData 'ViewDuration') + ClientRegion = $clientRegion + CopilotLogVersion = $copilotLogVersion + TargetId = (Get-SafeProperty $auditData 'TargetId') + TeamName = (Get-SafeProperty $auditData 'TeamName') + TeamGuid = (Get-SafeProperty $auditData 'TeamGuid') + ResponseId = (Get-SafeProperty $auditData 'ResponseId') + IsAnonymous = (Get-SafeProperty $auditData 'IsAnonymous') + DeviceType = (Get-SafeProperty $auditData 'DeviceType') + ChannelName = (Get-SafeProperty $auditData 'ChannelName') + ChannelGuid = (Get-SafeProperty $auditData 'ChannelGuid') + ChannelType = (Get-SafeProperty $auditData 'ChannelType') + AppName = (Get-SafeProperty $auditData 'AppName') + EnvironmentName = (Get-SafeProperty $auditData 'EnvironmentName') + PlanId = (Get-SafeProperty $auditData 'PlanId') + PlanName = (Get-SafeProperty $auditData 'PlanName') + TaskId = (Get-SafeProperty $auditData 'TaskId') + TaskName = (Get-SafeProperty $auditData 'TaskName') + PercentComplete = (Get-SafeProperty $auditData 'PercentComplete') + CrossMailboxOperation = (Get-SafeProperty $auditData 'CrossMailboxOperation') + RecordTypeNum = $(try { [int]$Record.RecordType } catch { $Record.RecordType }) + ResultStatus_Audit = $resultStatusAudit + ModelId = $modelId + ModelProvider = $modelProvider + ModelFamily = $modelFamily + TokensTotal = $tokensTotal + TokensInput = $tokensInput + TokensOutput = $tokensOutput + DurationMs = $durationMs + OutcomeStatus = $outcomeStatus + ConversationId = $conversationId + TurnNumber = $turnNumber + RetryCount = $retryCount + ClientVersion = $clientVersion + ClientPlatform = $clientPlatform + AgentId = $agentId + AgentName = $agentName + AgentVersion = $agentVersion + AgentCategory = $agentCategory + ApplicationName = (Get-SafeProperty $auditData 'ApplicationName') + SensitivityLabel = $(if ($i -lt $sensitivityLabels.Count) { try { [string]$sensitivityLabels[$i] } catch { '' } } else { '' }) + AppHost = $appHost + ThreadId = $threadId + Context_Id = $(if ($i -lt $contexts.Count -and $contexts[$i]) { try { Get-SafeProperty $contexts[$i] 'Id' } catch { '' } } else { '' }) + Context_Type = $(if ($i -lt $contexts.Count -and $contexts[$i]) { try { Get-SafeProperty $contexts[$i] 'Type' } catch { '' } } else { '' }) + Message_Id = $(if ($i -lt $messages.Count) { $msg = $messages[$i]; if ($msg -is [psobject]) { try { Get-SafeProperty $msg 'Id' } catch { '' } } else { $msg } } else { '' }) + Message_isPrompt = $(if ($i -lt $messages.Count) { $msg = $messages[$i]; if ($msg -is [psobject]) { try { script:BoolTFFast (Get-SafeProperty $msg 'isPrompt') } catch { '' } } else { '' } } else { '' }) + AccessedResource_Action = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'Action' } catch { '' } } else { '' }) + AccessedResource_PolicyDetails = $(if ($i -lt $resources.Count -and $resources[$i]) { try { script:ToJsonIfObjectFast (Get-SafeProperty $resources[$i] 'PolicyDetails') } catch { '' } } else { '' }) + AccessedResource_SiteUrl = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'SiteUrl' } catch { '' } } else { '' }) + AISystemPlugin_Id = $(if ($i -lt $pluginsRaw.Count -and $pluginsRaw[$i]) { try { Get-SafeProperty $pluginsRaw[$i] 'Id' } catch { '' } } else { '' }) + AISystemPlugin_Name = $(if ($i -lt $pluginsRaw.Count -and $pluginsRaw[$i]) { try { Get-SafeProperty $pluginsRaw[$i] 'Name' } catch { '' } } else { '' }) + ModelTransparencyDetails_ModelName = $(if ($i -lt $modelDetRaw.Count -and $modelDetRaw[$i]) { try { Get-SafeProperty $modelDetRaw[$i] 'ModelName' } catch { '' } } else { '' }) + MessageIds = $(if ($messageIds.Count -gt 0) { $messageIds -join ';' } else { '' }) + AccessedResource_Name = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'Name' } catch { '' } } else { '' }) + AccessedResource_SensitivityLabel = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'SensitivityLabel' } catch { '' } } else { '' }) + AccessedResource_ResourceType = $(if ($i -lt $resources.Count -and $resources[$i]) { try { Get-SafeProperty $resources[$i] 'ResourceType' } catch { '' } } else { '' }) + Context_Item = $( + if ($activityType -eq 'CopilotInteraction') { + if ($PartialExplode) { + if ($i -lt $contexts.Count -and $contexts[$i]) { + try { + $items = script:GetArrayFast $contexts[$i] 'Items' + if ($items -and $items.Count -gt 0) { + ($items | ForEach-Object { try { script:ToJsonIfObjectFast $_ } catch { '' } }) -join ';' + } else { '' } + } catch { '' } + } else { '' } + } else { + try { + $foundItem = $null + foreach ($ctx in $contexts) { + if ($ctx) { + $items = script:GetArrayFast $ctx 'Items' + if ($items -and $i -lt $items.Count) { + $foundItem = $items[$i] + break + } + } + } + if ($foundItem) { script:ToJsonIfObjectFast $foundItem } else { '' } + } catch { '' } + } + } else { '' } + ) + } + + # Partial explosion mode: Preserve AuditData column (full JSON) for downstream processing + if ($PartialExplode) { + try { + Add-Member -InputObject $rowObj -NotePropertyName 'AuditData' -NotePropertyValue $Record.AuditData -Force + } catch {} + } + + # DSPM for AI: 2-level explosion for ConnectedAIAppInteraction (AppIdentity.Plugins[]) + if ($activityType -eq 'ConnectedAIAppInteraction' -and $plugins) { + try { + if ($PartialExplode) { + # Partial mode: Semi-colon-joined JSON for all plugins + $pluginsList = ($plugins | ForEach-Object { try { script:ToJsonIfObjectFast $_ } catch { '' } }) -join ';' + if (-not $rowObj.PSObject.Properties['AppIdentity_Plugins']) { + Add-Member -InputObject $rowObj -NotePropertyName 'AppIdentity_Plugins' -NotePropertyValue $pluginsList -Force + if (-not $script:DeepExtraColumns.Contains('AppIdentity_Plugins')) { [void]$script:DeepExtraColumns.Add('AppIdentity_Plugins') } + } + } else { + # Full mode: One plugin per row + if ($i -lt $plugins.Count) { + $plugin = $plugins[$i] + $pluginJson = try { script:ToJsonIfObjectFast $plugin } catch { '' } + if (-not $rowObj.PSObject.Properties['AppIdentity_Plugin']) { + Add-Member -InputObject $rowObj -NotePropertyName 'AppIdentity_Plugin' -NotePropertyValue $pluginJson -Force + if (-not $script:DeepExtraColumns.Contains('AppIdentity_Plugin')) { [void]$script:DeepExtraColumns.Add('AppIdentity_Plugin') } + } + } + } + } catch {} + } + + if ($Deep) { + if ($ced) { + $flat = ConvertTo-FlatColumns -Node $ced -Prefix '' -MaxDepth $FlatDepthDeep + foreach ($k in $flat.Keys) { if ($baseSet.Contains($k)) { continue }; if (-not $rowObj.PSObject.Properties[$k]) { if (-not $script:DeepExtraColumns.Contains($k)) { [void]$script:DeepExtraColumns.Add($k) }; try { Add-Member -InputObject $rowObj -NotePropertyName $k -NotePropertyValue $flat[$k] -Force } catch {} } } + } + if ($auditData) { + $auditDataClone = [PSCustomObject]@{} + foreach ($prop in $auditData.PSObject.Properties) { if ($prop.Name -ne 'CopilotEventData') { Add-Member -InputObject $auditDataClone -NotePropertyName $prop.Name -NotePropertyValue $prop.Value -Force } } + $flatAudit = ConvertTo-FlatColumns -Node $auditDataClone -Prefix '' -MaxDepth $FlatDepthDeep + foreach ($k in $flatAudit.Keys) { if ($baseSet.Contains($k)) { continue }; if (-not $rowObj.PSObject.Properties[$k]) { if (-not $script:DeepExtraColumns.Contains($k)) { [void]$script:DeepExtraColumns.Add($k) }; try { Add-Member -InputObject $rowObj -NotePropertyName $k -NotePropertyValue $flatAudit[$k] -Force } catch {} } } + } + } + $rows.Add($rowObj) | Out-Null + } + if (-not $SkipMetrics -and $rows.Count -gt 1) { try { $script:metrics.ExplosionEvents += 1; $script:metrics.ExplosionRowsFromEvents += ($rows.Count - 1); if ($rows.Count -gt $script:metrics.ExplosionMaxPerRecord) { $script:metrics.ExplosionMaxPerRecord = $rows.Count } } catch {} } + return $rows + } + catch { + if (-not $SkipMetrics) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringParseFailures++ + } + Write-LogHost "Failed Purview explosion: $($_.Exception.Message)" -ForegroundColor Red + return @() + } +} + +function Select-FirstNonNull { param([object[]]$Values) foreach ($v in $Values) { if ($null -ne $v -and ('' -ne [string]$v)) { return $v } } return $null } + +function Convert-ToStructuredRecord { + # Uses proven stable implementation for record conversion + param( + [Parameter(Mandatory = $true)] $Record, + [bool]$EnableExplosion = $false + ) + try { + function Local:Get-Num([object]$v) { if ($null -eq $v) { return $null }; try { if ($v -is [string] -and [string]::IsNullOrWhiteSpace($v)) { return $null }; return [double]$v } catch { return $null } } + function Local:Add-OrUpdate([pscustomobject]$obj, [string]$name, $value) { try { if ($obj.PSObject.Properties[$name]) { $obj.PSObject.Properties[$name].Value = $value } else { Add-Member -InputObject $obj -NotePropertyName $name -NotePropertyValue $value -Force } } catch {} } + # Use pre-parsed AuditData if available + $auditData = if ($Record.PSObject.Properties['_ParsedAuditData']) { $Record._ParsedAuditData } else { try { $Record.AuditData | ConvertFrom-Json -ErrorAction Stop } catch { $null } } + if (-not $auditData) { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringMissingAuditData++ + return @() + } + + # NON-EXPLOSION MODE: Return 8-column compact record matching Purview UI export schema + if (-not $EnableExplosion -and -not $ExplodeDeep) { + $compactRecord = [pscustomobject]@{ + RecordId = $(if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id }) + CreationDate = $Record.CreationDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + RecordType = $Record.RecordType + Operation = $(try { $auditData.Operation } catch { if ($Record.Operation) { $Record.Operation } else { $Record.Operations } }) + UserId = if ($Record.UserId) { $Record.UserId } elseif ($Record.UserIds) { $Record.UserIds } else { '' } + AuditData = $Record.AuditData + AssociatedAdminUnits = $(try { if ($auditData.AssociatedAdminUnits) { $auditData.AssociatedAdminUnits } elseif ($Record.AssociatedAdminUnits) { $Record.AssociatedAdminUnits } else { '' } } catch { '' }) + AssociatedAdminUnitsNames = $(try { if ($auditData.AssociatedAdminUnitsNames) { $auditData.AssociatedAdminUnitsNames } elseif ($Record.AssociatedAdminUnitsNames) { $Record.AssociatedAdminUnitsNames } else { '' } } catch { '' }) + } + return @($compactRecord) + } + + # EXPLOSION MODE: Extract and flatten all fields from AuditData (no raw JSON columns) + $ced = Get-SafeProperty $auditData 'CopilotEventData' + $modelId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelId'), (Get-SafeProperty $ced 'ModelID'), (Get-SafeProperty $auditData 'ModelId')) + $modelProvider = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelProvider'), (Get-SafeProperty $ced 'Provider'), (Get-SafeProperty $ced 'ModelVendor')) + $modelFamily = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ModelFamily'), (Get-SafeProperty $ced 'ModelType')) + $usageNode = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'Usage'), (Get-SafeProperty $ced 'TokenUsage'), (Get-SafeProperty $ced 'Tokens'), (Get-SafeProperty $auditData 'Usage')) + $tokensTotal = $null; $tokensInput = $null; $tokensOutput = $null + if ($usageNode) { + $tokensTotal = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Total'), (Get-SafeProperty $usageNode 'TotalTokens'), (Get-SafeProperty $usageNode 'TokensTotal'))) + $tokensInput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Input'), (Get-SafeProperty $usageNode 'Prompt'), (Get-SafeProperty $usageNode 'InputTokens'), (Get-SafeProperty $usageNode 'TokensInput'))) + $tokensOutput = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $usageNode 'Output'), (Get-SafeProperty $usageNode 'Completion'), (Get-SafeProperty $usageNode 'OutputTokens'), (Get-SafeProperty $usageNode 'TokensOutput'))) + } + if (-not $tokensTotal -and ($tokensInput -or $tokensOutput)) { try { $tokensTotal = ($tokensInput + $tokensOutput) } catch {} } + $durationMs = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'DurationMs'), (Get-SafeProperty $ced 'ElapsedMs'), (Get-SafeProperty $ced 'ProcessingTimeMs'), (Get-SafeProperty $ced 'LatencyMs'))) + $outcomeStatus = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'OutcomeStatus'), (Get-SafeProperty $ced 'Outcome'), (Get-SafeProperty $ced 'Result'), (Get-SafeProperty $ced 'Status')) + if ($outcomeStatus -is [bool]) { $outcomeStatus = if ($outcomeStatus) { 'Success' } else { 'Failure' } } + $conversationId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ConversationId'), (Get-SafeProperty $ced 'ConversationID'), (Get-SafeProperty $ced 'SessionId')) + $turnNumber = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'TurnNumber'), (Get-SafeProperty $ced 'TurnIndex'), (Get-SafeProperty $ced 'MessageIndex'))) + $retryCount = Local:Get-Num (Select-FirstNonNull -Values @((Get-SafeProperty $ced 'RetryCount'), (Get-SafeProperty $ced 'Retries'))) + $clientVersion = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientVersion'), (Get-SafeProperty $ced 'Version'), (Get-SafeProperty $ced 'Build')) + $clientPlatform = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ClientPlatform'), (Get-SafeProperty $ced 'Platform'), (Get-SafeProperty $ced 'OS')) + $agentId = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AgentId'), (Get-SafeProperty $ced 'AgentID'), (Get-SafeProperty $ced 'AssistantId')) + $agentName = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AgentName'), (Get-SafeProperty $ced 'AssistantName')) + $agentVersion = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AgentVersion'), (Get-SafeProperty $ced 'Version')) + + # Agent categorization based on AgentId pattern + $agentCategory = "" + if ($agentId) { + if ($agentId -like "CopilotStudio.Declarative.*") { + $agentCategory = "Declarative Agent" + } elseif ($agentId -like "CopilotStudio.CustomEngine.*") { + $agentCategory = "Custom Engine Agent" + } elseif ($agentId -like "P_*") { + $agentCategory = "Declarative Agent (Purview)" + } elseif ($agentId) { + $agentCategory = "Other Agent" + } + } + + # With -IncludeUserInfo, license data now appears only in EntraUsers output + # (or in combined mode via left join) + + $appIdentity = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'AppIdentity'), (Get-SafeProperty $ced 'ApplicationId'), (Get-SafeProperty $ced 'HostAppId')) + $applicationName = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'ApplicationName'), (Get-SafeProperty $ced 'HostAppName'), (Get-SafeProperty $ced 'ClientAppName')) + $suggestions = (Get-SafeProperty $ced 'Suggestions'); if (-not $suggestions) { $suggestions = Get-SafeProperty $ced 'SuggestionList' } + $actions = Get-SafeProperty $ced 'Actions' + $references = Select-FirstNonNull -Values @((Get-SafeProperty $ced 'References'), (Get-SafeProperty $ced 'Sources'), (Get-SafeProperty $ced 'Citations')) + $participants = Get-SafeProperty $ced 'Participants' + function Local:Measure-Collection($items, [string]$prefix) { + $result = @{}; if (-not $items) { return $result }; $arr = @($items); if ($arr.Count -eq 0) { return $result } + $result["${prefix}Count"] = $arr.Count; $types = New-Object System.Collections.Generic.HashSet[string]; $latencies = @(); $edits = @(); $accepted = 0; $success = 0; $failure = 0 + foreach ($s in $arr) { + foreach ($cand in @('Type', 'SuggestionType', 'Name', 'Kind', 'ActionType')) { try { if ($s.PSObject.Properties[$cand]) { [void]$types.Add([string]$s.$cand); break } } catch {} } + foreach ($lat in @('LatencyMs', 'DurationMs', 'ElapsedMs')) { try { if ($s.PSObject.Properties[$lat]) { $v = Local:Get-Num $s.$lat; if ($null -ne $v) { $latencies += $v; break } } } catch {} } + foreach ($ed in @('EditCount', 'Edits', 'EditsCount')) { try { if ($s.PSObject.Properties[$ed]) { $v = Local:Get-Num $s.$ed; if ($null -ne $v) { $edits += $v; break } } } catch {} } + foreach ($acc in @('Accepted', 'IsAccepted', 'Success', 'Succeeded')) { try { if ($s.PSObject.Properties[$acc]) { $val = $s.$acc; if ($val -is [bool]) { if ($val) { $accepted++ } } elseif ($val -match '^(?i:true|yes|1|success)') { $accepted++ } } } catch {} } + foreach ($succ in @('Success', 'Succeeded')) { try { if ($s.PSObject.Properties[$succ]) { $val = $s.$succ; if ($val -is [bool]) { if ($val) { $success++ } else { $failure++ } } elseif ($val -match '^(?i:true|yes|1|success)') { $success++ } else { $failure++ } } } catch {} } + } + if ($types.Count -gt 0) { $result["${prefix}Types"] = [string]::Join(';', [array]$types) } + if ($latencies.Count -gt 0) { $result["${prefix}AvgLatencyMs"] = [math]::Round(($latencies | Measure-Object -Average).Average, 2) } + if ($edits.Count -gt 0) { $result["${prefix}AvgEdits"] = [math]::Round(($edits | Measure-Object -Average).Average, 2); $result["${prefix}TotalEdits"] = ($edits | Measure-Object -Sum).Sum } + if ($accepted -gt 0) { $result["${prefix}Accepted"] = $accepted; $result["${prefix}AcceptanceRate"] = [math]::Round(($accepted / $arr.Count) * 100, 2) } + if ($success -gt 0 -or $failure -gt 0) { $result["${prefix}Success"] = $success; $result["${prefix}Failure"] = $failure } + return $result + } + $suggestAgg = Local:Measure-Collection $suggestions 'Suggestions' + $actionAgg = Local:Measure-Collection $actions 'Actions' + $refAgg = Local:Measure-Collection $references 'References' + $partAgg = Local:Measure-Collection $participants 'Participants' + $baseRecord = [pscustomobject]@{ + RecordId = $(if ($Record.RecordId) { $Record.RecordId } elseif ($Record.Identity) { $Record.Identity } elseif ($Record.Id) { $Record.Id } else { $auditData.Id }) + RecordType = $Record.RecordType + CreationDate = $Record.CreationDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) + ResultStatus = $Record.ResultStatus + ResultCount = $Record.ResultCount + Identity = $Record.Identity + IsValid = $Record.IsValid + ObjectState = $Record.ObjectState + Id = $auditData.Id + CreationTime = & { $ct = script:Parse-DateSafe $auditData.CreationTime; if ($ct) { $ct.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ', [System.Globalization.CultureInfo]::InvariantCulture) } else { $auditData.CreationTime } } + Operation = $auditData.Operation + OrganizationId = $auditData.OrganizationId + RecordTypeNum = $auditData.RecordType + ResultStatus_Audit = $auditData.ResultStatus + UserKey = $auditData.UserKey + UserType = $auditData.UserType + Version = $auditData.Version + Workload = $auditData.Workload + UserId = $auditData.UserId + AppId = $auditData.AppId + ClientAppId = $auditData.ClientAppId + CorrelationId = $auditData.CorrelationId + ModelId = $modelId + ModelProvider = $modelProvider + ModelFamily = $modelFamily + TokensTotal = $tokensTotal + TokensInput = $tokensInput + TokensOutput = $tokensOutput + DurationMs = $durationMs + OutcomeStatus = $outcomeStatus + ConversationId = $conversationId + TurnNumber = $turnNumber + RetryCount = $retryCount + ClientVersion = $clientVersion + ClientPlatform = $clientPlatform + AgentId = $agentId + AgentName = $agentName + AgentVersion = $agentVersion + AgentCategory = $agentCategory + AppIdentity = $appIdentity + ApplicationName = $applicationName + } + # Flatten AppAccessContext for Copilot/AI records as well + try { + $aac = Get-SafeProperty $auditData 'AppAccessContext' + if ($aac -and -not (Test-ScalarValue $aac)) { + $flatAac = ConvertTo-FlatColumns -Node $aac -Prefix 'AppAccessContext.' -MaxDepth $FlatDepthStandard + foreach ($k in $flatAac.Keys) { if (-not $baseRecord.PSObject.Properties[$k]) { Add-Member -InputObject $baseRecord -NotePropertyName $k -NotePropertyValue $flatAac[$k] -Force } } + if ($baseRecord.PSObject.Properties['AppAccessContext']) { $baseRecord.PSObject.Members.Remove('AppAccessContext') } + } + elseif ($aac -and (Test-ScalarValue $aac)) { + if (-not $baseRecord.PSObject.Properties['AppAccessContext']) { Add-Member -InputObject $baseRecord -NotePropertyName 'AppAccessContext' -NotePropertyValue $aac -Force } + } + } catch {} + foreach ($k in $suggestAgg.Keys) { Add-OrUpdate $baseRecord $k $suggestAgg[$k] } + foreach ($k in $actionAgg.Keys) { Add-OrUpdate $baseRecord $k $actionAgg[$k] } + foreach ($k in $refAgg.Keys) { Add-OrUpdate $baseRecord $k $refAgg[$k] } + foreach ($k in $partAgg.Keys) { Add-OrUpdate $baseRecord $k $partAgg[$k] } + + # If not doing array explosion, return base record now + if (-not $EnableExplosion) { return @($baseRecord) } + $rows = @($baseRecord) + $arraysToExplode = @( + @{ Name = 'Suggestions'; Data = $suggestions; Prefix = 'Suggestion'; Enabled = $suggestions }, + @{ Name = 'Actions'; Data = $actions; Prefix = 'Action'; Enabled = $actions }, + @{ Name = 'References'; Data = $references; Prefix = 'Reference'; Enabled = $references }, + @{ Name = 'Participants'; Data = $participants; Prefix = 'Participant'; Enabled = $participants } + ) + $maxRows = $ExplosionPerRecordRowCap + foreach ($entry in $arraysToExplode) { + if (-not $entry.Enabled) { continue } + $dataArr = @($entry.Data); if ($dataArr.Count -eq 0) { continue } + $newRows = New-Object System.Collections.ArrayList + foreach ($r in $rows) { + $idx = 0 + foreach ($el in $dataArr) { + $nr = [pscustomobject]@{} + foreach ($p in $r.PSObject.Properties) { Add-Member -InputObject $nr -NotePropertyName $p.Name -NotePropertyValue $p.Value -Force } + Add-OrUpdate $nr ("ArrayIndex_{0}" -f $entry.Name) $idx + if ($el) { + foreach ($prop in $el.PSObject.Properties) { + $pname = ("{0}_{1}" -f $entry.Prefix, $prop.Name) + if ($nr.PSObject.Properties[$pname]) { continue } + $val = $prop.Value + if (Test-ScalarValue $val) { Add-OrUpdate $nr $pname $val } else { try { Add-OrUpdate $nr $pname ($val | ConvertTo-Json -Depth $JsonDepth -Compress) } catch {} } + } + } + [void]$newRows.Add($nr); $idx++ + if ($newRows.Count -gt $maxRows) { break } + } + if ($newRows.Count -gt $maxRows) { break } + } + $rows = @($newRows) + if ($rows.Count -gt $maxRows) { break } + } + if ($rows.Count -gt $maxRows) { foreach ($r in $rows) { Add-OrUpdate $r 'ExplosionTruncated' $true }; $rows = $rows[0..($maxRows - 1)]; try { $script:metrics.ExplosionTruncated = $true } catch {} } + if ($ExplodeDeep -and $ced) { + for ($i = 0; $i -lt $rows.Count; $i++) { + $r = $rows[$i] + $flat = ConvertTo-FlatColumns -Node $ced -Prefix '' -MaxDepth $FlatDepthStandard + foreach ($ck in $flat.Keys) { if (-not $r.PSObject.Properties[$ck]) { Add-OrUpdate $r $ck $flat[$ck] } } + } + } + return $rows + } + catch { + $script:metrics.FilteringSkippedRecords++ + $script:metrics.FilteringParseFailures++ + Write-LogHost "Failed to process record: $($_.Exception.Message)" -ForegroundColor Red + return @() + } +} + +try { + # Unregister the early exit handler since catch/finally will handle Ctrl+C from this point + # This prevents duplicate "Script Interrupted" messages + Unregister-Event -SourceIdentifier PowerShell.Exiting -ErrorAction SilentlyContinue + + # ============================================================ + # RESUME MODE VALIDATION - Ensure no conflicting parameters + # ============================================================ + if ($ResumeSpecified) { + # Resume mode is standalone - only auth-related parameters allowed + $allowedWithResume = @( + 'Resume', + 'RemainingArgs', + 'Force', + 'UserIds', + 'GroupNames', + 'Auth', + 'TenantId', + 'ClientId', + 'ClientSecret', + 'Dashboard', + 'OutputPathDefenderUsage', + 'AppendDefenderUsage', + 'DisableAISIDDeltaCache', + 'ClearUncertainCreate', + 'ClearUncertainContract', + 'ClientCertificateThumbprint', + 'ClientCertificateStoreLocation', + 'ClientCertificatePath', + 'ClientCertificatePassword', + # Standard PowerShell common parameters + 'Verbose', + 'Debug', + 'ErrorAction', + 'WarningAction', + 'InformationAction', + 'ErrorVariable', + 'WarningVariable', + 'InformationVariable', + 'OutVariable', + 'OutBuffer', + 'PipelineVariable' + ) + + $invalidParams = @($PSBoundParameters.Keys | Where-Object { $_ -notin $allowedWithResume }) + + if ($invalidParams.Count -gt 0) { + Write-Host "" + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host " ERROR: Invalid parameters used with -Resume" -ForegroundColor Red + Write-Host "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-Host "" + Write-Host " Resume mode restores ALL settings from the checkpoint file." -ForegroundColor Yellow + Write-Host " You cannot specify other parameters (they would be ignored or cause inconsistency)." -ForegroundColor Yellow + Write-Host "" + Write-Host " Invalid parameters:" -ForegroundColor White + foreach ($p in $invalidParams) { + Write-Host " - $p" -ForegroundColor Red + } + Write-Host "" + Write-Host " ALLOWED with -Resume:" -ForegroundColor Green + Write-Host " -Resume [path] Checkpoint file (or auto-discover)" -ForegroundColor Gray + Write-Host " -Force Use most recent checkpoint without prompting" -ForegroundColor Gray + Write-Host " -Auth Override authentication method" -ForegroundColor Gray + Write-Host " -TenantId Tenant ID (for AppRegistration)" -ForegroundColor Gray + Write-Host " -ClientId Client ID (for AppRegistration)" -ForegroundColor Gray + Write-Host " -ClientSecret Client secret (for AppRegistration)" -ForegroundColor Gray + Write-Host " -UserIds Override the checkpoint's user scope (fail-closed)" -ForegroundColor Gray + Write-Host " -GroupNames Override the checkpoint's group scope (fail-closed)" -ForegroundColor Gray + Write-Host ' .\Script.ps1 -Resume' -ForegroundColor White + Write-Host ' .\Script.ps1 -Resume -Auth DeviceCode' -ForegroundColor White + Write-Host ' .\Script.ps1 -Resume "C:\path\.pax_checkpoint_xxx.json" -Force' -ForegroundColor White + Write-Host "" + exit 1 + } + } + + # ============================================================ + # RESUME MODE DETECTION - Check for checkpoint to resume + # ============================================================ + if ($ResumeSpecified) { + Write-LogHost "" + Write-LogHost "========================================" -ForegroundColor Cyan + Write-LogHost " RESUME MODE DETECTED" -ForegroundColor Cyan + Write-LogHost "========================================" -ForegroundColor Cyan + Write-LogHost "" + + if ($Resume -ne '') { + # Explicit checkpoint path provided + Write-LogHost "Loading checkpoint from explicit path: $Resume" -ForegroundColor Yellow + $checkpointLoadSuccess = Read-Checkpoint -CheckpointPath $Resume + if (-not $checkpointLoadSuccess) { + Write-LogHost "ERROR: Failed to load checkpoint file. Cannot resume." -ForegroundColor Red + exit 1 + } + $script:CheckpointPath = $Resume + # Lock the resumed checkpoint so a second replica cannot race on it. + try { script:Acquire-CheckpointLock -CheckpointPath $script:CheckpointPath } catch { + Write-LogHost ("ERROR: {0}" -f $_.Exception.Message) -ForegroundColor Red + exit 1 + } + # Read-Checkpoint sets $script:CheckpointData on success + $checkpointData = $script:CheckpointData + } + else { + # Auto-discover checkpoints in OutputPath + $searchPath = if ($OutputPath) { $OutputPath } else { (Get-Location).Path } + + # Fabric tier: resume artifacts are mirrored to OneLake Files/.pax_resume/ + # inside each Save-CheckpointToDisk call so a container restart can resume. + # Hydrate the local search path from the mirror before Find-Checkpoints runs. + # Local and SharePoint tiers keep artifacts local-only — those hosts are not + # ephemeral and a remote prefetch would be needless I/O. + if ($script:DestTier -and $script:DestTier['Purview'] -eq 'Fabric') { + try { + $restored = Restore-AllFabricResumeMirrors -LocalDir $searchPath + if ($restored -gt 0) { + Write-LogHost "Hydrated $restored resume artifact(s) from OneLake mirror" -ForegroundColor DarkGray + } + } catch { + # A hydration failure is an INTEGRITY signal, not a hint: the mirror + # exposed an authority record whose artifact set could not be proven, or + # it could not be read at all. Continuing to checkpoint discovery here + # would search a directory that either holds stale local state or holds + # nothing, and could silently resume the wrong state. Stop non-zero and + # distinguish this from the ordinary "no remote generation exists" case, + # which returns 0 and is not an error. + Write-LogHost ("ERROR: Resume mirror hydration failed - resume cannot continue: {0}" -f $_.Exception.Message) -ForegroundColor Red + Write-LogHost " No checkpoint was hydrated from the remote mirror, and no partial resume state was left behind." -ForegroundColor Gray + exit 1 + } + } + + Write-LogHost "Searching for checkpoints in: $searchPath" -ForegroundColor Yellow + + $checkpoints = Find-Checkpoints -OutputPath $searchPath + + if ($checkpoints.Count -eq 0) { + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost " NO CHECKPOINT FILES FOUND" -ForegroundColor Red + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + Write-LogHost " Searched in: $searchPath" -ForegroundColor White + Write-LogHost "" + Write-LogHost " Checkpoint files are named: .pax_checkpoint_YYYYMMDD_HHMMSS.json" -ForegroundColor Gray + Write-LogHost " They are saved in the same folder as the _PARTIAL.csv output file." -ForegroundColor Gray + Write-LogHost "" + Write-LogHost " COMMON LOCATIONS TO CHECK:" -ForegroundColor Yellow + Write-LogHost " • The 'output' folder where you typically save exports" -ForegroundColor White + Write-LogHost " • The folder shown in the Ctrl+C message when the run was interrupted" -ForegroundColor White + Write-LogHost " • Look for _PARTIAL.csv files - the checkpoint is in the same folder" -ForegroundColor White + Write-LogHost "" + Write-LogHost " HOW TO RESUME:" -ForegroundColor Cyan + Write-LogHost "" + Write-LogHost " Option 1: Specify the folder containing the checkpoint:" -ForegroundColor White + Write-LogHost " -Resume -OutputPath `"C:\path\to\output\folder`"" -ForegroundColor Green + Write-LogHost "" + Write-LogHost " Option 2: Specify the checkpoint file directly:" -ForegroundColor White + Write-LogHost " -Resume `"C:\path\to\.pax_checkpoint_20260120_123456.json`"" -ForegroundColor Green + Write-LogHost "" + Write-LogHost " Option 3: Run from the folder containing the checkpoint:" -ForegroundColor White + Write-LogHost " cd `"C:\path\to\output\folder`"" -ForegroundColor Green + Write-LogHost " pwsh -File `"...\PAX_Purview_Audit_Log_Processor.ps1`" -Resume" -ForegroundColor Green + Write-LogHost "" + Write-LogHost "════════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" + exit 1 + } + elseif ($checkpoints.Count -eq 1) { + $selectedCheckpoint = $checkpoints[0] + Write-LogHost "Found checkpoint: $($selectedCheckpoint.FileName)" -ForegroundColor Green + } + else { + # Multiple checkpoints found + if ($Force) { + # Use most recent without prompting + $selectedCheckpoint = $checkpoints | Sort-Object { $_.LastUpdated } -Descending | Select-Object -First 1 + Write-LogHost "Multiple checkpoints found. -Force specified, using most recent:" -ForegroundColor Yellow + Write-LogHost " $($selectedCheckpoint.FileName)" -ForegroundColor White + } + else { + # Prompt user to select + $selectedCheckpoint = Select-Checkpoint -Checkpoints $checkpoints + if (-not $selectedCheckpoint) { + Write-LogHost "No checkpoint selected. Exiting." -ForegroundColor Yellow + exit 0 + } + } + } + + $script:CheckpointPath = $selectedCheckpoint.Path + # Lock the resumed checkpoint so a second replica cannot race on it. + try { script:Acquire-CheckpointLock -CheckpointPath $script:CheckpointPath } catch { + Write-LogHost ("ERROR: {0}" -f $_.Exception.Message) -ForegroundColor Red + exit 1 + } + $checkpointLoadSuccess = Read-Checkpoint -CheckpointPath $script:CheckpointPath + if (-not $checkpointLoadSuccess) { + Write-LogHost "ERROR: Failed to load checkpoint file. Cannot resume." -ForegroundColor Red + exit 1 + } + # Read-Checkpoint sets $script:CheckpointData on success + $checkpointData = $script:CheckpointData + } + + # Note: $script:CheckpointData and $script:IsResumeMode already set by Read-Checkpoint + + # Operator-only SELECTIVE recovery: when -ClearUncertainCreate is supplied it carries an + # explicit list of partition indexes to recover. Validate the selection against the + # durable uncertain-create set NOW (pre-auth): an empty, duplicate, unknown, or + # out-of-range selection is REJECTED and nothing is cleared. A valid selection is only + # STAGED here - the durable marker for a selected partition is cleared selectively and + # atomically later, at the parent-create reconcile, and ONLY after a READ-ONLY + # exact-name reconcile confirms the query is not visible (a still-visible selected + # partition is adopted, never cleared into a duplicate POST). Durable uncertain state is + # never auto-cleared by elapsed time, a collection miss, a process restart, or a failure count. + $script:ClearUncertainSelection = [System.Collections.Generic.HashSet[int]]::new() + if ($PSBoundParameters.ContainsKey('ClearUncertainCreate')) { + $clearSelection = Clear-PaxUncertainCreateState -CheckpointData $script:CheckpointData -Requested:$true -SelectedIndexes $ClearUncertainCreate -ValidateOnly + if ($clearSelection.Rejected) { + Write-LogHost "" + Write-LogHost " ERROR: -ClearUncertainCreate selection rejected - $($clearSelection.Reason). No uncertain-create state was changed." -ForegroundColor Red + $script:ResumeValidationFailed = $true + exit 1 + } + foreach ($selIdx in @($clearSelection.Selected)) { [void]$script:ClearUncertainSelection.Add([int]$selIdx) } + Write-LogHost " [RESUME] -ClearUncertainCreate: $($script:ClearUncertainSelection.Count) partition(s) selected for recovery; each is reconciled READ-ONLY first and only cleared for a fresh create if not visible by exact name." -ForegroundColor Yellow + } + # EXPLICIT-CONTRACT companion (-ClearUncertainContract): each ":" + # token names EXACTLY one durable uncertain contract, disambiguating an index that maps + # to more than one. Validated NOW (pre-auth) against the durable set: a malformed, + # unknown, multiply-matching, or duplicate token is REJECTED and nothing is cleared. A + # valid selection is only STAGED here - the durable marker is cleared selectively and + # atomically later, at the reconcile, and ONLY after a READ-ONLY exact-name reconcile + # confirms the selected contract is not visible (a still-visible contract is adopted, + # never cleared into a duplicate POST). + $script:ClearUncertainContractSelection = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + if ($PSBoundParameters.ContainsKey('ClearUncertainContract')) { + $contractSelection = Clear-PaxUncertainCreateState -CheckpointData $script:CheckpointData -Requested:$true -SelectedContracts $ClearUncertainContract -ValidateOnly + if ($contractSelection.Rejected) { + Write-LogHost "" + Write-LogHost " ERROR: -ClearUncertainContract selection rejected - $($contractSelection.Reason). No uncertain-create state was changed." -ForegroundColor Red + $script:ResumeValidationFailed = $true + exit 1 + } + foreach ($selTok in @($contractSelection.SelectedContracts)) { [void]$script:ClearUncertainContractSelection.Add([string]$selTok) } + Write-LogHost " [RESUME] -ClearUncertainContract: $($script:ClearUncertainContractSelection.Count) contract(s) selected for recovery; each is reconciled READ-ONLY first and only cleared for a fresh create if not visible by exact name." -ForegroundColor Yellow + } + + # Display resume summary + $completedCount = if ($checkpointData.partitions.completed) { $checkpointData.partitions.completed.Count } else { 0 } + $queryCreatedCount = if ($checkpointData.partitions.queryCreated) { $checkpointData.partitions.queryCreated.Count } else { 0 } + $totalPartitions = if ($checkpointData.partitions.total) { $checkpointData.partitions.total } else { 0 } + + Write-LogHost "" + Write-LogHost "Checkpoint loaded successfully:" -ForegroundColor Green + Write-LogHost " Original Run: $($checkpointData.runTimestamp)" -ForegroundColor White + $cpStartDate = if ($checkpointData.parameters.startDate) { $d = script:Parse-DateSafe $checkpointData.parameters.startDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + $cpEndDate = if ($checkpointData.parameters.endDate) { $d = script:Parse-DateSafe $checkpointData.parameters.endDate; if ($d) { $d.ToLocalTime().ToString('yyyy-MM-dd') } else { 'Unknown' } } else { 'Unknown' } + Write-LogHost " Date Range: $cpStartDate to $cpEndDate" -ForegroundColor White + Write-LogHost " Total Partitions: $totalPartitions" -ForegroundColor White + Write-LogHost " Completed: $completedCount" -ForegroundColor Green + Write-LogHost " Query Created: $queryCreatedCount (will attempt data fetch)" -ForegroundColor Yellow + Write-LogHost " Remaining: $($totalPartitions - $completedCount)" -ForegroundColor Cyan + Write-LogHost "" + + # ============================================================ + # RESTORE ALL PARAMETERS FROM CHECKPOINT + # ============================================================ + Write-LogHost "Restoring parameters from checkpoint..." -ForegroundColor DarkGray + $cp = $checkpointData.parameters + $script:ResumeValidationFailed = $true + # ============================================================ + # AISID resume validation. Runs BEFORE the dashboard/window + # restore below so it sets $Dashboard authoritatively and stops the rollup-dashboard + # restore from overriding an AISID resume. Reads raw checkpoint fields and + # enforces conflicts/requirements NONINTERACTIVELY (exit 1; never Read-Host). + # $checkpointData / $cp are hashtables (ConvertFrom-Json -AsHashtable). + # ============================================================ + $script:ResumeIsAISID = $false + $script:AISIDResumeForceCacheMiss = $false + $script:ResumeAISIDClass = $null + $hasAisidWindow = [bool]$checkpointData.aisidWindow + $markerRaw = if ($checkpointData.Contains('aisidResumeStateVersion')) { $checkpointData.aisidResumeStateVersion } else { $null } + if ($null -ne $markerRaw -and [string]$markerRaw -ne '1') { Write-LogHost "ERROR: this checkpoint carries an unsupported AISID resume-state version ($markerRaw); this build supports version 1 only." -ForegroundColor Red; exit 1 } + $hasMarker = ($markerRaw -eq 1) + $cpDashboard = if ($cp.Contains('dashboard')) { [string]$cp.dashboard } else { $null } + $cpRollupDash = if ($cp.Contains('rollupDashboard')) { [string]$cp.rollupDashboard } else { $null } + $cliDash = if ($PSBoundParameters.ContainsKey('Dashboard')) { [string]$Dashboard } else { $null } + $cliOut = $PSBoundParameters.ContainsKey('OutputPathDefenderUsage') + $cliApp = $PSBoundParameters.ContainsKey('AppendDefenderUsage') + if ($hasMarker -and $cpDashboard -eq 'AISID') { + $Dashboard = 'AISID'; $script:ResumeIsAISID = $true; $script:ResumeAISIDClass = 'Complete' + if ($cliDash) { Write-LogHost "ERROR: resume dashboard conflict - this checkpoint already stores its dashboard; -Dashboard cannot be supplied on resume for it. Omit -Dashboard." -ForegroundColor Red; exit 1 } + if ($cliOut -or $cliApp) { Write-LogHost "ERROR: resume DefenderUsage destination conflict - this checkpoint already stores its AISID DefenderUsage destination; do not pass -OutputPathDefenderUsage or -AppendDefenderUsage on resume." -ForegroundColor Red; exit 1 } + if ($PSBoundParameters.ContainsKey('DisableAISIDDeltaCache')) { Write-LogHost "ERROR: resume cache conflict - this checkpoint already stores the AISID delta-cache setting; do not pass -DisableAISIDDeltaCache on resume." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: restoring the AISID dashboard, DefenderUsage destination, cache state, and frozen collection window from the checkpoint." -ForegroundColor Cyan + } + elseif ($hasAisidWindow -and -not $hasMarker -and ($null -eq $cpDashboard)) { + $Dashboard = 'AISID'; $script:ResumeIsAISID = $true; $script:ResumeAISIDClass = 'Transitional'; $script:AISIDResumeForceCacheMiss = $true + if ($cliDash -and $cliDash -ne 'AISID') { Write-LogHost "ERROR: resume dashboard conflict - this AISID checkpoint can only resume as AISID; -Dashboard $cliDash is not allowed." -ForegroundColor Red; exit 1 } + if (-not ($cliOut -xor $cliApp)) { Write-LogHost "ERROR: this AISID checkpoint has no stored DefenderUsage destination; supply EXACTLY ONE of -OutputPathDefenderUsage or -AppendDefenderUsage on the resume command line (not both, not neither)." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: this AISID checkpoint has no stored DefenderUsage destination; using the destination supplied on the command line and forcing a delta-cache miss." -ForegroundColor Cyan + } + elseif (-not $hasAisidWindow -and ($null -eq $cpDashboard) -and ($cpRollupDash -eq 'AIO')) { + if (-not $cliDash) { Write-LogHost "ERROR: this older checkpoint is ambiguous - it could be an AIO run or an AISID run. Re-run with -Dashboard AIO, or -Dashboard AISID plus exactly one of -OutputPathDefenderUsage or -AppendDefenderUsage." -ForegroundColor Red; exit 1 } + elseif ($cliDash -eq 'AIO') { + $Dashboard = 'AIO'; $script:ResumeIsAISID = $false + if ($cliOut -or $cliApp) { Write-LogHost "ERROR: -OutputPathDefenderUsage and -AppendDefenderUsage are not valid with -Dashboard AIO." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: dashboard resolved to AIO from the command line." -ForegroundColor Cyan + } + elseif ($cliDash -eq 'AISID') { + $Dashboard = 'AISID'; $script:ResumeIsAISID = $true; $script:ResumeAISIDClass = 'LegacyAISID'; $script:AISIDResumeForceCacheMiss = $true + if (-not ($cliOut -xor $cliApp)) { Write-LogHost "ERROR: resuming this checkpoint as AISID requires EXACTLY ONE of -OutputPathDefenderUsage or -AppendDefenderUsage (not both, not neither)." -ForegroundColor Red; exit 1 } + Write-LogHost " Resume: dashboard resolved to AISID from the command line; the collection window is reconstructed from the checkpoint dates and creation time, and a delta-cache miss is forced." -ForegroundColor Cyan + } + else { Write-LogHost "ERROR: -Dashboard $cliDash is not a valid choice for resuming this ambiguous checkpoint (use AIO or AISID)." -ForegroundColor Red; exit 1 } + } + + # ============================================================ + # AISID AVAILABILITY GATE - restored checkpoint state + # Runs immediately after the AISID resume classification above and before any + # further parameter restoration, destination re-resolution, remote probe, + # checkpoint mutation, sign-in, collection, or publication. A checkpoint that + # restores the AISID dashboard, an AISID DefenderUsage output or append + # destination, or the AISID delta-cache setting stops here, so stored state + # cannot reach a gated AISID execution path. The checkpoint lock is released + # first so a gated resume leaves nothing behind. $cp is the checkpoint + # parameter block that the restore below reads for these same values. + # ============================================================ + $aisidResumeDashboard = ($script:ResumeIsAISID -or (([string]$Dashboard).Trim().ToUpperInvariant() -eq 'AISID') -or (($null -ne $cpDashboard) -and (([string]$cpDashboard).Trim().ToUpperInvariant() -eq 'AISID'))) + $aisidResumeOutput = ($cp.Contains('outputPathDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$cp.outputPathDefenderUsage)) + $aisidResumeAppend = ($cp.Contains('appendDefenderUsage') -and -not [string]::IsNullOrWhiteSpace([string]$cp.appendDefenderUsage)) + $aisidResumeCache = ($cp.Contains('disableAISIDDeltaCache') -and [bool]$cp.disableAISIDDeltaCache) + if (script:Test-AISIDEntryPointGated -DashboardIsAISID $aisidResumeDashboard -DefenderUsageOutputRequested $aisidResumeOutput -DefenderUsageAppendRequested $aisidResumeAppend -DeltaCacheSwitchRequested $aisidResumeCache) { + try { script:Release-CheckpointLock } catch { } + exit 1 + } + + # Restore original run timestamp so incremental files use consistent naming + # This ensures all partition files (original run + resumes) share the same timestamp + if ($checkpointData.runTimestamp) { + $global:ScriptRunTimestamp = $checkpointData.runTimestamp + Write-LogHost " Restored original run timestamp: $($global:ScriptRunTimestamp)" -ForegroundColor DarkGray + } + + # Date range (required) - using locale-safe parsing + $parsedStart = script:Parse-DateSafe $cp.startDate + if (-not $parsedStart) { throw "Failed to parse checkpoint startDate: $($cp.startDate)" } + # Checkpoint stores the date as a UTC instant (local midnight -> ToUniversalTime at save). + # Parse-DateSafe returns a UTC-kind value, so convert back to local before formatting the + # calendar date; otherwise positive-UTC-offset hosts (e.g. IST) render the prior day, + # shifting the trim window by one day on resume. + $StartDate = $parsedStart.ToLocalTime().ToString('yyyy-MM-dd') + + $parsedEnd = script:Parse-DateSafe $cp.endDate + if (-not $parsedEnd) { throw "Failed to parse checkpoint endDate: $($cp.endDate)" } + # See StartDate note above: convert the UTC-kind parse result back to local before + # formatting so the restored end date matches the originally requested calendar date. + $EndDate = $parsedEnd.ToLocalTime().ToString('yyyy-MM-dd') + + # Activity/Record filtering + if ($cp.activityTypes -and $cp.activityTypes.Count -gt 0) { $ActivityTypes = $cp.activityTypes } + if ($cp.recordTypes -and $cp.recordTypes.Count -gt 0) { $RecordTypes = $cp.recordTypes } + if ($cp.serviceTypes -and $cp.serviceTypes.Count -gt 0) { $ServiceTypes = $cp.serviceTypes } + if (-not $PSBoundParameters.ContainsKey('UserIds') -and $cp.userIds -and $cp.userIds.Count -gt 0) { $UserIds = $cp.userIds } + if (-not $PSBoundParameters.ContainsKey('GroupNames') -and $cp.groupNames -and $cp.groupNames.Count -gt 0) { $GroupNames = $cp.groupNames } + + # Agent filtering + if ($cp.agentId -and $cp.agentId.Count -gt 0) { $AgentId = $cp.agentId } + if ($cp.agentsOnly) { $AgentsOnly = [switch]$true } + if ($cp.excludeAgents) { $ExcludeAgents = [switch]$true } + + # Prompt filtering + if ($cp.promptFilter) { $PromptFilter = $cp.promptFilter } + + # Schema/Explosion settings + if ($cp.explodeArrays) { $ExplodeArrays = [switch]$true } + if ($cp.explodeDeep) { $ExplodeDeep = [switch]$true } + if ($cp.flatDepth) { $FlatDepth = $cp.flatDepth } + if ($cp.streamingSchemaSample) { $StreamingSchemaSample = $cp.streamingSchemaSample } + if ($cp.streamingChunkSize) { $StreamingChunkSize = $cp.streamingChunkSize } + # Allow user to override explosion threads on resume (different machine/load) + if (-not $PSBoundParameters.ContainsKey('ExplosionThreads') -and $cp.explosionThreads) { $ExplosionThreads = $cp.explosionThreads } + + # M365/User info bundles + if ($cp.includeM365Usage) { $IncludeM365Usage = [switch]$true } + if ($cp.includeUserInfo) { $IncludeUserInfo = [switch]$true } + if ($cp.Contains('includeDSPMForAI') -and $cp.includeDSPMForAI) { + Write-LogHost "Resume checkpoint references legacy -IncludeDSPMForAI switch; ignoring (no longer supported)." -ForegroundColor Yellow + } + if ($cp.includeCopilotInteraction) { $IncludeCopilotInteraction = [switch]$true } + if ($cp.excludeCopilotInteraction) { $ExcludeCopilotInteraction = [switch]$true } + if ($cp.includeAgent365Info) { $IncludeAgent365Info = [switch]$true } + # Note: onlyAgent365Info is intentionally not restored on resume - it would imply + # no audit phase to resume from. Original validation blocks -OnlyAgent365Info + -Resume. + + # Partitioning + if ($cp.blockHours) { $BlockHours = $cp.blockHours } + if ($cp.partitionHours) { $PartitionHours = $cp.partitionHours } + if ($cp.maxPartitions) { $MaxPartitions = $cp.maxPartitions } + + # Output settings + if ($cp.outputPath) { $OutputPath = $cp.outputPath } + if ($cp.exportWorkbook) { $ExportWorkbook = [switch]$true } + if ($cp.combineOutput) { $CombineOutput = [switch]$true } + # Deidentify is checkpoint-driven on resume (the resume allow-list blocks it on the + # command line), so restore it here as the sole source of truth for a resumed run. + if ($cp.Contains('deidentify') -and $cp.deidentify) { $Deidentify = [switch]$true; $script:PaxDeidEnabled = $true } + if ($cp.Contains('verifyPartitionStability') -and $cp.verifyPartitionStability) { $VerifyPartitionStability = [switch]$true } + + # -UserInfoSupplement (hybrid directory enrichment) is checkpoint-driven on resume so the original + # supplemental input is reused. An explicit resume override with a DIFFERENT value fails clearly; a + # matching value is accepted. Only the source path/URL is restored - no supplemental values are stored. + if ($cp.Contains('userInfoSupplement') -and $cp.userInfoSupplement) { + $cpUis = [string]$cp.userInfoSupplement + if ($PSBoundParameters.ContainsKey('UserInfoSupplement') -and -not [string]::IsNullOrWhiteSpace($UserInfoSupplement) -and ($UserInfoSupplement.Trim() -ne $cpUis.Trim())) { + Write-Host "" + Write-Host "ERROR: -UserInfoSupplement was supplied on the resume command line with a value that differs from the original run." -ForegroundColor Red + Write-Host (" Original: {0}" -f $cpUis) -ForegroundColor Yellow + Write-Host (" Provided: {0}" -f $UserInfoSupplement.Trim()) -ForegroundColor Yellow + Write-Host " A resumed run reuses the original supplemental input; omit -UserInfoSupplement on resume." -ForegroundColor Yellow + Write-Host "" + exit 1 + } + $UserInfoSupplement = $cpUis + $script:UserInfoSupplementActive = $true + } + + # FillerLabel is checkpoint-driven on resume (the allow-list blocks it on the command + # line); restore the resolved hierarchy-filler mode + literal as the sole source of truth. + if ($cp.Contains('fillerLabelMode') -and $cp.fillerLabelMode) { $script:HierarchyFillMode = [string]$cp.fillerLabelMode } + if ($cp.Contains('fillerLabelText')) { $script:HierarchyFillLabel = [string]$cp.fillerLabelText } + + # Per-stream destinations and append targets. Persisted by New-Checkpoint; + # restored here so a -Resume run does not require the user to re-supply any + # destination switch. Resume-allowed-list already blocks the user from passing + # these on the resume command line, so checkpoint is the sole source of truth. + if ($cp.Contains('outputPathUserInfo') -and $cp.outputPathUserInfo) { $OutputPathUserInfo = [string]$cp.outputPathUserInfo } + if ($cp.Contains('outputPathAgent365Info') -and $cp.outputPathAgent365Info) { $OutputPathAgent365Info = [string]$cp.outputPathAgent365Info } + if ($cp.Contains('outputPathLog') -and $cp.outputPathLog) { $OutputPathLog = [string]$cp.outputPathLog } + if ($cp.Contains('appendFile') -and $cp.appendFile) { $AppendFile = [string]$cp.appendFile } + if ($cp.Contains('appendUserInfo') -and $cp.appendUserInfo) { $AppendUserInfo = [string]$cp.appendUserInfo } + if ($cp.Contains('appendAgent365Info') -and $cp.appendAgent365Info) { $AppendAgent365Info = [string]$cp.appendAgent365Info } + # AISID (Defender) output set + cache switch. Restored ONLY when the checkpoint stores the full + # AISID resume state (marker present). A checkpoint without it requires an explicit destination + # recovery instead REQUIRES a dedicated -OutputPathDefenderUsage/-AppendDefenderUsage on the + # resume CLI and forces a cache miss (handled in the classification block below). + if ($cp.Contains('outputPathDefenderUsage') -and $cp.outputPathDefenderUsage) { $OutputPathDefenderUsage = [string]$cp.outputPathDefenderUsage } + if ($cp.Contains('appendDefenderUsage') -and $cp.appendDefenderUsage) { $AppendDefenderUsage = [string]$cp.appendDefenderUsage } + if ($cp.Contains('disableAISIDDeltaCache') -and $cp.disableAISIDDeltaCache) { $DisableAISIDDeltaCache = [System.Management.Automation.SwitchParameter]::new($true) } + + # ============================================================ + # RESUME: Re-resolve destination state from RESTORED variable values. + # The parse-time destination-resolution block (search for "destSwitches") + # keys off $PSBoundParameters, but on a -Resume run the user MUST NOT pass + # -OutputPath* / -Append* (resume-allowed-list rejects them) — so at parse + # time every $script:DestIsBound[*] was false and $script:RemoteOutputMode + # stayed 'None'. Now that the checkpoint restore above repopulated $OutputPath, + # $OutputPathUserInfo, $OutputPathAgent365Info, $OutputPathLog, $AppendFile, + # $AppendUserInfo, $AppendAgent365Info, re-derive the destination hashtables + # so the end-of-run upload sweep + display helpers route correctly. Without + # this, $script:RemoteOutputMode stays 'None' and no files ever upload to SP/Fabric. + # "Bound-ness" is inferred from a non-empty value (the new-run path uses + # $PSBoundParameters because string parameter defaults are '' which is falsy here too). + # ============================================================ + $script:DestTier = @{} + $script:DestRaw = @{} + $script:DestIsBound = @{} + $resumeDestSwitches = @( + @{ Key = 'Purview' ; Name = 'OutputPath' ; Value = $OutputPath ; AllowFabricFilesOnly = $false }, + @{ Key = 'UserInfo' ; Name = 'OutputPathUserInfo' ; Value = $OutputPathUserInfo ; AllowFabricFilesOnly = $false }, + @{ Key = 'Agent365Info' ; Name = 'OutputPathAgent365Info' ; Value = $OutputPathAgent365Info ; AllowFabricFilesOnly = $false }, + @{ Key = 'DefenderUsage'; Name = 'OutputPathDefenderUsage' ; Value = $OutputPathDefenderUsage ; AllowFabricFilesOnly = $false }, + @{ Key = 'Log' ; Name = 'OutputPathLog' ; Value = $OutputPathLog ; AllowFabricFilesOnly = $true } + ) + foreach ($ds in $resumeDestSwitches) { + $hasValue = -not [string]::IsNullOrWhiteSpace($ds.Value) + $script:DestIsBound[$ds.Key] = $hasValue + if ($hasValue) { + $tier = script:Get-PathTier -Value $ds.Value -SwitchName $ds.Name -AllowFabricFilesOnly:$ds.AllowFabricFilesOnly + $script:DestTier[$ds.Key] = $tier + $script:DestRaw[$ds.Key] = $ds.Value.Trim() + } + } + $script:AppendIsBound = @{} + $script:AppendRaw = @{} + $script:AppendIsRemote = @{} + $resumeAppendSwitches = @( + @{ Key = 'Purview' ; Name = 'AppendFile' ; Value = $AppendFile }, + @{ Key = 'UserInfo' ; Name = 'AppendUserInfo' ; Value = $AppendUserInfo }, + @{ Key = 'Agent365Info' ; Name = 'AppendAgent365Info' ; Value = $AppendAgent365Info }, + @{ Key = 'DefenderUsage'; Name = 'AppendDefenderUsage' ; Value = $AppendDefenderUsage } + ) + foreach ($as in $resumeAppendSwitches) { + $hasValue = -not [string]::IsNullOrWhiteSpace($as.Value) + $script:AppendIsBound[$as.Key] = $hasValue + if (-not $hasValue) { continue } + $v = $as.Value.Trim() + $script:AppendRaw[$as.Key] = $v + $looksUrl = $v -match '^https?://' + $looksRooted = $v -match '^[A-Za-z]:[\\/]' -or $v -match '^/' + if ($looksUrl -or $looksRooted) { + $aTier = script:Get-PathTier -Value $v -SwitchName $as.Name + $script:AppendIsRemote[$as.Key] = ($aTier -ne 'Local') + if (-not $script:DestTier.ContainsKey($as.Key)) { + $script:DestTier[$as.Key] = $aTier + $script:DestRaw[$as.Key] = $v + } + } else { + $script:AppendIsRemote[$as.Key] = $false + } + } + # Promote legacy back-compat variables. + $script:RemoteOutputMode = 'None' + $script:RemoteOutputUrl = $null + if ($script:DestTier.ContainsKey('Purview')) { + switch ($script:DestTier['Purview']) { + 'SharePoint' { $script:RemoteOutputMode = 'SharePoint'; $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') } + 'Fabric' { $script:RemoteOutputMode = 'Fabric'; $script:RemoteOutputUrl = $script:DestRaw['Purview'].TrimEnd('/') } + } + } + if ($script:RemoteOutputMode -eq 'None') { + foreach ($k in @('UserInfo','Agent365Info','DefenderUsage')) { + if (-not $script:DestTier.ContainsKey($k)) { continue } + $t = $script:DestTier[$k] + if ($t -eq 'SharePoint' -or $t -eq 'Fabric') { + $script:RemoteOutputMode = $t + $script:RemoteOutputUrl = $script:DestRaw[$k].TrimEnd('/') + break + } + } + } + # Folder-URL normalization for DestParentUrl and RemoteOutputUrl (mirrors the + # parse-time block at $script:NormalizeFolderUrl). + $script:DestParentUrl = @{} + $resumeNormalizeFolderUrl = { + param([string]$u) + if ([string]::IsNullOrWhiteSpace($u)) { return $u } + if ($u -match '/[^/]+\.(csv|log|xlsx|json|parquet|txt)$') { + return ($u -replace '/[^/]+\.[A-Za-z0-9]+$','') + } + return $u + } + if ($script:RemoteOutputUrl) { $script:RemoteOutputUrl = & $resumeNormalizeFolderUrl $script:RemoteOutputUrl } + foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { + if (-not $script:DestRaw.ContainsKey($k)) { continue } + if (-not $script:DestTier.ContainsKey($k)) { continue } + if ($script:DestTier[$k] -eq 'Local') { continue } + $script:DestParentUrl[$k] = & $resumeNormalizeFolderUrl $script:DestRaw[$k] + } + $restoredTiers = @(); foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if ($script:DestTier.ContainsKey($k)) { $restoredTiers += $script:DestTier[$k] } }; $restoredTiers = @($restoredTiers | Select-Object -Unique); if ($restoredTiers.Count -gt 1) { Write-LogHost ("ERROR: resume destination conflict - restored destinations resolve to mixed storage tiers ({0}); all destinations in one run must share one tier." -f ($restoredTiers -join ', ')) -ForegroundColor Red; exit 1 } + $restoredNorm = @{}; foreach ($k in @('Purview','UserInfo','Agent365Info','DefenderUsage','Log')) { if (-not $script:DestRaw.ContainsKey($k)) { continue }; $rn = $script:DestRaw[$k].TrimEnd('/','\').ToLowerInvariant(); if ($rn -match '\.[a-z0-9]{2,5}$') { if ($restoredNorm.ContainsKey($rn)) { Write-LogHost ("ERROR: resume destination conflict - {0} and {1} resolve to the same fully qualified path: {2}" -f $restoredNorm[$rn], $k, $script:DestRaw[$k]) -ForegroundColor Red; exit 1 }; $restoredNorm[$rn] = $k } } + Write-LogHost " Resume: re-resolved destination state (RemoteOutputMode=$script:RemoteOutputMode, tiers=$(($script:DestTier.GetEnumerator() | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ', '))" -ForegroundColor DarkGray + + # Resume-side scratch-dir rewrite. Mirrors the parse-time scratch-dir rewrite + # (search for "Redirect OutputPath to a per-run scratch dir") for the -Resume + # code path. Once we know the destination is SharePoint / Fabric, downstream + # code must continue to treat $OutputPath as the LOCAL scratch directory + # (where _PARTIAL.csv and .pax_incremental live) — exactly as it did during + # the interrupted run. The scratch dir on a -Resume is the parent of the + # checkpoint file (every PAX_/ scratch is self-contained). Without this + # rewrite, downstream code would re-use the remote URL string as a local + # Join-Path target and crash / write nowhere. Tier-agnostic: applies + # identically to SharePoint and Fabric (OneLake) modes. + if ($script:RemoteOutputMode -ne 'None') { + $script:RemoteScratchDir = Split-Path -Parent $script:CheckpointPath + $sep = [System.IO.Path]::DirectorySeparatorChar + if (-not $script:RemoteScratchDir.EndsWith($sep)) { $script:RemoteScratchDir = $script:RemoteScratchDir + $sep } + $OutputPath = $script:RemoteScratchDir + Write-LogHost " Resume: remote mode active - scratch dir = $script:RemoteScratchDir (remote URL = $script:RemoteOutputUrl)" -ForegroundColor DarkGray + } + + # Restore rollup post-processor intent from checkpoint, with last-write-wins semantics: + # if the user passed -Rollup or -RollupPlusRaw on the resume command line, the resume + # switch wins and the checkpoint value is ignored. If the user passed neither, restore + # the original run's rollup mode from the checkpoint so a resumed run produces the same + # outputs as the original would have. + $resumeUserPassedRollup = $PSBoundParameters.ContainsKey('Rollup') -or $PSBoundParameters.ContainsKey('RollupPlusRaw') + if (-not $resumeUserPassedRollup -and $cp.rollupMode) { + switch ([string]$cp.rollupMode) { + 'Rollup' { $Rollup = [System.Management.Automation.SwitchParameter]::new($true); $RollupPlusRaw = [System.Management.Automation.SwitchParameter]::new($false) } + 'RollupPlusRaw' { $RollupPlusRaw = [System.Management.Automation.SwitchParameter]::new($true); $Rollup = [System.Management.Automation.SwitchParameter]::new($false) } + default { } + } + if ($cp.processorMode -and ($cp.processorMode -in @('CopilotInteraction','M365Bundle','None'))) { + $script:RollupProcessorMode = [string]$cp.processorMode + } + if ($Rollup -or $RollupPlusRaw) { + $restoredSwitch = if ($Rollup) { '-Rollup' } else { '-RollupPlusRaw' } + Write-LogHost " Restored rollup mode from checkpoint: $restoredSwitch (processorMode=$($script:RollupProcessorMode))" -ForegroundColor DarkGray + } + } + elseif ($resumeUserPassedRollup -and $cp.rollupMode -and $cp.rollupMode -ne 'None') { + # Resume's switch wins; just log the override for visibility. + $nowSwitch = if ($Rollup) { '-Rollup' } elseif ($RollupPlusRaw) { '-RollupPlusRaw' } else { '' } + Write-LogHost " Rollup mode overridden on resume: checkpoint='$($cp.rollupMode)' -> resume='$nowSwitch' (last-write-wins)." -ForegroundColor Yellow + } + + # Dashboard restore (independent last-write-wins): an explicit -Dashboard on the + # resume CLI wins; otherwise restore the original run's dashboard so a resumed + # CopilotInteraction run keeps its AIO/AIBV profile instead of defaulting to AIO. + # Guarded to valid values so the [ValidateSet] on $Dashboard never throws on assign. + $resumeUserPassedDashboard = $PSBoundParameters.ContainsKey('Dashboard') + if (-not $resumeUserPassedDashboard -and -not $script:ResumeIsAISID -and $cp.rollupDashboard -and ([string]$cp.rollupDashboard -in @('AIO', 'AIBV', 'M365'))) { + $Dashboard = if ([string]$cp.rollupDashboard -ieq 'AIBV') { 'ValueLens' } else { [string]$cp.rollupDashboard } + Write-LogHost " Restored rollup dashboard from checkpoint: $Dashboard" -ForegroundColor DarkGray + } + + # Re-derive $script:RollupProcessorMode AFTER all resume-restore lines above have + # merged checkpoint state (especially IncludeM365Usage). This guarantees the wire-up + # at end-of-run picks the right embedded processor even when the resume CLI omitted + # the modifier switches. The gating BLOCKERS are not re-run here — they were validated + # on the original run and the resume restore preserves all gating-relevant switches + # (ExportWorkbook, OnlyUserInfo, OnlyAgent365Info, AppendFile, + # RAWInputCSV, UseEOM); if any of those were set, the original run would have already + # hard-failed before producing a checkpoint to resume from. + if ($Rollup -or $RollupPlusRaw) { + if ($IncludeM365Usage) { + $script:RollupProcessorMode = 'M365Bundle' + $script:RollupDashboard = 'M365' + $script:RollupDashboardProfile = $null + } + else { + $script:RollupProcessorMode = 'CopilotInteraction' + # Honor the restored / resume-CLI dashboard so a resumed AIBV run is not + # silently downgraded to the AIO default. Only AIO/AIBV reach here + # (M365 implies -IncludeM365Usage -> the M365Bundle branch above). + $script:RollupDashboard = if ($Dashboard) { if ($Dashboard.ToUpperInvariant() -eq 'VALUELENS') { 'AIBV' } else { $Dashboard.ToUpperInvariant() } } else { 'AIO' } + if ($script:RollupDashboard -notin @('AIO', 'AIBV')) { $script:RollupDashboard = 'AIO' } + $script:RollupDashboardProfile = $script:RollupDashboard.ToLowerInvariant() + if (-not $IncludeUserInfo) { + $IncludeUserInfo = [System.Management.Automation.SwitchParameter]::new($true) + Write-LogHost " Resume: auto-enabled -IncludeUserInfo for CopilotInteraction-mode rollup." -ForegroundColor Cyan + } + } + } + + # Auth - only restore if user didn't override + if (-not $PSBoundParameters.ContainsKey('Auth') -and $cp.auth) { $Auth = $cp.auth } + if (-not $PSBoundParameters.ContainsKey('TenantId') -and $cp.tenantId) { $TenantId = $cp.tenantId } + if (-not $PSBoundParameters.ContainsKey('ClientId') -and $cp.clientId) { $ClientId = $cp.clientId } + + # Re-promote restored auth identifiers to script scope. The original L1808-L1812 + # promotion ran at parse time, BEFORE the checkpoint restored TenantId/ClientId + # from the snapshot. Without this re-promotion Connect-PurviewAudit would see + # the empty default values and fail to authenticate even when the user supplied + # -ClientSecret on the resume command line. + $script:TenantId = $TenantId + $script:ClientId = $ClientId + + # Resume preflight for AppRegistration: ClientSecret / cert material is never + # persisted in the checkpoint (by design). If the restored Auth is AppRegistration + # and no credential material was supplied on the resume command line, fail fast + # with an actionable error rather than crashing inside Connect-PurviewAudit on + # the first audit query attempt. + if (([string]$Auth) -ieq 'AppRegistration' ` + -and [string]::IsNullOrEmpty([string]$script:ClientSecret) ` + -and [string]::IsNullOrEmpty([string]$script:ClientCertificateThumbprint) ` + -and [string]::IsNullOrEmpty([string]$script:ClientCertificatePath)) { + Write-LogHost "" -ForegroundColor Red + Write-LogHost "═════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost " ERROR: -Resume against an AppRegistration checkpoint requires credential material" -ForegroundColor Red + Write-LogHost "═════════════════════════════════════════════════════════════════════════════" -ForegroundColor Red + Write-LogHost "" -ForegroundColor Red + Write-LogHost (" The original run used -Auth AppRegistration (TenantId={0}, ClientId={1})." -f $script:TenantId, $script:ClientId) -ForegroundColor Yellow + Write-LogHost " Secrets / certificate material are NOT persisted in the checkpoint by design." -ForegroundColor Yellow + Write-LogHost " Re-supply one of the credential switches on the resume command line:" -ForegroundColor Yellow + Write-LogHost "" -ForegroundColor Yellow + Write-LogHost " -ClientSecret " -ForegroundColor Green + Write-LogHost " -ClientCertificateThumbprint " -ForegroundColor Green + Write-LogHost " -ClientCertificatePath [-ClientCertificatePassword ]" -ForegroundColor Green + Write-LogHost "" -ForegroundColor Yellow + Write-LogHost " Example:" -ForegroundColor Cyan + Write-LogHost (" pwsh -NoLogo -NoProfile -File `"