Fix KQL join pitfalls and adopt lookup for dimension enrichment

docs-mslearn/best-practices/compute.md

@@ -3,7 +3,7 @@ title: FinOps best practices for compute
 description: This article provides FinOps best practices for compute services, including cost optimization, efficiency improvements, and insights into Azure resources.
 author: flanakin
 ms.author: micflan
-ms.date: 04/01/2026
+ms.date: 08/03/2026
 ms.topic: concept-article
 ms.service: finops
 ms.subservice: finops-learning-resources
@@ -180,7 +180,7 @@ Costs
 // Join with prices to filter out ineligible SKUs
 | extend tmp_MeterKey = strcat(substring(ChargePeriodStart, 0, 7), x_SkuMeterId)
 | project tmp_MeterKey, EffectiveCost, PricingCategory, CommitmentDiscountCategory, ResourceName, x_ResourceGroupName, SubAccountName, BillingCurrency
-| join kind=leftouter (
+| lookup kind=leftouter (
     Prices
     | where x_SkuMeterCategory startswith 'Virtual Machines'
     | summarize sp = countif(x_SkuPriceType == 'SavingsPlan'), ri = countif(x_SkuPriceType == 'ReservedInstance')
@@ -216,7 +216,7 @@ Costs
 // Join with prices to filter out ineligible SKUs
 | extend tmp_MeterKey = strcat(substring(ChargePeriodStart, 0, 7), x_SkuMeterId)
 | project tmp_MeterKey, EffectiveCost, PricingCategory, CommitmentDiscountCategory, ResourceName, x_ResourceGroupName, SubAccountName, BillingCurrency
-| join kind=leftouter (
+| lookup kind=leftouter (
     Prices
     | where x_SkuMeterCategory startswith 'Virtual Machines'
     | summarize sp = countif(x_SkuPriceType == 'SavingsPlan'), ri = countif(x_SkuPriceType == 'ReservedInstance')
@@ -366,7 +366,7 @@ resourcecontainers
 | where type =~ 'Microsoft.Resources/subscriptions'
 | where tostring(properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01'
 | project SubscriptionName = name, subscriptionId
-| join (
+| join kind=inner (
     resources
     | where type =~ 'microsoft.compute/virtualmachines'
         or type =~ 'microsoft.compute/virtualMachineScaleSets'
@@ -422,28 +422,28 @@ resourcecontainers
 | where type =~ 'Microsoft.Resources/subscriptions'
 | where tostring(properties.subscriptionPolicies.quotaId) !has 'MSDNDevTest_2014-09-01'
 | project SubscriptionName = name, subscriptionId
-| join (
+| join kind=inner (
     resources
     | where type =~ 'Microsoft.SqlVirtualMachine/SqlVirtualMachines'
         and tostring(properties.['sqlServerLicenseType']) != 'AHUB'
     | project
         ResourceId = id,
         ResourceName = name,
+        VMResourceId = tolower(tostring(properties.virtualMachineResourceId)),
         LicenseType = tostring(properties.['sqlServerLicenseType']),
         SQLVersion = tostring(properties.['sqlImageOffer']),
         SQLSKU = tostring(properties.['sqlImageSku']),
         Region = location,
         ResourceGroupName = resourceGroup,
         subscriptionId
 ) on subscriptionId
-| join (
+| join kind=inner (
     resources
     | where type =~ 'Microsoft.Compute/virtualMachines'
     | project
-        ResourceName = tolower(name),
-        VMSize = tostring(properties.hardwareProfile.vmSize),
-        subscriptionId
-) on ResourceName
+        VMResourceId = tolower(id),
+        VMSize = tostring(properties.hardwareProfile.vmSize)
+) on VMResourceId
 | where SQLSKU != 'Developer' and SQLSKU != 'Express'
 | project
     ResourceId,

docs-mslearn/best-practices/networking.md

@@ -3,7 +3,7 @@ title: FinOps best practices for Networking
 description: This article outlines proven FinOps practices for networking services, focusing on cost optimization, efficiency improvements, and resource insights.
 author: flanakin
 ms.author: micflan
-ms.date: 04/01/2026
+ms.date: 08/03/2026
 ms.topic: concept-article
 ms.service: finops
 ms.subservice: finops-learning-resources
@@ -134,7 +134,7 @@ resources
     backendPools = properties.backendAddressPools,
     resourceGroup = strcat('/subscriptions/',subscriptionId,'/resourceGroups/',resourceGroup)
 | project id, name, SKUName, SKUTier, SKUCapacity, resourceGroup, subscriptionId
-| join (
+| join kind=inner (
     resources
     | where type =~ 'Microsoft.Network/applicationGateways'
     | mvexpand backendPools = properties.backendAddressPools
@@ -578,8 +578,8 @@ resources
     | mv-expand IPconfig
     | extend PublicIpId= tostring(IPconfig.properties.publicIPAddress.id)
     | project PublicIpId
-    | join (
-        resource
+    | join kind=inner (
+        resources
         | where type =~ 'Microsoft.Network/publicIPAddresses'
         | extend
             PublicIpId = id,

docs-mslearn/toolkit/changelog.md

@@ -3,7 +3,7 @@ title: FinOps toolkit changelog
 description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more.
 author: MSBrett
 ms.author: brettwil
-ms.date: 07/30/2026
+ms.date: 08/03/2026
 ms.topic: reference
 ms.service: finops
 ms.subservice: finops-toolkit
@@ -34,6 +34,19 @@ The following section lists features and enhancements that are currently in deve
   - Replaced whole-term `contains` matches with `has` across hub KQL and the query catalog (resource ID paths, licensing phrases, SKU description terms) and added a per-row operator-equivalence regression harness with unit test coverage ([#2220](https://github.com/microsoft/finops-toolkit/pull/2220)).
 - **Fixed**
   - Fixed the `ContractedCost` recompute guard to compare with a null-safe tolerance instead of exact float equality, eliminating millions of no-op rewrites that polluted the `x_SourceValues` audit trail while preserving the null-cost backfill and no longer overwriting an existing cost when the unit price is missing ([#2216](https://github.com/microsoft/finops-toolkit/issues/2216)).
+  - Fixed the SQL VMs without Azure Hybrid Benefit recommendation query to join on the SQL VM `virtualMachineResourceId` instead of a case-sensitive VM name match that skipped VMs with uppercase names and dropped duplicate names, and made all Azure Resource Graph join kinds explicit so no query relies on the `innerunique` default ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
+  - Switched dimension enrichment in the v1_0/v1_2 ingestion transforms (`PricingUnits`, `Regions`, `ResourceTypes`, `Services`) from `join` to the broadcast-optimized `lookup` operator and deduplicated the `Services` mapping per resource type to prevent cost row fan-out ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
+
+### [FinOps workbooks](workbooks/finops-workbooks-overview.md)
+
+- **Fixed**
+  - Fixed the savings plan workbook summary and details queries silently dropping all but one recommendation per subscription due to an implicit `innerunique` join ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
+  - Fixed the AHB workbook "VM Latest Change Last 7 days" tile, which joined the `resourcechanges` record id against the resource id and always returned no rows, and the SQL VM queries, which never matched VMs with uppercase names ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
+
+### [FinOps alerts](alerts/finops-alerts-overview.md)
+
+- **Fixed**
+  - Made the idle application gateway and idle public IP query join kinds explicit so they no longer rely on the `innerunique` default ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
 
 ### [Power BI reports](power-bi/reports.md)
 

docs-mslearn/toolkit/workbooks/customize-workbooks.md

@@ -3,7 +3,7 @@ title: Use and customize FinOps workbooks
 description: Learn how to install and customize FinOps workbooks to achieve FinOps goals, including cost recommendations, idle resource identification, and more.
 author: flanakin
 ms.author: micflan
-ms.date: 04/01/2026
+ms.date: 08/03/2026
 ms.topic: how-to
 ms.service: finops
 ms.subservice: finops-toolkit
@@ -122,7 +122,7 @@ In the last section, the query joins the `ResourceContainerstable` with the `res
 
 ```kusto
 ResourceContainers | "Insert first code section go here"
-| join (
+| join kind=inner (
     resources  "Insert second code section here"
 ) on subscriptionId
 | order by type asc
@@ -137,7 +137,7 @@ Here's the full code example that you use to insert into the workbook.
 
 ```kusto
 ResourceContainers | where type =~ 'Microsoft.Resources/subscriptions' | where tostring (properties.subscriptionPolicies.quotaId) !has "MSDNDevTest_2014-09-01"  | extend SubscriptionName=name
-| join (
+| join kind=inner (
   resources
   | where resourceGroup in ({ResourceGroup})
   | where type == 'microsoft.azurestackhci/clusters'

src/optimization-engine/runbooks/recommendations/Recommend-SqlDbOptimizationsToBlobStorage.ps1

@@ -162,7 +162,7 @@ $baseQuery = @"
     | where ResourceId in (CandidateDatabaseIds) and MetricNames_s == 'dtu_consumption_percent' and AggregationType_s == 'Maximum'
     | summarize P99DTUPercentage = percentile(todouble(MetricValue_s), dtuPercentPercentile) by ResourceId
     | where P99DTUPercentage < DTUPercentageThreshold
-    | join (
+    | join kind=inner (
         $sqlDbsTableName
         | where TimeGenerated > ago(1d)
         | project ResourceId = InstanceId_s, DBName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, SkuName_s, ServiceObjectiveName_s, Tags_s, Cloud_s
@@ -311,7 +311,7 @@ $baseQuery = @"
     | where ResourceId in (CandidateDatabaseIds) and MetricNames_s == 'dtu_consumption_percent' and AggregationType_s == 'Average' and AggregationOfType_s == 'Maximum'
     | summarize AvgDTUPercentage = avg(todouble(MetricValue_s)) by ResourceId
     | where AvgDTUPercentage > DTUPercentageThreshold
-    | join (
+    | join kind=inner (
         $sqlDbsTableName
         | where TimeGenerated > ago(1d)
         | project ResourceId = InstanceId_s, DBName_s, ResourceGroupName_s, SubscriptionGuid_g, TenantGuid_g, SkuName_s, ServiceObjectiveName_s, Tags_s, Cloud_s

src/powershell/Tests/Lint/KqlJoinKinds.Tests.ps1

@@ -0,0 +1,114 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+<#
+    Lint rule: every KQL join must state an explicit kind (PR #2225).
+
+    A bare `| join (...)` defaults to kind=innerunique, which deduplicates the left side
+    on the join key and silently drops rows. This has caused real data loss (savings plan
+    recommendations collapsing to one row per subscription, SQL VMs with duplicate names
+    disappearing). In Azure Resource Graph the same default applies and `lookup` is not
+    available, so an explicit kind is the only way to state intent.
+
+    The rule scans every surface that carries KQL: hub database scripts, the query catalog,
+    ARG recommendation queries, the ADX dashboard, the finops-alerts logic app, workbooks,
+    optimization engine runbooks and views, and the published docs examples.
+
+    Known pre-existing bare joins are baselined per file below. The baseline is a ratchet:
+    - Fixing a bare join REQUIRES lowering the count here (the test fails on stale entries).
+    - Adding a new bare join is never allowed; write `join kind=...` explicitly.
+
+    ARG-only surfaces (workbooks, recommendation queries, the alerts logic app) are additionally
+    checked for operators Azure Resource Graph rejects: the lookup operator and the semi/anti
+    join flavors. Verified live against ARG (2026-08): supported kinds are inner, innerunique,
+    leftouter, rightouter, fullouter; lookup, leftsemi, leftanti, rightsemi, rightanti, and
+    `in`/`!in` with a subquery are all rejected with InvalidQuery. Exclusion joins in ARG must
+    therefore use the leftouter + `where isempty(<right key>)` emulation (with a key-unique
+    right side) — the one place that pattern is acceptable.
+#>
+
+Describe 'KqlJoinKinds' {
+
+    BeforeDiscovery {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+
+        $scanTargets = @(
+            @{ Path = 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts'; Filter = '*.kql'; Recurse = $false }
+            @{ Path = 'src/queries/catalog'; Filter = '*.kql'; Recurse = $false }
+            @{ Path = 'src/powershell/Tests/assets'; Filter = '*.kql'; Recurse = $false }
+            @{ Path = 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries'; Filter = '*.json'; Recurse = $false }
+            @{ Path = 'src/templates/finops-hub'; Filter = 'dashboard.json'; Recurse = $false }
+            @{ Path = 'src/templates/finops-alerts/modules'; Filter = 'logicApp.bicep'; Recurse = $false }
+            @{ Path = 'src/workbooks'; Filter = '*.workbook'; Recurse = $true }
+            @{ Path = 'src/workbooks'; Filter = 'workbook.json'; Recurse = $true }
+            @{ Path = 'src/optimization-engine/runbooks'; Filter = '*.ps1'; Recurse = $true }
+            @{ Path = 'src/optimization-engine/views'; Filter = '*.json'; Recurse = $true }
+            @{ Path = 'docs-mslearn'; Filter = '*.md'; Recurse = $true }
+        )
+
+        $scanFiles = @($scanTargets | ForEach-Object {
+                $full = Join-Path $repoRoot $_.Path
+                Get-ChildItem -Path $full -Filter $_.Filter -Recurse:$_.Recurse -File -ErrorAction SilentlyContinue
+            } | Sort-Object FullName -Unique | ForEach-Object {
+                @{ Name = $_.Name; FullName = $_.FullName; RelPath = $_.FullName.Substring($repoRoot.Length + 1).Replace('\', '/') }
+            })
+
+        # Surfaces whose KQL runs on Azure Resource Graph. Workbook files may also contain the
+        # occasional Log Analytics query (queryType 0); if one legitimately needs lookup or a
+        # semi/anti join, add a per-file allowlist analogous to the bare-join baseline.
+        $argFiles = @($scanFiles | Where-Object {
+                $_.RelPath -like 'src/workbooks/*' -or
+                $_.RelPath -like 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Recommendations/queries/*' -or
+                $_.RelPath -eq 'src/templates/finops-alerts/modules/logicApp.bicep'
+            })
+    }
+
+    BeforeAll {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+        $scanFileCount = @(
+            (Join-Path $repoRoot 'src/workbooks'),
+            (Join-Path $repoRoot 'src/optimization-engine'),
+            (Join-Path $repoRoot 'src/queries/catalog'),
+            (Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts')
+        ) | ForEach-Object { Get-ChildItem -Path $_ -Recurse -Include '*.kql', '*.workbook', 'workbook.json', '*.ps1', '*.json' -File -ErrorAction SilentlyContinue } | Measure-Object | Select-Object -ExpandProperty Count
+
+        # Matches `| join` not followed by `kind=` before the right-table parenthesis.
+        # Catches `| join (`, `| join(`, and `| join hint.x=y (`; ignores `| join kind=...` and `lookup`.
+        $bareJoinPattern = [regex]'\|\s*join\b(?![^(\r\n]*\bkind\s*=)'
+
+        # Pre-existing bare joins, counted per repo-relative path. Ratchet only: lower on fix, never raise.
+        # All remaining entries are benign today (left side unique on the join key) but rely on the
+        # innerunique default implicitly. Convert to an explicit kind when touching these queries.
+        $baseline = @{
+            'src/workbooks/optimization/AHB/AHB.workbook'         = 24
+            'src/workbooks/optimization/Compute/AHB.workbook'     = 20
+            'src/workbooks/optimization/Networking/Networking.workbook' = 3
+            'src/workbooks/governance/workbook.json'              = 1
+        }
+    }
+
+    It 'Should scan at least one file per surface' {
+        $scanFileCount | Should -BeGreaterThan 100
+    }
+
+    It 'Should not add bare joins (no explicit kind): <RelPath>' -ForEach $scanFiles {
+        $content = Get-Content -Path $FullName -Raw
+        $bareJoins = @($bareJoinPattern.Matches($content))
+        $allowed = if ($baseline.ContainsKey($RelPath)) { $baseline[$RelPath] } else { 0 }
+
+        $bareJoins.Count | Should -BeLessOrEqual $allowed -Because ('a bare "| join" defaults to kind=innerunique, which deduplicates the left side on the join key and silently drops rows (see PR #2225). State the kind explicitly: kind=inner for lookups/filters, kind=leftouter for enrichment, kind=leftanti for exclusion. In ADX/Log Analytics, prefer the lookup operator for small dimension tables.')
+
+        if ($bareJoins.Count -le $allowed)
+        {
+            # Ratchet: if bare joins were removed, the baseline must be lowered so they cannot return.
+            $bareJoins.Count | Should -Be $allowed -Because ("the bare-join count in this file dropped below the baseline ($allowed); lower the baseline entry for '$RelPath' in KqlJoinKinds.Tests.ps1 to $($bareJoins.Count) (or remove it if 0) so the fix is locked in.")
+        }
+    }
+
+    It 'Should not use operators ARG rejects (lookup, semi/anti joins): <RelPath>' -ForEach $argFiles {
+        $content = Get-Content -Path $FullName -Raw
+        $rejected = @([regex]::Matches($content, '\|\s*lookup\b|join\s+kind\s*=\s*(leftanti|leftsemi|rightanti|rightsemi|anti|semi|leftantisemi|rightantisemi)\b'))
+
+        @($rejected | ForEach-Object { $_.Value }) -join '; ' | Should -BeNullOrEmpty -Because ('Azure Resource Graph rejects the lookup operator and all semi/anti join flavors with InvalidQuery (verified live; supported kinds are inner, innerunique, leftouter, rightouter, fullouter). For exclusions in ARG, use join kind=leftouter + where isempty(<right key>) with a key-unique right side. If this file contains a Log Analytics query that legitimately needs the operator, add a per-file allowlist to this test.')
+    }
+}

src/queries/catalog/macc-consumption-vs-commitment.kql

@@ -47,11 +47,11 @@ let accountCommitment =
     // switch to max() or scan for the most-recent MACC event per period.
     | summarize AccountCommitment = todouble(sum(x_MonetaryCommitment)) by BillingCurrency, BillingAccountId, ReportingMonth;
 profileConsumption
-| join kind=leftouter (
+| lookup kind=leftouter (
     profileConsumption
     | summarize AccountConsumption = sum(ProfileConsumption) by BillingCurrency, BillingAccountId, ReportingMonth
 ) on BillingCurrency, BillingAccountId, ReportingMonth
-| join kind=leftouter accountCommitment on BillingCurrency, BillingAccountId, ReportingMonth
+| lookup kind=leftouter (accountCommitment) on BillingCurrency, BillingAccountId, ReportingMonth
 | extend AccountCommitment = todouble(coalesce(AccountCommitment, 0.0))
 | extend ProfileShare = iff(AccountConsumption == 0.0, 0.0, ProfileConsumption / AccountConsumption)
 | extend ConsumptionAmount = ProfileConsumption

src/queries/catalog/storage-tier-distribution.kql

@@ -48,7 +48,7 @@ bucketedCosts
     EffectiveCost = todouble(sum(EffectiveCost)),
     GBMonths = todouble(sum(GBMonths))
     by BillingCurrency, TierBucket
-| join kind=inner totalsByCurrency on BillingCurrency
+| lookup kind=inner (totalsByCurrency) on BillingCurrency
 | project
     BillingCurrency,
     TierBucket,

src/queries/catalog/tagging-policy-compliance.kql

@@ -47,7 +47,7 @@ let tag_compliance = materialize(
     | project tmp_TagsString, tmp_IsCompliant
 );
 base
-| join kind=leftouter tag_compliance on tmp_TagsString
+| lookup kind=leftouter (tag_compliance) on tmp_TagsString
 | summarize
     CompliantEffectiveCost = todouble(sumif(EffectiveCost, tmp_IsCompliant)),
     TotalEffectiveCost = todouble(sum(EffectiveCost))

src/queries/finops-hub-database-guide.md

@@ -126,17 +126,16 @@ This example demonstrates how to report costs using the full financial hierarchy
 
 ```kusto
 let numberOfMonths = 1; // Set to desired reporting period
+let GrandTotal = toscalar(
+    Costs()
+    | where ChargePeriodStart >= monthsago(numberOfMonths)
+    | summarize sum(EffectiveCost)
+);
 Costs()
 | where ChargePeriodStart >= monthsago(numberOfMonths)
 | extend Team = tostring(Tags['team']), Product = tostring(Tags['product']), Application = tostring(Tags['application'])
 | summarize TotalCost = sum(EffectiveCost)
     by x_BillingProfileName, x_InvoiceSectionName, Team, Product, Application
-| join kind=leftouter (
-    Costs()
-    | where ChargePeriodStart >= monthsago(numberOfMonths)
-    | summarize GrandTotal = sum(EffectiveCost)
-)
-on 1 == 1
 | extend PercentOfTotal = 100.0 * TotalCost / GrandTotal
 | project x_BillingProfileName, x_InvoiceSectionName, Team, Product, Application, TotalCost, PercentOfTotal
 | order by TotalCost desc