diff --git a/docs-mslearn/TOC.yml b/docs-mslearn/TOC.yml index 4d48e0aff..caafb6f84 100644 --- a/docs-mslearn/TOC.yml +++ b/docs-mslearn/TOC.yml @@ -128,6 +128,8 @@ href: toolkit/hubs/finops-hubs-overview.md - name: Create or upgrade hubs href: toolkit/hubs/deploy.md + - name: Run hubs locally + href: toolkit/hubs/run-hubs-locally.md - name: Configure private networking href: toolkit/hubs/private-networking.md - name: Configure scopes @@ -246,6 +248,8 @@ href: toolkit/powershell/hubs/deploy-finopshub.md - name: Initialize-FinOpsHubDeployment href: toolkit/powershell/hubs/initialize-finopshubdeployment.md + - name: Initialize-FinOpsHubLocal + href: toolkit/powershell/hubs/initialize-finopshublocal.md - name: Register-FinOpsHubProviders href: toolkit/powershell/hubs/register-finopshubproviders.md - name: Remove-FinOpsHub diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index c7c27e96c..2cce54a9d 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 07/30/2026 +ms.date: 08/02/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -74,6 +74,9 @@ _Released June 2026_ ### [FinOps hubs](hubs/finops-hubs-overview.md) v15 +- **Added** + - Added a [Run hubs locally](hubs/run-hubs-locally.md) guide to stand up a FinOps hub in a local Kusto emulator container and ingest cost data using the same KQL, transforms, and open data as a deployed hub. + - Added a build-generated `finops-hub-local-opendata.kql` release artifact that loads the open data reference tables from CSV, so the local hub guide stays in sync with published open data instead of hard-coding schemas. - **Changed** - Added a callout to the `config_RunBackfillJob` backfill option clarifying that it isn't supported on Microsoft Customer Agreement (MCA) billing accounts or billing profiles ([#2113](https://github.com/microsoft/finops-toolkit/issues/2113)). - **Fixed** @@ -100,6 +103,8 @@ _Released June 2026_ ### [PowerShell module](powershell/powershell-commands.md) v15 +- **Added** + - Added [Initialize-FinOpsHubLocal](powershell/hubs/initialize-finopshublocal.md) to set up a local FinOps hub in a running Kusto emulator with one command. - **Changed** - Updated PowerShell test compatibility with Pester 6 ([#2204](https://github.com/microsoft/finops-toolkit/pull/2204)). - **Fixed** diff --git a/docs-mslearn/toolkit/hubs/deploy.md b/docs-mslearn/toolkit/hubs/deploy.md index 77451bd5f..2e4f17445 100644 --- a/docs-mslearn/toolkit/hubs/deploy.md +++ b/docs-mslearn/toolkit/hubs/deploy.md @@ -3,7 +3,7 @@ title: How to create and update FinOps hubs description: This tutorial helps you create a new or update an existing FinOps hubs instance in Azure or Microsoft Fabric. author: flanakin ms.author: micflan -ms.date: 07/30/2026 +ms.date: 08/02/2026 ms.topic: tutorial ms.service: finops ms.subservice: finops-toolkit @@ -136,6 +136,9 @@ Configuring Microsoft Fabric is a manual process and requires explicit steps bef The core engine for FinOps hubs is deployed via an Azure Resource Manager deployment template. The template is available in [bicep](/azure/azure-resource-manager/bicep/overview). The template includes a storage account, Azure Data Factory, Azure Data Explorer, and other supporting resources. To learn more about the template and least-privileged access requirements, refer to the [FinOps hub template details](template.md). +> [!TIP] +> Want to explore the hub's KQL and data model without deploying Azure resources? See [Run FinOps hubs locally](run-hubs-locally.md) to stand up a hub on your own hardware. + ### [Azure portal](#tab/azure-portal) 1. Open the desired template in the Azure portal: diff --git a/docs-mslearn/toolkit/hubs/finops-hubs-overview.md b/docs-mslearn/toolkit/hubs/finops-hubs-overview.md index 5292e0c9d..fdc1a4280 100644 --- a/docs-mslearn/toolkit/hubs/finops-hubs-overview.md +++ b/docs-mslearn/toolkit/hubs/finops-hubs-overview.md @@ -3,7 +3,7 @@ title: FinOps hubs overview description: FinOps hubs provide a reliable platform for cost analytics, insights, and optimization, supporting large accounts and organizations. author: flanakin ms.author: micflan -ms.date: 04/01/2026 +ms.date: 08/02/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -137,6 +137,9 @@ To create a new FinOps hub, follow these steps: For more detailed instructions, see [Create and update FinOps hubs](deploy.md). If you run into any issues, refer to the [Troubleshooting guide](../help/troubleshooting.md). +> [!TIP] +> Want to explore the hub's KQL and data model before deploying Azure resources? See [Run FinOps hubs locally](run-hubs-locally.md). +
## Get started with hubs diff --git a/docs-mslearn/toolkit/hubs/run-hubs-locally.md b/docs-mslearn/toolkit/hubs/run-hubs-locally.md new file mode 100644 index 000000000..45a694882 --- /dev/null +++ b/docs-mslearn/toolkit/hubs/run-hubs-locally.md @@ -0,0 +1,289 @@ +--- +title: Run FinOps hubs locally +description: Stand up a FinOps hub on your own hardware in a local container and ingest cost data, using the same KQL and open data as a deployed hub. +author: MSBrett +ms.author: brettwil +ms.date: 08/02/2026 +ms.topic: how-to +ms.service: finops +ms.subservice: finops-toolkit +ms.reviewer: brettwil +#customer intent: As a FinOps user, I want to run a FinOps hub locally so I can explore my cost data without deploying Azure resources. +--- + + + +# Run FinOps hubs locally + +This article shows how to run a FinOps hub on your own hardware using the [Kusto emulator](/azure/data-explorer/kusto-emulator-overview), a free local container. It uses the same analytics KQL, transforms, and open data as a deployed [FinOps hub](finops-hubs-overview.md) — only the host changes from Azure Data Explorer to a local container. Run the PowerShell blocks below in order; each is a step you (or an agent) can run one after another. + +A local hub is useful when you need full hub analysis without managed daily refresh: exploring the data model, validating large customer datasets, supporting consulting deliveries, or working from on-premises, other-cloud, or disconnected environments where you can bring exported data to the machine. It isn't a replacement for a deployed hub — there's no scheduled ingestion, networking, security, or sharing. + +
+ +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/), to run the Kusto emulator container, with at least 16 GB of memory available to the container. Start Docker before you run the commands and verify `docker info` works in the same PowerShell session. +- [PowerShell 7](/powershell/scripting/install/installing-powershell) — every command in this article is PowerShell. +- [Azure PowerShell](/powershell/azure/install-azure-powershell) (the `Az` modules), only needed to download cost data from an Azure storage account. Sign in first with `Connect-AzAccount` and select the subscription that holds your cost data — this article never runs a sign-in command. With `-UseConnectedAccount`, your identity also needs storage data-plane access, such as **Storage Blob Data Reader**, on the account or container. +- FOCUS cost exports as Parquet, available on the local machine. The emulator reads cost data from a local folder, so the files must be on disk — but you can download them from a [FinOps hub storage account](configure-scopes.md) or [Cost Management exports](/cost-management-billing/costs/tutorial-improved-exports) (shown below), or copy them from any other source into the same folder structure. + +For background on the emulator and its platform requirements, see the [Kusto emulator overview](/azure/data-explorer/kusto-emulator-overview) and [installation guide](/azure/data-explorer/kusto-emulator-install). + +
+ +## Start the emulator + +Create a working folder outside the repository, then start the emulator. Two folders are mounted into the container: exports (read-only at `/data/export`) and a data folder where the engine persists its databases. Mounting the data folder means your databases survive container restarts and removals. For other ways to run the container, see [Install the Kusto emulator](/azure/data-explorer/kusto-emulator-install). + +```powershell +$exportPath = '../exports/local-hub' +$dataPath = '../exports/local-hub-data' + +New-Item -ItemType Directory -Force -Path $exportPath | Out-Null +New-Item -ItemType Directory -Force -Path $dataPath | Out-Null +docker run -d --name finops-hub-local --platform linux/amd64 ` + -p 8082:8080 -m 16g -e ACCEPT_EULA=Y ` + -v "$((Resolve-Path $exportPath).Path):/data/export:ro" ` + -v "$((Resolve-Path $dataPath).Path):/kustodata" ` + mcr.microsoft.com/azuredataexplorer/kustainer-linux:latest +``` + +To restart after stopping the container, run `docker start finops-hub-local`. Your databases reload automatically from the data folder — no need to repeat the setup steps. + +All later steps talk to the emulator's HTTP endpoint, so define a small helper to send a command. Management commands (those starting with `.`) go to `/v1/rest/mgmt`; queries go to `/v1/rest/query`. The endpoint has no authentication. + +```powershell +$hub = 'http://localhost:8082/v1/rest' +function Invoke-Kusto { + param([string]$Database, [string]$Command, [ValidateSet('mgmt','query')][string]$Endpoint = 'mgmt') + Invoke-RestMethod -Uri "$hub/$Endpoint" -Method Post -ContentType 'application/json' ` + -Body (@{ db = $Database; csl = $Command } | ConvertTo-Json) +} +``` + +Wait for the engine to answer before continuing: + +```powershell +do { + Start-Sleep -Seconds 3 + $ready = try { Invoke-Kusto NetDefaultDB '.show version'; $true } catch { Write-Host 'waiting for emulator...'; $false } +} until ($ready) +Write-Host 'emulator ready' +``` + +
+ +## Set up the hub + +Set up the databases, schema, and open data in one step with the [FinOps toolkit PowerShell module](../powershell/powershell-commands.md), or follow the next three sections to do it by hand. Both run the same released scripts and produce the same result. + +To use the module, stage the open data CSVs in the mounted folder first (the emulator reads them from there), then run one command: + +```powershell +New-Item -ItemType Directory -Force -Path "$exportPath/open-data" | Out-Null +foreach ($f in 'PricingUnits', 'Regions', 'ResourceTypes', 'Services') { + Invoke-WebRequest "https://github.com/microsoft/finops-toolkit/releases/latest/download/$f.csv" -OutFile "$exportPath/open-data/$f.csv" +} + +Install-Module -Name FinOpsToolkit -Scope CurrentUser +Initialize-FinOpsHubLocal -ClusterUri 'http://localhost:8082' -RawRetentionInDays 90 +``` + +`Initialize-FinOpsHubLocal` creates the `Ingestion` and `Hub` databases, applies the released setup scripts, and loads open data from `/data/export/open-data`. It doesn't manage the container or ingest cost data — start the emulator first and ingest your exports later. Add `-SkipOpenData` to skip open data. When it finishes, skip ahead to [Download your cost data](#download-your-cost-data). + +To set everything up by hand instead, continue with the steps below. + +
+ +## Create the two databases + +A FinOps hub uses two databases: `Ingestion` for raw and transformed data, and `Hub` for the view functions you query. + +```powershell +foreach ($db in 'Ingestion', 'Hub') { + Invoke-Kusto NetDefaultDB ".create-merge database $db persist (@`"/kustodata/dbs/$db/md`", @`"/kustodata/dbs/$db/data`")" | Out-Null +} +``` + +
+ +## Load the schema + +Download the hub's setup scripts from the latest toolkit release and load each into its database. These are the same `finops-hub-fabric-setup-*.kql` files used to set up a [Microsoft Fabric hub](deploy.md#optional-set-up-microsoft-fabric) — each is a single `.execute database script` that creates every table, mapping, transform function, and update policy. + +The Ingestion script has one placeholder, `$$rawRetentionInDays$$` (how long to keep raw data). Replace it with a number of days before loading. + +```powershell +$release = 'https://github.com/microsoft/finops-toolkit/releases/latest/download' +New-Item -ItemType Directory -Force -Path "$exportPath/setup" | Out-Null +Invoke-WebRequest "$release/finops-hub-fabric-setup-Ingestion.kql" -OutFile "$exportPath/setup/setup-Ingestion.kql" +Invoke-WebRequest "$release/finops-hub-fabric-setup-Hub.kql" -OutFile "$exportPath/setup/setup-Hub.kql" + +# Ingestion: raw tables, transforms, final tables, update policies +$ingestion = (Get-Content "$exportPath/setup/setup-Ingestion.kql" -Raw) -replace '\$\$rawRetentionInDays\$\$', '90' +Invoke-Kusto Ingestion $ingestion + +# Hub: the Costs/Prices/Transactions view functions +Invoke-Kusto Hub (Get-Content "$exportPath/setup/setup-Hub.kql" -Raw) +``` + +Each response is a table with one row per statement. The Hub functions reference `database('Ingestion').*`, which resolves because both databases exist. + +
+ +## Load the open data + +FinOps hubs enrich cost data with open-data reference tables (regions, services, resource types, and pricing units). Download the reference CSVs and the matching load script from the toolkit release, then run the script. The load script (`finops-hub-local-opendata.kql`) is generated by the toolkit build from the same open data a deployed hub uses, so its schema always matches the published CSVs. + +The emulator's first external-data read right after setup sometimes returns no rows without raising an error, so load the open data, check the tables filled, and retry if they didn't: + +```powershell +New-Item -ItemType Directory -Force -Path "$exportPath/open-data" | Out-Null +foreach ($f in 'PricingUnits', 'Regions', 'ResourceTypes', 'Services') { + Invoke-WebRequest "$release/$f.csv" -OutFile "$exportPath/open-data/$f.csv" +} +Invoke-WebRequest "$release/finops-hub-local-opendata.kql" -OutFile "$exportPath/setup/load-open-data.kql" + +# Point the load script at the mounted open-data folder. +$openData = (Get-Content "$exportPath/setup/load-open-data.kql" -Raw) -replace '\$\$openDataPath\$\$', '/data/export/open-data' + +# Load, verify the tables filled, and retry if the first read came back empty. +foreach ($attempt in 1..5) { + Invoke-Kusto Ingestion $openData | Out-Null + $empty = 'PricingUnits', 'Regions', 'ResourceTypes', 'Services' | Where-Object { + (Invoke-Kusto Ingestion "$_ | count" -Endpoint query).Tables[0].Rows[0][0] -eq 0 + } + if (-not $empty) { Write-Host 'open data loaded'; break } + if ($attempt -eq 5) { throw "Open data tables still empty: $($empty -join ', ')" } + Start-Sleep -Seconds 2 +} +``` + +
+ +## Download your cost data + +Download FOCUS cost and price exports from your storage account into the export folder, using your existing Azure sign-in. This works against either the **msexports** or **ingestion** container of a FinOps hub, or any container holding Cost Management exports. Only the data files (`manifest.json` and `*.parquet`) are downloaded. + +Set these to match your storage account, then run the download: + +> [!NOTE] +> This download step is specific to Azure Storage. The local hub uses the same FOCUS transforms as a deployed hub, which are aligned to [FOCUS](../../focus/what-is-focus.md) and validated against Microsoft Cost Management cost data. If your FOCUS exports already live elsewhere, copy each dataset's `manifest.json` and `*.parquet` files into `$exportPath` and skip to [Ingest the data](#ingest-the-data). + +```powershell +$account = '' +$container = 'ingestion' # or: msexports +$prefix = '' # optional, e.g. 'billingAccounts/00000000' for msexports + +$ctx = New-AzStorageContext -StorageAccountName $account -UseConnectedAccount +Get-AzStorageBlob -Container $container -Context $ctx -Prefix $prefix | + Where-Object { $_.Name -match '(manifest\.json|\.parquet)$' } | + ForEach-Object { + Get-AzStorageBlobContent -Container $container -Context $ctx -Blob $_.Name ` + -Destination $exportPath -Force | Out-Null + } +``` + +> [!NOTE] +> FinOps hub storage accounts use a hierarchical namespace, where each folder is also a zero-byte marker blob. Downloading only `manifest.json` and `*.parquet` skips those markers, which otherwise collide with same-named folders on disk. `-UseConnectedAccount` reuses your existing Azure sign-in. + +
+ +## Ingest the data + +Ingest each Parquet file into the matching raw table. As rows land, the update policy created by the schema runs the FOCUS transform automatically and appends the result to the final table — the same mechanism a deployed hub uses. Cost exports go to `Costs_raw`; price sheets go to `Prices_raw`. + +For large exports, use a small amount of parallelism. Start with about one ingestion thread per 8 GB of emulator memory. For the 16 GB container below, use two threads; if you increase the container to 32 GB, four threads is a good starting point. + +Ingest price sheets before cost data. The cost transform enriches rows with missing prices from `Prices_final_v1_2`, and the update policy runs as each file lands, so any prices must already be in place when costs are ingested. + +```powershell +$ingestConcurrency = 2 + +# Group exports by manifest type, then ingest each dataset's Parquet files +$ingestJobs = Get-ChildItem $exportPath -Recurse -Filter manifest.json | ForEach-Object { + # Default to FocusCost; treat as PriceSheet only when the manifest type or path says so. + $manifest = Get-Content $_.FullName -Raw + $isPrice = $manifest -match '"type"\s*:\s*"PriceSheet"' -or $_.FullName -match '(?i)price' + $target = if ($isPrice) { 'Prices_raw', 'Prices_raw_mapping' } else { 'Costs_raw', 'Costs_raw_mapping' } + Get-ChildItem $_.Directory -Filter *.parquet | ForEach-Object { + [PSCustomObject]@{ + Table = $target[0] + Mapping = $target[1] + File = $_.FullName + # Prices first (0) then costs (1): the cost transform enriches from Prices_final_v1_2. + Phase = if ($isPrice) { 0 } else { 1 } + } + } +} + +# Ingest prices, then costs. Each phase finishes before the next starts. +foreach ($phase in $ingestJobs | Group-Object Phase | Sort-Object Name) { + $phase.Group | ForEach-Object -Parallel { + $hub = $using:hub + $exportPath = $using:exportPath + function Invoke-Kusto { + param([string]$Database, [string]$Command, [ValidateSet('mgmt','query')][string]$Endpoint = 'mgmt') + Invoke-RestMethod -Uri "$hub/$Endpoint" -Method Post -ContentType 'application/json' ` + -Body (@{ db = $Database; csl = $Command } | ConvertTo-Json) + } + function Invoke-Ingest { + param([string]$Table, [string]$Mapping, [string]$File) + $rel = [IO.Path]::GetRelativePath((Resolve-Path $exportPath), $File) -replace '\\', '/' + $path = "/data/export/$rel" + Invoke-Kusto Ingestion ".ingest into table $Table (h@'$path') with (format='parquet', ingestionMappingReference='$Mapping')" | Out-Null + Write-Host " ingested $(Split-Path $File -Leaf)" + } + Invoke-Ingest $_.Table $_.Mapping $_.File + } -ThrottleLimit $ingestConcurrency +} +``` + +If the emulator exits with code 137, it ran out of memory. Reduce `$ingestConcurrency`, increase the container memory, or ingest one month at a time and restart the emulator between batches. For very large imports, keep a per-file log so you can retry failed files without restarting the entire load. + +Confirm the data landed and the final tables filled in: + +```powershell +foreach ($t in 'Costs_raw', 'Costs_final_v1_2', 'Prices_raw', 'Prices_final_v1_2') { + $count = (Invoke-Kusto Ingestion "$t | count" -Endpoint query).Tables[0].Rows[0][0] + Write-Host "$t`: $count" +} +``` + +> [!TIP] +> Ingesting raw data is all you need — the final tables populate themselves through the update policy. Keep the policy enabled so each file's transform stays small; it scales to tens of millions of rows without running out of memory. + +
+ +## Query your data + +The `Hub` database exposes the same view functions a deployed hub does. Query them directly: + +```powershell +$result = Invoke-Kusto Hub 'Costs_v1_2 | summarize EffectiveCost = round(sum(EffectiveCost), 2) by ServiceCategory | top 10 by EffectiveCost' -Endpoint query +$result.Tables[0].Rows +``` + +You now have a working local FinOps hub. To explore it visually, connect the [Azure Data Explorer web UI](https://dataexplorer.azure.com) to `http://localhost:8082`, or point Power BI and other tools at the same endpoint. + +
+ +## Clean up + +Remove the container when you're done. Your downloaded data stays in the export folder you created (`$exportPath`), but removing the container deletes its databases — re-creating it means re-loading the schema and open data and re-ingesting. To keep the loaded data between sessions, use `docker stop finops-hub-local` instead and restart it later with `docker start finops-hub-local`. + +```powershell +docker rm -f finops-hub-local +``` + +
+ +## Related content + +- [FinOps hubs overview](finops-hubs-overview.md) +- [Create and update FinOps hubs](deploy.md) +- [FinOps hub data model](data-model.md) +- [How data is processed](data-processing.md) + +
diff --git a/docs-mslearn/toolkit/powershell/hubs/initialize-finopshublocal.md b/docs-mslearn/toolkit/powershell/hubs/initialize-finopshublocal.md new file mode 100644 index 000000000..6b0c53b54 --- /dev/null +++ b/docs-mslearn/toolkit/powershell/hubs/initialize-finopshublocal.md @@ -0,0 +1,82 @@ +--- +title: Initialize-FinOpsHubLocal command +description: Set up a local FinOps hub in a running Kusto emulator using the Initialize-FinOpsHubLocal command in the FinOpsToolkit module. +author: MSBrett +ms.author: brettwil +ms.date: 08/02/2026 +ms.topic: reference +ms.service: finops +ms.subservice: finops-toolkit +ms.reviewer: brettwil +#customer intent: As a FinOps user, I want to set up a local FinOps hub with one command so I can explore my cost data without deploying Azure resources. +--- + +# Initialize-FinOpsHubLocal command + +The **Initialize-FinOpsHubLocal** command sets up a local FinOps hub in a running [Kusto emulator](/azure/data-explorer/kusto-emulator-overview). It creates the `Ingestion` and `Hub` databases, then downloads and applies the released FinOps hub setup scripts and open data load script, so the local hub uses the same KQL, transforms, and open data as a deployed hub. + +The command doesn't install Docker or manage the emulator container, and it doesn't ingest cost data. Start the emulator first, then run this command against its endpoint. For the full walkthrough, see [Run FinOps hubs locally](../../hubs/run-hubs-locally.md). + +
+ +## Syntax + +```powershell +Initialize-FinOpsHubLocal ` + [-ClusterUri ] ` + [-ReleaseUri ] ` + [-RawRetentionInDays ] ` + [-OpenDataPath ] ` + [-SkipOpenData] ` + [-Destination ] ` + [-TimeoutSec ] ` + [-WhatIf] +``` + +
+ +## Parameters + +| Name | Description | +| --------------------- | --------------------------------------------------------------------------------------------------------------- | +| `‑ClusterUri` | Optional. Base URI of the running Kusto emulator. Default = `http://localhost:8082`. | +| `‑ReleaseUri` | Optional. Base URI to download the setup scripts and open data load script from. Default = the latest FinOps toolkit GitHub release. Point this at a local file server or a specific release to run offline or pin a version. | +| `‑RawRetentionInDays` | Optional. Number of days to keep raw data in the `Ingestion` database. Default = `90`. | +| `‑OpenDataPath` | Optional. Path, as seen by the emulator, to the folder that holds the open data CSV files. Default = `/data/export/open-data`. | +| `‑SkipOpenData` | Optional. Skips loading the open data reference tables. Default = `false`. | +| `‑Destination` | Optional. Local folder used to download the setup scripts. Default = temp folder. | +| `‑TimeoutSec` | Optional. Maximum number of seconds to wait for each emulator request or asset download. Default = `30`. Use `0` to wait indefinitely (for example, on slow hardware). | +| `‑WhatIf` | Optional. Shows what would happen if the command runs without actually running it. | + +
+ +## Examples + +### Set up a local hub + +```powershell +Initialize-FinOpsHubLocal +``` + +Sets up a local FinOps hub in the emulator at `http://localhost:8082` using the latest release. + +### Set up the schema only + +```powershell +Initialize-FinOpsHubLocal ` + -RawRetentionInDays 30 ` + -SkipOpenData +``` + +Creates the databases and applies the schema with 30-day raw retention, and skips loading open data. + +
+ +## Related content + +Related solutions: + +- [Run FinOps hubs locally](../../hubs/run-hubs-locally.md) +- [FinOps hubs](../../hubs/finops-hubs-overview.md) + +
diff --git a/docs-mslearn/toolkit/powershell/powershell-commands.md b/docs-mslearn/toolkit/powershell/powershell-commands.md index d80414150..6975fa3fe 100644 --- a/docs-mslearn/toolkit/powershell/powershell-commands.md +++ b/docs-mslearn/toolkit/powershell/powershell-commands.md @@ -3,7 +3,7 @@ title: FinOps toolkit PowerShell module description: Automate and scale your FinOps efforts using the FinOps toolkit PowerShell module, which includes commands to manage FinOps solutions. author: flanakin ms.author: micflan -ms.date: 04/01/2026 +ms.date: 08/02/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -64,6 +64,7 @@ The FinOps toolkit PowerShell module includes commands to manage FinOps solution - [Deploy-FinOpsHub](hubs/Deploy-FinOpsHub.md) – Deploy your first hub or update to the latest version. - [Get-FinOpsHub](hubs/Get-FinOpsHub.md) – Get details about your FinOps hub instance. - [Initialize-FinOpsHubDeployment](hubs/Initialize-FinOpsHubDeployment.md) – Initializes the deployment for FinOps hubs. +- [Initialize-FinOpsHubLocal](hubs/initialize-finopshublocal.md) – Set up a local FinOps hub in a running Kusto emulator. - [Register-FinOpsHubProviders](hubs/Register-FinOpsHubProviders.md) – Registers resource providers for FinOps hubs. - [Remove-FinOpsHub](hubs/Remove-FinOpsHub.md) – Deletes a FinOps hub instance. diff --git a/src/powershell/Private/Invoke-FinOpsHubLocalCommand.ps1 b/src/powershell/Private/Invoke-FinOpsHubLocalCommand.ps1 new file mode 100644 index 000000000..ef40ce383 --- /dev/null +++ b/src/powershell/Private/Invoke-FinOpsHubLocalCommand.ps1 @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Sends a single command or query to a local FinOps hub (Kusto emulator) HTTP endpoint. + + .DESCRIPTION + Posts a management command or query to a running Kusto emulator over its REST API. Used by Initialize-FinOpsHubLocal to set up a local FinOps hub. This command does not manage the container; the emulator must already be running. + + .PARAMETER ClusterUri + Required. Base URI of the running Kusto emulator (for example, http://localhost:8082). + + .PARAMETER Database + Required. Name of the database to run the command against. + + .PARAMETER Command + Required. The management command (starting with '.') or query to run. + + .PARAMETER Endpoint + Optional. The REST endpoint to use: 'mgmt' for management commands (default) or 'query' for queries. + + .PARAMETER TimeoutSec + Optional. Maximum number of seconds to wait for a response. Default = 0 (wait indefinitely). +#> +function Invoke-FinOpsHubLocalCommand +{ + [Diagnostics.CodeAnalysis.SuppressMessage("PSAvoidUsingEmptyCatchBlock", "", Justification="Not all failures have a JSON error body (for example, a connection failure); ignore parse errors and fall through to rethrow the original exception.")] + [CmdletBinding()] + param + ( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $ClusterUri, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $Database, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] + $Command, + + [Parameter()] + [ValidateSet('mgmt', 'query')] + [string] + $Endpoint = 'mgmt', + + [Parameter()] + [ValidateRange(0, [int]::MaxValue)] + [int] + $TimeoutSec = 0 + ) + + $uri = '{0}/v1/rest/{1}' -f $ClusterUri.TrimEnd('/'), $Endpoint + $body = @{ db = $Database; csl = $Command } | ConvertTo-Json -Compress + + try + { + return Invoke-RestMethod -Uri $uri -Method 'Post' -ContentType 'application/json' -Body $body -TimeoutSec $TimeoutSec -ErrorAction 'Stop' + } + catch + { + # Kusto returns a structured JSON error body. Surface its message/code when present; + # otherwise rethrow the original exception (for example, a connection failure) as-is. + $content = $null + if ($null -ne $_.ErrorDetails -and -not [string]::IsNullOrWhiteSpace($_.ErrorDetails.Message)) + { + try + { + $content = $_.ErrorDetails.Message | ConvertFrom-Json -Depth 10 + } + catch {} + } + + if ($null -ne $content -and $content.PSObject.Properties['error']) + { + $errorBody = $content.error + throw ($script:LocalizedData.Common_ErrorResponse -f $errorBody.message, $errorBody.code) + } + + throw + } +} diff --git a/src/powershell/Public/Initialize-FinOpsHubLocal.ps1 b/src/powershell/Public/Initialize-FinOpsHubLocal.ps1 new file mode 100644 index 000000000..079f457a8 --- /dev/null +++ b/src/powershell/Public/Initialize-FinOpsHubLocal.ps1 @@ -0,0 +1,213 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Sets up a local FinOps hub in a running Kusto emulator. + + .DESCRIPTION + The Initialize-FinOpsHubLocal command configures a local FinOps hub in a running Kusto emulator. It creates the Ingestion and Hub databases, then downloads and applies the released FinOps hub setup scripts and open data load script so the local hub uses the same KQL, transforms, and open data as a deployed hub. + + This command does not install Docker or manage the emulator container. Start the emulator first, then run this command against its endpoint. It does not ingest cost data; stage and ingest your own exports after setup. + + Database storage paths assume the emulator's default data root (/kustodata), matching the container setup in the "Run FinOps hubs locally" guide. If your emulator mounts data elsewhere, create the Ingestion and Hub databases manually before running this command. + + .PARAMETER ClusterUri + Optional. Base URI of the running Kusto emulator. Default = http://localhost:8082. + + .PARAMETER ReleaseUri + Optional. Base URI to download the setup scripts and open data load script from. Default = the latest FinOps toolkit GitHub release. Point this at a local file server or a specific release to run offline or pin a version. + + .PARAMETER RawRetentionInDays + Optional. Number of days to keep raw data in the Ingestion database. Default = 90. + + .PARAMETER OpenDataPath + Optional. Path, as seen by the emulator, to the folder that holds the open data CSV files. Default = /data/export/open-data. + + .PARAMETER SkipOpenData + Optional. Skips loading the open data reference tables. Default = false. + + .PARAMETER Destination + Optional. Local folder used to download the setup scripts. Default = temp folder. + + .PARAMETER TimeoutSec + Optional. Maximum number of seconds to wait for each emulator request or asset download. Default = 30. Use 0 to wait indefinitely (for example, on slow hardware). + + .EXAMPLE + Initialize-FinOpsHubLocal + + Sets up a local FinOps hub in the emulator at http://localhost:8082 using the latest release. + + .EXAMPLE + Initialize-FinOpsHubLocal -ClusterUri 'http://localhost:8082' -RawRetentionInDays 30 -SkipOpenData + + Sets up the databases and schema with 30-day raw retention and skips loading open data. + + .LINK + https://aka.ms/ftk/Initialize-FinOpsHubLocal +#> +function Initialize-FinOpsHubLocal +{ + [CmdletBinding(SupportsShouldProcess)] + param + ( + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] + $ClusterUri = 'http://localhost:8082', + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] + $ReleaseUri = 'https://github.com/microsoft/finops-toolkit/releases/latest/download', + + [Parameter()] + [ValidateRange(1, [int]::MaxValue)] + [int] + $RawRetentionInDays = 90, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] + $OpenDataPath = '/data/export/open-data', + + [Parameter()] + [switch] + $SkipOpenData, + + [Parameter()] + [ValidateNotNullOrEmpty()] + [string] + $Destination = [System.IO.Path]::GetTempPath(), + + [Parameter()] + [ValidateRange(0, [int]::MaxValue)] + [int] + $TimeoutSec = 30 + ) + + $progress = $ProgressPreference + $ProgressPreference = 'SilentlyContinue' + + try + { + # Verify the emulator is reachable. This command does not start the container. + try + { + $null = Invoke-FinOpsHubLocalCommand -ClusterUri $ClusterUri -Database 'NetDefaultDB' -Command '.show version' -TimeoutSec $TimeoutSec + } + catch + { + throw ($script:LocalizedData.HubLocal_Initialize_NotReachable -f $ClusterUri) + } + + # Download the required release assets by name from the release URI. + $assetNames = @('finops-hub-fabric-setup-Ingestion.kql', 'finops-hub-fabric-setup-Hub.kql') + if (-not $SkipOpenData) + { + $assetNames += 'finops-hub-local-opendata.kql' + } + + $scripts = @{} + if ($PSCmdlet.ShouldProcess($Destination, 'Download release assets')) + { + New-Directory -Path $Destination + foreach ($assetName in $assetNames) + { + $filePath = Join-Path -Path $Destination -ChildPath $assetName + try + { + $null = Invoke-WebRequest -Uri "$($ReleaseUri.TrimEnd('/'))/$assetName" -OutFile $filePath -TimeoutSec $TimeoutSec -Verbose:$false -ErrorAction 'Stop' + } + catch + { + throw ($script:LocalizedData.HubLocal_Initialize_DownloadFailed -f $assetName, $ReleaseUri) + } + + $scripts[$assetName] = Get-Content -Path $filePath -Raw + if ([string]::IsNullOrWhiteSpace($scripts[$assetName])) + { + throw ($script:LocalizedData.HubLocal_Initialize_AssetEmpty -f $assetName, $ReleaseUri) + } + } + } + + # Create the Ingestion and Hub databases. Uses create-merge so re-running this command + # (for example, after a failed setup step) does not fail if the databases already exist. + foreach ($database in @('Ingestion', 'Hub')) + { + $createCommand = '.create-merge database {0} persist (@"/kustodata/dbs/{0}/md", @"/kustodata/dbs/{0}/data")' -f $database + if ($PSCmdlet.ShouldProcess($database, 'Create database')) + { + $null = Invoke-FinOpsHubLocalCommand -ClusterUri $ClusterUri -Database 'NetDefaultDB' -Command $createCommand -TimeoutSec $TimeoutSec + } + } + + # Apply the Ingestion setup with the requested raw retention. Uses .Replace() (not -replace) + # since the replacement value flows into the replacement string, where -replace would treat + # sequences like $1 or $& as regex substitution tokens instead of literal text. + if ($PSCmdlet.ShouldProcess('Ingestion', 'Apply setup script')) + { + if (-not $scripts.ContainsKey('finops-hub-fabric-setup-Ingestion.kql')) + { + throw ($script:LocalizedData.HubLocal_Initialize_ScriptsNotDownloaded -f 'Ingestion setup') + } + + $ingestionScript = $scripts['finops-hub-fabric-setup-Ingestion.kql'].Replace('$$rawRetentionInDays$$', $RawRetentionInDays.ToString()) + $null = Invoke-FinOpsHubLocalCommand -ClusterUri $ClusterUri -Database 'Ingestion' -Command $ingestionScript -TimeoutSec $TimeoutSec + } + + # Apply the Hub setup + if ($PSCmdlet.ShouldProcess('Hub', 'Apply setup script')) + { + if (-not $scripts.ContainsKey('finops-hub-fabric-setup-Hub.kql')) + { + throw ($script:LocalizedData.HubLocal_Initialize_ScriptsNotDownloaded -f 'Hub setup') + } + + $null = Invoke-FinOpsHubLocalCommand -ClusterUri $ClusterUri -Database 'Hub' -Command $scripts['finops-hub-fabric-setup-Hub.kql'] -TimeoutSec $TimeoutSec + } + + # Load the open data reference tables + if (-not $SkipOpenData) + { + if ($PSCmdlet.ShouldProcess('Ingestion', 'Load open data')) + { + if (-not $scripts.ContainsKey('finops-hub-local-opendata.kql')) + { + throw ($script:LocalizedData.HubLocal_Initialize_ScriptsNotDownloaded -f 'open data load') + } + + $openDataScript = $scripts['finops-hub-local-opendata.kql'].Replace('$$openDataPath$$', $OpenDataPath) + + # The emulator's first external data read after setup can return no rows without + # raising an error. Load, verify the tables filled, and retry before giving up. + $openDataTables = @('PricingUnits', 'Regions', 'ResourceTypes', 'Services') + $maxAttempts = 5 + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) + { + $null = Invoke-FinOpsHubLocalCommand -ClusterUri $ClusterUri -Database 'Ingestion' -Command $openDataScript -TimeoutSec $TimeoutSec + $empty = @($openDataTables | Where-Object { + [int64](Invoke-FinOpsHubLocalCommand -ClusterUri $ClusterUri -Database 'Ingestion' -Command "$_ | count" -Endpoint 'query' -TimeoutSec $TimeoutSec).Tables[0].Rows[0][0] -eq 0 + }) + + if ($empty.Count -eq 0) + { + break + } + + if ($attempt -eq $maxAttempts) + { + throw ($script:LocalizedData.HubLocal_Initialize_OpenDataEmpty -f $maxAttempts, ($empty -join ', ')) + } + + Start-Sleep -Seconds 2 + } + } + } + } + finally + { + $ProgressPreference = $progress + } +} diff --git a/src/powershell/Tests/Integration/Initialize-FinOpsHubLocal.Tests.ps1 b/src/powershell/Tests/Integration/Initialize-FinOpsHubLocal.Tests.ps1 new file mode 100644 index 000000000..92b4f66df --- /dev/null +++ b/src/powershell/Tests/Integration/Initialize-FinOpsHubLocal.Tests.ps1 @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Integration tests for Initialize-FinOpsHubLocal. These run the command against a real +# running Kusto emulator and download real release artifacts over HTTP -- no mocks. +# +# They are skipped unless the following environment variables point at live infrastructure: +# FTK_LOCAL_EMULATOR_URI Base URI of a running Kusto emulator (for example, http://localhost:8087). +# FTK_LOCAL_RELEASE_URI Base URI serving the release artifacts (for example, a local file server). +# FTK_LOCAL_EXPORT_PATH Host path mounted into the emulator at /data/export, holding FOCUS exports +# and an open-data subfolder. + +if (-not (Get-Module -Name 'FinOpsToolkit')) +{ + $rootDirectory = ((Get-Item -Path $PSScriptRoot).Parent.Parent).FullName + $modulePath = (Get-ChildItem -Path $rootDirectory -Include 'FinOpsToolKit.psm1' -Recurse).FullName + Import-Module -FullyQualifiedName $modulePath +} + +$script:emulatorUri = $env:FTK_LOCAL_EMULATOR_URI +$script:releaseUri = $env:FTK_LOCAL_RELEASE_URI +$script:exportPath = $env:FTK_LOCAL_EXPORT_PATH +$script:ready = [bool]$script:emulatorUri -and [bool]$script:releaseUri -and [bool]$script:exportPath + +Describe 'Initialize-FinOpsHubLocal' { + BeforeAll { + $script:hub = $env:FTK_LOCAL_EMULATOR_URI + $script:release = $env:FTK_LOCAL_RELEASE_URI + $script:exports = $env:FTK_LOCAL_EXPORT_PATH + # Recompute readiness from the environment: discovery-scope variables do not flow into BeforeAll. + $script:ready = [bool]$script:hub -and [bool]$script:release -and [bool]$script:exports + + function Invoke-Q + { + param([string]$Database, [string]$Command, [ValidateSet('mgmt', 'query')][string]$Endpoint = 'query') + Invoke-RestMethod -Uri "$script:hub/v1/rest/$Endpoint" -Method 'Post' -ContentType 'application/json' -Body (@{ db = $Database; csl = $Command } | ConvertTo-Json) + } + + function Get-Count + { + param([string]$Database, [string]$Table) + [int64](Invoke-Q -Database $Database -Command "$Table | count" -Endpoint 'query').Tables[0].Rows[0][0] + } + + if ($script:ready) + { + # Real setup: download real artifacts over HTTP, create databases, apply schema, load open data. + Initialize-FinOpsHubLocal -ClusterUri $script:hub -ReleaseUri $script:release -RawRetentionInDays 90 -OpenDataPath '/data/export/open-data' -Destination (Join-Path $TestDrive 'setup') + + # Real ordered ingest: prices first, then costs (the documented pattern). + $jobs = Get-ChildItem $script:exports -Recurse -Filter manifest.json | ForEach-Object { + $manifest = Get-Content $_.FullName -Raw + $isPrice = $manifest -match '"type"\s*:\s*"PriceSheet"' -or $_.FullName -match '(?i)price' + $tableInfo = if ($isPrice) { 'Prices_raw', 'Prices_raw_mapping' } else { 'Costs_raw', 'Costs_raw_mapping' } + Get-ChildItem $_.Directory -Filter *.parquet | ForEach-Object { + [PSCustomObject]@{ Table = $tableInfo[0]; Mapping = $tableInfo[1]; File = $_.FullName; Phase = [int](-not $isPrice) } + } + } + + $script:pricesBeforeCostsPhase = $null + foreach ($phase in $jobs | Group-Object Phase | Sort-Object Name) + { + if ($phase.Name -eq '1') { $script:pricesBeforeCostsPhase = Get-Count -Database 'Ingestion' -Table 'Prices_raw' } + foreach ($job in $phase.Group) + { + $rel = [System.IO.Path]::GetRelativePath((Resolve-Path $script:exports), $job.File) -replace '\\', '/' + Invoke-Q -Database 'Ingestion' -Endpoint 'mgmt' -Command ".ingest into table $($job.Table) (h@'/data/export/$rel') with (format='parquet', ingestionMappingReference='$($job.Mapping)')" | Out-Null + } + } + } + } + + It 'creates the Ingestion and Hub databases' -Skip:(-not $script:ready) { + $databases = (Invoke-Q -Database 'NetDefaultDB' -Command '.show databases' -Endpoint 'mgmt').Tables[0].Rows | ForEach-Object { $_[0] } + $databases | Should -Contain 'Ingestion' + $databases | Should -Contain 'Hub' + } + + It 'loads all four open data tables with rows' -Skip:(-not $script:ready) { + Get-Count -Database 'Ingestion' -Table 'PricingUnits' | Should -BeGreaterThan 0 + Get-Count -Database 'Ingestion' -Table 'Regions' | Should -BeGreaterThan 0 + Get-Count -Database 'Ingestion' -Table 'ResourceTypes' | Should -BeGreaterThan 0 + Get-Count -Database 'Ingestion' -Table 'Services' | Should -BeGreaterThan 0 + } + + It 'applies the requested raw retention to Costs_raw' -Skip:(-not $script:ready) { + $policy = (Invoke-Q -Database 'Ingestion' -Command '.show table Costs_raw policy retention' -Endpoint 'mgmt').Tables[0].Rows[0][2] | ConvertFrom-Json + $policy.SoftDeletePeriod | Should -Be '90.00:00:00' + } + + It 'ingests prices before costs' -Skip:(-not $script:ready) { + # Captured just before the costs phase started: prices were already present. + $script:pricesBeforeCostsPhase | Should -BeGreaterThan 0 + } + + It 'fills the final cost and price tables through the update policy' -Skip:(-not $script:ready) { + $costsRaw = Get-Count -Database 'Ingestion' -Table 'Costs_raw' + $costsFinal = Get-Count -Database 'Ingestion' -Table 'Costs_final_v1_2' + $costsRaw | Should -BeGreaterThan 0 + $costsFinal | Should -Be $costsRaw + } + + It 'enriches costs with open data and prices' -Skip:(-not $script:ready) { + $total = Get-Count -Database 'Hub' -Table 'Costs_v1_2' + $enriched = Get-Count -Database 'Hub' -Table 'Costs_v1_2 | where isnotempty(ServiceCategory)' + $priced = Get-Count -Database 'Hub' -Table 'Costs_v1_2 | where isnotempty(ListUnitPrice) and ListUnitPrice > 0' + $total | Should -BeGreaterThan 0 + $enriched | Should -Be $total + $priced | Should -BeGreaterThan 0 + } + + It 'throws a clear error when the emulator is unreachable' -Skip:(-not $script:ready) { + # Real dead port -- no mock. + { Initialize-FinOpsHubLocal -ClusterUri 'http://localhost:1' -ReleaseUri $script:release -Destination (Join-Path $TestDrive 'unreachable') } | + Should -Throw '*Could not reach the Kusto emulator*' + } +} diff --git a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 index c154dc5c6..6a43b02dd 100644 --- a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 +++ b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 @@ -49,6 +49,7 @@ Describe 'Get-FinOpsToolkitVersion' { CheckFile "finops-hub-dashboard.json" '0.8' $null CheckFile "finops-hub-fabric-setup-Hub.kql" '0.10' $null CheckFile "finops-hub-fabric-setup-Ingestion.kql" '0.10' $null + CheckFile "finops-hub-local-opendata.kql" '15.0' $null CheckFile "finops-hub-v$verStr.zip" $null $null CheckFile "finops-workbooks-v$verStr.zip" '0.6' $null CheckFile "governance-workbook-v$verStr.zip" '0.1' '0.5' diff --git a/src/powershell/Tests/Unit/Initialize-FinOpsHubLocal.Tests.ps1 b/src/powershell/Tests/Unit/Initialize-FinOpsHubLocal.Tests.ps1 new file mode 100644 index 000000000..4605b4d4d --- /dev/null +++ b/src/powershell/Tests/Unit/Initialize-FinOpsHubLocal.Tests.ps1 @@ -0,0 +1,272 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +& "$PSScriptRoot/../Initialize-Tests.ps1" + +InModuleScope 'FinOpsToolkit' { + Describe 'Initialize-FinOpsHubLocal' { + BeforeAll { + function newQueryResult + { + param + ( + [Parameter(Mandatory = $true)] + [int] + $Count + ) + + # Shape matches the real Kusto REST response the function reads: + # (Invoke-FinOpsHubLocalCommand ...).Tables[0].Rows[0][0] + return @{ Tables = @(@{ Rows = @(, @($Count)) }) } + } + + $clusterUri = 'http://localhost:8082' + $destination = 'TestDrive:\hub-local' + + Mock -CommandName 'New-Directory' + Mock -CommandName 'Start-Sleep' + Mock -CommandName 'Invoke-WebRequest' + Mock -CommandName 'Get-Content' -MockWith { + if ($Path -like '*opendata*') + { + return 'opendata script $$openDataPath$$' + } + elseif ($Path -like '*Hub.kql') + { + return 'hub script content' + } + + return 'ingestion script $$rawRetentionInDays$$ days' + } + + # Default: every command call succeeds with an empty response. + Mock -CommandName 'Invoke-FinOpsHubLocalCommand' -MockWith { return @{} } + + # Every open data table count check reports rows present (no retry needed). + Mock -CommandName 'Invoke-FinOpsHubLocalCommand' -ParameterFilter { $Endpoint -eq 'query' } -MockWith { + return (newQueryResult -Count 5) + } + } + + Context 'Happy path' { + It 'Should verify reachability, then create the Ingestion and Hub databases' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'NetDefaultDB' -and $Command -eq '.show version' + } + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'NetDefaultDB' -and $Command -like '*create-merge database Ingestion*' + } + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'NetDefaultDB' -and $Command -like '*create-merge database Hub*' + } + } + + It 'Should apply the Ingestion and Hub setup scripts' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'Ingestion' -and $Command -like '*ingestion script*' + } + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'Hub' -and $Command -eq 'hub script content' + } + } + + It 'Should download the ingestion, hub, and open data setup scripts' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 3 + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 1 -ParameterFilter { $Uri -like '*finops-hub-fabric-setup-Ingestion.kql' } + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 1 -ParameterFilter { $Uri -like '*finops-hub-fabric-setup-Hub.kql' } + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 1 -ParameterFilter { $Uri -like '*finops-hub-local-opendata.kql' } + } + + It 'Should replace the raw retention placeholder before applying the Ingestion setup script' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -RawRetentionInDays 30 -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'Ingestion' -and $Command -eq 'ingestion script 30 days' + } + } + + It 'Should replace the open data path placeholder before loading open data' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -OpenDataPath '/custom/path' -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'Ingestion' -and $Command -eq 'opendata script /custom/path' + } + } + + It 'Should not mangle an open data path containing regex replacement metacharacters' { + # Act -- '$&' and '$1' are special in -replace's replacement string; .Replace() must treat them literally. + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -OpenDataPath '$&/$1data' -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { + $Database -eq 'Ingestion' -and $Command -eq 'opendata script $&/$1data' + } + } + + It 'Should thread the requested timeout through every emulator request and download' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -TimeoutSec 45 -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 0 -ParameterFilter { $TimeoutSec -ne 45 } + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 0 -ParameterFilter { $TimeoutSec -ne 45 } + } + + It 'Should default TimeoutSec to a finite value rather than waiting indefinitely' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 0 -ParameterFilter { $TimeoutSec -eq 0 } + } + + It 'Should skip downloading and loading open data when requested' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -SkipOpenData -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 0 -ParameterFilter { $Uri -like '*opendata*' } + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 0 -ParameterFilter { $Endpoint -eq 'query' } + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 0 -ParameterFilter { $Command -like '*opendata script*' } + } + } + + Context 'Emulator unreachable' { + It 'Should throw a clear error naming the cluster URI and stop before downloading anything' { + # Arrange + Mock -CommandName 'Invoke-FinOpsHubLocalCommand' -MockWith { throw 'connection refused' } -ParameterFilter { + $Database -eq 'NetDefaultDB' -and $Command -eq '.show version' + } + + # Act / Assert + { Initialize-FinOpsHubLocal -ClusterUri 'http://localhost:1' -Destination $destination } | + Should -Throw "*Could not reach the Kusto emulator at 'http://localhost:1'*" + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 0 + } + } + + Context 'Download failures' { + It 'Should throw a clear error naming the asset and release URI' { + # Arrange + Mock -CommandName 'Invoke-WebRequest' -MockWith { throw 'network error' } -ParameterFilter { + $Uri -like '*finops-hub-fabric-setup-Ingestion.kql' + } + + # Act / Assert + { Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination } | + Should -Throw "*Could not download asset 'finops-hub-fabric-setup-Ingestion.kql'*" + } + } + + Context 'Empty downloaded assets' { + It 'Should throw a clear error when a downloaded asset is empty' { + # Arrange + Mock -CommandName 'Get-Content' -MockWith { return '' } + + # Act / Assert + { Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination } | + Should -Throw "*Downloaded asset 'finops-hub-fabric-setup-Ingestion.kql'*was empty*" + } + } + + Context 'Open data retry logic' { + It 'Should retry loading open data until every table reports rows' { + # Arrange + $script:queryCallCount = 0 + Mock -CommandName 'Invoke-FinOpsHubLocalCommand' -ParameterFilter { $Endpoint -eq 'query' } -MockWith { + $script:queryCallCount++ + # First attempt's 4 table checks report empty; second attempt's report filled. + if ($script:queryCallCount -le 4) + { + return (newQueryResult -Count 0) + } + + return (newQueryResult -Count 5) + } + + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination + + # Assert + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 2 -ParameterFilter { + $Database -eq 'Ingestion' -and $Command -like '*opendata script*' + } + Should -Invoke -CommandName 'Start-Sleep' -Times 1 + } + + It 'Should throw after the maximum number of attempts if the tables remain empty' { + # Arrange + Mock -CommandName 'Invoke-FinOpsHubLocalCommand' -ParameterFilter { $Endpoint -eq 'query' } -MockWith { + return (newQueryResult -Count 0) + } + + # Act / Assert + { Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination } | + Should -Throw '*Open data tables were still empty after 5 attempts*' + Should -Invoke -CommandName 'Start-Sleep' -Times 4 + } + } + + Context 'ShouldProcess (-WhatIf)' { + It 'Should not create databases, apply setup scripts, or load open data' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination -WhatIf + + # Assert -- only the reachability check runs; every ShouldProcess-gated call is skipped. + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 + Should -Invoke -CommandName 'Invoke-FinOpsHubLocalCommand' -Times 1 -ParameterFilter { $Command -eq '.show version' } + } + + It 'Should not create the destination folder or download any release assets' { + # Act + Initialize-FinOpsHubLocal -ClusterUri $clusterUri -Destination $destination -WhatIf + + # Assert + Should -Invoke -CommandName 'New-Directory' -Times 0 + Should -Invoke -CommandName 'Invoke-WebRequest' -Times 0 + } + } + + Context 'Parameter validation' { + It 'Should require a non-empty ClusterUri' { + { Initialize-FinOpsHubLocal -ClusterUri '' } | Should -Throw + } + + It 'Should require a non-empty ReleaseUri' { + { Initialize-FinOpsHubLocal -ReleaseUri '' } | Should -Throw + } + + It 'Should reject a RawRetentionInDays of 0' { + { Initialize-FinOpsHubLocal -RawRetentionInDays 0 } | Should -Throw + } + + It 'Should require a non-empty OpenDataPath' { + { Initialize-FinOpsHubLocal -OpenDataPath '' } | Should -Throw + } + + It 'Should require a non-empty Destination' { + { Initialize-FinOpsHubLocal -Destination '' } | Should -Throw + } + + It 'Should reject a negative TimeoutSec' { + { Initialize-FinOpsHubLocal -TimeoutSec -1 } | Should -Throw + } + } + } +} diff --git a/src/powershell/Tests/Unit/Invoke-FinOpsHubLocalCommand.Tests.ps1 b/src/powershell/Tests/Unit/Invoke-FinOpsHubLocalCommand.Tests.ps1 new file mode 100644 index 000000000..8cfda627c --- /dev/null +++ b/src/powershell/Tests/Unit/Invoke-FinOpsHubLocalCommand.Tests.ps1 @@ -0,0 +1,127 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +& "$PSScriptRoot/../Initialize-Tests.ps1" + +InModuleScope FinOpsToolkit { + Describe 'Invoke-FinOpsHubLocalCommand' { + BeforeAll { + function newKustoErrorRecord($code, $message) + { + $body = @{ error = @{ code = $code; message = $message } } | ConvertTo-Json -Compress + $exception = New-Object System.Exception('Response status code does not indicate success') + $errorRecord = New-Object System.Management.Automation.ErrorRecord( + $exception, 'WebCmdletWebResponseException', [System.Management.Automation.ErrorCategory]::InvalidOperation, $null + ) + $errorRecord.ErrorDetails = [System.Management.Automation.ErrorDetails]::new($body) + return $errorRecord + } + } + + Context 'Successful call' { + It 'Should post to the mgmt endpoint by default and return the response' { + # Arrange + Mock -CommandName 'Invoke-RestMethod' -MockWith { return @{ Tables = @() } } + + # Act + $result = Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command '.show version' + + # Assert + $result | Should -Not -BeNullOrEmpty + Should -Invoke -CommandName 'Invoke-RestMethod' -Times 1 -ParameterFilter { + $Uri -eq 'http://localhost:8082/v1/rest/mgmt' -and + $Method -eq 'Post' -and + $TimeoutSec -eq 0 + } + } + + It 'Should post to the query endpoint when requested' { + # Arrange + Mock -CommandName 'Invoke-RestMethod' -MockWith { return @{ Tables = @() } } + + # Act + Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command 'Costs_v1_2 | count' -Endpoint 'query' + + # Assert + Should -Invoke -CommandName 'Invoke-RestMethod' -Times 1 -ParameterFilter { $Uri -eq 'http://localhost:8082/v1/rest/query' } + } + + It 'Should trim a trailing slash from the cluster URI' { + # Arrange + Mock -CommandName 'Invoke-RestMethod' -MockWith { return @{ Tables = @() } } + + # Act + Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082/' -Database 'Hub' -Command '.show version' + + # Assert + Should -Invoke -CommandName 'Invoke-RestMethod' -Times 1 -ParameterFilter { $Uri -eq 'http://localhost:8082/v1/rest/mgmt' } + } + + It 'Should pass the requested timeout through to Invoke-RestMethod' { + # Arrange + Mock -CommandName 'Invoke-RestMethod' -MockWith { return @{ Tables = @() } } + + # Act + Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command '.show version' -TimeoutSec 45 + + # Assert + Should -Invoke -CommandName 'Invoke-RestMethod' -Times 1 -ParameterFilter { $TimeoutSec -eq 45 } + } + } + + Context 'Structured Kusto errors' { + It 'Should surface the Kusto error message and code' { + # Arrange + Mock -CommandName 'Invoke-RestMethod' -MockWith { throw (newKustoErrorRecord 'BadRequest_SyntaxError' 'Query could not be parsed') } + + # Act / Assert + { Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command 'invalid |||' } | + Should -Throw '*Query could not be parsed*BadRequest_SyntaxError*' + } + } + + Context 'Unstructured errors' { + It 'Should rethrow the original exception when there is no JSON error body' { + # Arrange + Mock -CommandName 'Invoke-RestMethod' -MockWith { throw 'Unable to connect to the remote server' } + + # Act / Assert + { Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command '.show version' } | + Should -Throw 'Unable to connect to the remote server' + } + + It 'Should rethrow the original exception when the error payload is blank' { + # Arrange + $errorRecord = New-Object System.Management.Automation.ErrorRecord( + (New-Object System.Exception('Response status code does not indicate success')), + 'WebCmdletWebResponseException', + [System.Management.Automation.ErrorCategory]::InvalidOperation, + $null) + $errorRecord.ErrorDetails = [System.Management.Automation.ErrorDetails]::new([string]::Empty) + Mock -CommandName 'Invoke-RestMethod' -MockWith { throw $errorRecord } + + # Act / Assert + { Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command '.show version' } | + Should -Throw 'Response status code does not indicate success' + } + } + + Context 'Parameter validation' { + It 'Should require a non-empty ClusterUri' { + { Invoke-FinOpsHubLocalCommand -ClusterUri '' -Database 'Hub' -Command '.show version' } | Should -Throw + } + + It 'Should require a non-empty Database' { + { Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database '' -Command '.show version' } | Should -Throw + } + + It 'Should require a non-empty Command' { + { Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command '' } | Should -Throw + } + + It 'Should reject a negative TimeoutSec' { + { Invoke-FinOpsHubLocalCommand -ClusterUri 'http://localhost:8082' -Database 'Hub' -Command '.show version' -TimeoutSec -1 } | Should -Throw + } + } + } +} diff --git a/src/powershell/en-US/FinOpsToolkit.strings.psd1 b/src/powershell/en-US/FinOpsToolkit.strings.psd1 index bf003b56b..0bba99f77 100644 --- a/src/powershell/en-US/FinOpsToolkit.strings.psd1 +++ b/src/powershell/en-US/FinOpsToolkit.strings.psd1 @@ -16,6 +16,12 @@ ConvertFrom-StringData -StringData @' Hub_Remove_Failed = FinOps hub could not be deleted. {0}. Hub_Remove_NotFound = FinOps hub '{0}' not found. + HubLocal_Initialize_AssetEmpty = Downloaded asset '{0}' from '{1}' was empty. The release may be incomplete or the URI may not point to a valid FinOps toolkit release. + HubLocal_Initialize_DownloadFailed = Could not download asset '{0}' from '{1}'. Check the release URI and network connectivity. + HubLocal_Initialize_NotReachable = Could not reach the Kusto emulator at '{0}'. Start the local hub container before running this command. See https://aka.ms/finops/hubs/local. + HubLocal_Initialize_OpenDataEmpty = Open data tables were still empty after {0} attempts: {1}. The emulator's first external data read after setup can return no rows; rerun the command to retry. + HubLocal_Initialize_ScriptsNotDownloaded = Cannot apply the {0} because release assets were not downloaded. Confirm the "Download release assets" prompt, or run without -Confirm. + HubProviders_Register_AlreadyRegistered = Resource provider {0} is already registered. HubProviders_Register_Register = Registering resource provider {0}. HubProviders_Register_RegisterError = Error registering resource provider: {0}. diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 2dd495e60..3b4f76c15 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -45,6 +45,12 @@ "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_v1_2.kql", "modules/Microsoft.FinOpsHubs/Analytics/scripts/HubSetup_Latest.kql" ] + }, + { + "name": "finops-hub-local-opendata.kql", + "files": [ + "modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_OpenDataExternal.kql" + ] } ] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_OpenDataExternal.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_OpenDataExternal.kql new file mode 100644 index 000000000..22ca67d06 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_OpenDataExternal.kql @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//====================================================================================================================== +// Ingestion database / Open data load (local emulator) +// Populates the open data tables (PricingUnits, Regions, ResourceTypes, Services) from CSV files staged on disk. +// A deployed hub loads these tables via Data Factory; this script is the local/emulator stand-in for that step and is +// published as a standalone release artifact (finops-hub-local-opendata.kql) for the "Run hubs locally" guide. +// The $$openDataPath$$ placeholder is replaced with the directory that holds the open data CSVs before the script runs. +//====================================================================================================================== + +// For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script + +.set-or-replace PricingUnits <| externaldata(x_PricingUnitDescription: string, AccountTypes: string, x_PricingBlockSize: real, PricingUnit: string)[@'$$openDataPath$$/PricingUnits.csv'] with (format='csv', ignoreFirstRecord=true) | project-away AccountTypes +.set-or-replace Regions <| externaldata(ResourceLocation: string, RegionId: string, RegionName: string)[@'$$openDataPath$$/Regions.csv'] with (format='csv', ignoreFirstRecord=true) +.set-or-replace ResourceTypes <| externaldata(x_ResourceType: string, SingularDisplayName: string, PluralDisplayName: string, LowerSingularDisplayName: string, LowerPluralDisplayName: string, IsPreview: bool, Description: string, IconUri: string, Links: string)[@'$$openDataPath$$/ResourceTypes.csv'] with (format='csv', ignoreFirstRecord=true) | project-away Links +.set-or-replace Services <| externaldata(x_ConsumedService: string, x_ResourceType: string, ServiceName: string, ServiceCategory: string, ServiceSubcategory: string, PublisherName: string, x_PublisherCategory: string, x_Environment: string, x_ServiceModel: string)[@'$$openDataPath$$/Services.csv'] with (format='csv', ignoreFirstRecord=true)