diff --git a/.github/workflows/front-end-tests.yml b/.github/workflows/front-end-tests.yml index ce74a69..4f03d7f 100644 --- a/.github/workflows/front-end-tests.yml +++ b/.github/workflows/front-end-tests.yml @@ -8,8 +8,14 @@ on: - 'task/**' - 'sql_queries/**' - 'linux_host/**' + - 'avd_host/**' - 'custom_script_extensions/**' - 'deploy/bicep/**' + - 'deploy/Update-AvdHostBrokerScript.ps1' + - 'deploy/Enable-HostMonitoring.ps1' + - 'deploy/Post-Provision.ps1' + - 'deploy/Migrate-ExistingEnvironment.ps1' + - 'deploy/Initialize-DeploymentEnvironment.ps1' - '.github/workflows/front-end-tests.yml' push: paths: @@ -18,8 +24,14 @@ on: - 'task/**' - 'sql_queries/**' - 'linux_host/**' + - 'avd_host/**' - 'custom_script_extensions/**' - 'deploy/bicep/**' + - 'deploy/Update-AvdHostBrokerScript.ps1' + - 'deploy/Enable-HostMonitoring.ps1' + - 'deploy/Post-Provision.ps1' + - 'deploy/Migrate-ExistingEnvironment.ps1' + - 'deploy/Initialize-DeploymentEnvironment.ps1' - '.github/workflows/front-end-tests.yml' jobs: @@ -198,6 +210,49 @@ jobs: - name: Run the host script tests run: bash linux_host/tests/run.sh + avd-scripts: + name: AVD session host scripts + # Connect-LinuxBroker.ps1 runs in Windows PowerShell 5.1 on the session hosts. + runs-on: windows-latest + permissions: + contents: read + + steps: + - uses: actions/checkout@v4 + + - name: Install Pester and PSScriptAnalyzer + shell: powershell + run: | + Install-Module Pester -RequiredVersion 5.7.1 -Force -Scope CurrentUser -SkipPublisherCheck + Install-Module PSScriptAnalyzer -RequiredVersion 1.24.0 -Force -Scope CurrentUser -SkipPublisherCheck + + # Also runs Update-AvdHostBrokerScript.ps1, Enable-HostMonitoring.ps1, Post-Provision.ps1's + # call to it, Migrate-ExistingEnvironment.ps1 and the autoscale and monitoring checks of + # Initialize-DeploymentEnvironment.ps1 with Azure mocked. + - name: Run the Pester tests + shell: powershell + run: | + Import-Module Pester -RequiredVersion 5.7.1 + $configuration = New-PesterConfiguration + $configuration.Run.Path = 'avd_host/tests' + $configuration.Run.Exit = $true + $configuration.Output.Verbosity = 'Detailed' + Invoke-Pester -Configuration $configuration + + - name: Lint the scripts + shell: powershell + run: | + Import-Module PSScriptAnalyzer -RequiredVersion 1.24.0 + $findings = @( + 'avd_host/broker/Connect-LinuxBroker.ps1', + 'deploy/Update-AvdHostBrokerScript.ps1', + 'deploy/Enable-HostMonitoring.ps1' + ) | ForEach-Object { Invoke-ScriptAnalyzer -Path $_ -Settings avd_host/tests/PSScriptAnalyzerSettings.psd1 } + if (@($findings).Count -gt 0) { + $findings | Format-Table -Property ScriptName, Line, Severity, RuleName, Message -AutoSize -Wrap | Out-String -Width 240 | Write-Output + throw "PSScriptAnalyzer reported $(@($findings).Count) finding(s)." + } + bicep-build: name: Bicep templates runs-on: ubuntu-latest diff --git a/README.md b/README.md index b8b323f..c1555db 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The solution consists of the following components: - **Azure Function for Scaling Tasks**: An Azure Function that runs on a schedule to manage scaling of Linux hosts based on the scaling rules. It updates VM network statuses, turns VMs on or off, performs health checks on the Linux hosts, returns released hosts, advances rolling maintenance runs, and purges old audit entries and checkout events. -- **Service Management Portal**: A front-end web application that allows administrators to manage VMs, scaling rules, and monitor the system. It provides functionalities such as finding and acting on hosts in bulk, importing hosts from Azure, helping users with their sessions, scheduling scaling, patching hosts in rolling maintenance runs, charting capacity and unmet demand, and viewing logs. It is a React 18 and TypeScript single-page app built with Vite and Tailwind CSS, served by a Flask backend-for-frontend that holds the Entra ID token server-side and calls the Broker API on the administrator's behalf. +- **Service Management Portal**: A front-end web application that allows administrators to manage VMs, scaling rules, and monitor the system. It provides functionalities such as finding and acting on hosts in bulk, importing hosts from Azure, helping users with their sessions, scheduling scaling, patching hosts in rolling maintenance runs, charting capacity and unmet demand, and viewing logs. It is a React 18 and TypeScript single-page app built with Vite and Tailwind CSS, served by a Flask backend-for-frontend that holds the Entra ID token server-side and calls the Broker API on the administrator's behalf. By default, the sign-in sessions live in Redis, which every instance of the portal shares. - **Azure Key Vault**: Stores sensitive information such as SSH keys and database passwords, accessed securely by the Broker API using managed identity. A second vault holds the key that unlocks each user's login keyring. @@ -55,6 +55,7 @@ The architecture ensures secure, efficient, and scalable management of Linux hos - **Broker Database**: Azure SQL Database for storing VM and scaling data. - **Azure Function for Scaling Tasks**: Manages scaling of Linux hosts. - **Service Management Portal**: React and TypeScript front-end application for administrators, served by a Flask backend-for-frontend. +- **Portal Session Cache**: Azure Managed Redis, or Azure Cache for Redis in Azure Government, which keeps the portal's sign-ins so that the portal can run on more than one instance. - **Azure Key Vault**: Secure storage for SSH keys, passwords and each user's login keyring key. - **Managed Identities**: Used for secure authentication between components. - **Security Groups**: Controls access permissions for managed identities. @@ -65,11 +66,11 @@ The architecture ensures secure, efficient, and scalable management of Linux hos 2. **Selects Linux Host Connection**: The user opens the **Linux Desktop** RemoteApp, which the `azd` deployment publishes, for a full RDP session to a Linux host. 3. **Broker Agent Initiates Connection**: - The Broker Agent script (`Connect-LinuxBroker.ps1`) connects to the Broker API using the AVD host's managed identity. - - It checks out an available Linux VM for the user. + - It checks out an available Linux VM for the user. When none is ready, the broker can start a stopped one, and the user waits a minute or two in a small window that says their desktop is starting. - The user's ID is added to the Linux host with a unique 25-character password. - The user is added to appropriate user groups on the Linux host for RDP access. - The host also receives the key that unlocks the user's login keyring, so applications that save passwords do not ask for one. -4. **User Connects to Linux Host**: The user is connected to the Linux host via RDP and can work as needed. +4. **User Connects to Linux Host**: The user is connected to the Linux host via RDP, full screen across every monitor unless the deployment chose a window or one monitor, and can work as needed. 5. **Session Management**: - If the user disconnects or logs off, the Session Release Agent on the Linux host reconciles the XRDP/Xorg session state immediately when possible and otherwise on the next safety-net poll. - A reconnect timer is initiated, 20 minutes by default and configurable from the portal. @@ -101,18 +102,20 @@ What an administrator can do depends on their role (see [RBAC Permissions](#rbac 4. **Manage Scaling**: - **Edit the default rule**: Set the minimum and maximum number of running VMs, the scale-up and scale-down thresholds and increments, and whether scale-down powers VMs off or deallocates them. - **Add schedule windows**: Override the default rule on chosen days and times in one time zone, for example business hours, and see what the next run would do before saving. + - **Start on demand**: Let a checkout that finds no ready host start one, limit how many hosts start at once for waiting users, and see which AVD session hosts run a broker script that can wait. 5. **Manage Linux Host Settings**: - **Edit the fleet-wide profile**: Change the reconnect grace period, whether disconnected sessions are kept alive, the reconcile interval, watcher timings, idle session timeout, and screen lock policy without editing or redeploying any script. - **Apply Now**: Optionally push the profile to hosts immediately instead of waiting for them to pick it up. - **Review drift**: See which hosts have applied the current settings version. - **Version history**: See what changed in each saved version of the profile, and who saved it. 6. **Monitor System**: - - **Overview**: Capacity and checkout health over a day or a week, including checkouts that found no host, and an **Attention** panel for what needs an operator now. + - **Overview**: Capacity and checkout health over a day or a week, including checkouts that found no host and users who waited for one to start, and an **Attention** panel for what needs an operator now. - **View VM Details**: Access detailed information about VMs, including what the host's agent last reported. - **Fleet health**: See each host's last heartbeat, agent version, OS, desktop, xrdp and NFS state, load, memory, disk and sessions, and which hosts need attention, without SSH. - **Audit log**: See who did what: every portal action, every denied attempt, and the changes the broker makes on its own, with CSV export. - **View Scaling Activity Logs**: Monitor scaling activities and history, including why each run did or did not act. - **View VM History**: Track the usage and status changes of VMs. + - **Host logs and alerts**: Search the Linux hosts' and AVD session hosts' logs in Log Analytics, follow the fleet in the **Linux Broker fleet** workbook, and get alerts by email. See [Host Monitoring](deploy/DEPLOYMENT.md#host-monitoring). ## Service Management Portal @@ -202,6 +205,7 @@ These scripts: - **Install xrdp**: Set up xrdp for full desktop access over RDP, enabling users to connect via AVD. The host firewall allows only SSH and RDP. - **Start the desktop**: xrdp starts every session through `xrdp-startwm.sh`, which unlocks the user's login keyring and runs the desktop the deployment chose. GNOME's file indexer is turned off, because it would crawl the home directories on the NFS share. +- **Keep caches local**: `install-host-config.sh` keeps each user's cache on the host's own disk rather than in the home directory on the NFS share, sets the NFS read-ahead Microsoft recommends for Azure Files, and rotates the broker's logs. See [Home Directory Share](deploy/DEPLOYMENT.md#home-directory-share) for how to size the share. - **Configure Authentication**: Sets up authentication mechanisms for secure user access. - **Deploy the Linux Session Release Agent**: Installs the timer-based reconciliation service plus a `systemd-logind` watcher that can trigger early reconciliations. The timer remains the fallback path so the system still converges even if event delivery is delayed or unavailable. - **Install the Host Settings Agent**: Installs `apply-host-settings.sh` and seeds the settings profile, so screen lock policy and session timings are applied consistently on every supported distribution rather than only on RHEL 8. `LINUXBROKER_DISABLE_SCREEN_LOCK` still chooses the screen lock posture that is seeded; from then on the values are managed from the portal. @@ -213,12 +217,13 @@ These scripts: - **One default rule**: the rule with the lowest ID. Creating a second rule is refused; edit the existing one. - **Schedule windows**: Optional windows override the default rule on chosen days and times, read in one policy time zone, for example Monday to Friday 08:00–18:00 with a higher minimum. A window can run past midnight, enabled windows cannot overlap, and outside every window the default rule applies. A window takes effect at the next scaling run, about every five minutes. -- **Minimum VMs Running**: At least 1. Whenever fewer serviceable hosts are running (powered on, not in maintenance, and reachable or still booting), hosts are started to reach it. -- **Maximum VMs Running**: The maximum number of Linux VMs allowed to be powered on, including hosts in maintenance. When the two conflict, the minimum wins: a scale-down never leaves fewer serviceable hosts than the minimum. -- **Scale-Up Ratio**: The utilization at which more VMs are started (e.g., 80%). Utilization is hosts in use (checked out, released within their grace period, or pending cleanup) divided by serviceable hosts. +- **Minimum VMs Running**: At least 1, or 0 while start on demand is on, so that idle hosts stop until someone needs one and the first user to arrive waits for a host to start. That user then fills the pool, so the next scaling run starts **Scale-Up Increment** more hosts, as it does for any full pool. If start on demand is turned off later, scaling keeps one host on instead. Whenever fewer serviceable hosts are running (powered on, not in maintenance, and reachable or still booting), hosts are started to reach it. +- **Maximum VMs Running**: The maximum number of Linux VMs allowed to be powered on, including hosts in maintenance. When the two conflict, the minimum wins: a scale-down never leaves fewer serviceable hosts than the minimum, or, while users wait for a host to start, fewer than the hosts in use plus the users waiting. +- **Scale-Up Ratio**: The utilization at which more VMs are started (e.g., 80%). Utilization is hosts in use (checked out, released within their grace period, or pending cleanup), plus users waiting for a host to start, divided by serviceable hosts. - **Scale-Up Increment**: The number of VMs to start when scaling up. - **Scale-Down Ratio**: The utilization at or below which VMs are stopped (e.g., 30%). - **Scale-Down Increment**: The number of VMs to stop when scaling down. Only idle, reachable, unassigned hosts that have stayed in their current power state for at least 10 minutes are stopped, highest VMID first. +- **Start on demand**: When a checkout finds no ready host, the broker starts a stopped one for the user, up to the maximum, instead of refusing. It is on by default, and the **Scaling** page turns it off and sets how many hosts may start at once for waiting users, 2 by default. See [Upgrading To Start On Demand](deploy/DEPLOYMENT.md#upgrading-to-start-on-demand). - **Stop mode**: **Power off** (the default) keeps the VM's compute allocation, so it starts quickly but compute is still billed. **Deallocate** stops compute billing, but starts take longer, and in a capacity-constrained region or VM size a start can fail with `AllocationFailed` (the broker records the host as off and retries on a later run). Deallocation also wipes the temporary disk; private IP addresses and host names are kept. Every run first reads each host's power state from Azure and corrects the broker's record, runs never overlap, and each run writes an activity log entry whose notes explain the decision. Before this release, scaling decisions were recorded but never sent to Azure, so upgrading makes scaling start and stop VMs for the first time. @@ -274,7 +279,7 @@ Because the profile is versioned, the portal shows which hosts have applied the - **Azure Key Vault**: Stores SSH keys and database passwords securely, accessed via managed identities. - **Login keyring**: The Linux password changes at every checkout, so it cannot protect a user's GNOME login keyring. The broker keeps a separate random key for each user in a vault of its own, where the API can write secrets but cannot change the deployment's, and `xrdp-startwm.sh` unlocks the keyring with it before the desktop starts. The key reaches the host over the checkout's SSH session and stays in memory-backed storage under `/run`, readable only by the user, until the host is returned. - **API Permissions**: Specific API permissions are granted to components to restrict access based on roles. -- **Logging and Monitoring**: All activities are logged to Azure Application Insights and Log Analytics Workspace. +- **Logging and Monitoring**: The portal, the API and the task function log to Azure Application Insights and a Log Analytics workspace. Unless `deployHostMonitoring` is `false`, the Linux hosts' broker logs, xrdp logs and syslog, and the **Linux Desktop** events of the AVD session hosts, are collected in the same workspace, and a workbook and alerts show the state of the Linux host fleet. See [Host Monitoring](deploy/DEPLOYMENT.md#host-monitoring). ### AVD Host Sizing Recommendations @@ -300,7 +305,7 @@ Given that AVD acts as a pass-through in this solution, starting with **light to - Avoid using VMs with more than 24 vCPUs to prevent diminishing returns due to increased synchronization overhead. 4. **Optimize for Multi-Session Workloads**: - Use multiple smaller VMs (e.g., 8-core instances) rather than fewer large VMs. This allows for better load balancing and resource management. - - Smaller VMs can be shut down when not in use, conserving resources and reducing costs. Use Azure autoscale to manage VM power states based on demand. + - Smaller VMs can be shut down when not in use, conserving resources and reducing costs. The deployment's [scaling plan](deploy/DEPLOYMENT.md#avd-autoscale) starts and stops the AVD session hosts on a schedule. ## Getting Started @@ -329,13 +334,13 @@ azd env new azd up ``` -For existing environments that need in-place rollout instead of new-environment provisioning, use [deploy/Migrate-ExistingEnvironment.ps1](deploy/Migrate-ExistingEnvironment.ps1) from the `deploy/` directory. `azd up` remains the supported greenfield path. +For existing environments that need in-place rollout instead of new-environment provisioning, use [deploy/Migrate-ExistingEnvironment.ps1](deploy/Migrate-ExistingEnvironment.ps1) from the `deploy/` directory. It updates the apps, the database, the Linux hosts' agent and the AVD session hosts' broker script. `azd up` remains the supported greenfield path. The deployment targets Azure commercial by default. Set `azureCloudName` to `AzureUSGovernment` or `AzureCustom` to deploy elsewhere; commercial and Government resolve their endpoints automatically, while custom and sovereign clouds require their own authority, Graph, STS, and App Service FQDNs. Air-gapped environments should also set `scriptSourceRoot` to a reachable mirror of this repository, because the Linux hosts download their agent scripts from it during bootstrap. The Service Management Portal serves all of its front-end assets from its own container under `front_end/static/dist/`. The bundle is compiled during the container build, uses the system font stack, and draws its icons as inline SVG, so it makes no requests to a public CDN and renders correctly in Government, sovereign and air-gapped environments where outbound internet access is blocked. Note that building the portal image does require access to the npm registry, so a disconnected build host needs an internal npm mirror. See [front_end/README.md](front_end/README.md). -The deployment defaults the App Service plan to Premium v3 `P2mv3`, which provides the minimum supported baseline of 4 vCPUs and 32 GB memory for the frontend, API, and task apps. +The deployment defaults the App Service plan to Premium v3 `P2mv3`, which provides the minimum supported baseline of 4 vCPUs and 32 GB memory for the frontend, API, and task apps. It runs one instance unless you set `appServicePlanCapacity`; see [Portal And API Scale-Out](deploy/DEPLOYMENT.md#portal-and-api-scale-out) before you run more. Before running `azd up`, review the detailed guide and set any environment-specific values you need, especially networking, host counts, VM sizes, App Service plan sizing, and SQL firewall access. The deployment scripts under `deploy/` now handle the Entra bootstrap, SSH key flow, App Service health checks on `/health`, Application Insights wiring for the frontend and API, post-provision role assignment, container image builds, SQL initialization, and Linux host SQL registration used by this solution. @@ -361,9 +366,21 @@ The distribution and desktop support release needs `azd provision` and agent 1.2 - **Review the idle timeout.** It had never disconnected anyone before this release, and migrated Ubuntu hosts now enforce any timeout already set. - **Replace or bootstrap again any Ubuntu hosts.** Earlier releases deployed them with no desktop and without the packages NFS homes need. +The start on demand release needs `azd provision`, for the AVD scaling plan and its role on the subscription, for the host monitoring and for the portal's session cache, the AVD session hosts need `Connect-LinuxBroker.ps1` 2.0.0, and the Linux hosts need agent 1.3.0. See [Upgrading To Start On Demand](deploy/DEPLOYMENT.md#upgrading-to-start-on-demand). + +- **A checkout can start a host.** When no host is ready, the broker starts a stopped one and the user's **Linux Desktop** waits for it, for up to 10 minutes. It is on after the upgrade, and the **Scaling** page turns it off or limits how many hosts start at once. +- **Update the AVD session hosts.** `deploy/Migrate-ExistingEnvironment.ps1` now also runs `deploy/Update-AvdHostBrokerScript.ps1`, which replaces the script on every running session host. An older script tells its user that no host is available while one starts for them. +- **A pool can scale to zero.** With start on demand on, a rule or schedule window may keep a minimum of 0 hosts. Set one only after the **Scaling** page shows that every session host runs a script that can wait. +- **The Linux desktop opens full screen across every monitor.** Set `avdLinuxDesktopFullScreen` or `avdLinuxDesktopMultiMonitor` to `false` to open it in a window or on one monitor, but only after every session host runs the new script. See [Linux Desktop Display](deploy/DEPLOYMENT.md#linux-desktop-display). +- **The AVD session hosts start and stop on a schedule.** `azd provision` adds a scaling plan, turns on Start VM on Connect and gives the Azure Virtual Desktop service principal **Desktop Virtualization Power On Off Contributor** on the subscription, which needs Owner or User Access Administrator there. Update the session hosts' script first, while they all still run, and review the schedule before you provision, or set `avdScalingPlanEnabled` to `false`. See [AVD Autoscale](deploy/DEPLOYMENT.md#avd-autoscale). +- **Update the Linux hosts to agent 1.3.0.** Fleet health flags every host as outdated until `deploy/Migrate-LinuxHostReleaseAgent.ps1` from this release, which `deploy/Migrate-ExistingEnvironment.ps1` runs, has updated it. Hosts that are not migrated keep working as before. +- **Caches move off the NFS share.** The migration installs `install-host-config.sh` on the running Linux hosts. Each user's cache then stays on the host's own disk, NFS mounts read ahead 15 MiB, and the broker's logs rotate. Caches already in `~/.cache` on the share can be deleted. See [Home Directory Share](deploy/DEPLOYMENT.md#home-directory-share). +- **Host logs, a fleet workbook and alerts.** `azd provision` adds data collection rules, a workbook and alerts, and `postprovision` installs the Azure Monitor agent on the running hosts, so their logs reach Log Analytics. Set `alertEmailAddresses` to get the alerts by email, and once each host that `postprovision` names as not running is started, connect it with `deploy/Enable-HostMonitoring.ps1 -HostNames`. See [Host Monitoring](deploy/DEPLOYMENT.md#host-monitoring). +- **Portal sessions move to Redis.** `azd provision` creates a cache for the portal's sign-ins, Azure Managed Redis or, in Azure Government, Azure Cache for Redis. Moving the sessions there signs every administrator out once. The cache is billed by the hour; set `portalSessionStore` to `filesystem` to keep the sessions on the portal's disk. With the cache, `appServicePlanCapacity` can run the portal and the API on more than one instance. See [Portal And API Scale-Out](deploy/DEPLOYMENT.md#portal-and-api-scale-out). + ## Roadmap -Planned work beyond this release, including RHEL 10 and Ubuntu 26.04 support, starting a host on demand, golden images and multi-session hosts, is described in [docs/ROADMAP.md](docs/ROADMAP.md). +Planned work beyond this release, including RHEL 10 and Ubuntu 26.04 support, multi-session hosts, host pools, golden images and GPU-accelerated OpenGL, is described in [docs/ROADMAP.md](docs/ROADMAP.md). ## Contributing diff --git a/api/README.md b/api/README.md index 591c7fe..ffaf2f4 100644 --- a/api/README.md +++ b/api/README.md @@ -19,7 +19,7 @@ Role groups used below: **READ** = `Reader`, `Operator`, `FullAccess`; **OPERATE | GET | `/api/me` | any valid token | The caller's app roles and `permissions` (`read`, `operate`, `admin`), plus `legacyScopeAccess`. The portal uses it to adapt its interface. | | GET | `/api/vms` | READ, `ScheduledTask` | Lists all broker VM records, including `ReleasedDate`, `CleanupPending`, `CleanupUsername`, `PowerStateChangedDate`, `DrainRequested`, and `DrainRequestedDate`. With any of `page`, `per_page`, `q`, `status`, `sort` or `dir` it answers one page of the host list instead; see [Host List and Import](#host-list-and-import). | | GET | `/api/vms/summary` | READ, `ScheduledTask` | Returns dashboard counters: `TotalVMs`, `Available`, `CheckedOut`, `Maintenance`, `Released`, `PoweredOn`, `PoweredOff`, `Unreachable`, `Ready`, `CleanupPending`, and `Draining`. | -| POST | `/api/vms/checkout` | `AvdHost`, `FullAccess`, or `AVD_HOST_GROUP_ID` membership | Checks out a ready Linux host and provisions the user in one SSH session. | +| POST | `/api/vms/checkout` | `AvdHost`, `FullAccess`, or `AVD_HOST_GROUP_ID` membership | Checks out a ready Linux host and provisions the user in one SSH session. With no host ready it starts one on demand and answers `202` with `Retry-After`; see [Start on Demand](#start-on-demand). | | POST | `/api/vms//update-attributes` | ADMIN, `ScheduledTask` | Repairs the broker's record of a VM's power, network, or broker status. It does not start or stop anything. `ScheduledTask` keeps access for task builds older than `/network-status`. | | POST | `/api/vms//network-status` | `ScheduledTask`, `FullAccess` | Records a reachability probe result; writes only when the status changes. | | POST | `/api/vms//maintenance` | OPERATE | Moves an unassigned host between `Available` and `Maintenance`, clearing any drain flag. | @@ -47,8 +47,8 @@ Role groups used below: **READ** = `Reader`, `Operator`, `FullAccess`; **OPERATE | POST | `/api/scaling/rules//update` | ADMIN | Updates the scaling rule; the resulting rule is validated. | | POST | `/api/scaling/rules//delete` | ADMIN | Deletes a scaling rule. | | POST | `/api/scaling/rules/history` | READ | Returns scaling rule history, optionally paged with `page` and `per_page`. | -| GET | `/api/scaling/policy` | READ | The policy time zone, the default rule, every schedule window, and what applies now and next; `404` until the database has the scaling policy procedures. | -| POST | `/api/scaling/policy/update` | ADMIN | Sets the time zone every schedule window is read in (`{"timezone": ""}`). | +| GET | `/api/scaling/policy` | READ | The policy time zone, the default rule, every schedule window, what applies now and next, the start on demand settings, and which broker script the AVD hosts run; `404` until the database has the scaling policy procedures. | +| POST | `/api/scaling/policy/update` | ADMIN | Sets any of the time zone every schedule window is read in (`timezone`, a Windows zone name), `startondemandenabled` and `maxpendingstarts` (1–20), together or not at all. | | GET | `/api/scaling/timezones` | READ | The time zones the policy can use, from `sys.time_zone_info`. | | POST | `/api/scaling/schedules/create` | ADMIN | Adds a schedule window; `409` when it overlaps another enabled window. | | POST | `/api/scaling/schedules//update` | ADMIN | Replaces a schedule window. | @@ -133,7 +133,7 @@ Fleet health reports only. Checkout readiness is still decided by the task funct ## Audit Log -`@audited` records every call a portal user or an administrator principal makes to a mutating route, with the outcome taken from the status (`success`, `failure`, or `denied` for `403`), the target (the hostname where known), and a curated detail: the fields changed and their previous values, the result, and the error envelope's message. It never records a response body, so a password or lease ID cannot reach the log. `token_required` records authorization denials on mutating routes from any caller, up to 30 a minute per caller in each worker process; past that it only logs them, so a caller retrying a refused call cannot fill the table. The broker's own changes are recorded where they happen: scaling power actions and their failures, power states corrected from Azure, expired releases, completed cleanups and drains, and each purge. Routine agent calls are not audited: AVD checkouts, Linux host releases, acknowledgements and heartbeats, and the task's probes are high volume, and `VirtualMachinesHistory` already records what they change. +`@audited` records every call a portal user or an administrator principal makes to a mutating route, with the outcome taken from the status (`success`, `failure`, or `denied` for `403`), the target (the hostname where known), and a curated detail: the fields changed and their previous values, the result, and the error envelope's message. It never records a response body, so a password or lease ID cannot reach the log. `token_required` records authorization denials on mutating routes from any caller, up to 30 a minute per caller in each worker process; past that it only logs them, so a caller retrying a refused call cannot fill the table. The broker's own changes are recorded where they happen: scaling power actions and their failures, hosts started on demand for a waiting user, power states corrected from Azure, expired releases, completed cleanups and drains, and each purge. Routine agent calls are not audited: AVD checkouts, Linux host releases, acknowledgements and heartbeats, and the task's probes are high volume, and `VirtualMachinesHistory` already records what they change. The actor comes from the validated token: a portal user's sign-in name, or for a managed identity the VM or function app named in `xms_mirid`. The correlation ID is the request's OpenTelemetry trace ID when it is traced. Writing an entry never fails the operation: a failure is logged and the call carries on. Every entry is also written to the `linuxbroker.api.audit` logger with `audit_*` attributes, so Application Insights keeps a copy with its own retention, for example for export to a SIEM. @@ -161,9 +161,25 @@ A profile reset is requested and then applied at the user's next new assignment, Schedule windows override the default rule on chosen days and times, read in one policy time zone (a Windows zone name, `UTC` until one is chosen): for example Monday to Friday 08:00–18:00 keeping at least five hosts ready. A window may run past midnight; outside every enabled window the default rule applies. Overlapping enabled windows are refused in SQL under a lock, and in the API with a message naming the clash. `TriggerScalingLogic` takes its values from the active phase, and `GET /api/scaling/preview` runs the same decision as a dry run against the current counts, so the portal can show what the next run would do, and why, before anything is saved. `MinVMs` wins over `MaxVMs`: draining and maintenance hosts count toward the maximum, and a scale-down never takes serviceable hosts below the minimum. Windows take effect at the next five-minute scaling run. +## Start on Demand + +When a checkout finds no ready host, the API asks `ReserveVmForStart` what to do instead of refusing it. Under the scaling application lock, so it never races a scaling run, the procedure starts a stopped, free host for the user, or tells them to wait for one already starting, and the checkout answers `202`: + +```json +{"status": "Starting", "reason": "Started", "retryAfterSeconds": 60, "message": "A Linux host is starting for you. Ask again in 60 seconds."} +``` + +with a `Retry-After` header of the same number of seconds, 5–300. The AVD host asks again then, and gets the host once the reachability probe, or the API's own probe, reaches it. `reason` is `Started`, `AlreadyStarting` (enough hosts are starting for the users waiting, or `MaxPendingStarts` are starting), `Busy` (the scaling lock was not free within five seconds) or `ReadyNow` (a host became ready but was taken again). The wait is the median host start time over the last week, less how long the oldest starting host has been starting, kept between 30 and 120 seconds. While a user waits, each request probes the starting hosts on port 22 (`START_ON_DEMAND_PROBE_*`) and records the ones that answer as reachable, so the user need not wait for the task's next probe, which runs every two minutes. + +A start is recorded exactly as a scaling start is, with a `Start On Demand` row in the scaling activity log and a `vm.start_on_demand` audit entry. If Azure refuses it, the host is recorded as off again and the checkout is refused with `409`, as it is when start on demand is off, the phase's `MaxVMs` hosts are all on, or no stopped host is free. A database without `ReserveVmForStart` refuses as before, logged once. + +Scaling counts waiting users (`fnWaitingCheckoutUsers`: users whose latest checkout event in the last five minutes is `Starting`) as demand, and while anyone waits, a scale-down keeps at least as many serviceable hosts as there are users in use and waiting. With start on demand on, a rule or window may set `MinVMs` to `0`, so idle hosts stop until someone needs one; the API and `SaveScalingSchedule` refuse `0` while it is off. Turning it off while such rules exist is allowed: scaling then keeps one host running for them, and the update's message says so. `no-ready-hosts` is raised in that idle state only while a user is waiting. + +`POST /api/scaling/policy/update` takes `startondemandenabled` and `maxpendingstarts` with or without `timezone`, and applies them in one transaction. `GET /api/scaling/policy` returns `StartOnDemandEnabled`, `MaxPendingStarts`, `ZeroMinimumCount` (rules and enabled windows with a minimum of `0`), and `AvdHostScripts`: of the AVD hosts that asked for a checkout in the last seven days, how many run each broker script version and which report none (`Outdated`, `OutdatedHostnames`). A script that reports no version predates start on demand: it treats `202` as a failure, so its users see an error while a host starts for them. Update it with [`deploy/Update-AvdHostBrokerScript.ps1`](../deploy/Update-AvdHostBrokerScript.ps1). The checkout body's optional `clientVersion` (at most 32 letters, digits, `.`, `_`, `+` or `-`) is recorded with each checkout event; `AVD_HOST_SCRIPT_VERSION` in [`config.py`](config.py) is the current one, and a unit test requires it to match the script's `$ScriptVersion`. + ## Trends and Attention -Every checkout records its outcome (`Assigned`, `Reused`, `NoneAvailable`, `ProvisionFailed` or `Error`) and duration in `CheckoutEvents`. Recording never fails the checkout, and a missing procedure is logged once. `GET /api/metrics/utilization` combines the five-minute scaling runs with those events: running, in use, serviceable and the scaling maximum per bucket (15 minutes over a day, an hour over a week), denied checkouts, checkout time percentiles, and how long started hosts took to become reachable. `GET /api/metrics/attention` lists what needs an operator now: no ready host, denied checkouts in the last hour, hosts unreachable for 10 minutes, stuck in cleanup for 15, or checked out 30 minutes ago without the user connecting, hosts a maintenance run could not patch, and the fleet health flags, grouped. The daily audit purge also removes events older than `CHECKOUT_EVENT_RETENTION_DAYS`. Both endpoints answer `404` against a database without the procedures, and the portal then hides the charts. +Every checkout records its outcome (`Assigned`, `Reused`, `NoneAvailable`, `ProvisionFailed`, `Error`, or `Starting` each time a user is told to wait for a host) and duration in `CheckoutEvents`. Recording never fails the checkout, and a missing procedure is logged once. `GET /api/metrics/utilization` combines the five-minute scaling runs with those events: running, in use, serviceable and the scaling maximum per bucket (15 minutes over a day, an hour over a week), denied checkouts, the users told to wait (`Waited`), checkout time percentiles, and how long started hosts took to become reachable. `Total` and the denied share count checkouts that ended, so `Starting` events are counted apart, with `Waits` (waits that began in the period), `WaitsServed` (those that ended in a checkout within 20 minutes), `WaitP50Seconds` and `WaitP95Seconds`, and `WaitingNow`. `GET /api/metrics/attention` lists what needs an operator now: no ready host, denied checkouts in the last hour, hosts unreachable for 10 minutes, stuck in cleanup for 15, or checked out 30 minutes ago without the user connecting, hosts a maintenance run could not patch, and the fleet health flags, grouped. The daily audit purge also removes events older than `CHECKOUT_EVENT_RETENTION_DAYS`. Both endpoints answer `404` against a database without the procedures, and the portal then hides the charts. ## Rolling Maintenance @@ -276,8 +292,11 @@ The API reads environment variables directly; it does not load `.env` files by i | `NFS_SHARE` | required for checkout provisioning | NFS share argument passed to `create-user.sh`; used by code but not currently listed in `env.example`. The `azd` deployment sets it to the Azure Files NFS share it provisions unless you supply `nfsShare` or set `deployNfsShare` to `false`. | | `ALLOW_LEGACY_SCOPE_ACCESS` | optional | `true` treats the portal's `access_as_user` scope as `FullAccess` while roles are assigned during an upgrade. Defaults to `false`; set through the `allowLegacyScopeAccess` deployment value. | | `GUNICORN_CMD_ARGS` | optional | Overrides the image default of `--workers 2 --threads 8 --timeout 120 --graceful-timeout 30 --keep-alive 5`. | -| `DB_MAX_CONCURRENCY` | optional | Maximum SQL connections per worker process. Defaults to `6`, which keeps two workers inside the Basic tier's 30 concurrent workers; raise it with the database tier. | +| `DB_MAX_CONCURRENCY` | optional | Maximum SQL connections per worker process, idle pooled connections included. Defaults to `6`, which keeps two workers inside the Basic tier's 30 concurrent workers; raise it with the database tier. | | `DB_ACQUIRE_TIMEOUT_SECONDS` | optional | How long a request waits for a free connection slot before answering `503`. Defaults to `15`. | +| `DB_POOL_ENABLED` | optional | `false` opens a new SQL connection for every `db_connection()` block and closes it afterwards, as before the pool. Defaults to `true`; see [Database Access](#database-access). | +| `DB_POOL_IDLE_SECONDS` | optional | A pooled connection unused this long is closed rather than reused. Defaults to `120` (5–3600): App Service's outbound load balancer forgets a connection after four minutes without traffic. | +| `DB_POOL_MAX_LIFETIME_SECONDS` | optional | A connection this old is closed rather than pooled again, so connections sign in again regularly with the password as last read from Key Vault. Defaults to `1800` (60–86400). | | `JWKS_CACHE_SECONDS` | optional | How long token signing keys are cached. Defaults to `3600`. | | `SSH_KEY_CACHE_SECONDS` | optional | How long the SSH private key read from Key Vault is reused. Defaults to `3600`. | | `SWEEP_CONCURRENCY`, `SWEEP_DEADLINE_SECONDS` | optional | Parallel cleanups in the released-VM sweep (default `8`), and the time after which it stops starting new ones (default `40`); the rest are retried on the next run. | @@ -291,7 +310,11 @@ The API reads environment variables directly; it does not load `.env` files by i ## Database Access -Handlers call stored procedures rather than embedding schema logic in Python. `db_connection()` wraps `get_db_connection()` as a context manager so every acquired connection is closed on success or exception, and bounds the connections each worker process holds at once (`DB_MAX_CONCURRENCY`). Never nest `db_connection()` blocks: a thread holding one slot while waiting for another can starve the pool. +Handlers call stored procedures rather than embedding schema logic in Python. `db_connection()` is a context manager that takes one of the worker process's `DB_MAX_CONCURRENCY` slots and yields a connection. Never nest `db_connection()` blocks: a thread holding one slot while waiting for another can starve the pool. + +The connection comes from a pool (`db_pool.py`) of connections the same worker process has finished with, or is opened when none is idle. Reusing a connection saves a TLS handshake and a login, and each connection closed on App Service holds an outbound SNAT port for four minutes. When the block ends, whatever it left uncommitted is rolled back and the connection returns to the pool with no transaction open. When the block raises, the connection is closed instead. It begins a new transaction only as the next block takes it, because system-versioned tables such as `VirtualMachines` stamp each change with its transaction's start time: a transaction begun while the connection sat idle would be older than changes other connections made meanwhile, and SQL Server refuses such a change (error 13535, which `CheckoutVm` then turns into error 266). A connection that stopped answering while idle is closed and replaced without failing the request. + +Pooling is safe because nothing the API does outlives a transaction on its connection: every `sp_getapplock` is owned by the transaction, and the API sets no session options, temporary tables or `SESSION_CONTEXT`. Keep it that way, or set `DB_POOL_ENABLED=false`. The integration tests in `tests_integration` run with the pool on. pymssql runs every statement inside its own transaction, so a procedure must never roll that outer transaction back on a normal path: SQL Server then raises error 266 when the procedure returns, and the handler's commit fails. Procedures that need their own transaction either commit what they opened or use a savepoint when `@@TRANCOUNT > 0` (see `059_alter_procedure-TriggerScalingLogic.sql`). diff --git a/api/app.py b/api/app.py index a8cfeb3..d2b19b6 100644 --- a/api/app.py +++ b/api/app.py @@ -32,6 +32,7 @@ from flask_caching import Cache from azure.keyvault.secrets import SecretClient from config import * +from db_pool import ConnectionPool try: # Installed with azure-monitor-opentelemetry. The trace id ties an audit entry to the @@ -44,7 +45,7 @@ # Flask App app = Flask(__name__) -app.config['VERSION'] = '0.171' +app.config['VERSION'] = '0.172' # Backs is_member_of_group_cached, which keeps token validation off the Graph API on # every request. @@ -317,6 +318,9 @@ def reset_caches(): with _keyring_lock: _keyring_state.update({'client': None, 'unavailable_until': 0.0}) _checkout_event_state['missing_logged'] = False + _checkout_event_state['client_version_retry_at'] = 0.0 + _start_on_demand_state['missing_logged'] = False + _fleet_snapshot_state['missing_logged'] = False cache.clear() def is_duplicate_key_error(error) -> bool: @@ -435,14 +439,24 @@ class GroupCheckUnavailable(Exception): # cannot exceed the database tier's worker limit during a login storm. _db_slots = threading.BoundedSemaphore(DB_MAX_CONCURRENCY) +# Idle connections kept for the next request. They count against DB_MAX_CONCURRENCY along +# with the busy ones: db_connection() opens a connection only while holding a slot, and +# puts it back or closes it before giving the slot up. +_db_pool = ConnectionPool( + max_idle=DB_MAX_CONCURRENCY, + idle_seconds=DB_POOL_IDLE_SECONDS, + max_lifetime_seconds=DB_POOL_MAX_LIFETIME_SECONDS, +) if DB_POOL_ENABLED else None + @contextmanager def db_connection(): - """Yield a database connection that is always closed. + """Yield a database connection that is always put back in the pool or closed. Most handlers previously called get_db_connection() and then conn.close() on the success path only, so any exception in between leaked the connection until the - pool was exhausted. Using this as a context manager makes the close unconditional. + pool was exhausted. Using this as a context manager makes the cleanup unconditional. + A connection whose block raised is closed rather than reused. Raises DatabaseUnavailable when a connection cannot be established, so callers do not have to repeat the `if not conn` check. Never nest these: a thread holding one @@ -451,16 +465,28 @@ def db_connection(): if not _db_slots.acquire(timeout=DB_ACQUIRE_TIMEOUT_SECONDS): raise DatabaseBusy("Timed out waiting for a database connection slot.") try: - conn = get_db_connection() - if not conn: - raise DatabaseUnavailable("Could not establish a database connection.") - try: - yield conn - finally: + pool = _db_pool + if pool is None: + conn = get_db_connection() + if not conn: + raise DatabaseUnavailable("Could not establish a database connection.") try: - conn.close() - except Exception: - logger.exception("Failed to close database connection.") + yield conn + finally: + try: + conn.close() + except Exception: + logger.exception("Failed to close database connection.") + else: + pooled = pool.acquire(get_db_connection) + if pooled is None: + raise DatabaseUnavailable("Could not establish a database connection.") + reusable = False + try: + yield pooled.connection + reusable = True + finally: + pool.release(pooled, reusable) finally: _db_slots.release() @@ -2053,8 +2079,34 @@ def get_vm_summary(): logger.exception("Failed to build the VM summary.") return error_response("Unable to retrieve the virtual machine summary.", 500) -CHECKOUT_OUTCOMES = ('Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Error') -_checkout_event_state = {'missing_logged': False} +CHECKOUT_OUTCOMES = ('Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Error', 'Starting') +CLIENT_VERSION_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._+-]*$') +_checkout_event_state = {'missing_logged': False, 'client_version_retry_at': 0.0} +_start_on_demand_state = {'missing_logged': False} +# How long to leave out @ClientVersion after a database that predates it refused it. +CLIENT_VERSION_RECHECK_SECONDS = 600 + + +def parse_client_version(value): + """The version the AVD host script reports, or None when it reports none or an unusable one.""" + if not isinstance(value, str): + return None + value = value.strip() + if not value or len(value) > CLIENT_VERSION_MAX_CHARS or not CLIENT_VERSION_RE.match(value): + return None + return value + + +def _write_checkout_event(params, client_version): + sql = "EXEC RecordCheckoutEvent @Username = %s, @AvdHost = %s, @Outcome = %s, @DurationMs = %s, @Hostname = %s" + if client_version is not None: + sql += ", @ClientVersion = %s" + params = params + (client_version,) + with db_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(sql, params) + cursor.fetchone() + conn.commit() def record_checkout_event(event, started): @@ -2062,18 +2114,25 @@ def record_checkout_event(event, started): Never raises: a checkout must not fail because its statistics could not be written. A database without dbo.RecordCheckoutEvent, while the API is upgraded ahead of SQL, is - logged once per process rather than on every checkout. + logged once per process rather than on every checkout. One whose procedure predates + @ClientVersion (146) gets the event without it. """ duration_ms = max(0, int((time.monotonic() - started) * 1000)) + params = (event.get('username'), event.get('avdhost'), event['outcome'], duration_ms, event.get('hostname')) + client_version = event.get('clientVersion') + if client_version is not None and time.monotonic() < _checkout_event_state['client_version_retry_at']: + client_version = None try: - with db_connection() as conn: - with conn.cursor() as cursor: - cursor.execute( - "EXEC RecordCheckoutEvent @Username = %s, @AvdHost = %s, @Outcome = %s, @DurationMs = %s, @Hostname = %s", - (event.get('username'), event.get('avdhost'), event['outcome'], duration_ms, event.get('hostname')) - ) - cursor.fetchone() - conn.commit() + try: + _write_checkout_event(params, client_version) + except DatabaseUnavailable: + raise + except Exception as e: + if client_version is None or 'ClientVersion' not in str(e): + raise + _checkout_event_state['client_version_retry_at'] = time.monotonic() + CLIENT_VERSION_RECHECK_SECONDS + logger.warning("RecordCheckoutEvent does not take @ClientVersion yet; AVD host script versions are not recorded until the SQL upgrade.") + _write_checkout_event(params, None) except Exception as e: if not is_missing_procedure_error(e): logger.warning("Could not record the checkout event (%s).", event.get('outcome'), exc_info=True) @@ -2082,6 +2141,164 @@ def record_checkout_event(event, started): logger.warning("RecordCheckoutEvent is not deployed yet; checkouts are not counted until the SQL upgrade.") +def call_checkout_vm(username, avdhost): + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.callproc('CheckoutVm', (username, avdhost)) + rows = cursor.fetchall() + conn.commit() + return rows + + +def found_no_host(rows): + return not rows or 'Message' in rows[0] + + +def reserve_vm_for_start(username, avdhost): + """dbo.ReserveVmForStart's decision for a user whose checkout found no ready host. + + The procedure holds the scaling lock until the caller commits, so this commits at once, + before anything asks Azure. Returns None when the database made no decision: a database + that predates start on demand is logged once per process, any other failure every time, + and either way the checkout is refused as it was before start on demand. + """ + try: + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC ReserveVmForStart @Username = %s, @AvdHost = %s", (username, avdhost)) + row = cursor.fetchone() + conn.commit() + return row or None + except DatabaseUnavailable: + raise + except Exception as e: + if not is_missing_procedure_error(e): + logger.exception("Could not decide whether to start a host for %s.", username) + elif not _start_on_demand_state['missing_logged']: + _start_on_demand_state['missing_logged'] = True + logger.warning("ReserveVmForStart is not deployed yet; a checkout that finds no ready host is refused until the SQL upgrade.") + return None + + +def probe_ssh(ip_address): + """Whether a host accepts a connection on the port the reachability probe uses.""" + try: + with socket.create_connection((ip_address, START_ON_DEMAND_PROBE_PORT), timeout=START_ON_DEMAND_PROBE_TIMEOUT_SECONDS): + return True + except (OSError, ValueError): + return False + + +def mark_started_hosts_reachable(hosts): + """Probe the hosts starting for waiting users, and record the ones that answer as Reachable. + + Returns how many answered. The scheduled task probes every host each minute; this hands a + waiting user a host as soon as it is up. Never raises, except for an unreachable database. + """ + candidates = [host for host in hosts or [] if isinstance(host, dict) and host.get('VMID') and host.get('IPAddress')] + if not candidates: + return 0 + + with ThreadPoolExecutor(max_workers=min(START_ON_DEMAND_PROBE_CONCURRENCY, len(candidates))) as pool: + answers = list(pool.map(lambda host: probe_ssh(str(host['IPAddress'])), candidates)) + answered = [host for host, up in zip(candidates, answers) if up] + if not answered: + return 0 + + try: + with db_connection() as conn: + for host in answered: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC SetVmNetworkStatus @VMID = %s, @NetworkStatus = %s", (host['VMID'], 'Reachable')) + cursor.fetchone() + conn.commit() + except DatabaseUnavailable: + raise + except Exception: + logger.exception("Could not record the started hosts that answered as reachable.") + return 0 + + for host in answered: + logger.info("%s answered while starting for a waiting user, so it was recorded as reachable.", host.get('Hostname')) + return len(answered) + + +def _retry_after(value): + try: + seconds = int(value) + except (TypeError, ValueError): + return START_ON_DEMAND_RETRY_SECONDS + return min(300, max(5, seconds)) + + +def start_host_on_demand(username, avdhost, client_version): + """What to do for a user whose checkout found no ready host. + + Returns {'retry': bool, 'wait': None or {'reason', 'retryAfterSeconds', 'hostname'}}. retry + means a host may now be ready, so the checkout should be tried once more; wait means the + user should ask again after retryAfterSeconds, rather than be refused. + """ + row = reserve_vm_for_start(username, avdhost) or {} + result = row.get('Result') + retry_after = _retry_after(row.get('RetryAfterSeconds')) + + if result == 'ReadyNow': + return {'retry': True, 'wait': {'reason': 'ReadyNow', 'retryAfterSeconds': START_ON_DEMAND_RETRY_SECONDS}} + + if result == 'Busy': + return {'retry': False, 'wait': {'reason': 'Busy', 'retryAfterSeconds': retry_after}} + + if result == 'AlreadyStarting': + answered = mark_started_hosts_reachable(_json_column(row.get('BootingHostsJson'), list)) + return {'retry': answered > 0, 'wait': {'reason': 'AlreadyStarting', 'retryAfterSeconds': retry_after}} + + if result == 'Started': + vm_name = row.get('VMName') + detail = {key: value for key, value in ( + ('username', username), ('avdhost', avdhost), ('clientVersion', client_version), + ('activityId', row.get('ActivityID')), ('waiting', row.get('Waiting')), ('starting', row.get('Booting')), + ) if value is not None} + try: + if not VM_SUBSCRIPTION_ID or not VM_RESOURCE_GROUP: + raise RuntimeError("The Azure subscription or resource group of the Linux hosts is not configured.") + get_compute_client().virtual_machines.begin_start(VM_RESOURCE_GROUP, vm_name) + except Exception: + logger.exception("Azure refused to start %s for %s.", vm_name, username) + revert_power_action(row, 'Off') + append_scaling_note(row.get('ActivityID'), f"Starting {vm_name} failed, so it was recorded as off again.") + audit('vm.start_on_demand', 'vm', vm_name, AUDIT_FAILURE, + dict(detail, error='The Azure start operation could not be requested.')) + return {'retry': False, 'wait': None} + + logger.info("Started %s for %s, who found no ready host.", vm_name, username) + audit('vm.start_on_demand', 'vm', vm_name, AUDIT_SUCCESS, detail) + return {'retry': False, 'wait': {'reason': 'Started', 'retryAfterSeconds': retry_after, 'hostname': vm_name}} + + if result in ('AtMaximum', 'NoCandidate'): + logger.warning("No host could be started for %s (%s).", username, result) + return {'retry': False, 'wait': None} + + +START_ON_DEMAND_MESSAGES = { + 'Started': "A Linux host is starting for you.", + 'AlreadyStarting': "A Linux host is starting.", +} + + +def starting_response(wait): + """202: no host is ready yet, so the AVD host should ask again after Retry-After seconds.""" + seconds = wait['retryAfterSeconds'] + lead = START_ON_DEMAND_MESSAGES.get(wait['reason'], "No Linux host is ready yet.") + response = jsonify({ + 'status': 'Starting', + 'reason': wait['reason'], + 'retryAfterSeconds': seconds, + 'message': f"{lead} Ask again in {seconds} seconds.", + }) + response.headers['Retry-After'] = str(seconds) + return response, 202 + + @app.route('/api/vms/checkout', methods=['POST']) @token_required([ROLE_AVD_HOST, ROLE_ADMIN], required_group_ids=[AVD_HOST_GROUP_ID]) @audited('vm.checkout', target_type='vm') @@ -2111,23 +2328,38 @@ def _checkout_vm(event): if not username: return error_response("The username contains no characters a Linux account can use.", 400) - event.update(username=username, avdhost=avdhost, outcome='Error') + client_version = parse_client_version(req_body.get('clientVersion')) + event.update(username=username, avdhost=avdhost, outcome='Error', clientVersion=client_version) g.audit_detail = {'username': username, 'avdhost': avdhost} + if client_version: + g.audit_detail['clientVersion'] = client_version user_password = generate_secure_password() - with db_connection() as conn: - with conn.cursor(as_dict=True) as cursor: - cursor.callproc('CheckoutVm', (username, avdhost)) - rows = cursor.fetchall() - conn.commit() - + rows = call_checkout_vm(username, avdhost) if rows and is_procedure_error(rows[0]): logger.error("CheckoutVm failed with SQL error %s.", rows[0].get('ErrorNumber')) return error_response("Unable to check out a virtual machine.", 500) - if not rows or 'Message' in rows[0]: - event['outcome'] = 'NoneAvailable' - return error_response("No available VM found. Please try again.", 409) + if found_no_host(rows): + # Start on demand: start a host for this user, or wait for one already starting, + # instead of refusing them. + decision = start_host_on_demand(username, avdhost, client_version) + if decision['retry']: + rows = call_checkout_vm(username, avdhost) + if rows and is_procedure_error(rows[0]): + logger.error("CheckoutVm failed with SQL error %s.", rows[0].get('ErrorNumber')) + return error_response("Unable to check out a virtual machine.", 500) + + if found_no_host(rows): + wait = decision['wait'] + if wait: + event['outcome'] = 'Starting' + event['hostname'] = wait.get('hostname') + g.audit_detail['startOnDemand'] = wait['reason'] + return starting_response(wait) + + event['outcome'] = 'NoneAvailable' + return error_response("No available VM found. Please try again.", 409) checked_out_vm = rows[0] vmid = checked_out_vm.get("VMID") @@ -3174,6 +3406,47 @@ def append_scaling_note(activity_id, note): except Exception: logger.exception("Could not append a note to scaling activity %s.", activity_id) +FLEET_SNAPSHOT_FIELDS = ( + 'ReadyHosts', 'PoweredOn', 'Serviceable', 'InUse', 'Waiting', 'Booting', 'StaleHeartbeats', + 'NfsUnreachable', 'XrdpInactive', 'Draining', 'Maintenance', 'TotalHosts', + 'EffectiveMinVMs', 'MaxVMs', 'StartOnDemandEnabled', 'StaleAfterSeconds', +) +_fleet_snapshot_state = {'missing_logged': False} + + +def log_fleet_snapshot(): + """Log dbo.GetFleetSnapshot as "fleet snapshot", each figure a custom dimension. + + The scaling trigger runs every five minutes, and the monitoring workbook and alerts read + these from AppTraces. Returns the figures logged, or None. Never raises: a scaling run must + not fail because its snapshot could not be read. A database without dbo.GetFleetSnapshot, + while the API is upgraded ahead of SQL, is logged once per process. + """ + try: + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC GetFleetSnapshot") + row = cursor.fetchone() + except Exception as e: + if not is_missing_procedure_error(e): + logger.warning("Could not read the fleet snapshot.", exc_info=True) + elif not _fleet_snapshot_state['missing_logged']: + _fleet_snapshot_state['missing_logged'] = True + logger.warning("GetFleetSnapshot is not deployed yet; fleet snapshots are not logged until the SQL upgrade.") + return None + if not row: + return None + # Integers, so the queries' toint() reads them; a figure the database left NULL, such as the + # minimum with no scaling rule configured, is left out. + snapshot = {} + for field in FLEET_SNAPSHOT_FIELDS: + try: + snapshot[field] = int(row[field]) + except (KeyError, TypeError, ValueError): + continue + logger.info("fleet snapshot", extra=snapshot) + return snapshot + @app.route('/api/scaling/trigger', methods=['POST']) @token_required([ROLE_SCHEDULED_TASK, ROLE_ADMIN]) @audited('scaling.trigger', target_type='fleet') @@ -3263,6 +3536,8 @@ def trigger_scaling_logic(): 'corrections': len(corrections), } + log_fleet_snapshot() + return jsonify({ 'PoweredOnVMs': powered_on_vms, 'PoweredOffVMs': powered_off_vms, @@ -3340,14 +3615,31 @@ def parse_stop_mode(value): raise RuleValidationError("stopmode must be PowerOff or Deallocate.") return STOP_MODES[value.strip().lower()] -def validate_rule(rule): +def start_on_demand_enabled(): + """Whether start on demand is on. A database that predates it (144, 149) has it off.""" + try: + with db_connection() as conn: + with conn.cursor(as_dict=True) as cursor: + cursor.execute("EXEC GetScalingPolicy") + row = cursor.fetchone() + except DatabaseUnavailable: + raise + except Exception as e: + if is_missing_procedure_error(e): + return False + raise + return bool((row or {}).get('StartOnDemandEnabled')) + +def validate_rule(rule, zero_minimum_allowed=None): """Check a complete rule. Mirrors the CHECK constraints on dbo.VmScalingRules. - A minimum of zero is rejected because a pool with no running host cannot recover: - nothing starts a VM until a checkout succeeds, and no checkout can succeed. + A minimum of zero is accepted only while start on demand is on: without it, a pool with no + running host cannot recover, because nothing starts a VM until a checkout succeeds and no + checkout can succeed. zero_minimum_allowed says whether start on demand is on; when it is + None the policy is read, and only for a minimum of zero. """ - if rule['minvms'] < 1: - raise RuleValidationError("minvms must be at least 1.") + if rule['minvms'] < 0: + raise RuleValidationError("minvms must be at least 0.") if rule['maxvms'] <= rule['minvms']: raise RuleValidationError("maxvms must be greater than minvms.") for field in ('scaleupratio', 'scaledownratio'): @@ -3358,6 +3650,10 @@ def validate_rule(rule): for field in ('scaleupincrement', 'scaledownincrement'): if rule[field] < 1: raise RuleValidationError(f"{field} must be at least 1.") + if rule['minvms'] == 0: + allowed = start_on_demand_enabled() if zero_minimum_allowed is None else zero_minimum_allowed + if not allowed: + raise RuleValidationError("minvms can be 0 only while start on demand is on.") def _rule_from_row(row): return { @@ -3777,6 +4073,40 @@ def _phase_from_policy(row): }) +def _optional_bool(value): + return None if value is None else bool(value) + + +def avd_host_scripts(policy): + """Which broker script the AVD hosts that asked for a checkout in the last seven days run. + + None when the database predates the summary (149). Outdated counts the hosts whose script + reports no version: it predates start on demand and gives up instead of waiting for a host. + """ + if 'AvdHostsSeen' not in policy: + return None + current = version_tuple(AVD_HOST_SCRIPT_VERSION) + versions = [] + for entry in _json_column(policy.get('AvdClientVersionsJson'), list) or []: + if isinstance(entry, dict) and entry.get('ClientVersion'): + reported = version_tuple(entry['ClientVersion']) + versions.append({ + 'ClientVersion': entry['ClientVersion'], + 'AvdHosts': int(entry.get('AvdHosts') or 0), + 'Current': reported is not None and current is not None and reported >= current, + }) + return { + 'Seen': int(policy.get('AvdHostsSeen') or 0), + 'Outdated': int(policy.get('AvdHostsOutdated') or 0), + 'OutdatedHostnames': [ + entry['AvdHost'] for entry in _json_column(policy.get('OutdatedAvdHostsJson'), list) or [] + if isinstance(entry, dict) and entry.get('AvdHost') + ], + 'Versions': versions, + 'CurrentVersion': AVD_HOST_SCRIPT_VERSION, + } + + @app.route('/api/scaling/policy', methods=['GET']) @token_required(READ_ROLES) def get_scaling_policy(): @@ -3805,6 +4135,11 @@ def get_scaling_policy(): 'Schedules': schedules, 'NextChange': next_phase_change(schedules, policy.get('LocalTime')), 'LastRun': last_run, + # None on a database that predates start on demand (144, 149). + 'StartOnDemandEnabled': _optional_bool(policy.get('StartOnDemandEnabled')), + 'MaxPendingStarts': policy.get('MaxPendingStarts'), + 'ZeroMinimumCount': policy.get('ZeroMinimumCount'), + 'AvdHostScripts': avd_host_scripts(policy), })), 200 except DatabaseUnavailable as e: @@ -3842,34 +4177,118 @@ def get_time_zones(): return error_response("Unable to list time zones.", 500) +def _policy_update_fields(body): + """The policy fields a request sets. Raises RuleValidationError.""" + fields = {} + zone = body.get('timezone') + if zone is not None: + if not isinstance(zone, str) or not zone.strip() or len(zone) > 64: + raise RuleValidationError("Provide the timezone as a name from /api/scaling/timezones.") + fields['timezone'] = zone.strip() + + enabled = body.get('startondemandenabled') + if enabled is not None: + if not isinstance(enabled, bool): + raise RuleValidationError("startondemandenabled must be true or false.") + fields['enabled'] = enabled + + pending = body.get('maxpendingstarts') + if pending is not None: + if isinstance(pending, str) and re.fullmatch(r'[0-9]{1,3}', pending.strip()): + pending = int(pending.strip()) + if isinstance(pending, bool) or not isinstance(pending, int) or not 1 <= pending <= 20: + raise RuleValidationError("maxpendingstarts must be a whole number from 1 to 20.") + fields['maxpendingstarts'] = pending + + if not fields: + raise RuleValidationError("Provide timezone, startondemandenabled, maxpendingstarts, or any of them.") + return fields + + +def _start_on_demand_message(row): + if not row.get('StartOnDemandEnabled'): + zero = int(row.get('ZeroMinimumCount') or 0) + if zero: + places = 'rule or window has' if zero == 1 else 'rules and windows have' + return (f"Start on demand is off. {zero} scaling {places} a minimum of 0, so scaling keeps one " + "host running for them.") + return "Start on demand is off. A user who finds no ready host is refused." + pending = row.get('MaxPendingStarts') + return f"Start on demand is on. Up to {pending} host{'s' if pending != 1 else ''} may start at once for waiting users." + + @app.route('/api/scaling/policy/update', methods=['POST']) @token_required(ADMIN_ROLES) @audited('scaling.policy_update', target_type='scaling') def update_scaling_policy(): - """Set the time zone every schedule window is read in.""" + """Set the time zone schedule windows are read in, and start on demand. + + Any of timezone, startondemandenabled and maxpendingstarts; they are applied together or + not at all. + """ try: g.audit_target_id = 'Policy' body = request.get_json(silent=True) body = body if isinstance(body, dict) else {} - zone = body.get('timezone') - if not isinstance(zone, str) or not zone.strip() or len(zone) > 64: - return error_response("Provide the timezone as a name from /api/scaling/timezones.", 400) + try: + fields = _policy_update_fields(body) + except RuleValidationError as e: + return error_response(e.client_message, 400) _, updated_by, _ = audit_actor() + zone_row = None + start_row = None + wants_start = 'enabled' in fields or 'maxpendingstarts' in fields with db_connection() as conn: with conn.cursor(as_dict=True) as cursor: - cursor.execute("EXEC SetScalingPolicyTimeZone @TimeZone = %s, @UpdatedBy = %s", (zone.strip(), updated_by)) - row = cursor.fetchone() or {} + if 'timezone' in fields: + cursor.execute("EXEC SetScalingPolicyTimeZone @TimeZone = %s, @UpdatedBy = %s", + (fields['timezone'], updated_by)) + zone_row = cursor.fetchone() or {} + if zone_row.get('Result') == 'InvalidTimeZone': + # Returning before the commit applies nothing else in the request either. + return error_response( + f"'{fields['timezone']}' is not a time zone SQL Server knows. Choose one from the list.", 400) + if wants_start: + try: + cursor.execute( + "EXEC SetScalingPolicyStartOnDemand @Enabled = %s, @MaxPendingStarts = %s, @UpdatedBy = %s", + (fields.get('enabled'), fields.get('maxpendingstarts'), updated_by)) + except Exception as e: + if is_missing_procedure_error(e): + logger.warning("SetScalingPolicyStartOnDemand is not deployed yet; start on demand cannot be changed.") + return error_response("Start on demand is not available until the database is upgraded.", 404) + raise + start_row = cursor.fetchone() or {} + if start_row.get('Result') == 'Invalid': + return error_response(str(start_row.get('Message') or 'The start on demand settings are not valid.'), 400) conn.commit() - result = row.get('Result') - if result == 'InvalidTimeZone': - return error_response(f"'{zone.strip()}' is not a time zone SQL Server knows. Choose one from the list.", 400) + results = [row.get('Result') for row in (zone_row, start_row) if row is not None] + result = 'Updated' if 'Updated' in results else 'Unchanged' + detail = {'result': result} + response = {'Result': result} + messages = [] + if zone_row is not None: + zone = zone_row.get('TimeZone') + detail.update({'from': zone_row.get('PreviousTimeZone'), 'to': zone}) + response['TimeZone'] = zone + messages.append(f"Schedules are now read in {zone}." if zone_row.get('Result') == 'Updated' + else f"Schedules were already read in {zone}.") + if start_row is not None: + enabled = _optional_bool(start_row.get('StartOnDemandEnabled')) + detail['startOnDemand'] = {'from': _optional_bool(start_row.get('PreviousStartOnDemandEnabled')), 'to': enabled} + detail['maxPendingStarts'] = {'from': start_row.get('PreviousMaxPendingStarts'), 'to': start_row.get('MaxPendingStarts')} + response.update({ + 'StartOnDemandEnabled': enabled, + 'MaxPendingStarts': start_row.get('MaxPendingStarts'), + 'ZeroMinimumCount': start_row.get('ZeroMinimumCount'), + }) + messages.append(_start_on_demand_message(start_row)) - g.audit_detail = {'from': row.get('PreviousTimeZone'), 'to': row.get('TimeZone'), 'result': result} - return jsonify({'TimeZone': row.get('TimeZone'), 'Result': result, - 'message': f"Schedules are now read in {row.get('TimeZone')}." if result == 'Updated' - else f"Schedules were already read in {row.get('TimeZone')}."}), 200 + g.audit_detail = detail + response['message'] = ' '.join(messages) + return jsonify(serialize_for_json(response)), 200 except DatabaseUnavailable as e: logger.error("Database connection failed while updating the scaling policy.") @@ -4100,8 +4519,11 @@ def preview_scaling(): 'Serviceable': row.get('Serviceable'), 'InUse': row.get('InUse'), 'Draining': row.get('Draining'), + # Users waiting for a host to start (4.1); counted as demand. + 'Waiting': row.get('Waiting'), 'Utilization': float(utilization) if utilization is not None else None, }, + 'StartOnDemandEnabled': _optional_bool(row.get('StartOnDemandEnabled')), 'TimeZone': row.get('TimeZone'), 'LocalTime': row.get('LocalTime'), 'AtUtc': row.get('AtUtc'), @@ -4412,7 +4834,7 @@ def get_host_settings_history(): HEARTBEAT_SCRIPTS = ( 'release-session.sh', 'logind-session-watcher.sh', 'xrdp-who-xorg.sh', 'create-user.sh', 'manage-lease.sh', 'apply-host-settings.sh', 'session-control.sh', - 'patch-host.sh', 'xrdp-startwm.sh', + 'patch-host.sh', 'xrdp-startwm.sh', 'install-host-config.sh', ) HEARTBEAT_DESKTOPS = ('gnome', 'xfce', 'mate', 'kde', 'other', 'none', 'unknown') HEARTBEAT_SESSION_STATES = ('active', 'disconnected', 'unknown') @@ -4837,6 +5259,9 @@ def get_host_health(): ATTENTION_MAX_HOSTNAMES = 10 CHECKOUT_STAT_COUNTS = ( 'Total', 'Assigned', 'Reused', 'NoneAvailable', 'ProvisionFailed', 'Errors', 'DeniedLastHour', 'HostStarts', + # Start on demand (154): Starting answers, the users who waited, those a host was then + # given to, and those waiting now. Starting answers are not in Total. + 'Starting', 'Waits', 'WaitsServed', 'WaitingNow', ) @@ -4875,6 +5300,7 @@ def utilization_point(row): 'Checkouts': int(row.get('Checkouts') or 0), 'Denied': int(row.get('Denied') or 0), 'Failed': int(row.get('Failed') or 0), + 'Waited': int(row.get('Waited') or 0), } @@ -4885,6 +5311,8 @@ def checkout_stats(row): 'P95Ms': _optional_int(row.get('P95Ms')), 'StartP50Seconds': _optional_int(row.get('StartP50Seconds')), 'StartP95Seconds': _optional_int(row.get('StartP95Seconds')), + 'WaitP50Seconds': _optional_int(row.get('WaitP50Seconds')), + 'WaitP95Seconds': _optional_int(row.get('WaitP95Seconds')), 'LastDeniedUtc': row.get('LastDeniedUtc'), 'DeniedPercent': round(stats['NoneAvailable'] * 100.0 / stats['Total'], 1) if stats['Total'] else None, }) diff --git a/api/config.py b/api/config.py index e4ea3bb..1f7e123 100644 --- a/api/config.py +++ b/api/config.py @@ -110,6 +110,15 @@ def env_int(name, default, minimum=None, maximum=None): DB_MAX_CONCURRENCY = env_int('DB_MAX_CONCURRENCY', 6, minimum=1, maximum=200) DB_ACQUIRE_TIMEOUT_SECONDS = env_int('DB_ACQUIRE_TIMEOUT_SECONDS', 15, minimum=1, maximum=120) +# Connections a worker process has finished with are kept for its next request (db_pool.py). +# The API reaches Azure SQL through App Service's outbound load balancer, which forgets a +# connection after four minutes without traffic, so idle connections are closed well before +# then. The lifetime limit makes connections sign in again regularly, with the password +# refresh_db_password last read from Key Vault. +DB_POOL_ENABLED = env_bool('DB_POOL_ENABLED', True) +DB_POOL_IDLE_SECONDS = env_int('DB_POOL_IDLE_SECONDS', 120, minimum=5, maximum=3600) +DB_POOL_MAX_LIFETIME_SECONDS = env_int('DB_POOL_MAX_LIFETIME_SECONDS', 1800, minimum=60, maximum=86400) + SWEEP_CONCURRENCY = env_int('SWEEP_CONCURRENCY', 8, minimum=1, maximum=64) SWEEP_DEADLINE_SECONDS = env_int('SWEEP_DEADLINE_SECONDS', 40, minimum=5, maximum=100) APPLY_CONCURRENCY = env_int('APPLY_CONCURRENCY', 10, minimum=1, maximum=64) @@ -197,6 +206,28 @@ def env_int(name, default, minimum=None, maximum=None): ATTENTION_NOT_CONNECTED_MINUTES = 30 ATTENTION_DENIED_MINUTES = 60 +# =============================== +# Start on demand +# +# When a checkout finds no ready host and start on demand is on, dbo.ReserveVmForStart starts a +# stopped host for the user, and the API answers 202 with how long to wait before asking again. +# A waiting user's next request first probes the hosts starting for it on the port the scheduled +# task's reachability probe uses, so a host that is up is handed out without waiting for the +# task's next run. The probes run in parallel and each waits at most the timeout. +START_ON_DEMAND_PROBE_PORT = 22 +START_ON_DEMAND_PROBE_TIMEOUT_SECONDS = 2 +START_ON_DEMAND_PROBE_CONCURRENCY = 10 +# The wait the API asks for when it could not decide: scaling held the lock, or the host that +# became ready went to another user. dbo.ReserveVmForStart asks for 30 to 120 seconds otherwise. +START_ON_DEMAND_RETRY_SECONDS = 30 +# The AVD host script reports its version with each checkout. It is recorded with the checkout +# event, so the scaling policy page can list the AVD hosts whose script cannot wait for a host. +CLIENT_VERSION_MAX_CHARS = 32 +# The version avd_host/broker/Connect-LinuxBroker.ps1 declares as $ScriptVersion. Bump it with +# any change to that script; api/tests checks they agree. The scaling policy page shows the AVD +# hosts that report an older one. +AVD_HOST_SCRIPT_VERSION = '2.0.0' + # =============================== # Linux host agent # @@ -204,7 +235,7 @@ def env_int(name, default, minimum=None, maximum=None): # with any change to those scripts; api/tests checks they agree. Fleet health flags a host # whose reported agent or scripts are older. The override exists so an operator can silence # the flag during a staged rollout. -HOST_AGENT_VERSION = '1.2.0' +HOST_AGENT_VERSION = '1.3.0' EXPECTED_HOST_AGENT_VERSION = (os.environ.get('EXPECTED_HOST_AGENT_VERSION') or '').strip() or HOST_AGENT_VERSION HEARTBEAT_MAX_BYTES = 32 * 1024 diff --git a/api/db_pool.py b/api/db_pool.py new file mode 100644 index 0000000..1be77f7 --- /dev/null +++ b/api/db_pool.py @@ -0,0 +1,176 @@ +"""Reuse SQL connections within a Broker API worker process. + +Opening a connection to Azure SQL costs a TCP and TLS handshake and a login, and on App +Service every closed connection also holds one of the instance's outbound SNAT ports for +four minutes. The pool keeps the connections a worker process has finished with for its +next request. + +A connection is only reused in a clean state. When it comes back it is rolled back, so +no uncommitted work or lock outlives the request, and it waits in the pool with no +transaction open. It begins a new transaction only as it is handed out again, which also +proves it still answers, so the transaction starts when the request does; see _reset +for why that matters. Nothing else the API does outlives a transaction on its +connection: application locks are owned by the transaction, and the API sets no session +options, temporary tables or session context. Keep it that way, or set +DB_POOL_ENABLED=false. + +db_connection() still bounds how many connections a process holds. A connection is +opened only by a thread holding one of its slots, and is back in the pool or closed +before that slot is released, so idle and busy connections together stay within +DB_MAX_CONCURRENCY. +""" + +import logging +import os +import threading +import time +from dataclasses import dataclass + +# A child of the logger the API sends to Application Insights. +logger = logging.getLogger('linuxbroker.api.db_pool') + + +@dataclass +class PooledConnection: + """A connection handed out by the pool, with what the pool needs to take it back.""" + + connection: object + opened_at: float + pid: int + returned_at: float = 0.0 + + +class ConnectionPool: + """Idle connections of one process, the most recently returned reused first. + + Reusing the most recent keeps the rest idle, so when load drops they reach the idle + limit and are closed rather than kept open for nobody. + """ + + def __init__(self, max_idle, idle_seconds, max_lifetime_seconds, + clock=time.monotonic, getpid=os.getpid): + self.max_idle = max_idle + self.idle_seconds = idle_seconds + self.max_lifetime_seconds = max_lifetime_seconds + self._clock = clock + self._getpid = getpid + self._lock = threading.Lock() + self._idle = [] + self._pid = getpid() + # Connections a forked process inherited from its parent. They share the + # parent's sockets, so this process neither uses nor closes them, and the + # reference keeps garbage collection from closing them either. + self._inherited = [] + + @property + def idle_count(self): + with self._lock: + return len(self._idle) + + def acquire(self, connect): + """Hand out an idle connection that still answers, or a new one from connect(). + + Returns None when connect() does. + """ + while True: + pooled, expired = self._take_idle() + for stale in expired: + self._close(stale.connection) + if pooled is None: + break + if self._begin(pooled.connection): + return pooled + logger.warning("A pooled database connection stopped answering. It was closed " + "and another connection is being used.") + self._close(pooled.connection) + + connection = connect() + if connection is None: + return None + return PooledConnection(connection, opened_at=self._clock(), pid=self._getpid()) + + def release(self, pooled, reusable=True): + """Take a connection back, or close it when it cannot be reused.""" + if pooled.pid != self._getpid(): + # Opened before this process was forked, so it belongs to the parent. + with self._lock: + self._inherited.append(pooled) + return + + if reusable and self._reset(pooled.connection): + now = self._clock() + with self._lock: + if (pooled.pid == self._pid and len(self._idle) < self.max_idle + and not self._too_old(pooled, now)): + pooled.returned_at = now + self._idle.append(pooled) + return + self._close(pooled.connection) + + def close_idle(self): + """Close every idle connection.""" + with self._lock: + idle, self._idle = self._idle, [] + for pooled in idle: + self._close(pooled.connection) + + def _take_idle(self): + now = self._clock() + with self._lock: + pid = self._getpid() + if pid != self._pid: + self._inherited.extend(self._idle) + self._idle = [] + self._pid = pid + expired = [pooled for pooled in self._idle if self._expired(pooled, now)] + if expired: + self._idle = [pooled for pooled in self._idle if not self._expired(pooled, now)] + pooled = self._idle.pop() if self._idle else None + return pooled, expired + + def _expired(self, pooled, now): + return now - pooled.returned_at >= self.idle_seconds or self._too_old(pooled, now) + + def _too_old(self, pooled, now): + return now - pooled.opened_at >= self.max_lifetime_seconds + + @staticmethod + def _begin(connection): + # pymssql begins a transaction when autocommit is turned off, as it does when a + # connection opens. The request's transaction therefore starts now, not when the + # connection went idle; see _reset. The round trip also shows the connection + # still answers. + try: + connection.autocommit(False) + return True + except Exception: + return False + + @staticmethod + def _reset(connection): + # Rolls back whatever the request left uncommitted, releasing its locks, and ends + # the transaction pymssql otherwise keeps open. System-versioned tables stamp each + # change with its transaction's start time, and SQL Server refuses a change stamped + # earlier than the row's current version (error 13535). A transaction begun when + # the connection went idle would be older than every change made meanwhile. + try: + connection.autocommit(True) + return True + except Exception: + pass + try: + # The transaction was already gone, as after an error that aborts the batch, so + # its rollback failed. rollback() tolerates that and begins a new one to end. + connection.rollback() + connection.autocommit(True) + return True + except Exception as error: + logger.info("A database connection could not be rolled back, so it was closed: %s", error) + return False + + @staticmethod + def _close(connection): + try: + connection.close() + except Exception: + logger.debug("Closing a database connection failed.", exc_info=True) diff --git a/api/env.example b/api/env.example index f748571..92f7741 100644 --- a/api/env.example +++ b/api/env.example @@ -59,6 +59,9 @@ ALLOW_LEGACY_SCOPE_ACCESS="false" # Optional tuning; see api/README.md for the defaults. # DB_MAX_CONCURRENCY="6" +# DB_POOL_ENABLED="true" +# DB_POOL_IDLE_SECONDS="120" +# DB_POOL_MAX_LIFETIME_SECONDS="1800" # GUNICORN_CMD_ARGS="--workers 2 --threads 8 --timeout 120" # Audit entries older than this many days are purged daily by the scheduled task diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 9b4f010..446e4ff 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -25,6 +25,9 @@ "DB_PASSWORD_NAME": "db-password", "MICROSOFT_PROVIDER_AUTHENTICATION_SECRET": "provider-secret", "NFS_SHARE": "/mnt/test", + # Each test counts the connections its requests open. test_db_pool.py covers the pool, + # and the integration tests run with it on. + "DB_POOL_ENABLED": "false", } for key, value in _ENV.items(): os.environ.setdefault(key, value) @@ -160,6 +163,8 @@ class FakeConnection: def __init__(self, db): self.db = db self.closed = False + self.rollbacks = 0 + self.autocommit_calls = [] self.db.connections.append(self) def cursor(self, as_dict=False): @@ -168,6 +173,12 @@ def cursor(self, as_dict=False): def commit(self): self.db.commits += 1 + def rollback(self): + self.rollbacks += 1 + + def autocommit(self, status): + self.autocommit_calls.append(status) + def close(self): self.closed = True diff --git a/api/tests/test_db_pool.py b/api/tests/test_db_pool.py new file mode 100644 index 0000000..f4ff597 --- /dev/null +++ b/api/tests/test_db_pool.py @@ -0,0 +1,376 @@ +"""The per-process SQL connection pool behind db_connection().""" + +import logging +import threading +import types + +import pytest + +from db_pool import ConnectionPool + + +class Clock: + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + + def advance(self, seconds): + self.now += seconds + + +class Connection: + """Records what the pool does with a connection, as pymssql would see it.""" + + def __init__(self, opener, name): + self.opener = opener + self.name = name + self.begins = 0 # autocommit(False): BEGIN TRAN + self.ends = 0 # autocommit(True): ROLLBACK TRAN + self.rollbacks = 0 + self.transaction_open = True # pymssql begins one as the connection opens + self.closed = False + self.dead = False + + def autocommit(self, status): + if self.dead: + raise RuntimeError("DBPROCESS is dead or not enabled") + if status: + if not self.transaction_open: + raise RuntimeError("The ROLLBACK TRANSACTION request has no corresponding BEGIN TRANSACTION.") + self.ends += 1 + self.transaction_open = False + else: + self.begins += 1 + self.transaction_open = True + + def rollback(self): + # pymssql ignores a rollback with no transaction to roll back, then begins one. + if self.dead: + raise RuntimeError("DBPROCESS is dead or not enabled") + self.rollbacks += 1 + self.transaction_open = True + + def close(self): + if not self.closed: + self.closed = True + self.opener.closed_one() + + +class Opener: + """Stands in for get_db_connection and tracks how many connections are open at once.""" + + def __init__(self): + self.opened = [] + self.open_now = 0 + self.most_open = 0 + self._lock = threading.Lock() + + def __call__(self): + with self._lock: + connection = Connection(self, f"connection-{len(self.opened) + 1}") + self.opened.append(connection) + self.open_now += 1 + self.most_open = max(self.most_open, self.open_now) + return connection + + def closed_one(self): + with self._lock: + self.open_now -= 1 + + +@pytest.fixture +def clock(): + return Clock() + + +@pytest.fixture +def opener(): + return Opener() + + +@pytest.fixture +def pool(clock): + return ConnectionPool(max_idle=3, idle_seconds=120, max_lifetime_seconds=1800, clock=clock) + + +# ================================================================== reuse + + +def test_a_returned_connection_waits_with_no_transaction_and_begins_one_when_reused(pool, opener): + first = pool.acquire(opener) + pool.release(first) + + # Rolled back, and no transaction left open while it waits. + assert first.connection.ends == 1 + assert not first.connection.transaction_open + + again = pool.acquire(opener) + + assert again.connection is first.connection + assert len(opener.opened) == 1 + # The request's transaction begins as it is handed out, never while it waited. + assert first.connection.begins == 1 + assert first.connection.transaction_open + assert not first.connection.closed + + +def test_the_most_recently_returned_connection_is_reused_first(pool, opener): + older, newer = pool.acquire(opener), pool.acquire(opener) + pool.release(older) + pool.release(newer) + + assert pool.acquire(opener).connection is newer.connection + + +def test_no_connection_is_handed_out_when_none_can_be_opened(pool): + assert pool.acquire(lambda: None) is None + + +def test_no_more_than_max_idle_connections_are_kept(pool, opener): + busy = [pool.acquire(opener) for _ in range(4)] + for pooled in busy: + pool.release(pooled) + + assert pool.idle_count == 3 + assert [pooled.connection.closed for pooled in busy] == [False, False, False, True] + + +def test_close_idle_closes_every_idle_connection(pool, opener): + busy = [pool.acquire(opener) for _ in range(2)] + for pooled in busy: + pool.release(pooled) + + pool.close_idle() + + assert pool.idle_count == 0 + assert all(pooled.connection.closed for pooled in busy) + + +# ========================================================= discarding + + +def test_a_connection_whose_block_failed_is_closed_rather_than_reused(pool, opener): + first = pool.acquire(opener) + + pool.release(first, reusable=False) + + assert first.connection.closed + assert first.connection.ends == 0 + assert pool.acquire(opener).connection is not first.connection + + +def test_a_connection_that_cannot_roll_back_is_closed(pool, opener): + first = pool.acquire(opener) + first.connection.dead = True + + pool.release(first) + + assert first.connection.closed + assert pool.idle_count == 0 + + +def test_a_connection_whose_transaction_was_already_rolled_back_is_kept(pool, opener): + first = pool.acquire(opener) + # An error that aborts the batch rolls the transaction back on the server. + first.connection.transaction_open = False + + pool.release(first) + + assert not first.connection.closed + assert first.connection.rollbacks == 1 + assert not first.connection.transaction_open + assert pool.idle_count == 1 + + +def test_a_connection_that_stopped_answering_is_replaced_without_failing_the_request(pool, opener, caplog): + first = pool.acquire(opener) + pool.release(first) + first.connection.dead = True # a failover or a gateway restart while it sat idle + + with caplog.at_level(logging.WARNING, logger="linuxbroker.api.db_pool"): + replacement = pool.acquire(opener) + + assert replacement.connection is opener.opened[1] + assert first.connection.closed + assert "stopped answering" in caplog.text + + +# ============================================================ time limits + + +def test_idle_connections_are_closed_before_the_outbound_load_balancer_forgets_them(pool, opener, clock): + first = pool.acquire(opener) + pool.release(first) + clock.advance(119) + reused = pool.acquire(opener) + assert reused.connection is first.connection + pool.release(reused) + + clock.advance(120) + replacement = pool.acquire(opener) + + assert replacement.connection is not first.connection + assert first.connection.closed + # It was closed without being tried: an idle connection past the limit may no longer answer at all. + assert first.connection.begins == 1 + + +def test_an_idle_connection_past_its_lifetime_is_not_reused(pool, opener, clock): + first = pool.acquire(opener) + clock.advance(1750) # one long request + pool.release(first) + assert pool.idle_count == 1 + + clock.advance(60) # idle for only a minute, but 1810 seconds old + replacement = pool.acquire(opener) + + assert replacement.connection is not first.connection + assert first.connection.closed + + +def test_a_connection_past_its_lifetime_is_closed_when_it_comes_back(pool, opener, clock): + first = pool.acquire(opener) + clock.advance(1800) + + pool.release(first) + + assert first.connection.closed + assert pool.idle_count == 0 + + +# ================================================================ forking + + +def forked_pool(clock): + process = {"pid": 100} + return ConnectionPool(max_idle=3, idle_seconds=120, max_lifetime_seconds=1800, clock=clock, + getpid=lambda: process["pid"]), process + + +def test_a_forked_process_leaves_its_parents_idle_connections_alone(clock, opener): + pool, process = forked_pool(clock) + parents = pool.acquire(opener) + pool.release(parents) + + process["pid"] = 200 # gunicorn --preload forked a worker after the pool was used + childs = pool.acquire(opener) + + assert childs.connection is not parents.connection + # The parent's connection shares its socket: never tried, never closed. + assert parents.connection.begins == 0 + assert not parents.connection.closed + + pool.release(childs) + assert pool.acquire(opener).connection is childs.connection + + +def test_a_connection_opened_before_a_fork_is_neither_pooled_nor_closed_after_it(clock, opener): + pool, process = forked_pool(clock) + parents = pool.acquire(opener) + + process["pid"] = 200 + pool.release(parents) + + assert not parents.connection.closed + assert pool.idle_count == 0 + + +# ======================================================== db_connection() + + +@pytest.fixture +def pooled_api(app_module, monkeypatch, clock): + opener = Opener() + pool = ConnectionPool(max_idle=2, idle_seconds=120, max_lifetime_seconds=1800, clock=clock) + slots = threading.BoundedSemaphore(2) + monkeypatch.setattr(app_module, "_db_pool", pool) + monkeypatch.setattr(app_module, "_db_slots", slots) + monkeypatch.setattr(app_module, "get_db_connection", opener) + return types.SimpleNamespace(pool=pool, opener=opener, slots=slots) + + +def slots_free(slots, count): + taken = [slots.acquire(timeout=0) for _ in range(count)] + for ok in taken: + if ok: + slots.release() + return all(taken) + + +def test_db_connection_reuses_one_connection_across_blocks(app_module, pooled_api): + with app_module.db_connection() as first: + pass + with app_module.db_connection() as second: + pass + + assert second is first + assert len(pooled_api.opener.opened) == 1 + assert slots_free(pooled_api.slots, 2) + + +def test_db_connection_closes_the_connection_when_its_block_raises(app_module, pooled_api): + with pytest.raises(RuntimeError, match="boom"): + with app_module.db_connection() as conn: + raise RuntimeError("boom") + + assert conn.closed + assert pooled_api.pool.idle_count == 0 + assert slots_free(pooled_api.slots, 2) + + +def test_db_connection_frees_its_slot_when_sql_is_unreachable(app_module, pooled_api, monkeypatch): + monkeypatch.setattr(app_module, "get_db_connection", lambda: None) + + with pytest.raises(app_module.DatabaseUnavailable): + with app_module.db_connection(): + pass + + assert slots_free(pooled_api.slots, 2) + + +def test_idle_and_busy_connections_stay_within_the_slot_count(app_module, pooled_api): + errors = [] + + def work(): + try: + for _ in range(50): + with app_module.db_connection() as conn: + assert not conn.closed + except Exception as error: # pragma: no cover - reported below + errors.append(error) + + threads = [threading.Thread(target=work) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert errors == [] + assert pooled_api.opener.most_open <= 2 + assert slots_free(pooled_api.slots, 2) + + +def test_requests_share_pooled_connections(app_module, client, fake_db, monkeypatch): + monkeypatch.setattr(app_module, "_db_pool", ConnectionPool(max_idle=2, idle_seconds=120, + max_lifetime_seconds=1800)) + + assert client.get("/api/vms").status_code == 200 + assert client.get("/api/vms").status_code == 200 + + assert len(fake_db.connections) == 1 + # Ended after the first request, begun for the second, ended after it. + assert fake_db.connections[0].autocommit_calls == [True, False, True] + assert not fake_db.connections[0].closed + + +def test_the_pool_can_be_turned_off(app_module, client, fake_db): + # The unit tests run with DB_POOL_ENABLED=false, as an operator can. + assert app_module._db_pool is None + + client.get("/api/vms") + client.get("/api/vms") + + assert len(fake_db.connections) == 2 + assert all(connection.closed for connection in fake_db.connections) diff --git a/api/tests/test_fleet_snapshot.py b/api/tests/test_fleet_snapshot.py new file mode 100644 index 0000000..2e603d9 --- /dev/null +++ b/api/tests/test_fleet_snapshot.py @@ -0,0 +1,86 @@ +"""4.7 fleet snapshot: every scaling run logs the fleet's figures for the monitoring workbook and +alerts, and a snapshot that cannot be read never fails the run.""" + +import logging +import re +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MISSING = "(2812, b\"Could not find stored procedure '{}'.\")" +SNAPSHOT = { + "ReadyHosts": 2, "PoweredOn": 5, "Serviceable": 4, "InUse": 2, "Waiting": 1, "Booting": 1, + "StaleHeartbeats": 0, "NfsUnreachable": 0, "XrdpInactive": 1, "Draining": 1, "Maintenance": 0, + "TotalHosts": 8, "EffectiveMinVMs": None, "MaxVMs": None, "StartOnDemandEnabled": True, + "StaleAfterSeconds": 180, +} + + +def snapshots(caplog): + return [record for record in caplog.records if record.getMessage() == "fleet snapshot"] + + +def trigger(client): + return client.post("/api/scaling/trigger", json={}) + + +def test_each_scaling_run_logs_the_fleet_after_scaling(client, fake_db, caplog): + fake_db.fetchall_rows["GetVms"] = [] + fake_db.fetchone_rows["GetFleetSnapshot"] = dict(SNAPSHOT) + caplog.set_level(logging.INFO, logger="linuxbroker.api") + + assert trigger(client).status_code == 200 + + procs = [call["proc"] for call in fake_db.calls] + assert procs.index("TriggerScalingLogic") < procs.index("GetFleetSnapshot") + [record] = snapshots(caplog) + assert record.levelno == logging.INFO + assert (record.ReadyHosts, record.Waiting, record.XrdpInactive, record.TotalHosts) == (2, 1, 1, 8) + assert record.StartOnDemandEnabled == 1 and type(record.StartOnDemandEnabled) is int + # With no scaling rule the database has no minimum or maximum, and a dimension is never empty. + assert not hasattr(record, "EffectiveMinVMs") and not hasattr(record, "MaxVMs") + + +def test_the_logged_figures_are_the_procedures_columns(app_module): + sql = (REPO_ROOT / "sql_queries" / "157_create_procedure-GetFleetSnapshot.sql").read_text() + final_select = sql[sql.rindex("SELECT"):sql.rindex("FROM dbo.VirtualMachines")] + + assert tuple(re.findall(r"\bAS (\w+)", final_select)) == app_module.FLEET_SNAPSHOT_FIELDS + + +def test_a_database_without_the_snapshot_is_logged_once(client, fake_db, caplog): + fake_db.fetchall_rows["GetVms"] = [] + fake_db.raise_on_execute["GetFleetSnapshot"] = MISSING.format("GetFleetSnapshot") + + assert trigger(client).status_code == 200 + assert trigger(client).status_code == 200 + assert caplog.text.count("GetFleetSnapshot is not deployed yet") == 1 + assert snapshots(caplog) == [] + + +def test_a_snapshot_that_cannot_be_read_never_fails_the_scaling_run(client, fake_db, caplog): + fake_db.fetchall_rows["GetVms"] = [] + fake_db.fetchall_rows["TriggerScalingLogic"] = [ + {"ActionType": "PowerOn", "VMName": "lnx-01", "VMID": 1, "StopMode": None, "ActivityID": 3}, + ] + fake_db.raise_on_execute["GetFleetSnapshot"] = "connection reset" + + first = trigger(client) + trigger(client) + + assert first.status_code == 200 and first.get_json()["PoweredOnVMs"] == ["lnx-01"] + assert caplog.text.count("Could not read the fleet snapshot.") == 2 + + +def test_an_unreachable_database_skips_the_snapshot(app_module, monkeypatch, caplog): + monkeypatch.setattr(app_module, "get_db_connection", lambda: None) + + assert app_module.log_fleet_snapshot() is None + assert "Could not read the fleet snapshot." in caplog.text + + +def test_no_row_logs_nothing(app_module, fake_db, caplog): + caplog.set_level(logging.INFO, logger="linuxbroker.api") + + assert app_module.log_fleet_snapshot() is None + assert snapshots(caplog) == [] diff --git a/api/tests/test_host_list.py b/api/tests/test_host_list.py index d197974..8bbe7b3 100644 --- a/api/tests/test_host_list.py +++ b/api/tests/test_host_list.py @@ -10,7 +10,7 @@ def page_row(hostname, **values): row = {"VMID": 1, "Hostname": hostname, "IPAddress": "10.0.0.4", "PowerState": "On", "NetworkStatus": "Reachable", "VmStatus": "CheckedOut", "Username": "alice", "Ready": False, "CleanupPending": False, "DrainRequested": False, - "SettingsVersion": 3, "CurrentSettingsVersion": 3, "AgentVersion": "1.2.0", "HeartbeatAgeSeconds": 20, + "SettingsVersion": 3, "CurrentSettingsVersion": 3, "AgentVersion": "1.3.0", "HeartbeatAgeSeconds": 20, "ReconcileIntervalSeconds": 60, "SessionsJson": json.dumps([{"username": "Alice", "state": "disconnected"}]), "TotalCount": 41} row.update(values) diff --git a/api/tests/test_lifecycle_scaling_settings.py b/api/tests/test_lifecycle_scaling_settings.py index 1cb6705..f34b310 100644 --- a/api/tests/test_lifecycle_scaling_settings.py +++ b/api/tests/test_lifecycle_scaling_settings.py @@ -403,7 +403,7 @@ def test_a_refused_power_operation_restores_the_recorded_state(app_module, clien @pytest.mark.parametrize("override,field", [ - ({"minvms": 0}, "minvms"), + ({"minvms": -1}, "minvms"), ({"maxvms": 2}, "maxvms"), ({"scaleupratio": 101}, "scaleupratio"), ({"scaleupratio": 20}, "scaleupratio"), @@ -419,6 +419,33 @@ def test_rule_creation_names_the_invalid_field(client, fake_db, override, field) assert fake_db.calls == [] +def test_a_rule_scales_to_zero_only_while_start_on_demand_is_on(client, fake_db): + zero = {**VALID_RULE, "minvms": 0} + + # Off, and on a database that predates start on demand, a pool with no host cannot recover. + for policy in ({"TimeZone": "UTC"}, {"TimeZone": "UTC", "StartOnDemandEnabled": False}): + fake_db.fetchone_rows["GetScalingPolicy"] = policy + response = client.post("/api/scaling/rules/create", json=zero) + assert response.status_code == 400 + assert response.get_json()["error"] == "minvms can be 0 only while start on demand is on." + fake_db.fetchone_rows.pop("GetScalingPolicy") + fake_db.raise_on_execute["GetScalingPolicy"] = "(2812, b\"Could not find stored procedure 'GetScalingPolicy'.\")" + assert client.post("/api/scaling/rules/create", json=zero).status_code == 400 + assert "CreateScalingRule" not in [call["proc"] for call in fake_db.calls] + + del fake_db.raise_on_execute["GetScalingPolicy"] + fake_db.fetchone_rows["GetScalingPolicy"] = {"TimeZone": "UTC", "StartOnDemandEnabled": True} + fake_db.fetchone_rows["CreateScalingRule"] = {"NewRuleID": 4, "ActiveRuleID": 4, "Message": None} + assert client.post("/api/scaling/rules/create", json=zero).status_code == 201 + assert fake_db.latest_call("CreateScalingRule")["params"][:2] == (0, 10) + + +def test_a_rule_above_zero_never_reads_the_policy(client, fake_db): + fake_db.fetchone_rows["CreateScalingRule"] = {"NewRuleID": 5, "ActiveRuleID": 5, "Message": None} + assert client.post("/api/scaling/rules/create", json=VALID_RULE).status_code == 201 + assert "GetScalingPolicy" not in [call["proc"] for call in fake_db.calls] + + def test_rule_creation_passes_the_stop_mode(client, fake_db): fake_db.fetchone_rows["CreateScalingRule"] = {"NewRuleID": 3, "ActiveRuleID": 3, "Message": None} response = client.post("/api/scaling/rules/create", json={**VALID_RULE, "stopmode": "deallocate"}) diff --git a/api/tests/test_phase2_foundations.py b/api/tests/test_phase2_foundations.py index e95e315..6237f1f 100644 --- a/api/tests/test_phase2_foundations.py +++ b/api/tests/test_phase2_foundations.py @@ -731,8 +731,8 @@ def health_row(hostname, **values): "VMID": 1, "Hostname": hostname, "PowerState": "On", "NetworkStatus": "Reachable", "VmStatus": "Available", "DrainRequested": False, "CleanupPending": False, "Username": None, "AppliedSettingsVersion": 7, "CurrentSettingsVersion": 7, "ReconcileIntervalSeconds": 60, "LastHeartbeatUtc": "2026-09-24T12:00:00Z", - "HeartbeatAgeSeconds": 30, "AgentVersion": "1.2.0", - "ScriptVersionsJson": '{"release-session.sh": "1.2.0", "create-user.sh": "1.2.0"}', + "HeartbeatAgeSeconds": 30, "AgentVersion": "1.3.0", + "ScriptVersionsJson": '{"release-session.sh": "1.3.0", "create-user.sh": "1.3.0"}', "ReportedSettingsVersion": 7, "OsId": "rhel", "OsVersion": "9.4", "OsName": "RHEL 9.4", "KernelVersion": "5.14", "Desktop": "gnome", "XrdpVersion": "0.10.1", "XrdpActive": True, "NfsReachable": True, "NfsMountCount": 1, "LoadAverage": 0.5, "CpuCount": 4, "MemoryAvailableMb": 8000, "MemoryTotalMb": 16000, "RootDiskFreePct": 60, @@ -750,7 +750,7 @@ def test_fleet_health_flags_what_an_operator_must_act_on(client, fake_db): health_row("off", PowerState="Off", HeartbeatAgeSeconds=9000, AppliedSettingsVersion=3), health_row("broken", XrdpActive=False, NfsReachable=False, RootDiskFreePct=4), health_row("old", AgentVersion="0.9.0"), - health_row("half", ScriptVersionsJson='{"release-session.sh": "1.2.0", "manage-lease.sh": null}'), + health_row("half", ScriptVersionsJson='{"release-session.sh": "1.3.0", "manage-lease.sh": null}'), health_row("drift", AppliedSettingsVersion=None), ] @@ -776,8 +776,8 @@ def test_fleet_health_flags_what_an_operator_must_act_on(client, fake_db): } healthy = body["Hosts"][0] assert healthy["Sessions"] == [{"username": "alice", "state": "active"}] - assert healthy["ScriptVersions"]["create-user.sh"] == "1.2.0" - assert body["ExpectedAgentVersion"] == "1.2.0" and body["StaleAfterSeconds"] == 180 + assert healthy["ScriptVersions"]["create-user.sh"] == "1.3.0" + assert body["ExpectedAgentVersion"] == "1.3.0" and body["StaleAfterSeconds"] == 180 def test_fleet_health_for_one_host(client, fake_db): diff --git a/api/tests/test_scaling_policy.py b/api/tests/test_scaling_policy.py index 2271196..2dac20f 100644 --- a/api/tests/test_scaling_policy.py +++ b/api/tests/test_scaling_policy.py @@ -124,7 +124,8 @@ def test_a_window_is_saved_after_validation(auth_client, fake_db, audit_entries) ({"days": []}, "Choose at least one day."), ({"start": "8:00"}, "start must be a time as HH:MM, from 00:00 to 23:59."), ({"end": "08:00"}, "start and end must differ."), - ({"minvms": 0}, "minvms must be at least 1."), + ({"minvms": -1}, "minvms must be at least 0."), + ({"minvms": 0}, "minvms can be 0 only while start on demand is on."), ({"maxvms": 3}, "maxvms must be greater than minvms."), ({"scaleupratio": 10}, "scaleupratio must be greater than scaledownratio."), ({"stopmode": "Hibernate"}, "stopmode must be PowerOff or Deallocate."), diff --git a/api/tests/test_start_on_demand.py b/api/tests/test_start_on_demand.py new file mode 100644 index 0000000..703c0c7 --- /dev/null +++ b/api/tests/test_start_on_demand.py @@ -0,0 +1,576 @@ +"""4.1 start on demand and scale to zero: a checkout that finds no ready host starts one and +answers 202, a waiting user's next request probes the hosts starting for it, the AVD host +script's version is recorded, and the policy, preview and metrics carry the new settings.""" + +import json +import re +import types +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +LEASE_ID = "8ff6eb09-90ca-4efa-8ea1-695761f950f7" +MISSING = "(2812, b\"Could not find stored procedure '{}'.\")" +NO_HOST = [{"Message": "No available VM found"}] + + +def checkout_row(**values): + row = {"VMID": 5, "Hostname": "lnx-05", "IPAddress": "10.0.0.5", "LeaseId": LEASE_ID, "CheckoutType": "Assigned", + "ProfileResetRequested": False} + row.update(values) + return row + + +def reservation(result, **values): + row = {"Result": result, "VMID": None, "VMName": None, "PreviousNetworkStatus": None, "ActivityID": None, + "RetryAfterSeconds": None, "BootingHostsJson": "[]", "Waiting": 1, "Booting": 0} + row.update(values) + return row + + +def started(**values): + row = {"VMID": 7, "VMName": "lnx-07", "PreviousNetworkStatus": "Unreachable", "ActivityID": 41, + "RetryAfterSeconds": 75, "Booting": 1} + row.update(values) + return reservation("Started", **row) + + +def procs(fake_db): + return [call["proc"] for call in fake_db.calls] + + +def events(fake_db): + return [call["params"] for call in fake_db.calls if call["proc"] == "RecordCheckoutEvent"] + + +def checkout(client, **extra): + return client.post("/api/vms/checkout", json={"username": "a.lice", "avdhost": "avd-01", **extra}) + + +@pytest.fixture +def provisioned(app_module, monkeypatch): + calls = [] + monkeypatch.setattr(app_module, "create_or_update_remote_user", lambda *args: calls.append(args) or True) + monkeypatch.setattr(app_module, "release_vm_assignment", lambda vmid, lease_id: None) + return calls + + +@pytest.fixture +def azure(app_module, fake_db, monkeypatch): + state = {"starts": [], "refuse": False, "commits": None} + + def begin_start(resource_group, name): + state["commits"] = fake_db.commits + if state["refuse"]: + raise RuntimeError("Operation 'start' is not allowed on VM 'lnx-07' in region xyz.") + state["starts"].append((resource_group, name)) + + client = types.SimpleNamespace(virtual_machines=types.SimpleNamespace(begin_start=begin_start)) + monkeypatch.setattr(app_module, "get_compute_client", lambda: client) + return state + + +@pytest.fixture +def probes(app_module, monkeypatch): + state = {"up": set(), "asked": []} + + def probe(ip_address): + state["asked"].append(ip_address) + return ip_address in state["up"] + + monkeypatch.setattr(app_module, "probe_ssh", probe) + return state + + +def booting(*hosts): + return json.dumps([{"VMID": vmid, "Hostname": name, "IPAddress": ip} for vmid, name, ip in hosts]) + + +# ------------------------------------------------------------------ starting a host + + +def test_no_ready_host_starts_one_and_asks_the_avd_host_to_wait(client, fake_db, provisioned, azure, audit_entries): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started() + + response = checkout(client, clientVersion="2.0.0") + + assert response.status_code == 202 + assert response.headers["Retry-After"] == "75" + assert response.get_json() == { + "status": "Starting", "reason": "Started", "retryAfterSeconds": 75, + "message": "A Linux host is starting for you. Ask again in 75 seconds.", + } + assert azure["starts"] == [("resource-group", "lnx-07")] + assert fake_db.latest_call("ReserveVmForStart")["params"] == ("alice", "avd-01") + assert provisioned == [] + + [entry] = [entry for entry in audit_entries if entry["action"] == "vm.start_on_demand"] + assert (entry["targetType"], entry["targetId"], entry["outcome"]) == ("vm", "lnx-07", "success") + assert json.loads(entry["detailJson"]) == {"username": "alice", "avdhost": "avd-01", "clientVersion": "2.0.0", + "activityId": 41, "waiting": 1, "starting": 1} + + [params] = events(fake_db) + assert params[:3] == ("alice", "avd-01", "Starting") and params[4:] == ("lnx-07", "2.0.0") + # The reservation holds the scaling lock, so it is committed before Azure is asked: + # the checkout's commit, then the reservation's. + assert azure["commits"] == 2 + + +def test_azure_refusing_the_start_puts_the_host_back_and_refuses_the_checkout(client, fake_db, provisioned, azure, audit_entries): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started() + azure["refuse"] = True + + response = checkout(client) + + assert response.status_code == 409 + assert response.get_json()["error"] == "No available VM found. Please try again." + assert "region xyz" not in response.get_data(as_text=True) + assert fake_db.latest_call("UpdateVmAttributes")["params"] == (7, "Off", "Unreachable", None) + assert fake_db.latest_call("AppendScalingActivityNote")["params"] == (41, "Starting lnx-07 failed, so it was recorded as off again.") + [entry] = [entry for entry in audit_entries if entry["action"] == "vm.start_on_demand"] + assert entry["outcome"] == "failure" + assert json.loads(entry["detailJson"])["error"] == "The Azure start operation could not be requested." + assert events(fake_db)[0][2] == "NoneAvailable" + + +def test_a_missing_subscription_is_treated_as_a_refused_start(client, fake_db, provisioned, azure, app_module, monkeypatch): + monkeypatch.setattr(app_module, "VM_RESOURCE_GROUP", "") + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started() + + assert checkout(client).status_code == 409 + assert azure["starts"] == [] + assert fake_db.latest_call("UpdateVmAttributes")["params"][:2] == (7, "Off") + + +@pytest.mark.parametrize("seconds,expected", [(None, 30), (1, 5), (500, 300), ("abc", 30), (45, 45)]) +def test_the_wait_is_kept_between_five_seconds_and_five_minutes(client, fake_db, provisioned, azure, seconds, expected): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = started(RetryAfterSeconds=seconds) + response = checkout(client) + assert response.headers["Retry-After"] == str(expected) and response.get_json()["retryAfterSeconds"] == expected + + +# ------------------------------------------------------------------ hosts already starting + + +def test_a_waiting_user_is_given_a_starting_host_as_soon_as_it_answers(client, fake_db, provisioned, azure, probes): + fake_db.fetchall_sequence["CheckoutVm"] = [NO_HOST, [checkout_row(VMID=7, Hostname="lnx-07", IPAddress="10.0.0.7")]] + fake_db.fetchone_rows["ReserveVmForStart"] = reservation( + "AlreadyStarting", RetryAfterSeconds=40, Booting=2, + BootingHostsJson=booting((7, "lnx-07", "10.0.0.7"), (8, "lnx-08", "10.0.0.8"))) + probes["up"] = {"10.0.0.7"} + + response = checkout(client) + + assert response.status_code == 200 + assert response.get_json()["Hostname"] == "lnx-07" + assert sorted(probes["asked"]) == ["10.0.0.7", "10.0.0.8"] + marked = [call["params"] for call in fake_db.calls if call["proc"] == "SetVmNetworkStatus"] + assert marked == [(7, "Reachable")] + assert procs(fake_db).count("CheckoutVm") == 2 + assert azure["starts"] == [] + assert events(fake_db)[0][2:5:2] == ("Assigned", "lnx-07") + + +def test_a_waiting_user_waits_again_while_no_starting_host_answers(client, fake_db, provisioned, probes): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation( + "AlreadyStarting", RetryAfterSeconds=40, BootingHostsJson=booting((7, "lnx-07", "10.0.0.7"))) + + response = checkout(client) + + assert response.status_code == 202 and response.headers["Retry-After"] == "40" + assert response.get_json()["message"] == "A Linux host is starting. Ask again in 40 seconds." + assert probes["asked"] == ["10.0.0.7"] + assert "SetVmNetworkStatus" not in procs(fake_db) + assert procs(fake_db).count("CheckoutVm") == 1 + [params] = events(fake_db) + assert params[2] == "Starting" and params[4] is None + + +def test_a_host_that_answered_but_went_to_someone_else_means_waiting_again(client, fake_db, provisioned, probes): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation( + "AlreadyStarting", RetryAfterSeconds=30, BootingHostsJson=booting((7, "lnx-07", "10.0.0.7"))) + probes["up"] = {"10.0.0.7"} + + assert checkout(client).status_code == 202 + assert procs(fake_db).count("CheckoutVm") == 2 + + +def test_probing_skips_unusable_entries_and_survives_a_database_failure(app_module, fake_db, probes): + hosts = [{"VMID": 7, "Hostname": "lnx-07", "IPAddress": "10.0.0.7"}, {"VMID": None, "IPAddress": "10.0.0.9"}, + {"VMID": 8, "Hostname": "lnx-08"}, "junk"] + probes["up"] = {"10.0.0.7"} + assert app_module.mark_started_hosts_reachable(hosts) == 1 + assert probes["asked"] == ["10.0.0.7"] + assert app_module.mark_started_hosts_reachable(None) == 0 + + fake_db.raise_on_execute["SetVmNetworkStatus"] = "connection reset" + assert app_module.mark_started_hosts_reachable(hosts) == 0 + + +def test_the_probe_connects_to_the_ssh_port_with_a_timeout(app_module, monkeypatch): + attempts = [] + + class Connection: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def connect(address, timeout): + attempts.append((address, timeout)) + if address[0] == "10.0.0.9": + raise OSError("timed out") + return Connection() + + monkeypatch.setattr(app_module.socket, "create_connection", connect) + assert app_module.probe_ssh("10.0.0.7") is True + assert app_module.probe_ssh("10.0.0.9") is False + assert attempts == [(("10.0.0.7", 22), 2), (("10.0.0.9", 22), 2)] + + +# ------------------------------------------------------------------ the other decisions + + +def test_a_host_that_became_ready_is_checked_out_at_once(client, fake_db, provisioned): + fake_db.fetchall_sequence["CheckoutVm"] = [NO_HOST, [checkout_row()]] + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("ReadyNow") + + response = checkout(client) + + assert response.status_code == 200 and response.get_json()["Hostname"] == "lnx-05" + assert len(provisioned) == 1 + + +def test_a_ready_host_taken_by_someone_else_means_waiting(client, fake_db, provisioned): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("ReadyNow") + + response = checkout(client) + + assert response.status_code == 202 and response.headers["Retry-After"] == "30" + assert response.get_json()["reason"] == "ReadyNow" + assert response.get_json()["message"] == "No Linux host is ready yet. Ask again in 30 seconds." + + +def test_busy_scaling_means_asking_again_shortly(client, fake_db, provisioned): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("Busy", RetryAfterSeconds=30) + + response = checkout(client) + assert response.status_code == 202 and response.get_json()["reason"] == "Busy" + assert procs(fake_db).count("CheckoutVm") == 1 + + +@pytest.mark.parametrize("row", [reservation("Disabled"), reservation("AtMaximum"), reservation("NoCandidate"), + reservation("Unexpected"), None]) +def test_no_host_to_start_refuses_the_checkout_as_before(client, fake_db, provisioned, azure, row): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + if row is not None: + fake_db.fetchone_rows["ReserveVmForStart"] = row + + response = checkout(client) + + assert response.status_code == 409 + assert response.get_json()["error"] == "No available VM found. Please try again." + assert azure["starts"] == [] and events(fake_db)[0][2] == "NoneAvailable" + + +def test_a_database_without_start_on_demand_is_logged_once(client, fake_db, provisioned, caplog): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + fake_db.raise_on_execute["ReserveVmForStart"] = MISSING.format("ReserveVmForStart") + + assert checkout(client).status_code == 409 + assert checkout(client).status_code == 409 + assert caplog.text.count("ReserveVmForStart is not deployed yet") == 1 + + fake_db.raise_on_execute["ReserveVmForStart"] = "connection reset" + checkout(client) + checkout(client) + assert caplog.text.count("Could not decide whether to start a host for alice") == 2 + + +def test_an_unreachable_database_while_deciding_is_not_recorded(client, fake_db, provisioned, app_module, monkeypatch): + fake_db.fetchall_rows["CheckoutVm"] = NO_HOST + connections = {"count": 0} + real_connect = fake_db.connect + + def connect(): + connections["count"] += 1 + return real_connect() if connections["count"] == 1 else None + + monkeypatch.setattr(app_module, "get_db_connection", connect) + response = checkout(client) + assert response.get_json()["error"] == "Database connection failed." + assert connections["count"] == 2 + + +def test_a_failing_second_checkout_is_a_server_error(client, fake_db, provisioned): + fake_db.fetchall_sequence["CheckoutVm"] = [NO_HOST, [{"ErrorNumber": 1205, "Message": "Deadlock"}]] + fake_db.fetchone_rows["ReserveVmForStart"] = reservation("ReadyNow") + + assert checkout(client).status_code == 500 + assert events(fake_db)[0][2] == "Error" + + +# ------------------------------------------------------------------ the script's version + + +@pytest.mark.parametrize("value,recorded", [ + ("2.0.0", "2.0.0"), (" 2.1.0-beta+7 ", "2.1.0-beta+7"), ("", None), ("x" * 33, None), ("2.0; DROP", None), + (200, None), (None, None), ("-2", None), +]) +def test_only_a_plain_client_version_is_recorded(client, fake_db, provisioned, value, recorded): + fake_db.fetchall_rows["CheckoutVm"] = [checkout_row()] + + assert checkout(client, clientVersion=value).status_code == 200 + + [call] = [call for call in fake_db.calls if call["proc"] == "RecordCheckoutEvent"] + if recorded is None: + assert len(call["params"]) == 5 and "@ClientVersion" not in call["sql"] + else: + assert call["params"][5] == recorded and "@ClientVersion = %s" in call["sql"] + + +def test_an_admin_checkout_audit_names_the_client_version(auth_client, fake_db, provisioned, audit_entries): + fake_db.fetchall_rows["CheckoutVm"] = [checkout_row()] + client = auth_client({"roles": ["FullAccess"], "scp": "access_as_user"}) + + response = client.post("/api/vms/checkout", json={"username": "alice", "avdhost": "avd-01", "clientVersion": "2.0.0"}, + headers=auth_client.headers) + + assert response.status_code == 200 + [entry] = [entry for entry in audit_entries if entry["action"] == "vm.checkout"] + assert json.loads(entry["detailJson"])["clientVersion"] == "2.0.0" + + +def test_a_database_that_predates_the_client_version_still_records_the_event(client, fake_db, provisioned, app_module, monkeypatch, caplog): + fake_db.fetchall_rows["CheckoutVm"] = [checkout_row()] + rejected = [] + + class Cursor: + def __init__(self, inner): + self.inner = inner + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def execute(self, sql, params=None): + if "@ClientVersion" in sql: + rejected.append(params) + raise RuntimeError("(8145, b'@ClientVersion is not a parameter for procedure RecordCheckoutEvent.')") + return self.inner.execute(sql, params) + + def __getattr__(self, name): + return getattr(self.inner, name) + + real_connect = fake_db.connect + + def connect(): + connection = real_connect() + real_cursor = connection.cursor + connection.cursor = lambda as_dict=False: Cursor(real_cursor(as_dict=as_dict)) + return connection + + monkeypatch.setattr(app_module, "get_db_connection", connect) + + assert checkout(client, clientVersion="2.0.0").status_code == 200 + assert checkout(client, clientVersion="2.0.0").status_code == 200 + + # The first event is written again without the version; the next leaves it out at once. + assert len(rejected) == 1 + recorded = events(fake_db) + assert len(recorded) == 2 and all(len(params) == 5 for params in recorded) + assert caplog.text.count("does not take @ClientVersion yet") == 1 + + +def test_the_current_script_version_is_the_one_the_avd_host_script_sends(app_module): + script = (REPO_ROOT / "avd_host" / "broker" / "Connect-LinuxBroker.ps1").read_text(encoding="ascii") + + [version] = re.findall(r"^\$ScriptVersion = '([^']+)'", script, re.MULTILINE) + + assert version == app_module.AVD_HOST_SCRIPT_VERSION + assert app_module.CLIENT_VERSION_RE.match(version) and len(version) <= app_module.CLIENT_VERSION_MAX_CHARS + + +# ------------------------------------------------------------------ policy + + +def policy_row(**values): + row = {"TimeZone": "UTC", "LocalTime": "2026-09-24T10:15:00", "StartOnDemandEnabled": True, "MaxPendingStarts": 2, + "ZeroMinimumCount": 1, "AvdHostsSeen": 4, "AvdHostsOutdated": 1, + "OutdatedAvdHostsJson": json.dumps([{"AvdHost": "avd-03"}]), + "AvdClientVersionsJson": json.dumps([{"ClientVersion": "1.9.0", "AvdHosts": 1}, {"ClientVersion": "2.0.0", "AvdHosts": 2}])} + row.update(values) + return row + + +def test_the_policy_reports_start_on_demand_and_the_avd_host_scripts(client, fake_db): + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row() + + policy = client.get("/api/scaling/policy").get_json() + + assert (policy["StartOnDemandEnabled"], policy["MaxPendingStarts"], policy["ZeroMinimumCount"]) == (True, 2, 1) + assert policy["AvdHostScripts"] == { + "Seen": 4, "Outdated": 1, "OutdatedHostnames": ["avd-03"], "CurrentVersion": "2.0.0", + "Versions": [{"ClientVersion": "1.9.0", "AvdHosts": 1, "Current": False}, + {"ClientVersion": "2.0.0", "AvdHosts": 2, "Current": True}], + } + + +def test_the_policy_of_an_older_database_has_no_start_on_demand(client, fake_db): + fake_db.fetchone_rows["GetScalingPolicy"] = {"TimeZone": "UTC"} + policy = client.get("/api/scaling/policy").get_json() + assert policy["StartOnDemandEnabled"] is None and policy["MaxPendingStarts"] is None + assert policy["AvdHostScripts"] is None + + +def test_no_avd_host_asked_means_an_empty_summary(client, fake_db): + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row(AvdHostsSeen=0, AvdHostsOutdated=0, OutdatedAvdHostsJson=None, + AvdClientVersionsJson=None) + scripts = client.get("/api/scaling/policy").get_json()["AvdHostScripts"] + assert scripts["Seen"] == 0 and scripts["OutdatedHostnames"] == [] and scripts["Versions"] == [] + + +def start_on_demand_row(**values): + row = {"Result": "Updated", "Message": None, "StartOnDemandEnabled": True, "MaxPendingStarts": 4, + "PreviousStartOnDemandEnabled": False, "PreviousMaxPendingStarts": 2, "ZeroMinimumCount": 0} + row.update(values) + return row + + +ADMIN_USER = {"roles": ["FullAccess"], "scp": "access_as_user", "oid": "oid-alice", "preferred_username": "alice@contoso.com"} + + +def test_start_on_demand_is_set_and_audited(auth_client, fake_db, audit_entries): + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row() + client = auth_client(ADMIN_USER) + + response = client.post("/api/scaling/policy/update", json={"startondemandenabled": True, "maxpendingstarts": 4}, + headers=auth_client.headers) + + assert response.status_code == 200 + assert response.get_json() == { + "Result": "Updated", "StartOnDemandEnabled": True, "MaxPendingStarts": 4, "ZeroMinimumCount": 0, + "message": "Start on demand is on. Up to 4 hosts may start at once for waiting users.", + } + assert fake_db.latest_call("SetScalingPolicyStartOnDemand")["params"] == (True, 4, "alice@contoso.com") + assert "SetScalingPolicyTimeZone" not in procs(fake_db) and fake_db.commits == 1 + [entry] = [entry for entry in audit_entries if entry["action"] == "scaling.policy_update"] + assert json.loads(entry["detailJson"]) == {"result": "Updated", "startOnDemand": {"from": False, "to": True}, + "maxPendingStarts": {"from": 2, "to": 4}, "status": 200} + + +def test_one_setting_at_a_time_leaves_the_other_alone(client, fake_db): + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row(MaxPendingStarts=3) + client.post("/api/scaling/policy/update", json={"maxpendingstarts": "3"}) + assert fake_db.latest_call("SetScalingPolicyStartOnDemand")["params"][:2] == (None, 3) + + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row( + StartOnDemandEnabled=False, PreviousStartOnDemandEnabled=True, ZeroMinimumCount=2) + body = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert fake_db.latest_call("SetScalingPolicyStartOnDemand")["params"][:2] == (False, None) + assert body["message"] == ("Start on demand is off. 2 scaling rules and windows have a minimum of 0, so scaling keeps " + "one host running for them.") + + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row( + StartOnDemandEnabled=False, ZeroMinimumCount=0, Result="Unchanged") + body = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert body["Result"] == "Unchanged" + assert body["message"] == "Start on demand is off. A user who finds no ready host is refused." + + +def test_the_zone_and_start_on_demand_are_applied_together(client, fake_db): + fake_db.fetchone_rows["SetScalingPolicyTimeZone"] = {"Result": "Unchanged", "TimeZone": "UTC", "PreviousTimeZone": "UTC"} + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row(MaxPendingStarts=1) + + body = client.post("/api/scaling/policy/update", + json={"timezone": "UTC", "startondemandenabled": True, "maxpendingstarts": 1}).get_json() + + assert body["Result"] == "Updated" and body["TimeZone"] == "UTC" + assert body["message"] == "Schedules were already read in UTC. Start on demand is on. Up to 1 host may start at once for waiting users." + assert fake_db.commits == 1 and len(fake_db.connections) == 1 + + +@pytest.mark.parametrize("payload,message", [ + ({}, "Provide timezone, startondemandenabled, maxpendingstarts, or any of them."), + ({"startondemandenabled": "yes"}, "startondemandenabled must be true or false."), + ({"maxpendingstarts": 0}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": 21}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": True}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": 2.5}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": "\u00b2"}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"maxpendingstarts": "9" * 5000}, "maxpendingstarts must be a whole number from 1 to 20."), + ({"timezone": " ", "startondemandenabled": True}, "Provide the timezone as a name from /api/scaling/timezones."), +]) +def test_invalid_policy_settings_are_refused_before_the_database(client, fake_db, payload, message): + response = client.post("/api/scaling/policy/update", json=payload) + assert response.status_code == 400 and response.get_json()["error"] == message + assert fake_db.calls == [] + + +def test_nothing_is_applied_when_any_part_is_refused(client, fake_db): + fake_db.fetchone_rows["SetScalingPolicyTimeZone"] = {"Result": "InvalidTimeZone", "TimeZone": "UTC"} + response = client.post("/api/scaling/policy/update", json={"timezone": "Mars", "startondemandenabled": True}) + assert response.status_code == 400 and "Mars" in response.get_json()["error"] + assert "SetScalingPolicyStartOnDemand" not in procs(fake_db) and fake_db.commits == 0 + + fake_db.fetchone_rows["SetScalingPolicyTimeZone"] = {"Result": "Updated", "TimeZone": "UTC", "PreviousTimeZone": "Tokyo Standard Time"} + fake_db.fetchone_rows["SetScalingPolicyStartOnDemand"] = start_on_demand_row( + Result="Invalid", Message="maxpendingstarts must be between 1 and 20.") + response = client.post("/api/scaling/policy/update", json={"timezone": "UTC", "maxpendingstarts": 5}) + assert response.status_code == 400 and response.get_json()["error"] == "maxpendingstarts must be between 1 and 20." + assert fake_db.commits == 0 + + +def test_start_on_demand_needs_the_upgraded_database(client, fake_db): + fake_db.raise_on_execute["SetScalingPolicyStartOnDemand"] = MISSING.format("SetScalingPolicyStartOnDemand") + response = client.post("/api/scaling/policy/update", json={"startondemandenabled": True}) + assert response.status_code == 404 + assert response.get_json()["error"] == "Start on demand is not available until the database is upgraded." + assert fake_db.commits == 0 + + +# ------------------------------------------------------------------ schedules and the preview + + +SCHEDULE = {"name": "Nights", "days": ["mon"], "start": "20:00", "end": "23:00", "minvms": 0, "maxvms": 4, + "scaleupratio": 60, "scaleupincrement": 1, "scaledownratio": 20, "scaledownincrement": 1} + + +def test_a_window_scales_to_zero_while_start_on_demand_is_on(client, fake_db): + fake_db.fetchall_rows["GetScalingSchedules"] = [] + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row() + fake_db.fetchone_rows["SaveScalingSchedule"] = {"Result": "Created", "ScheduleID": 3} + + assert client.post("/api/scaling/schedules/create", json=SCHEDULE).status_code == 201 + assert fake_db.latest_call("SaveScalingSchedule")["params"][6] == 0 + + +def test_the_preview_counts_waiting_users_and_checks_a_zero_minimum(client, fake_db): + fake_db.fetchone_rows["TriggerScalingLogic"] = { + "Action": "PowerOn", "RequestCount": 1, "CandidatesJson": json.dumps([{"Hostname": "lnx-09"}]), + "Reason": "Serviceable hosts are below the minimum. 1 user is waiting for a host to start.", "PhaseSource": "Rule", + "MinVMs": 0, "MaxVMs": 4, "PoweredOn": 0, "Serviceable": 0, "InUse": 0, "Draining": 0, "Utilization": 100, + "Waiting": 1, "StartOnDemandEnabled": True, + } + + body = client.get("/api/scaling/preview").get_json() + assert body["Counts"]["Waiting"] == 1 and body["StartOnDemandEnabled"] is True + + fake_db.fetchone_rows["GetScalingPolicy"] = policy_row(StartOnDemandEnabled=False) + proposed = {key: value for key, value in SCHEDULE.items() if key not in ("days", "start", "end")} + response = client.post("/api/scaling/preview", json={"rule": proposed}) + assert response.status_code == 400 + assert response.get_json()["error"] == "minvms can be 0 only while start on demand is on." diff --git a/api/tests/test_trends.py b/api/tests/test_trends.py index 3aab1c6..c362f39 100644 --- a/api/tests/test_trends.py +++ b/api/tests/test_trends.py @@ -130,7 +130,8 @@ def test_utilization_returns_the_series_and_checkout_health(client, fake_db, app monkeypatch.setattr(app_module, "utilization_window", lambda hours: (start, end, 15)) fake_db.fetchall_rows["GetUtilizationSeries"] = [ {"BucketStartUtc": "2026-03-03T10:45:00Z", "Runs": 3, "PoweredOn": Decimal("4.0"), "InUse": Decimal("2.3"), - "Serviceable": Decimal("3.7"), "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0}, + "Serviceable": Decimal("3.7"), "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0, + "Waited": 2}, {"BucketStartUtc": "2026-03-03T11:00:00Z", "Runs": 0, "PoweredOn": None, "InUse": None, "Serviceable": None, "PeakInUse": None, "MinVMs": None, "MaxVMs": None, "Checkouts": 0, "Denied": 0, "Failed": 0}, ] @@ -138,6 +139,7 @@ def test_utilization_returns_the_series_and_checkout_health(client, fake_db, app "Total": 40, "Assigned": 20, "Reused": 15, "NoneAvailable": 3, "ProvisionFailed": 1, "Errors": 1, "P50Ms": 2100, "P95Ms": 8400, "DeniedLastHour": 1, "LastDeniedUtc": "2026-03-04T10:02:11.120Z", "HostStarts": 4, "StartP50Seconds": 95, "StartP95Seconds": 180, + "Starting": 6, "Waits": 3, "WaitsServed": 2, "WaitP50Seconds": 84, "WaitP95Seconds": 150, "WaitingNow": 1, } response = client.get("/api/metrics/utilization") @@ -146,9 +148,13 @@ def test_utilization_returns_the_series_and_checkout_health(client, fake_db, app body = response.get_json() assert (body["Hours"], body["BucketMinutes"], body["FromUtc"], body["ToUtc"]) == (24, 15, "2026-03-03T10:45:00Z", "2026-03-04T10:45:00Z") assert body["Series"][0] == {"BucketStartUtc": "2026-03-03T10:45:00Z", "Runs": 3, "PoweredOn": 4.0, "InUse": 2.3, - "Serviceable": 3.7, "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0} - assert body["Series"][1]["PoweredOn"] is None and body["Series"][1]["Runs"] == 0 + "Serviceable": 3.7, "PeakInUse": 3, "MinVMs": 2, "MaxVMs": 6, "Checkouts": 5, "Denied": 1, "Failed": 0, + "Waited": 2} + assert body["Series"][1]["PoweredOn"] is None and body["Series"][1]["Runs"] == 0 and body["Series"][1]["Waited"] == 0 assert body["Checkouts"]["DeniedPercent"] == 7.5 and body["Checkouts"]["StartP95Seconds"] == 180 + # Starting answers are not checkouts: the denial rate is still NoneAvailable over Total. + assert {key: body["Checkouts"][key] for key in ("Starting", "Waits", "WaitsServed", "WaitP50Seconds", "WaitP95Seconds", "WaitingNow")} == { + "Starting": 6, "Waits": 3, "WaitsServed": 2, "WaitP50Seconds": 84, "WaitP95Seconds": 150, "WaitingNow": 1} assert fake_db.latest_call("GetUtilizationSeries")["params"] == (start, end, 15) assert fake_db.latest_call("GetCheckoutStats")["params"] == (start, end) @@ -158,6 +164,8 @@ def test_utilization_with_no_checkouts_has_no_denial_rate(client, fake_db): body = client.get("/api/metrics/utilization?hours=168").get_json() assert body["BucketMinutes"] == 60 and body["Series"] == [] assert body["Checkouts"]["DeniedPercent"] is None and body["Checkouts"]["P50Ms"] is None + # A database that predates start on demand (154) reports no waits. + assert body["Checkouts"]["Waits"] == 0 and body["Checkouts"]["WaitP50Seconds"] is None @pytest.mark.parametrize("hours", ["12", "abc", "0", "720"]) @@ -178,7 +186,7 @@ def test_utilization_before_the_database_upgrade_answers_404(client, fake_db): def health_row(hostname, **values): row = {"VMID": 1, "Hostname": hostname, "PowerState": "On", "NetworkStatus": "Reachable", "VmStatus": "Available", - "HeartbeatAgeSeconds": 20, "ReconcileIntervalSeconds": 60, "AgentVersion": "1.2.0", "ScriptVersionsJson": None, + "HeartbeatAgeSeconds": 20, "ReconcileIntervalSeconds": 60, "AgentVersion": "1.3.0", "ScriptVersionsJson": None, "XrdpActive": True, "NfsReachable": True, "RootDiskFreePct": 50, "CurrentSettingsVersion": 3, "AppliedSettingsVersion": 3} row.update(values) diff --git a/api/tests_integration/conftest.py b/api/tests_integration/conftest.py index 302c099..c64a308 100644 --- a/api/tests_integration/conftest.py +++ b/api/tests_integration/conftest.py @@ -147,7 +147,7 @@ def db(sql): "DELETE FROM dbo.VmScalingActivityLog", "DELETE FROM dbo.VmScalingRules", "DELETE FROM dbo.ScalingSchedules", - "UPDATE dbo.ScalingPolicy SET TimeZone=N'UTC', UpdatedBy=NULL", + "UPDATE dbo.ScalingPolicy SET TimeZone=N'UTC', UpdatedBy=NULL, StartOnDemandEnabled=1, MaxPendingStarts=2", "UPDATE dbo.LinuxHostSettings SET GracePeriodSeconds=1200, ReconcileIntervalSeconds=60, " "IdleTimeoutSeconds=0, IdleWarningSeconds=120, ScreenLockEnabled=0, DisableLockScreen=1, " "PreserveSessionsOnDisconnect=0 WHERE SettingsScope='Global'", diff --git a/api/tests_integration/test_api_against_sql.py b/api/tests_integration/test_api_against_sql.py index b233cfb..8eec039 100644 --- a/api/tests_integration/test_api_against_sql.py +++ b/api/tests_integration/test_api_against_sql.py @@ -1,6 +1,7 @@ """End-to-end checks of the handlers against the real stored procedures and driver.""" import json +import logging import types from pathlib import Path @@ -14,9 +15,13 @@ def _checkout(client, username="alice", avdhost="avd-01"): def test_checkout_with_no_ready_host_is_a_conflict_not_a_server_error(client, db, remote): + # With start on demand on, the stopped host would be started instead: see + # test_start_on_demand_against_sql.py. + db.run("UPDATE dbo.ScalingPolicy SET StartOnDemandEnabled = 0") db.add_vm("lnxhost-01", power="Off", network="Unreachable") response = _checkout(client) assert response.status_code == 409, response.get_json() + assert db.vm("lnxhost-01")["PowerState"] == "Off" def test_checkout_provisions_the_user_and_records_the_assignment(client, db, remote): @@ -202,6 +207,23 @@ def test_scaling_starts_hosts_to_reach_the_minimum(app_module, client, db, remot assert log["ActionTaken"] == "Scale Up" and log["VMsPoweredOn"] == 1 +def test_each_scaling_run_logs_the_fleet_as_the_database_sees_it(app_module, client, db, remote, monkeypatch, caplog): + db.add_vm("lnxhost-01") + db.add_vm("lnxhost-02", power="Off", network="Unreachable") + compute = _Compute({"lnxhost-01": "PowerState/running", "lnxhost-02": "PowerState/deallocated"}) + monkeypatch.setattr(app_module, "ComputeManagementClient", lambda *a, **k: compute) + caplog.set_level(logging.INFO, logger="linuxbroker.api") + + assert client.post("/api/scaling/trigger", json={}).status_code == 200 + + [record] = [r for r in caplog.records if r.getMessage() == "fleet snapshot"] + # The run started lnxhost-02 to reach the minimum of two, so the snapshot sees it booting; + # lnxhost-01 has been on longer than the allowance and its agent has never reported. + assert (record.ReadyHosts, record.PoweredOn, record.Booting, record.Serviceable) == (1, 2, 1, 2) + assert (record.StaleHeartbeats, record.TotalHosts, record.EffectiveMinVMs, record.MaxVMs) == (1, 2, 2, 10) + assert record.StartOnDemandEnabled == 1 and record.StaleAfterSeconds == 180 + + def test_scaling_down_honours_the_deallocate_stop_mode(app_module, client, db, remote, monkeypatch): for index in range(1, 6): db.add_vm(f"lnxhost-0{index}") diff --git a/api/tests_integration/test_db_pool_against_sql.py b/api/tests_integration/test_db_pool_against_sql.py new file mode 100644 index 0000000..97d2bf7 --- /dev/null +++ b/api/tests_integration/test_db_pool_against_sql.py @@ -0,0 +1,51 @@ +"""The API's connection pool against the real driver and stored procedures. + +System-versioned tables (VirtualMachines, VmScalingRules, ScalingSchedules and +LinuxHostSettings) stamp each change with its transaction's start time, and SQL Server +refuses a change stamped earlier than the row's current version (error 13535). A pooled +connection must therefore begin its transaction when a request takes it, not when the +previous request gave it back. +""" + + +def _checkout(client, username="alice", avdhost="avd-01"): + return client.post("/api/vms/checkout", json={"username": username, "avdhost": avdhost}) + + +def test_the_integration_tests_run_with_the_pool_on(app_module): + assert app_module._db_pool is not None + + +def test_a_pooled_connection_can_change_a_row_changed_while_it_sat_idle(client, db, remote, app_module): + db.run("UPDATE dbo.ScalingPolicy SET StartOnDemandEnabled = 0") + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + assert _checkout(client).status_code == 409 + assert app_module._db_pool.idle_count >= 1 + + # Another worker changes the host while the connection waits in the pool. + db.run("UPDATE dbo.VirtualMachines SET PowerState = 'On', NetworkStatus = 'Reachable' " + "WHERE Hostname = 'lnxhost-01'") + + response = _checkout(client) + + assert response.status_code == 200, response.get_json() + assert db.vm("lnxhost-01")["VmStatus"] == "CheckedOut" + + +def test_a_pooled_connection_waits_with_no_transaction_open(client, db, app_module): + db.add_vm("lnxhost-01") + assert client.get("/api/vms").status_code == 200 + + pooled = app_module._db_pool.acquire(app_module.get_db_connection) + try: + # Taken back out, it has begun exactly one transaction: the one this request would use. + cursor = pooled.connection.cursor() + cursor.execute("SELECT @@TRANCOUNT") + assert cursor.fetchone()[0] == 1 + finally: + app_module._db_pool.release(pooled) + + idle = app_module._db_pool._idle[-1].connection + cursor = idle.cursor() + cursor.execute("SELECT @@TRANCOUNT") + assert cursor.fetchone()[0] == 0 diff --git a/api/tests_integration/test_start_on_demand_against_sql.py b/api/tests_integration/test_start_on_demand_against_sql.py new file mode 100644 index 0000000..6fa26f7 --- /dev/null +++ b/api/tests_integration/test_start_on_demand_against_sql.py @@ -0,0 +1,263 @@ +"""4.1 start on demand and scale to zero against the real stored procedures and driver.""" + +import json +import types + +import pytest + + +RULE = {"minvms": 2, "maxvms": 6, "scaleupratio": 70, "scaleupincrement": 1, "scaledownratio": 30, "scaledownincrement": 1} +EVERY_DAY = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] + + +class Compute: + def __init__(self, states=None, fail=()): + self.operations = [] + self.states = states or {} + self.fail = set(fail) + compute = self + + class VirtualMachines: + def _operate(self, operation, name): + if operation in compute.fail: + raise RuntimeError("refused") + compute.operations.append((operation, name)) + + def instance_view(self, resource_group, name): + return types.SimpleNamespace(statuses=[types.SimpleNamespace(code=compute.states.get(name, "PowerState/running"))]) + + def begin_start(self, resource_group, name): + self._operate("start", name) + + def begin_power_off(self, resource_group, name): + self._operate("power_off", name) + + def begin_deallocate(self, resource_group, name): + self._operate("deallocate", name) + + self.virtual_machines = VirtualMachines() + + +@pytest.fixture +def compute(app_module, monkeypatch): + fake = Compute() + monkeypatch.setattr(app_module, "ComputeManagementClient", lambda *a, **k: fake) + return fake + + +@pytest.fixture +def probes(app_module, monkeypatch): + """The hosts that answer the start on demand probe, by IP address.""" + answering = set() + monkeypatch.setattr(app_module, "probe_ssh", lambda ip: ip in answering) + return answering + + +def _checkout(client, username, avdhost="avd-01", **extra): + return client.post("/api/vms/checkout", json={"username": username, "avdhost": avdhost, **extra}) + + +def _events(db): + return [(row["Username"], row["Outcome"], row["Hostname"], row["ClientVersion"]) + for row in db.run("SELECT Username, Outcome, Hostname, ClientVersion FROM dbo.CheckoutEvents ORDER BY EventID")] + + +def _audits(db, action): + return db.run("SELECT TargetId, Outcome, DetailJson FROM dbo.AuditLog WHERE Action = %s ORDER BY AuditId", (action,)) + + +def test_a_checkout_that_finds_no_ready_host_starts_one_and_is_told_to_wait(client, db, remote, compute, probes): + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + + response = _checkout(client, "alice", clientVersion="2.0.0") + + assert response.status_code == 202, response.get_json() + body = response.get_json() + # With no start history the procedure expects a start to take 60 seconds. + assert body == {"status": "Starting", "reason": "Started", "retryAfterSeconds": 60, + "message": "A Linux host is starting for you. Ask again in 60 seconds."} + assert response.headers["Retry-After"] == "60" + assert compute.operations == [("start", "lnxhost-01")] + vm = db.vm("lnxhost-01") + assert (vm["PowerState"], vm["NetworkStatus"], vm["VmStatus"], vm["Username"]) == ("On", "Unreachable", "Available", None) + assert vm["StartRequestedAt"] is not None + + log = db.one("SELECT TOP 1 * FROM dbo.VmScalingActivityLog ORDER BY ActivityID DESC") + assert log["ActionTaken"] == "Start On Demand" and log["VMsPoweredOn"] == 1 + assert "alice" in log["Notes"] and "avd-01" in log["Notes"] + assert _events(db) == [("alice", "Starting", "lnxhost-01", "2.0.0")] + [entry] = _audits(db, "vm.start_on_demand") + detail = json.loads(entry["DetailJson"]) + assert (entry["TargetId"], entry["Outcome"]) == ("lnxhost-01", "success") + assert (detail["username"], detail["avdhost"], detail["clientVersion"], detail["activityId"]) == ( + "alice", "avd-01", "2.0.0", log["ActivityID"]) + assert remote.calls == [], "no user is created until a host is ready" + + +def test_the_waiting_user_is_given_the_host_as_soon_as_it_answers(client, db, remote, compute, probes): + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + assert _checkout(client, "alice", clientVersion="2.0.0").status_code == 202 + # The host was asked to start 90 seconds ago, and has not answered yet. + db.run("UPDATE dbo.VirtualMachines SET StartRequestedAt = DATEADD(SECOND, -90, SYSUTCDATETIME()), " + "PowerStateChangedDate = DATEADD(SECOND, -90, GETDATE())") + db.run("UPDATE dbo.CheckoutEvents SET OccurredAt = DATEADD(SECOND, -90, OccurredAt)") + + still_starting = _checkout(client, "alice", clientVersion="2.0.0") + assert still_starting.status_code == 202, still_starting.get_json() + assert still_starting.get_json()["reason"] == "AlreadyStarting" + assert still_starting.get_json()["retryAfterSeconds"] == 30, "a start expected in 60 seconds is already overdue" + assert compute.operations == [("start", "lnxhost-01")], "a user already waiting does not start a second host" + + probes.add("10.0.0.4") + ready = _checkout(client, "alice", clientVersion="2.0.0") + + assert ready.status_code == 200, ready.get_json() + assert ready.get_json()["Hostname"] == "lnxhost-01" + vm = db.vm("lnxhost-01") + assert (vm["NetworkStatus"], vm["VmStatus"], vm["Username"]) == ("Reachable", "CheckedOut", "alice") + assert [kind for _, kind in remote.calls] == ["create"] + start = db.one("SELECT Hostname, Seconds FROM dbo.HostStartEvents") + assert start["Hostname"] == "lnxhost-01" and 85 <= start["Seconds"] <= 120 + + stats = client.get("/api/metrics/utilization").get_json()["Checkouts"] + assert (stats["Total"], stats["Assigned"], stats["Starting"]) == (1, 1, 2) + assert (stats["Waits"], stats["WaitsServed"], stats["WaitingNow"]) == (1, 1, 0) + assert 85 <= stats["WaitP50Seconds"] <= 120 + # Alice asked twice while she waited and counts once in each bucket she asked in: one bucket, + # or two when a 15-minute boundary fell during her 90-second wait. + series = client.get("/api/metrics/utilization").get_json()["Series"] + assert [point["Waited"] for point in series if point["Waited"]] in ([1], [1, 1]) + + +def test_starts_for_waiting_users_stop_at_the_pending_limit(client, db, remote, compute, probes): + for index in range(1, 4): + db.add_vm(f"lnxhost-0{index}", power="Off", network="Unreachable") + + first = _checkout(client, "alice") + second = _checkout(client, "bob") + third = _checkout(client, "carol") + + assert [r.get_json()["reason"] for r in (first, second, third)] == ["Started", "Started", "AlreadyStarting"] + assert compute.operations == [("start", "lnxhost-01"), ("start", "lnxhost-02")] + assert db.vm("lnxhost-03")["PowerState"] == "Off" + assert client.get("/api/metrics/utilization").get_json()["Checkouts"]["WaitingNow"] == 3 + + preview = client.get("/api/scaling/preview").get_json() + assert preview["Counts"]["Waiting"] == 3 and preview["StartOnDemandEnabled"] is True + + +def test_a_refused_start_puts_the_host_back_off_and_refuses_the_checkout(app_module, client, db, remote, monkeypatch): + db.add_vm("lnxhost-01", power="Off", network="Unreachable") + monkeypatch.setattr(app_module, "ComputeManagementClient", lambda *a, **k: Compute(fail={"start"})) + + response = _checkout(client, "alice", clientVersion="2.0.0") + + assert response.status_code == 409, response.get_json() + vm = db.vm("lnxhost-01") + assert (vm["PowerState"], vm["NetworkStatus"]) == ("Off", "Unreachable") + log = db.one("SELECT TOP 1 Notes FROM dbo.VmScalingActivityLog ORDER BY ActivityID DESC") + assert "Starting lnxhost-01 failed, so it was recorded as off again." in log["Notes"] + [entry] = _audits(db, "vm.start_on_demand") + assert entry["Outcome"] == "failure" + assert json.loads(entry["DetailJson"])["error"] == "The Azure start operation could not be requested." + assert _events(db) == [("alice", "NoneAvailable", None, "2.0.0")] + + +def test_the_policy_update_sets_start_on_demand_with_the_time_zone_or_not_at_all(app_module, client, db, monkeypatch): + updated = client.post("/api/scaling/policy/update", json={"startondemandenabled": False, "maxpendingstarts": "5"}) + assert updated.status_code == 200, updated.get_json() + assert updated.get_json() == { + "Result": "Updated", "StartOnDemandEnabled": False, "MaxPendingStarts": 5, "ZeroMinimumCount": 0, + "message": "Start on demand is off. A user who finds no ready host is refused.", + } + policy = client.get("/api/scaling/policy").get_json() + assert (policy["StartOnDemandEnabled"], policy["MaxPendingStarts"], policy["ZeroMinimumCount"]) == (False, 5, 0) + + again = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert again["Result"] == "Unchanged" + + both = client.post("/api/scaling/policy/update", json={"timezone": "Tokyo Standard Time", "startondemandenabled": True}) + assert both.status_code == 200 and both.get_json()["Result"] == "Updated" + assert both.get_json()["message"] == ("Schedules are now read in Tokyo Standard Time. " + "Start on demand is on. Up to 5 hosts may start at once for waiting users.") + + refused = client.post("/api/scaling/policy/update", json={"timezone": "Nowhere", "startondemandenabled": False}) + assert refused.status_code == 400 + assert client.post("/api/scaling/policy/update", json={"maxpendingstarts": 21}).status_code == 400 + policy = client.get("/api/scaling/policy").get_json() + assert (policy["TimeZone"], policy["StartOnDemandEnabled"], policy["MaxPendingStarts"]) == ("Tokyo Standard Time", True, 5) + + # The database refusing the start on demand half rolls back the time zone set before it. + monkeypatch.setattr(app_module, "_policy_update_fields", lambda body: {"timezone": "UTC", "maxpendingstarts": 99}) + invalid = client.post("/api/scaling/policy/update", json={}) + assert invalid.status_code == 400 and invalid.get_json()["error"] == "maxpendingstarts must be between 1 and 20." + assert client.get("/api/scaling/policy").get_json()["TimeZone"] == "Tokyo Standard Time" + + +def test_a_minimum_of_zero_needs_start_on_demand(client, db, compute): + rule_id = client.get("/api/scaling/rules").get_json()[0]["RuleID"] + assert client.post(f"/api/scaling/rules/{rule_id}/update", json={"minvms": 0}).status_code == 200 + assert client.get("/api/scaling/policy").get_json()["ZeroMinimumCount"] == 1 + + off = client.post("/api/scaling/policy/update", json={"startondemandenabled": False}).get_json() + assert off["message"] == ("Start on demand is off. 1 scaling rule or window has a minimum of 0, " + "so scaling keeps one host running for them.") + preview = client.get("/api/scaling/preview").get_json() + assert preview["Phase"]["MinVMs"] == 1 and preview["StartOnDemandEnabled"] is False + + window = {"name": "Nights", "days": EVERY_DAY, "start": "20:00", "end": "06:00", **RULE, "minvms": 0} + refused = client.post("/api/scaling/schedules/create", json=window) + assert refused.status_code == 400 and refused.get_json()["error"] == "minvms can be 0 only while start on demand is on." + refused_rule = client.post(f"/api/scaling/rules/{rule_id}/update", json={"minvms": 0, "maxvms": 8}) + assert refused_rule.status_code == 400 + + assert client.post("/api/scaling/policy/update", json={"startondemandenabled": True}).status_code == 200 + assert client.post("/api/scaling/schedules/create", json=window).status_code == 201 + assert client.get("/api/scaling/policy").get_json()["ZeroMinimumCount"] == 2 + + +def test_scale_to_zero_stops_the_last_idle_host_and_the_next_user_starts_it(client, db, remote, compute, probes): + db.add_vm("lnxhost-01", PowerStateChangedDate="2000-01-01") + rule_id = client.get("/api/scaling/rules").get_json()[0]["RuleID"] + assert client.post(f"/api/scaling/rules/{rule_id}/update", json={"minvms": 0}).status_code == 200 + + scaled = client.post("/api/scaling/trigger", json={}) + + assert scaled.status_code == 200, scaled.get_json() + assert scaled.get_json()["PoweredOffVMs"] == ["lnxhost-01"] + assert compute.operations == [("power_off", "lnxhost-01")] + assert db.vm("lnxhost-01")["PowerState"] == "Off" + + compute.states["lnxhost-01"] = "PowerState/stopped" + response = _checkout(client, "alice") + assert response.status_code == 202 and response.get_json()["reason"] == "Started" + assert compute.operations[-1] == ("start", "lnxhost-01") + + # While alice waits, scaling keeps the host she is waiting for. + compute.states["lnxhost-01"] = "PowerState/running" + again = client.post("/api/scaling/trigger", json={}) + assert again.status_code == 200 and again.get_json()["PoweredOffVMs"] == [] + assert db.vm("lnxhost-01")["PowerState"] == "On" + + +def test_the_policy_reports_which_broker_script_each_avd_host_runs(client, db, remote): + db.run("UPDATE dbo.ScalingPolicy SET StartOnDemandEnabled = 0") + assert _checkout(client, "alice", "avd-01", clientVersion="2.0.0").status_code == 409 + assert _checkout(client, "bob", "avd-02").status_code == 409 + assert _checkout(client, "carol", "avd-03", clientVersion="1.9.0").status_code == 409 + assert _checkout(client, "dave", "avd-04", clientVersion="not a version!").status_code == 409 + + scripts = client.get("/api/scaling/policy").get_json()["AvdHostScripts"] + + assert scripts == { + "Seen": 4, "Outdated": 2, "OutdatedHostnames": ["avd-02", "avd-04"], + "Versions": [{"ClientVersion": "1.9.0", "AvdHosts": 1, "Current": False}, + {"ClientVersion": "2.0.0", "AvdHosts": 1, "Current": True}], + "CurrentVersion": "2.0.0", + } + + # Only each AVD host's latest checkout counts, so an updated host stops being outdated. + assert _checkout(client, "bob", "avd-02", clientVersion="2.0.0").status_code == 409 + scripts = client.get("/api/scaling/policy").get_json()["AvdHostScripts"] + assert scripts["Outdated"] == 1 and scripts["OutdatedHostnames"] == ["avd-04"] + assert {"ClientVersion": "2.0.0", "AvdHosts": 2, "Current": True} in scripts["Versions"] diff --git a/api/tests_integration/test_trends_against_sql.py b/api/tests_integration/test_trends_against_sql.py index 3f8c95e..f59cd66 100644 --- a/api/tests_integration/test_trends_against_sql.py +++ b/api/tests_integration/test_trends_against_sql.py @@ -66,7 +66,7 @@ def test_attention_combines_broker_items_with_host_health(client, db): db.run("UPDATE dbo.VirtualMachines SET PowerStateChangedDate = DATEADD(MINUTE, -30, GETDATE()) WHERE Hostname = 'lnxhost-02'") db.run("UPDATE dbo.VirtualMachines SET SettingsVersion = (SELECT TOP 1 SettingsVersion FROM dbo.LinuxHostSettings)") heartbeat = client.post("/api/hosts/lnxhost-01/heartbeat", data=json.dumps({ - "agentVersion": "1.2.0", "xrdp": {"active": False}, "sessions": [{"username": "dave", "state": "active"}], + "agentVersion": "1.3.0", "xrdp": {"active": False}, "sessions": [{"username": "dave", "state": "active"}], }), content_type="application/json") assert heartbeat.status_code == 200, heartbeat.get_json() diff --git a/avd_host/broker/Connect-LinuxBroker.ps1 b/avd_host/broker/Connect-LinuxBroker.ps1 index b191745..6a735d5 100644 --- a/avd_host/broker/Connect-LinuxBroker.ps1 +++ b/avd_host/broker/Connect-LinuxBroker.ps1 @@ -1,26 +1,76 @@ +<# +.SYNOPSIS + Checks out a Linux host from the Linux Broker and opens a Remote Desktop session to it. + +.DESCRIPTION + The Linux Desktop RemoteApp runs this script on the AVD session host, in the user's session. + The session host's managed identity authenticates it to the broker API. + + When no host is ready, the broker can start one and answer 202. The script then shows a + window saying the desktop is starting, with Cancel, and asks again when the broker says to, + until a host is ready, the user cancels, or MaxWaitSeconds pass. + + The desktop opens full screen across every monitor, unless FullScreen or MultiMonitor is + Off. The script starts mstsc with switches rather than a connection file, because since the + April 2026 update Remote Desktop Connection asks the user about every unsigned .rdp file and + turns its redirections off, the clipboard included. + + Configure-AVD-Host.ps1 installs the script at C:\Temp\Connect-LinuxBroker.ps1 when the + session host is provisioned, and deploy/Update-AvdHostBrokerScript.ps1 replaces it on + existing session hosts. Both fill in the broker API's URL and client ID. +#> param ( [Parameter(Mandatory = $false, HelpMessage = "Only 'desktop' is supported, which opens a Remote Desktop session to a Linux host. Any other value opens the desktop too.")] - [string]$Mode = "desktop" + [string]$Mode = "desktop", + + [Parameter(Mandatory = $false, HelpMessage = "How long to wait, in seconds, for a Linux host that the broker starts because none is ready. 0 does not wait.")] + [ValidateRange(0, 3600)] + [int]$MaxWaitSeconds = 600, + + [Parameter(Mandatory = $false, HelpMessage = "On opens the Linux desktop full screen. Off opens it in a window on one monitor, whatever MultiMonitor says.")] + [ValidateSet('On', 'Off')] + [string]$FullScreen = 'On', + + [Parameter(Mandatory = $false, HelpMessage = "On spreads a full-screen Linux desktop across every monitor. Off keeps it on one monitor.")] + [ValidateSet('On', 'Off')] + [string]$MultiMonitor = 'On' ) -$ProgressPreference = 'SilentlyContinue' +# Sent with every checkout, so the scaling policy page can list the session hosts whose script +# cannot wait for a host to start. AVD_HOST_SCRIPT_VERSION in api/config.py must match: bump +# both with any change to this script. +$ScriptVersion = '2.0.0' -# Automatically collect local Windows hostname and username -$localHostname = $env:COMPUTERNAME -$localUsername = $env:USERNAME -replace '[^a-zA-Z0-9_]', '' +$ProgressPreference = 'SilentlyContinue' -# Define the API endpoints +# Filled in when the script is installed on the session host. $apiBaseUrl = "https://your_linuxbroker_api_base_url/api" -$checkoutVmUrl = "$apiBaseUrl/vms/checkout" - -# Define the maximum number of update attempts -$maxAttempts = 3 -$attemptCount = 0 -$hasExistingCheckedInVM = $false +$apiAppIdUri = "api://your_linuxbroker_api_client_id" $sourceName = "LinuxBrokerScript" # The source name for your event log. $logName = "Application" # The log where your source will write events. Commonly "Application". +# A checkout that provisions the user on the Linux host can take a while. +$RequestTimeoutSeconds = 120 +# A request that gets no answer, 408, 429 or a 5xx is tried again, until this many fail in a row. +$MaxTransientFailures = 3 +$TransientRetrySeconds = 5 +# The broker counts a user as waiting for five minutes after its last 202, so the script always +# asks again well within that. +$MinRetrySeconds = 5 +$MaxRetrySeconds = 120 +# Used when a 202 does not say how long to wait: 30, 30, then 60 seconds from then on. +$FallbackRetrySeconds = @(30, 30, 60) + +$OutcomeMessages = @{ + NoHost = @{ Text = "No Linux host is available right now. Try again in a few minutes."; Icon = "Warning" } + Starting = @{ Text = "A Linux host is starting for you. Try again in a few minutes."; Icon = "Information" } + TimedOut = @{ Text = "Your Linux desktop is taking longer than usual to start. Try again in a few minutes."; Icon = "Warning" } + AuthFailed = @{ Text = "The Linux Broker could not authenticate this session host. Contact your administrator."; Icon = "Error" } + Unavailable = @{ Text = "The Linux Broker is not responding. Try again in a few minutes, or contact your administrator."; Icon = "Error" } + Rejected = @{ Text = "The Linux Broker could not give you a Linux host. Contact your administrator."; Icon = "Error" } +} + function Write-Log { param ( [string]$Message, @@ -36,10 +86,11 @@ function Write-Log { } try { - Write-EventLog -LogName $logName -Source $sourceName -EntryType $entryType -EventId 1 -Message $Message + Write-EventLog -LogName $logName -Source $sourceName -EntryType $entryType -EventId 1 -Message $Message -ErrorAction Stop } catch { # Logging must never stop the connection attempt. + $null = $_ } } @@ -91,75 +142,492 @@ function Get-AccessToken { } } -# Earlier releases kept every other value for starting a single application through xpra, -# which was never implemented and has been removed. A RemoteApp that still passes one gets the -# desktop rather than nothing. -if ($Mode -ine "desktop") { - Write-Log "Mode '$Mode' is not supported, so the desktop is opened instead." "WARNING" +# Seconds on a clock that is not moved by changes to the system time. +function Get-MonotonicTime { + return [System.Diagnostics.Stopwatch]::GetTimestamp() / [double][System.Diagnostics.Stopwatch]::Frequency +} + +# The whole number of seconds in a Retry-After value, or $null when it holds none. +function ConvertTo-RetryAfter { + param ( + $Value + ) + + if ($Value -is [array]) { + $Value = $Value | Select-Object -First 1 + } + if ($null -eq $Value) { + return $null + } + + $seconds = 0 + if ([int]::TryParse(([string]$Value).Trim(), [ref]$seconds)) { + return $seconds + } + return $null +} + +# How long to wait before asking again: what the broker asked for, within limits, or the +# fallback schedule when it asked for nothing. +function Get-RetryDelay { + param ( + $RetryAfter, + [int]$WaitCount = 0 + ) + + if ($null -ne $RetryAfter) { + return [int][Math]::Min($MaxRetrySeconds, [Math]::Max($MinRetrySeconds, [int]$RetryAfter)) + } + + $index = [Math]::Min([Math]::Max($WaitCount, 0), $FallbackRetrySeconds.Count - 1) + return [int]$FallbackRetrySeconds[$index] +} + +function Test-TransientFailure { + param ( + [int]$StatusCode + ) + + return ($StatusCode -eq 0 -or $StatusCode -eq 408 -or $StatusCode -eq 429 -or ($StatusCode -ge 500 -and $StatusCode -le 599)) +} + +function New-BrokerHttpClient { + Add-Type -AssemblyName System.Net.Http + $client = New-Object System.Net.Http.HttpClient + $client.Timeout = [TimeSpan]::FromSeconds($RequestTimeoutSeconds) + $client.DefaultRequestHeaders.ExpectContinue = $false + return $client +} + +# A small window that tells the user their desktop is starting, with Cancel. It is modeless: +# the script keeps working and pumps its messages. Returns $null when it cannot be shown, and +# the wait goes on without it. +function Open-WaitWindow { + try { + Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop + Add-Type -AssemblyName System.Drawing -ErrorAction Stop + [System.Windows.Forms.Application]::EnableVisualStyles() + + $state = [pscustomobject]@{ + Form = $null + Cancelled = $false + Closing = $false + } + + $form = New-Object System.Windows.Forms.Form + $form.Text = "Linux Desktop" + $form.FormBorderStyle = [System.Windows.Forms.FormBorderStyle]::FixedDialog + $form.StartPosition = [System.Windows.Forms.FormStartPosition]::CenterScreen + $form.MaximizeBox = $false + $form.ShowInTaskbar = $true + $form.AutoSize = $true + $form.AutoSizeMode = [System.Windows.Forms.AutoSizeMode]::GrowAndShrink + $form.Font = [System.Drawing.SystemFonts]::MessageBoxFont + + $panel = New-Object System.Windows.Forms.FlowLayoutPanel + $panel.FlowDirection = [System.Windows.Forms.FlowDirection]::TopDown + $panel.WrapContents = $false + $panel.AutoSize = $true + $panel.AutoSizeMode = [System.Windows.Forms.AutoSizeMode]::GrowAndShrink + $panel.Padding = New-Object System.Windows.Forms.Padding -ArgumentList 16 + + $heading = New-Object System.Windows.Forms.Label + $heading.AutoSize = $true + $heading.MaximumSize = New-Object System.Drawing.Size -ArgumentList 360, 0 + $heading.Font = New-Object System.Drawing.Font -ArgumentList $form.Font, ([System.Drawing.FontStyle]::Bold) + $heading.Text = "Your Linux desktop is starting" + [char]0x2026 + + $detail = New-Object System.Windows.Forms.Label + $detail.AutoSize = $true + $detail.MaximumSize = New-Object System.Drawing.Size -ArgumentList 360, 0 + $detail.Margin = New-Object System.Windows.Forms.Padding -ArgumentList 3, 8, 3, 12 + $detail.Text = "This can take a few minutes. It opens by itself when it is ready." + + $progress = New-Object System.Windows.Forms.ProgressBar + $progress.Style = [System.Windows.Forms.ProgressBarStyle]::Marquee + $progress.MarqueeAnimationSpeed = 30 + $progress.Width = 360 + $progress.AccessibleName = "Waiting for your Linux desktop" + + $cancel = New-Object System.Windows.Forms.Button + $cancel.Text = "Cancel" + $cancel.AutoSize = $true + $cancel.Anchor = [System.Windows.Forms.AnchorStyles]::Right + $cancel.Margin = New-Object System.Windows.Forms.Padding -ArgumentList 3, 12, 3, 3 + $cancel.Add_Click({ $state.Cancelled = $true; $state.Form.Close() }.GetNewClosure()) + + $panel.Controls.AddRange(@($heading, $detail, $progress, $cancel)) + $form.Controls.Add($panel) + $form.CancelButton = $cancel + # Closing the window any way but from the script cancels the wait too. + $form.Add_FormClosing({ if (-not $state.Closing) { $state.Cancelled = $true } }.GetNewClosure()) + + $state.Form = $form + $form.Show() + $form.Activate() + [System.Windows.Forms.Application]::DoEvents() + return $state + } + catch { + Write-Log "The wait window could not be shown: $_" "WARNING" + return $null + } +} + +function Close-WaitWindow { + param ( + $WaitWindow + ) + + if ($null -eq $WaitWindow -or $null -eq $WaitWindow.Form) { + return + } + + try { + $WaitWindow.Closing = $true + if (-not $WaitWindow.Form.IsDisposed) { + $WaitWindow.Form.Close() + $WaitWindow.Form.Dispose() + } + } + catch { + Write-Log "The wait window could not be closed: $_" "WARNING" + } } -# Define the API's Application ID URI (use the updated valid URL) -$apiAppIdUri = "api://your_linuxbroker_api_client_id" # Replace with your API's actual Application ID URI +# Handles the wait window's pending messages, so it repaints and Cancel works. +function Invoke-WindowMessagePump { + param ( + $WaitWindow + ) -# Obtain the access token using Managed Identity -$accessToken = Get-AccessToken -Resource $apiAppIdUri + if ($null -eq $WaitWindow) { + return + } -if (-not $accessToken) { - Write-Log "Unable to obtain access token. Exiting script." "ERROR" - Show-UserMessage "The Linux Broker could not authenticate this session host. Contact your administrator." "Error" - exit 1 + try { + [System.Windows.Forms.Application]::DoEvents() + } + catch { + Write-Log "The wait window stopped responding: $_" "WARNING" + } } -# Prepare the Authorization header -$authHeader = @{ - "Authorization" = "Bearer $accessToken" +# Waits the given number of seconds, keeping the wait window responsive. Returns $false as soon +# as the user cancels. +function Wait-CheckoutInterval { + param ( + [double]$Seconds, + $WaitWindow + ) + + $until = (Get-MonotonicTime) + $Seconds + while ((Get-MonotonicTime) -lt $until) { + if ($null -ne $WaitWindow -and $WaitWindow.Cancelled) { + return $false + } + Invoke-WindowMessagePump -WaitWindow $WaitWindow + Start-Sleep -Milliseconds 100 + } + + return -not ($null -ne $WaitWindow -and $WaitWindow.Cancelled) } -# Attempt to get an available VM or a checked-out VM -while ($attemptCount -lt $maxAttempts) { - $attemptCount++ - Write-Log "Attempt $attemptCount of $maxAttempts Checking for an available or already checked-out VM..." "INFO" +# Posts one checkout request. Returns its status code (0 when no answer came), the parsed body, +# the wait a Retry-After asks for, and the error. While the wait window is open it stays +# responsive, and Cancel abandons the request. +function Invoke-BrokerCheckout { + param ( + [Parameter(Mandatory = $true)] + $Client, + + [Parameter(Mandatory = $true)] + [string]$Url, + + [Parameter(Mandatory = $true)] + [string]$AccessToken, + + [Parameter(Mandatory = $true)] + [hashtable]$Body, + + $WaitWindow + ) + + $result = [pscustomobject]@{ + StatusCode = 0 + Body = $null + RetryAfter = $null + Error = $null + Cancelled = $false + } + $request = $null + $response = $null + $cancellation = New-Object System.Threading.CancellationTokenSource try { - # Prepare the payload - $checkoutPayload = @{ - "username" = $localUsername - "avdhost" = $localHostname + $request = New-Object System.Net.Http.HttpRequestMessage -ArgumentList ([System.Net.Http.HttpMethod]::Post), $Url + $request.Headers.Authorization = New-Object System.Net.Http.Headers.AuthenticationHeaderValue -ArgumentList "Bearer", $AccessToken + $request.Content = New-Object System.Net.Http.StringContent -ArgumentList ($Body | ConvertTo-Json -Compress), ([System.Text.Encoding]::UTF8), "application/json" + + $task = $Client.SendAsync($request, $cancellation.Token) + while (-not $task.IsCompleted) { + if ($null -ne $WaitWindow -and $WaitWindow.Cancelled) { + $cancellation.Cancel() + $result.Cancelled = $true + return $result + } + Invoke-WindowMessagePump -WaitWindow $WaitWindow + try { + [void]$task.Wait(100) + } + catch { + # A failed request is reported from the task below. + $null = $_ + } + } + + if ($task.IsCanceled) { + $result.Error = "The broker did not answer within $RequestTimeoutSeconds seconds." + return $result + } + if ($task.IsFaulted) { + $result.Error = $task.Exception.GetBaseException().Message + return $result + } + + $response = $task.Result + $result.StatusCode = [int]$response.StatusCode + $retryHeader = $response.Headers.RetryAfter + if ($null -ne $retryHeader -and $null -ne $retryHeader.Delta) { + $result.RetryAfter = [int][Math]::Ceiling($retryHeader.Delta.TotalSeconds) } - # Invoke the API to checkout a VM with authentication - Write-Log "Attempting to checkout a VM via API with Managed Identity authentication..." "INFO" - $checkoutResponse = Invoke-RestMethod -Uri $checkoutVmUrl -Method POST ` - -ContentType "application/json" ` - -Body ($checkoutPayload | ConvertTo-Json) ` - -Headers $authHeader - - if ($checkoutResponse.VMID) { - $hasExistingCheckedInVM = $true - Write-Log "Successfully checked out or retrieved an existing VM (VMID: $($checkoutResponse.VMID), Hostname: $($checkoutResponse.Hostname))." "INFO" - break + $text = $response.Content.ReadAsStringAsync().Result + if (-not [string]::IsNullOrWhiteSpace($text)) { + try { + $result.Body = $text | ConvertFrom-Json + } + catch { + $result.Error = "The broker's answer was not JSON." + } } - else { - Write-Log "Failed to retrieve VM information from API response." "WARNING" + + if ($null -ne $result.Body) { + $bodyRetryAfter = ConvertTo-RetryAfter $result.Body.retryAfterSeconds + if ($null -ne $bodyRetryAfter) { + $result.RetryAfter = $bodyRetryAfter + } + if ($result.Body.error) { + $result.Error = [string]$result.Body.error + } } + + return $result } catch { - Write-Log "API request failed: $_" "ERROR" + $result.Error = $_.Exception.Message + return $result + } + finally { + if ($null -ne $response) { + $response.Dispose() + } + if ($null -ne $request) { + $request.Dispose() + } + $cancellation.Dispose() + } +} + +function New-CheckoutResult { + param ( + [Parameter(Mandatory = $true)] + [ValidateSet("Assigned", "NoHost", "Starting", "TimedOut", "Cancelled", "AuthFailed", "Unavailable", "Rejected")] + [string]$Outcome, + + $Checkout = $null + ) + + return [pscustomobject]@{ + Outcome = $Outcome + Checkout = $Checkout + } +} + +# Checks out a Linux host for the user. While the broker answers 202 because a host is starting, +# shows the wait window and asks again when the broker says to, until a host is ready, the user +# cancels, or MaxWaitSeconds pass. Returns the outcome, with the checkout when a host was assigned. +function Request-LinuxHost { + param ( + [Parameter(Mandatory = $true)] + [string]$CheckoutUrl, + + [Parameter(Mandatory = $true)] + [string]$Resource, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Username, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$AvdHost, + + [int]$MaxWaitSeconds = 600 + ) + + $accessToken = Get-AccessToken -Resource $Resource + if (-not $accessToken) { + Write-Log "Unable to obtain access token. Exiting script." "ERROR" + return New-CheckoutResult -Outcome AuthFailed + } + + $body = @{ + username = $Username + avdhost = $AvdHost + clientVersion = $ScriptVersion + } + + $client = New-BrokerHttpClient + $waitWindow = $null + $deadline = $null + $waits = 0 + $failures = 0 + $tokenRenewed = $false + $attempt = 0 + + try { + while ($true) { + $attempt++ + Write-Log "Checkout attempt $attempt for $Username from $AvdHost." "INFO" + $response = Invoke-BrokerCheckout -Client $client -Url $CheckoutUrl -AccessToken $accessToken -Body $body -WaitWindow $waitWindow + + if ($response.Cancelled) { + Write-Log "$Username cancelled the wait for a Linux host." "INFO" + return New-CheckoutResult -Outcome Cancelled + } + + $status = $response.StatusCode + $checkout = $response.Body + + if ($status -eq 200) { + if ($null -ne $checkout -and $checkout.VMID -and $checkout.IPAddress) { + Write-Log "Successfully checked out or retrieved an existing VM (VMID: $($checkout.VMID), Hostname: $($checkout.Hostname))." "INFO" + return New-CheckoutResult -Outcome Assigned -Checkout $checkout + } + Write-Log "The checkout answered without a VMID or IP address." "ERROR" + return New-CheckoutResult -Outcome Rejected + } + + if ($status -eq 202) { + $failures = 0 + Write-Log "No Linux host is ready yet ($($checkout.reason)): $($checkout.message)" "INFO" + if ($MaxWaitSeconds -le 0) { + return New-CheckoutResult -Outcome Starting + } + + $now = Get-MonotonicTime + if ($null -eq $deadline) { + $deadline = $now + $MaxWaitSeconds + } + $remaining = $deadline - $now + if ($remaining -le 0) { + Write-Log "No Linux host was ready within $MaxWaitSeconds seconds." "WARNING" + return New-CheckoutResult -Outcome TimedOut + } + + $delay = [Math]::Min((Get-RetryDelay -RetryAfter $response.RetryAfter -WaitCount $waits), [Math]::Ceiling($remaining)) + $waits++ + if ($null -eq $waitWindow) { + $waitWindow = Open-WaitWindow + } + Write-Log "Asking again in $delay seconds." "INFO" + if (-not (Wait-CheckoutInterval -Seconds $delay -WaitWindow $waitWindow)) { + Write-Log "$Username cancelled the wait for a Linux host." "INFO" + return New-CheckoutResult -Outcome Cancelled + } + continue + } + + if ($status -eq 409) { + Write-Log "No available or checked-out VM found: $($response.Error)" "WARNING" + return New-CheckoutResult -Outcome NoHost + } + + # A token can expire while the user waits, so a refused one is renewed once. + if ($status -eq 401 -and -not $tokenRenewed) { + $tokenRenewed = $true + Write-Log "The broker did not accept the access token, so a new one is requested." "WARNING" + $accessToken = Get-AccessToken -Resource $Resource + if ($accessToken) { + continue + } + return New-CheckoutResult -Outcome AuthFailed + } + + if ($status -eq 401 -or $status -eq 403) { + Write-Log "The broker refused this session host ($status): $($response.Error)" "ERROR" + return New-CheckoutResult -Outcome AuthFailed + } + + if (Test-TransientFailure -StatusCode $status) { + $failures++ + Write-Log "API request failed ($status): $($response.Error)" "ERROR" + if ($failures -ge $MaxTransientFailures) { + return New-CheckoutResult -Outcome Unavailable + } + + $delay = $TransientRetrySeconds * $failures + if ($null -ne $response.RetryAfter) { + $delay = Get-RetryDelay -RetryAfter $response.RetryAfter + } + if ($null -ne $deadline) { + $remaining = $deadline - (Get-MonotonicTime) + if ($remaining -le 0) { + Write-Log "No Linux host was ready within $MaxWaitSeconds seconds." "WARNING" + return New-CheckoutResult -Outcome TimedOut + } + $delay = [Math]::Min($delay, [Math]::Ceiling($remaining)) + } + if (-not (Wait-CheckoutInterval -Seconds $delay -WaitWindow $waitWindow)) { + Write-Log "$Username cancelled the wait for a Linux host." "INFO" + return New-CheckoutResult -Outcome Cancelled + } + continue + } + + Write-Log "The broker refused the checkout ($status): $($response.Error)" "ERROR" + return New-CheckoutResult -Outcome Rejected + } + } + finally { + Close-WaitWindow -WaitWindow $waitWindow + if ($client -is [System.IDisposable]) { + $client.Dispose() + } } } -# If a VM was checked out or found, connect to it -if ($hasExistingCheckedInVM -and $checkoutResponse.IPAddress) { - $hostname = $checkoutResponse.Hostname - $ipAddress = $checkoutResponse.IPAddress +function Save-LinuxHostCredential { + param ( + [Parameter(Mandatory = $true)] + $Checkout, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Username + ) + + $hostname = $Checkout.Hostname + $ipAddress = $Checkout.IPAddress # Store or update credentials in Credential Manager try { Write-Log "Updating Windows Credential Manager with credentials for $hostname..." "INFO" - + # Delete all existing credentials in Credential Manager Write-Log "Deleting all existing credentials in Credential Manager..." "INFO" - + cmdkey /list | ForEach-Object { if ($_ -match "Target: (.+)") { $target = $matches[1] @@ -168,21 +636,70 @@ if ($hasExistingCheckedInVM -and $checkoutResponse.IPAddress) { } } - New-StoredCredential -Target $hostname -UserName $localUsername -Password $checkoutResponse.password -Persist LocalMachine | Out-Null - New-StoredCredential -Target $ipAddress -UserName $localUsername -Password $checkoutResponse.password -Persist LocalMachine | Out-Null + New-StoredCredential -Target $hostname -UserName $Username -Password $Checkout.Password -Persist LocalMachine | Out-Null + New-StoredCredential -Target $ipAddress -UserName $Username -Password $Checkout.Password -Persist LocalMachine | Out-Null # mstsc only reuses a saved credential whose target carries the TERMSRV/ prefix. - New-StoredCredential -Target "TERMSRV/$ipAddress" -UserName $localUsername -Password $checkoutResponse.password -Type Generic -Persist LocalMachine | Out-Null - New-StoredCredential -Target "TERMSRV/$hostname" -UserName $localUsername -Password $checkoutResponse.password -Type Generic -Persist LocalMachine | Out-Null + New-StoredCredential -Target "TERMSRV/$ipAddress" -UserName $Username -Password $Checkout.Password -Type Generic -Persist LocalMachine | Out-Null + New-StoredCredential -Target "TERMSRV/$hostname" -UserName $Username -Password $Checkout.Password -Type Generic -Persist LocalMachine | Out-Null Write-Log "Credentials for $hostname updated successfully in Credential Manager." "INFO" } catch { Write-Log "Failed to update credentials in Credential Manager: $_" "ERROR" } +} + +# Returns the switches for mstsc.exe. mstsc reads anything else on its command line as another +# switch or a connection file, so the address can only be a host name or an IP address, with an +# optional port. +function Get-MstscArgumentList { + param ( + [Parameter(Mandatory = $true)] + [string]$IPAddress, + + [ValidateSet('On', 'Off')] + [string]$FullScreen = 'On', + + [ValidateSet('On', 'Off')] + [string]$MultiMonitor = 'On' + ) + + if ($IPAddress -cnotmatch '^[A-Za-z0-9][A-Za-z0-9.:-]*\z') { + throw "The broker gave '$IPAddress' as the Linux host's address, which is not a host name or an IP address." + } + + $arguments = @("/v:$IPAddress") + if ($FullScreen -eq 'On') { + $arguments += '/f' + # A session that uses every monitor opens full screen even without /f, so a window + # leaves /multimon out. + if ($MultiMonitor -eq 'On') { + $arguments += '/multimon' + } + } + return $arguments +} + +function Open-RemoteDesktop { + param ( + [Parameter(Mandatory = $true)] + [string]$Hostname, + + [Parameter(Mandatory = $true)] + [string]$IPAddress, + + [ValidateSet('On', 'Off')] + [string]$FullScreen = 'On', - Write-Log "Connecting to $hostname (IP: $ipAddress) using Remote Desktop Connection..." "INFO" + [ValidateSet('On', 'Off')] + [string]$MultiMonitor = 'On' + ) + + Write-Log "Connecting to $Hostname (IP: $IPAddress) using Remote Desktop Connection..." "INFO" try { + $mstscArguments = @(Get-MstscArgumentList -IPAddress $IPAddress -FullScreen $FullScreen -MultiMonitor $MultiMonitor) + # xrdp presents a self-signed certificate, so skip the server authentication warning for this user. $rdpClientKey = "HKCU:\Software\Microsoft\Terminal Server Client" if (-not (Test-Path $rdpClientKey)) { @@ -190,17 +707,77 @@ if ($hasExistingCheckedInVM -and $checkoutResponse.IPAddress) { } New-ItemProperty -Path $rdpClientKey -Name "AuthenticationLevelOverride" -PropertyType DWord -Value 0 -Force | Out-Null - # Launch mstsc with the hostname or IP address - Start-Process mstsc.exe -ArgumentList "/v:$ipAddress" + Write-Log "Starting mstsc.exe $($mstscArguments -join ' ')." "INFO" + Start-Process mstsc.exe -ArgumentList $mstscArguments -ErrorAction Stop - Write-Log "Successfully connected to $hostname (IP: $ipAddress) using Remote Desktop Connection." "INFO" + Write-Log "Successfully connected to $Hostname (IP: $IPAddress) using Remote Desktop Connection." "INFO" + return $true } catch { - Write-Log "Failed to connect to $hostname (IP: $ipAddress) using Remote Desktop Connection: $_" "ERROR" - Show-UserMessage "Remote Desktop Connection could not be started for $hostname. Try again, or contact your administrator." "Error" + Write-Log "Failed to connect to $Hostname (IP: $IPAddress) using Remote Desktop Connection: $_" "ERROR" + Show-UserMessage "Remote Desktop Connection could not be started for $Hostname. Try again, or contact your administrator." "Error" + return $false + } +} + +# Returns the exit code: 0 when the desktop opened or the user cancelled, 1 otherwise. +function Invoke-ConnectLinuxBroker { + param ( + [string]$Mode = "desktop", + + [int]$MaxWaitSeconds = 600, + + [ValidateSet('On', 'Off')] + [string]$FullScreen = 'On', + + [ValidateSet('On', 'Off')] + [string]$MultiMonitor = 'On', + + [Parameter(Mandatory = $true)] + [string]$ApiBaseUrl, + + [Parameter(Mandatory = $true)] + [string]$Resource, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Username, + + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$AvdHost + ) + + # Earlier releases kept every other value for starting a single application through xpra, + # which was never implemented and has been removed. A RemoteApp that still passes one gets the + # desktop rather than nothing. + if ($Mode -ine "desktop") { + Write-Log "Mode '$Mode' is not supported, so the desktop is opened instead." "WARNING" + } + + Write-Log "Connect-LinuxBroker.ps1 $ScriptVersion is checking out a Linux host for $Username." "INFO" + $result = Request-LinuxHost -CheckoutUrl "$ApiBaseUrl/vms/checkout" -Resource $Resource -Username $Username -AvdHost $AvdHost -MaxWaitSeconds $MaxWaitSeconds + + if ($result.Outcome -eq "Assigned") { + Save-LinuxHostCredential -Checkout $result.Checkout -Username $Username + if (Open-RemoteDesktop -Hostname $result.Checkout.Hostname -IPAddress $result.Checkout.IPAddress -FullScreen $FullScreen -MultiMonitor $MultiMonitor) { + return 0 + } + return 1 + } + + if ($result.Outcome -eq "Cancelled") { + return 0 } + + $message = $OutcomeMessages[$result.Outcome] + Show-UserMessage $message.Text $message.Icon + return 1 } -else { - Write-Log "No available or checked-out VM found. Exiting script." "WARNING" - Show-UserMessage "No Linux host is available right now. Try again in a few minutes." "Warning" + +# Dot-sourcing the script, as its tests do, defines the functions without connecting. +if ($MyInvocation.InvocationName -ne '.') { + $exitCode = Invoke-ConnectLinuxBroker -Mode $Mode -MaxWaitSeconds $MaxWaitSeconds -FullScreen $FullScreen -MultiMonitor $MultiMonitor ` + -ApiBaseUrl $apiBaseUrl -Resource $apiAppIdUri -Username ($env:USERNAME -replace '[^a-zA-Z0-9_]', '') -AvdHost $env:COMPUTERNAME + exit ([int]($exitCode | Select-Object -Last 1)) } diff --git a/avd_host/tests/Connect-LinuxBroker.Tests.ps1 b/avd_host/tests/Connect-LinuxBroker.Tests.ps1 new file mode 100644 index 0000000..c4355a6 --- /dev/null +++ b/avd_host/tests/Connect-LinuxBroker.Tests.ps1 @@ -0,0 +1,913 @@ +# Pester 5 tests for avd_host/broker/Connect-LinuxBroker.ps1, which runs in Windows PowerShell 5.1 +# on the AVD session hosts. From the repository root, in Windows PowerShell: +# Invoke-Pester -Path avd_host/tests + +BeforeAll { + $script:BrokerScriptPath = (Resolve-Path (Join-Path $PSScriptRoot '..\broker\Connect-LinuxBroker.ps1')).Path + . $script:BrokerScriptPath + + Add-Type -AssemblyName System.Net.Http + if (-not ('LinuxBrokerTests.StubHandler' -as [type])) { + Add-Type -ReferencedAssemblies System.Net.Http -TypeDefinition @' +namespace LinuxBrokerTests +{ + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + + // Answers every request HttpClient sends with the response it is set up with. + public class StubHandler : HttpMessageHandler + { + public HttpStatusCode StatusCode = HttpStatusCode.OK; + public string ResponseBody = ""; + public string RetryAfter; + public string Failure; + public bool Cancel; + public bool Hang; + public int DelayMilliseconds; + public int Calls; + public HttpRequestMessage LastRequest; + public string LastBody; + public string LastContentType; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Calls++; + LastRequest = request; + if (request.Content != null) + { + LastBody = request.Content.ReadAsStringAsync().Result; + LastContentType = request.Content.Headers.ContentType.MediaType; + } + + var source = new TaskCompletionSource(); + if (Hang) + { + return source.Task; + } + if (Failure != null) + { + source.SetException(new HttpRequestException(Failure)); + return source.Task; + } + if (Cancel) + { + source.SetCanceled(); + return source.Task; + } + + var response = new HttpResponseMessage(StatusCode); + response.Content = new StringContent(ResponseBody ?? "", System.Text.Encoding.UTF8, "application/json"); + if (RetryAfter != null) + { + response.Headers.TryAddWithoutValidation("Retry-After", RetryAfter); + } + response.RequestMessage = request; + if (DelayMilliseconds > 0) + { + return Task.Delay(DelayMilliseconds).ContinueWith(delay => response); + } + source.SetResult(response); + return source.Task; + } + } +} +'@ + } + + function New-TestResponse { + param( + [int]$StatusCode, + $Body = $null, + $RetryAfter = $null, + [string]$Message = $null, + [switch]$Cancelled + ) + + [pscustomobject]@{ + StatusCode = $StatusCode + Body = $Body + RetryAfter = $RetryAfter + Error = $Message + Cancelled = [bool]$Cancelled + } + } + + function New-TestCheckout { + [pscustomobject]@{ VMID = 5; Hostname = 'lnx-05'; IPAddress = '10.0.0.5'; LeaseId = 'lease'; Password = 'from-the-broker' } + } + + function New-TestStarting { + param($RetryAfter = 60, [string]$Reason = 'Started') + + New-TestResponse -StatusCode 202 -RetryAfter $RetryAfter -Body ([pscustomobject]@{ + status = 'Starting' + reason = $Reason + retryAfterSeconds = $RetryAfter + message = "A Linux host is starting for you. Ask again in $RetryAfter seconds." + }) + } +} + +Describe 'Connect-LinuxBroker.ps1' { + BeforeAll { + $script:Content = [System.IO.File]::ReadAllText($script:BrokerScriptPath) + } + + It 'is plain ASCII, because it is installed without a byte order mark and Windows PowerShell reads it as ANSI' { + $bytes = [System.IO.File]::ReadAllBytes($script:BrokerScriptPath) + @($bytes | Where-Object { $_ -gt 127 }).Count | Should -Be 0 + } + + It 'parses' { + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseFile($script:BrokerScriptPath, [ref]$null, [ref]$errors) + $errors | Should -BeNullOrEmpty + } + + # Configure-AVD-Host.ps1 replaces every occurrence of each, so any other mention would be + # rewritten too. + It 'names each value filled in at installation exactly once' { + [regex]::Matches($script:Content, [regex]::Escape('https://your_linuxbroker_api_base_url/api')).Count | Should -Be 1 + [regex]::Matches($script:Content, [regex]::Escape('your_linuxbroker_api_client_id')).Count | Should -Be 1 + } + + It 'takes the API URL and client ID the way Configure-AVD-Host.ps1 fills them in' { + $installed = ($script:Content -replace 'https://your_linuxbroker_api_base_url/api', 'https://broker.example/api').Replace( + 'your_linuxbroker_api_client_id', '11111111-2222-3333-4444-555555555555') + + $installed | Should -Match ([regex]::Escape('$apiBaseUrl = "https://broker.example/api"')) + $installed | Should -Match ([regex]::Escape('$apiAppIdUri = "api://11111111-2222-3333-4444-555555555555"')) + $installed | Should -Not -Match 'your_linuxbroker' + $errors = $null + [void][System.Management.Automation.Language.Parser]::ParseInput($installed, [ref]$null, [ref]$errors) + $errors | Should -BeNullOrEmpty + } + + # The same rule as CLIENT_VERSION_RE and CLIENT_VERSION_MAX_CHARS in api/app.py. + It 'reports a version the broker records' { + $ScriptVersion | Should -Match '^[A-Za-z0-9][A-Za-z0-9._+-]*$' + $ScriptVersion.Length | Should -BeLessOrEqual 32 + } + + It 'waits ten minutes at most by default' { + $MaxWaitSeconds | Should -Be 600 + } + + It 'opens the desktop full screen across every monitor by default' { + $FullScreen | Should -Be 'On' + $MultiMonitor | Should -Be 'On' + } + + # The AVD module passes these on the RemoteApp's command line. + It 'takes only On or Off for ' -TestCases @( + @{ Name = 'FullScreen' } + @{ Name = 'MultiMonitor' } + ) { + $parameter = (Get-Command $script:BrokerScriptPath).Parameters[$Name] + $validateSet = $parameter.Attributes | Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } + $validateSet.ValidValues | Should -Be @('On', 'Off') + } + + # Scripts from before 2.0.0 refuse these parameters, so the RemoteApp passes one only to turn + # it Off, and a default deployment keeps the command line those scripts accept. + It 'is given a display parameter by the RemoteApp only to turn it Off' { + $bicep = [System.IO.File]::ReadAllText((Join-Path $PSScriptRoot '..\..\deploy\bicep\modules\AVD\main.bicep')) + $bicep | Should -Match ([regex]::Escape('-File C:\\Temp\\Connect-LinuxBroker.ps1${linuxDesktopDisplayArguments}''')) + $definition = [regex]::Match($bicep, '(?m)^var linuxDesktopDisplayArguments = (.+)$').Groups[1].Value + $passed = @([regex]::Matches($definition, '\? '''' : '' -(\w+) (\w+)''')) + + @($passed | ForEach-Object { $_.Groups[1].Value }) | Should -Be @('FullScreen', 'MultiMonitor') + foreach ($argument in $passed) { + $argument.Groups[2].Value | Should -BeExactly 'Off' + (Get-Command $script:BrokerScriptPath).Parameters.Keys | Should -Contain $argument.Groups[1].Value + } + } + + # Only starting the script runs the call, which no test does. + It 'passes - from its command line on' -TestCases @( + @{ Name = 'Mode' } + @{ Name = 'MaxWaitSeconds' } + @{ Name = 'FullScreen' } + @{ Name = 'MultiMonitor' } + ) { + $ast = [System.Management.Automation.Language.Parser]::ParseFile($script:BrokerScriptPath, [ref]$null, [ref]$null) + $calls = @($ast.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.CommandAst] -and $node.GetCommandName() -eq 'Invoke-ConnectLinuxBroker' + }, $true)) + + $calls.Count | Should -Be 1 + $calls[0].Extent.Text | Should -Match ('-{0} \${0}(\s|\z)' -f $Name) + } +} + +Describe 'ConvertTo-RetryAfter' { + It 'reads as ' -TestCases @( + @{ Value = 45; Expected = 45 } + @{ Value = '45'; Expected = 45 } + @{ Value = ' 30 '; Expected = 30 } + @{ Value = @('75', '10'); Expected = 75 } + ) { + ConvertTo-RetryAfter $Value | Should -Be $Expected + } + + It 'returns nothing for ' -TestCases @( + @{ Value = $null } + @{ Value = '' } + @{ Value = 'soon' } + @{ Value = '4.5' } + @{ Value = @() } + ) { + ConvertTo-RetryAfter $Value | Should -BeNullOrEmpty + } +} + +Describe 'Get-RetryDelay' { + It 'waits what the broker asks for, from 5 to 120 seconds: gives ' -TestCases @( + @{ RetryAfter = 60; Expected = 60 } + @{ RetryAfter = 1; Expected = 5 } + @{ RetryAfter = 0; Expected = 5 } + @{ RetryAfter = 600; Expected = 120 } + ) { + Get-RetryDelay -RetryAfter $RetryAfter -WaitCount 3 | Should -Be $Expected + } + + It 'waits 30, 30, then 60 seconds when the broker does not say' { + 0..5 | ForEach-Object { Get-RetryDelay -RetryAfter $null -WaitCount $_ } | Should -Be @(30, 30, 60, 60, 60, 60) + } +} + +Describe 'Test-TransientFailure' { + It 'retries ' -TestCases @( + @{ StatusCode = 0 }, @{ StatusCode = 408 }, @{ StatusCode = 429 }, @{ StatusCode = 500 }, @{ StatusCode = 502 }, @{ StatusCode = 503 } + ) { + Test-TransientFailure -StatusCode $StatusCode | Should -BeTrue + } + + It 'does not retry ' -TestCases @( + @{ StatusCode = 200 }, @{ StatusCode = 202 }, @{ StatusCode = 400 }, @{ StatusCode = 401 }, @{ StatusCode = 403 }, @{ StatusCode = 404 }, @{ StatusCode = 409 } + ) { + Test-TransientFailure -StatusCode $StatusCode | Should -BeFalse + } +} + +Describe 'Invoke-BrokerCheckout' { + BeforeEach { + Mock Write-Log {} + Mock Invoke-WindowMessagePump {} + $handler = New-Object LinuxBrokerTests.StubHandler + $client = New-Object System.Net.Http.HttpClient -ArgumentList $handler + $checkoutRequest = @{ + Client = $client + Url = 'https://broker.example/api/vms/checkout' + AccessToken = 'token-1' + Body = @{ username = 'alice'; avdhost = 'avd-0'; clientVersion = '2.0.0' } + } + } + + AfterEach { + $client.Dispose() + } + + It 'posts the checkout as JSON with the bearer token' { + $handler.ResponseBody = '{"VMID": 5, "Hostname": "lnx-05", "IPAddress": "10.0.0.5", "LeaseId": "lease"}' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 200 + $response.Body.Hostname | Should -Be 'lnx-05' + $response.Body.IPAddress | Should -Be '10.0.0.5' + $response.Error | Should -BeNullOrEmpty + $response.Cancelled | Should -BeFalse + $handler.Calls | Should -Be 1 + $handler.LastRequest.Method.Method | Should -Be 'POST' + $handler.LastRequest.RequestUri.AbsoluteUri | Should -Be 'https://broker.example/api/vms/checkout' + $handler.LastRequest.Headers.Authorization.Scheme | Should -Be 'Bearer' + $handler.LastRequest.Headers.Authorization.Parameter | Should -Be 'token-1' + $handler.LastContentType | Should -Be 'application/json' + $sent = $handler.LastBody | ConvertFrom-Json + $sent.username | Should -Be 'alice' + $sent.avdhost | Should -Be 'avd-0' + $sent.clientVersion | Should -Be '2.0.0' + } + + It 'reads the wait from a 202 body' { + $handler.StatusCode = [System.Net.HttpStatusCode]::Accepted + $handler.RetryAfter = '90' + $handler.ResponseBody = '{"status": "Starting", "reason": "Started", "retryAfterSeconds": 75, "message": "A Linux host is starting for you."}' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 202 + $response.RetryAfter | Should -Be 75 + $response.Body.reason | Should -Be 'Started' + } + + It 'reads the wait from the Retry-After header when the body has none' { + $handler.StatusCode = [System.Net.HttpStatusCode]::Accepted + $handler.RetryAfter = '40' + $handler.ResponseBody = '{"status": "Starting"}' + + (Invoke-BrokerCheckout @checkoutRequest).RetryAfter | Should -Be 40 + } + + It 'reports the error the broker gives' { + $handler.StatusCode = [System.Net.HttpStatusCode]::Conflict + $handler.ResponseBody = '{"error": "No available VM found. Please try again."}' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 409 + $response.Error | Should -Be 'No available VM found. Please try again.' + $response.RetryAfter | Should -BeNullOrEmpty + } + + It 'reports an answer that is not JSON' { + $handler.StatusCode = [System.Net.HttpStatusCode]::BadGateway + $handler.ResponseBody = 'Bad gateway' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 502 + $response.Body | Should -BeNullOrEmpty + $response.Error | Should -Match 'not JSON' + } + + It 'reports no status when the request fails' { + $handler.Failure = 'No such host is known.' + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 0 + $response.Error | Should -Match 'No such host is known' + } + + It 'reports no status when the broker does not answer in time' { + $handler.Cancel = $true + + $response = Invoke-BrokerCheckout @checkoutRequest + + $response.StatusCode | Should -Be 0 + $response.Error | Should -Match 'did not answer within 120 seconds' + $response.Cancelled | Should -BeFalse + } + + It 'keeps the wait window responsive while the request runs' { + $handler.DelayMilliseconds = 400 + $handler.ResponseBody = '{"VMID": 5, "Hostname": "lnx-05", "IPAddress": "10.0.0.5"}' + + $response = Invoke-BrokerCheckout @checkoutRequest -WaitWindow ([pscustomobject]@{ Cancelled = $false }) + + $response.StatusCode | Should -Be 200 + Should -Invoke Invoke-WindowMessagePump -Scope It + } + + It 'abandons the request when the user cancels' { + $handler.Hang = $true + + $response = Invoke-BrokerCheckout @checkoutRequest -WaitWindow ([pscustomobject]@{ Cancelled = $true }) + + $response.Cancelled | Should -BeTrue + $response.StatusCode | Should -Be 0 + } +} + +Describe 'Wait-CheckoutInterval' { + BeforeEach { + $script:Clock = 100.0 + Mock Get-MonotonicTime { $script:Clock } + Mock Start-Sleep { $script:Clock += $Milliseconds / 1000.0 } + Mock Invoke-WindowMessagePump {} + } + + It 'waits the whole interval, keeping the window responsive' { + Wait-CheckoutInterval -Seconds 2 -WaitWindow ([pscustomobject]@{ Cancelled = $false }) | Should -BeTrue + $script:Clock | Should -BeGreaterOrEqual 102 + $script:Clock | Should -BeLessThan 102.2 + Should -Invoke Invoke-WindowMessagePump -Scope It -Times 19 + } + + It 'stops as soon as the user cancels' { + $script:Window = [pscustomobject]@{ Cancelled = $false } + Mock Start-Sleep { + $script:Clock += $Milliseconds / 1000.0 + if ($script:Clock -ge 100.5) { $script:Window.Cancelled = $true } + } + + Wait-CheckoutInterval -Seconds 60 -WaitWindow $script:Window | Should -BeFalse + $script:Clock | Should -BeLessThan 101 + } + + It 'waits without a window' { + Wait-CheckoutInterval -Seconds 1 | Should -BeTrue + $script:Clock | Should -BeGreaterOrEqual 101 + } +} + +Describe 'Request-LinuxHost' { + BeforeEach { + $script:Responses = New-Object System.Collections.Queue + $script:Clock = 0.0 + $script:Waited = New-Object System.Collections.ArrayList + $script:Tokens = New-Object System.Collections.Queue + $script:Tokens.Enqueue('token-1') + $script:Tokens.Enqueue('token-2') + $script:Window = [pscustomobject]@{ Form = $null; Cancelled = $false; Closing = $false } + + Mock Write-Log {} + Mock Get-AccessToken { if ($script:Tokens.Count -gt 0) { $script:Tokens.Dequeue() } } + Mock New-BrokerHttpClient { [pscustomobject]@{ Name = 'client' } } + Mock Invoke-BrokerCheckout { $script:Responses.Dequeue() } + Mock Open-WaitWindow { $script:Window } + Mock Close-WaitWindow {} + Mock Get-MonotonicTime { $script:Clock } + Mock Wait-CheckoutInterval { + [void]$script:Waited.Add([double]$Seconds) + $script:Clock += $Seconds + $true + } + + $requestArguments = @{ + CheckoutUrl = 'https://broker.example/api/vms/checkout' + Resource = 'api://client' + Username = 'alice' + AvdHost = 'avd-0' + } + } + + It 'returns the host from a 200 without showing the wait window' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + $result = Request-LinuxHost @requestArguments + + $result.Outcome | Should -Be 'Assigned' + $result.Checkout.Hostname | Should -Be 'lnx-05' + Should -Invoke Get-AccessToken -Scope It -Times 1 -Exactly -ParameterFilter { $Resource -eq 'api://client' } + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly -ParameterFilter { + $Url -eq 'https://broker.example/api/vms/checkout' -and $AccessToken -eq 'token-1' -and + $Body.username -eq 'alice' -and $Body.avdhost -eq 'avd-0' -and $Body.clientVersion -eq $ScriptVersion + } + Should -Invoke Open-WaitWindow -Scope It -Times 0 -Exactly + Should -Invoke Wait-CheckoutInterval -Scope It -Times 0 -Exactly + } + + It 'does not ask the broker without an access token' { + $script:Tokens.Clear() + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'AuthFailed' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 0 -Exactly + } + + It 'shows the wait window and asks again when the broker says, until the host is ready' { + $script:Responses.Enqueue((New-TestStarting -RetryAfter 75)) + $script:Responses.Enqueue((New-TestStarting -RetryAfter 30 -Reason 'AlreadyStarting')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + $result = Request-LinuxHost @requestArguments + + $result.Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(75, 30) + Should -Invoke Open-WaitWindow -Scope It -Times 1 -Exactly + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 3 -Exactly + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 2 -Exactly -ParameterFilter { $null -ne $WaitWindow } + Should -Invoke Close-WaitWindow -Scope It -Times 1 -Exactly -ParameterFilter { $null -ne $WaitWindow } + } + + It 'waits 30, 30, then 60 seconds when the broker does not say how long' { + 1..5 | ForEach-Object { $script:Responses.Enqueue((New-TestResponse -StatusCode 202 -Body ([pscustomobject]@{ status = 'Starting' }))) } + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(30, 30, 60, 60, 60) + } + + It 'gives up after MaxWaitSeconds, asking one last time at the deadline' { + 1..10 | ForEach-Object { $script:Responses.Enqueue((New-TestStarting -RetryAfter 120)) } + + $result = Request-LinuxHost @requestArguments -MaxWaitSeconds 300 + + $result.Outcome | Should -Be 'TimedOut' + $script:Waited | Should -Be @(120, 120, 60) + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 4 -Exactly + Should -Invoke Close-WaitWindow -Scope It -Times 1 -Exactly -ParameterFilter { $null -ne $WaitWindow } + } + + It 'asks again within two minutes, however long the broker asks it to wait' { + $script:Responses.Enqueue((New-TestStarting -RetryAfter 900)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(120) + } + + It 'does not wait when MaxWaitSeconds is 0' { + $script:Responses.Enqueue((New-TestStarting)) + + (Request-LinuxHost @requestArguments -MaxWaitSeconds 0).Outcome | Should -Be 'Starting' + Should -Invoke Open-WaitWindow -Scope It -Times 0 -Exactly + Should -Invoke Wait-CheckoutInterval -Scope It -Times 0 -Exactly + } + + It 'stops when the user cancels the wait' { + $script:Responses.Enqueue((New-TestStarting)) + Mock Wait-CheckoutInterval { $false } + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Cancelled' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + Should -Invoke Close-WaitWindow -Scope It -Times 1 -Exactly -ParameterFilter { $null -ne $WaitWindow } + } + + It 'stops when the user cancels during a request' { + $script:Responses.Enqueue((New-TestStarting)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 0 -Cancelled)) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Cancelled' + } + + It 'waits with no window when it cannot be shown' { + Mock Open-WaitWindow { $null } + $script:Responses.Enqueue((New-TestStarting -RetryAfter 45)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(45) + } + + It 'reports that no host is available on a 409' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 409 -Message 'No available VM found. Please try again.')) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'NoHost' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + } + + It 'renews a refused access token once' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 401 -Message 'Token is invalid.')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + Should -Invoke Get-AccessToken -Scope It -Times 2 -Exactly + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly -ParameterFilter { $AccessToken -eq 'token-2' } + } + + It 'gives up when the renewed token is refused too' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 401)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 401)) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'AuthFailed' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 2 -Exactly + } + + It 'gives up when the session host is not allowed to check out' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 403 -Message 'Forbidden')) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'AuthFailed' + Should -Invoke Get-AccessToken -Scope It -Times 1 -Exactly + } + + It 'retries a failed request after 5, then 10 seconds' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 500 -Message 'Unable to check out a virtual machine.')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 0 -Message 'Unable to connect to the remote server')) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(5, 10) + Should -Invoke Open-WaitWindow -Scope It -Times 0 -Exactly + } + + It 'reports the broker as unavailable after three failures in a row' { + 1..3 | ForEach-Object { $script:Responses.Enqueue((New-TestResponse -StatusCode 503 -Message 'The broker is busy. Please retry shortly.')) } + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Unavailable' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 3 -Exactly + $script:Waited | Should -Be @(5, 10) + } + + It 'honours a Retry-After on a failure' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 429 -RetryAfter 20)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(20) + } + + It 'counts failures in a row only' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestStarting -RetryAfter 30)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body (New-TestCheckout))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Assigned' + $script:Waited | Should -Be @(5, 10, 30, 5, 10) + } + + It 'does not retry failures past the wait deadline' { + $script:Responses.Enqueue((New-TestStarting -RetryAfter 30)) + $script:Responses.Enqueue((New-TestResponse -StatusCode 500)) + + (Request-LinuxHost @requestArguments -MaxWaitSeconds 30).Outcome | Should -Be 'TimedOut' + $script:Waited | Should -Be @(30) + } + + It 'refuses a checkout that names no address' { + $script:Responses.Enqueue((New-TestResponse -StatusCode 200 -Body ([pscustomobject]@{ VMID = 5; Hostname = 'lnx-05'; IPAddress = $null }))) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Rejected' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + } + + It 'gives up on a request the broker rejects: ' -TestCases @( + @{ StatusCode = 400 }, @{ StatusCode = 404 } + ) { + $script:Responses.Enqueue((New-TestResponse -StatusCode $StatusCode -Message 'Rejected')) + + (Request-LinuxHost @requestArguments).Outcome | Should -Be 'Rejected' + Should -Invoke Invoke-BrokerCheckout -Scope It -Times 1 -Exactly + } +} + +Describe 'Invoke-ConnectLinuxBroker' { + BeforeEach { + Mock Write-Log {} + Mock Show-UserMessage {} + Mock Save-LinuxHostCredential {} + Mock Open-RemoteDesktop { $true } + $connectArguments = @{ + ApiBaseUrl = 'https://broker.example/api' + Resource = 'api://client' + Username = 'alice' + AvdHost = 'avd-0' + } + } + + It 'saves the credentials and opens the desktop on the assigned host' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 0 + Should -Invoke Request-LinuxHost -Scope It -Times 1 -Exactly -ParameterFilter { + $CheckoutUrl -eq 'https://broker.example/api/vms/checkout' -and $Resource -eq 'api://client' -and + $Username -eq 'alice' -and $AvdHost -eq 'avd-0' -and $MaxWaitSeconds -eq 600 + } + Should -Invoke Save-LinuxHostCredential -Scope It -Times 1 -Exactly -ParameterFilter { $Username -eq 'alice' -and $Checkout.Hostname -eq 'lnx-05' } + Should -Invoke Open-RemoteDesktop -Scope It -Times 1 -Exactly -ParameterFilter { + $Hostname -eq 'lnx-05' -and $IPAddress -eq '10.0.0.5' -and $FullScreen -eq 'On' -and $MultiMonitor -eq 'On' + } + Should -Invoke Show-UserMessage -Scope It -Times 0 -Exactly + } + + It 'passes MaxWaitSeconds on' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Cancelled } + + Invoke-ConnectLinuxBroker @connectArguments -MaxWaitSeconds 120 | Should -Be 0 + Should -Invoke Request-LinuxHost -Scope It -Times 1 -Exactly -ParameterFilter { $MaxWaitSeconds -eq 120 } + } + + # The test cases' names differ from the parameters', which the filter would see instead of an + # unbound parameter. + It 'passes FullScreen and MultiMonitor on' -TestCases @( + @{ Screen = 'Off'; Monitors = 'On' } + @{ Screen = 'On'; Monitors = 'Off' } + ) { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + + Invoke-ConnectLinuxBroker @connectArguments -FullScreen $Screen -MultiMonitor $Monitors | Should -Be 0 + Should -Invoke Open-RemoteDesktop -Scope It -Times 1 -Exactly -ParameterFilter { $FullScreen -eq $Screen -and $MultiMonitor -eq $Monitors } + } + + It 'fails when Remote Desktop Connection cannot be started' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + Mock Open-RemoteDesktop { $false } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 1 + } + + It 'closes quietly when the user cancels' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Cancelled } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 0 + Should -Invoke Show-UserMessage -Scope It -Times 0 -Exactly + Should -Invoke Open-RemoteDesktop -Scope It -Times 0 -Exactly + } + + It 'tells the user what happened: ' -TestCases @( + @{ Outcome = 'NoHost'; ExpectedText = 'No Linux host is available right now. Try again in a few minutes.'; ExpectedIcon = 'Warning' } + @{ Outcome = 'Starting'; ExpectedText = 'A Linux host is starting for you. Try again in a few minutes.'; ExpectedIcon = 'Information' } + @{ Outcome = 'TimedOut'; ExpectedText = 'Your Linux desktop is taking longer than usual to start. Try again in a few minutes.'; ExpectedIcon = 'Warning' } + @{ Outcome = 'AuthFailed'; ExpectedText = 'The Linux Broker could not authenticate this session host. Contact your administrator.'; ExpectedIcon = 'Error' } + @{ Outcome = 'Unavailable'; ExpectedText = 'The Linux Broker is not responding. Try again in a few minutes, or contact your administrator.'; ExpectedIcon = 'Error' } + @{ Outcome = 'Rejected'; ExpectedText = 'The Linux Broker could not give you a Linux host. Contact your administrator.'; ExpectedIcon = 'Error' } + ) { + $script:Outcome = $Outcome + Mock Request-LinuxHost { New-CheckoutResult -Outcome $script:Outcome } + + Invoke-ConnectLinuxBroker @connectArguments | Should -Be 1 + Should -Invoke Show-UserMessage -Scope It -Times 1 -Exactly -ParameterFilter { $Message -eq $ExpectedText -and $Icon -eq $ExpectedIcon } + Should -Invoke Open-RemoteDesktop -Scope It -Times 0 -Exactly + } + + It 'opens the desktop for any other mode, with a warning' { + Mock Request-LinuxHost { New-CheckoutResult -Outcome Assigned -Checkout (New-TestCheckout) } + + Invoke-ConnectLinuxBroker @connectArguments -Mode 'xpra' | Should -Be 0 + Should -Invoke Write-Log -Scope It -Times 1 -Exactly -ParameterFilter { $Level -eq 'WARNING' -and $Message -like "Mode 'xpra' is not supported*" } + Should -Invoke Open-RemoteDesktop -Scope It -Times 1 -Exactly + } +} + +Describe 'Save-LinuxHostCredential' { + BeforeAll { + # The CredentialManager module is installed on the session hosts only. + if (-not (Get-Command New-StoredCredential -ErrorAction SilentlyContinue)) { + function script:New-StoredCredential { + param($Target, $UserName, $Password, $Type, $Persist) + } + } + } + + BeforeEach { + Mock Write-Log {} + Mock cmdkey { 'Currently stored credentials:', '', ' Target: LegacyGeneric:target=lnx-01', ' Type: Generic' } + Mock New-StoredCredential {} + } + + It 'replaces every saved credential with the ones for the assigned host' { + Save-LinuxHostCredential -Checkout (New-TestCheckout) -Username 'alice' + + Should -Invoke cmdkey -Scope It -Times 1 -Exactly -ParameterFilter { $args[0] -eq '/delete:LegacyGeneric:target=lnx-01' } + foreach ($expected in 'lnx-05', '10.0.0.5') { + Should -Invoke New-StoredCredential -Scope It -Times 1 -Exactly -ParameterFilter { + $Target -eq $expected -and $UserName -eq 'alice' -and $Password -eq 'from-the-broker' -and $Persist -eq 'LocalMachine' -and -not $Type + } + } + foreach ($expected in 'TERMSRV/lnx-05', 'TERMSRV/10.0.0.5') { + Should -Invoke New-StoredCredential -Scope It -Times 1 -Exactly -ParameterFilter { + $Target -eq $expected -and $UserName -eq 'alice' -and $Password -eq 'from-the-broker' -and $Persist -eq 'LocalMachine' -and $Type -eq 'Generic' + } + } + } + + It 'still opens the desktop when the credentials cannot be saved' { + Mock New-StoredCredential { throw 'Access is denied.' } + + { Save-LinuxHostCredential -Checkout (New-TestCheckout) -Username 'alice' } | Should -Not -Throw + Should -Invoke Write-Log -Scope It -Times 1 -Exactly -ParameterFilter { $Level -eq 'ERROR' } + } +} + +Describe 'Get-MstscArgumentList' { + It 'opens full screen across every monitor by default' { + (Get-MstscArgumentList -IPAddress '10.0.0.5') -join ' ' | Should -BeExactly '/v:10.0.0.5 /f /multimon' + } + + # A session that uses every monitor opens full screen even without /f. + It 'builds for FullScreen and MultiMonitor ' -TestCases @( + @{ FullScreen = 'On'; MultiMonitor = 'On'; Expected = '/v:10.0.0.5 /f /multimon' } + @{ FullScreen = 'On'; MultiMonitor = 'Off'; Expected = '/v:10.0.0.5 /f' } + @{ FullScreen = 'Off'; MultiMonitor = 'On'; Expected = '/v:10.0.0.5' } + @{ FullScreen = 'Off'; MultiMonitor = 'Off'; Expected = '/v:10.0.0.5' } + ) { + (Get-MstscArgumentList -IPAddress '10.0.0.5' -FullScreen $FullScreen -MultiMonitor $MultiMonitor) -join ' ' | Should -BeExactly $Expected + } + + It 'returns a list even when the address is the only switch' { + $arguments = @(Get-MstscArgumentList -IPAddress '10.0.0.5' -FullScreen Off) + + $arguments.Count | Should -Be 1 + $arguments[0] | Should -BeExactly '/v:10.0.0.5' + } + + It 'takes the address
' -TestCases @( + @{ Address = '10.0.0.5' } + @{ Address = '10.0.0.5:3390' } + @{ Address = 'lnx-05' } + @{ Address = 'lnx-05.contoso.internal' } + ) { + (Get-MstscArgumentList -IPAddress $Address -MultiMonitor Off) -join ' ' | Should -BeExactly "/v:$Address /f" + } + + It 'refuses the address ' -TestCases @( + @{ Name = 'with another switch'; Address = '10.0.0.5 /admin' } + @{ Name = 'that is a switch'; Address = '/edit' } + @{ Name = 'that starts with a hyphen'; Address = '-10.0.0.5' } + @{ Name = 'with a quote'; Address = '10.0.0.5"' } + @{ Name = 'with a tab'; Address = "10.0.0.5`t" } + @{ Name = 'with a line break'; Address = "10.0.0.5`n" } + @{ Name = 'that is a connection file'; Address = 'C:\Users\Public\evil.rdp' } + @{ Name = 'on a share'; Address = '\\attacker\share\evil.rdp' } + @{ Name = 'with a Kelvin sign'; Address = "lnx-0$([char]0x212A)" } + @{ Name = 'with a typographic quote'; Address = "10.0.0.5$([char]0x201D)" } + ) { + { Get-MstscArgumentList -IPAddress $Address } | Should -Throw "*is not a host name or an IP address*" + } + + It 'refuses FullScreen or MultiMonitor other than On or Off' { + { Get-MstscArgumentList -IPAddress '10.0.0.5' -FullScreen 'Yes' } | Should -Throw + { Get-MstscArgumentList -IPAddress '10.0.0.5' -MultiMonitor 'All' } | Should -Throw + } +} + +Describe 'Open-RemoteDesktop' { + BeforeEach { + Mock Write-Log {} + Mock Show-UserMessage {} + Mock Test-Path { $true } + Mock New-Item {} + Mock New-ItemProperty {} + Mock Start-Process {} + } + + It 'turns off the server authentication warning and starts mstsc full screen across every monitor' { + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' | Should -BeTrue + + Should -Invoke New-ItemProperty -Scope It -Times 1 -Exactly -ParameterFilter { + $Path -eq 'HKCU:\Software\Microsoft\Terminal Server Client' -and $Name -eq 'AuthenticationLevelOverride' -and $Value -eq 0 + } + Should -Invoke Start-Process -Scope It -Times 1 -Exactly -ParameterFilter { $FilePath -eq 'mstsc.exe' -and (@($ArgumentList) -join ' ') -ceq '/v:10.0.0.5 /f /multimon' } + Should -Invoke Write-Log -Scope It -Times 1 -Exactly -ParameterFilter { $Message -eq 'Starting mstsc.exe /v:10.0.0.5 /f /multimon.' -and $Level -eq 'INFO' } + Should -Invoke New-Item -Scope It -Times 0 -Exactly + Should -Invoke Show-UserMessage -Scope It -Times 0 -Exactly + } + + It 'opens a window when FullScreen is Off' { + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' -FullScreen Off | Should -BeTrue + + Should -Invoke Start-Process -Scope It -Times 1 -Exactly -ParameterFilter { $FilePath -eq 'mstsc.exe' -and (@($ArgumentList) -join ' ') -ceq '/v:10.0.0.5' } + } + + It 'keeps a full-screen desktop on one monitor when MultiMonitor is Off' { + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' -MultiMonitor Off | Should -BeTrue + + Should -Invoke Start-Process -Scope It -Times 1 -Exactly -ParameterFilter { $FilePath -eq 'mstsc.exe' -and (@($ArgumentList) -join ' ') -ceq '/v:10.0.0.5 /f' } + } + + It 'creates the Remote Desktop client key when it is missing' { + Mock Test-Path { $false } + + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' | Should -BeTrue + Should -Invoke New-Item -Scope It -Times 1 -Exactly -ParameterFilter { $Path -eq 'HKCU:\Software\Microsoft\Terminal Server Client' } + } + + It 'tells the user when mstsc cannot be started' { + Mock Start-Process { throw 'The system cannot find the file specified.' } + + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5' | Should -BeFalse + Should -Invoke Show-UserMessage -Scope It -Times 1 -Exactly -ParameterFilter { + $Message -eq 'Remote Desktop Connection could not be started for lnx-05. Try again, or contact your administrator.' -and $Icon -eq 'Error' + } + } + + It 'refuses an address that is not a host name or an IP address before changing anything' { + Open-RemoteDesktop -Hostname 'lnx-05' -IPAddress '10.0.0.5 /admin' | Should -BeFalse + + Should -Invoke New-ItemProperty -Scope It -Times 0 -Exactly + Should -Invoke Start-Process -Scope It -Times 0 -Exactly + Should -Invoke Write-Log -Scope It -Times 1 -Exactly -ParameterFilter { $Level -eq 'ERROR' -and $Message -like '*is not a host name or an IP address*' } + Should -Invoke Show-UserMessage -Scope It -Times 1 -Exactly -ParameterFilter { + $Message -eq 'Remote Desktop Connection could not be started for lnx-05. Try again, or contact your administrator.' -and $Icon -eq 'Error' + } + } +} + +# Needs a desktop to show the window on. +Describe 'Open-WaitWindow' -Skip:(-not [Environment]::UserInteractive) { + BeforeEach { + Mock Write-Log {} + } + + It 'says the desktop is starting, and Cancel cancels the wait' { + $window = Open-WaitWindow + try { + $window | Should -Not -BeNullOrEmpty + $window.Form.Visible | Should -BeTrue + $window.Form.Text | Should -Be 'Linux Desktop' + $window.Cancelled | Should -BeFalse + + $window.Form.CancelButton.PerformClick() + + $window.Cancelled | Should -BeTrue + $window.Form.IsDisposed | Should -BeTrue + } + finally { + Close-WaitWindow -WaitWindow $window + } + } + + It 'counts closing the window as a cancel' { + $window = Open-WaitWindow + $window.Form.Close() + + $window.Cancelled | Should -BeTrue + } + + It 'does not count closing it from the script as a cancel' { + $window = Open-WaitWindow + Close-WaitWindow -WaitWindow $window + + $window.Cancelled | Should -BeFalse + $window.Form.IsDisposed | Should -BeTrue + { Close-WaitWindow -WaitWindow $window } | Should -Not -Throw + } +} diff --git a/avd_host/tests/Enable-HostMonitoring.Tests.ps1 b/avd_host/tests/Enable-HostMonitoring.Tests.ps1 new file mode 100644 index 0000000..c19edda --- /dev/null +++ b/avd_host/tests/Enable-HostMonitoring.Tests.ps1 @@ -0,0 +1,967 @@ +# Pester 5 tests for deploy/Enable-HostMonitoring.ps1, which connects the Linux hosts and the AVD +# session hosts to the host monitoring, and for how deploy/Post-Provision.ps1 runs it. From the +# repository root, in Windows PowerShell: +# Invoke-Pester -Path avd_host/tests + +BeforeAll { + $script:RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path + $script:MonitoringScriptPath = Join-Path $script:RepoRoot 'deploy\Enable-HostMonitoring.ps1' + . $script:MonitoringScriptPath + + # Mock can only replace a command that exists, and the machine running the tests may not have + # the Azure CLIs. + if (-not (Get-Command az -ErrorAction SilentlyContinue)) { + function script:az { } + } + if (-not (Get-Command azd -ErrorAction SilentlyContinue)) { + function script:azd { } + } + + $script:ResourceGroup = 'rg-linuxbroker-dev' + $script:SubscriptionId = '6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f' + $script:RulePrefix = "/subscriptions/$($script:SubscriptionId)/resourceGroups/$($script:ResourceGroup)/providers/Microsoft.Insights/dataCollectionRules" + $script:LinuxRuleId = "$($script:RulePrefix)/linuxbroker-dev-linux-hosts" + $script:AvdRuleId = "$($script:RulePrefix)/linuxbroker-dev-avd-hosts" + $script:WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $script:Utf8NoBom = New-Object System.Text.UTF8Encoding -ArgumentList $false + $script:AzureVariables = @('AZURE_ENV_NAME', 'AZURE_ENVIRONMENT_NAME', 'AZURE_SUBSCRIPTION_ID') + + function New-TestVm { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + + [AllowEmptyString()] + [string]$Role = 'linux-host', + + [string]$PowerState = 'VM running', + + [switch]$NoPowerState + ) + + $vm = [ordered]@{ + id = "/subscriptions/$($script:SubscriptionId)/resourceGroups/$($script:ResourceGroup)/providers/Microsoft.Compute/virtualMachines/$Name" + name = $Name + resourceGroup = $script:ResourceGroup + tags = $null + } + if ($Role.Length -gt 0) { + $vm['tags'] = @{ 'broker-role' = $Role } + } + if (-not $NoPowerState) { + $vm['powerState'] = $PowerState + } + return [pscustomobject]$vm + } + + function Set-TestVm { + param([object[]]$Vm) + + $script:VmListJson = ConvertTo-Json -InputObject @($Vm) -Depth 5 + # As the script reads them, with the tags as properties rather than hashtable keys. + $script:TestVms = @(ConvertFrom-Json -InputObject $script:VmListJson | ForEach-Object { $_ }) + } + + function Get-TestVmId { + param([string]$Name) + + return "/subscriptions/$($script:SubscriptionId)/resourceGroups/$($script:ResourceGroup)/providers/Microsoft.Compute/virtualMachines/$Name" + } + + # Queues what the listings of a VM's extensions report about the agent, one entry per listing: + # '' for no agent, a provisioning state, @{ State; Name } for the agent under another name, or + # @{ Error } for az failing. The last entry repeats. A VM with nothing queued has no agent + # until an install starts, and then has it. + function Set-AgentState { + param( + [Parameter(Mandatory = $true)] + [string]$Vm, + + [object[]]$State + ) + + $queue = New-Object System.Collections.Queue + foreach ($entry in $State) { + $queue.Enqueue($entry) + } + $script:AgentStates[$Vm] = $queue + } + + function Get-NextAgentState { + param([string]$Vm) + + if ($script:AgentStates.ContainsKey($Vm)) { + $queue = $script:AgentStates[$Vm] + if ($queue.Count -gt 1) { + return $queue.Dequeue() + } + return $queue.Peek() + } + if (@($script:Installs | Where-Object { $_.Vm -eq $Vm }).Count -gt 0) { + return 'Succeeded' + } + return '' + } + + function Get-TestAgentName { + param([string]$Vm) + + $testVm = @($script:TestVms | Where-Object { $_.name -eq $Vm }) | Select-Object -First 1 + if ($null -ne $testVm -and (Get-VmTagValue -Vm $testVm -Name 'broker-role') -eq 'avd-host') { + return 'AzureMonitorWindowsAgent' + } + return 'AzureMonitorLinuxAgent' + } + + # The extensions az lists for a VM: the one that set the host up, and the agent unless it has + # none, as az 2.83 lists them. + function ConvertTo-ExtensionListJson { + param( + [string]$Vm, + + $Agent + ) + + $extensions = @([ordered]@{ + name = 'host-setup' + publisher = 'Microsoft.Azure.Extensions' + type = 'Microsoft.Compute/virtualMachines/extensions' + typePropertiesType = 'CustomScript' + provisioningState = 'Succeeded' + }) + + if ($Agent -is [hashtable] -or ([string]$Agent).Length -gt 0) { + $agentName = Get-TestAgentName -Vm $Vm + $state = [string]$Agent + $name = $agentName + if ($Agent -is [hashtable]) { + $state = [string]$Agent['State'] + if ($Agent.ContainsKey('Name')) { + $name = $Agent['Name'] + } + } + $extensions += [ordered]@{ + name = $name + publisher = 'Microsoft.Azure.Monitor' + type = 'Microsoft.Compute/virtualMachines/extensions' + typePropertiesType = $agentName + typeHandlerVersion = '1.33' + enableAutomaticUpgrade = $true + provisioningState = $state + } + } + + return ConvertTo-Json -InputObject $extensions -Depth 5 + } + + function New-AzErrorRecord { + param([string]$Message) + + return New-Object System.Management.Automation.ErrorRecord -ArgumentList @( + (New-Object System.Exception -ArgumentList $Message), + 'NativeCommandError', + [System.Management.Automation.ErrorCategory]::NotSpecified, + $null) + } + + function Get-ArgumentValue { + param( + [string[]]$Arguments, + + [string]$Name + ) + + $index = [array]::IndexOf($Arguments, $Name) + if ($index -lt 0 -or $index + 1 -ge $Arguments.Count) { + return $null + } + return $Arguments[$index + 1] + } + + function Test-Utf8Bom { + param([byte[]]$Bytes) + + return ($Bytes.Length -ge 3 -and $Bytes[0] -eq 0xEF -and $Bytes[1] -eq 0xBB -and $Bytes[2] -eq 0xBF) + } + + function ConvertTo-TestLiteral { + param([string]$Value) + + return "'" + $Value.Replace("'", "''") + "'" + } + + function Invoke-ChildProcess { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + + [Parameter(Mandatory = $true)] + [string]$ScriptPath + ) + + $folder = Split-Path -Parent $ScriptPath + $stdoutPath = Join-Path $folder 'stdout.txt' + $stderrPath = Join-Path $folder 'stderr.txt' + $process = Start-Process -FilePath $FilePath ` + -ArgumentList @('-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', ('"{0}"' -f $ScriptPath)) ` + -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -Wait -PassThru -NoNewWindow + + return [pscustomobject]@{ + ExitCode = $process.ExitCode + StdOut = [System.IO.File]::ReadAllText($stdoutPath) + StdErr = [System.IO.File]::ReadAllText($stderrPath) + } + } + + function Save-AzureVariable { + $saved = @{} + foreach ($name in $script:AzureVariables) { + $saved[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') + [Environment]::SetEnvironmentVariable($name, $null, 'Process') + } + return $saved + } + + function Restore-AzureVariable { + param([hashtable]$Saved) + + foreach ($name in $Saved.Keys) { + [Environment]::SetEnvironmentVariable($name, $Saved[$name], 'Process') + } + } +} + +Describe 'Connecting the hosts to the host monitoring' { + BeforeEach { + Set-StrictMode -Version Latest + $global:LASTEXITCODE = 0 + $script:AzCalls = New-Object System.Collections.Generic.List[string] + $script:Associations = New-Object System.Collections.Generic.List[object] + $script:Installs = New-Object System.Collections.Generic.List[object] + $script:ExtensionLists = New-Object System.Collections.Generic.List[string] + $script:AgentStates = @{} + $script:AssociationErrors = @{} + $script:InstallErrors = @{} + $script:RawExtensionJson = $null + $script:AccountSetExitCode = 0 + $script:VmListExitCode = 0 + $script:TestVms = @() + $script:VmListJson = '[]' + $script:HostLines = New-Object System.Collections.Generic.List[string] + $script:Warnings = New-Object System.Collections.Generic.List[string] + $script:Slept = New-Object System.Collections.Generic.List[int] + $script:AzdValues = @{} + $script:AzdEnvironments = New-Object System.Collections.Generic.List[string] + + Mock az { + $arguments = [string[]]@($args | ForEach-Object { [string]$_ }) + $commandLine = $arguments -join ' ' + $script:AzCalls.Add($commandLine) + $global:LASTEXITCODE = 0 + + if ($commandLine -like 'account set *') { + $global:LASTEXITCODE = $script:AccountSetExitCode + return + } + + if ($commandLine -like 'vm list *') { + $global:LASTEXITCODE = $script:VmListExitCode + if ($script:VmListExitCode -ne 0) { + return (New-AzErrorRecord -Message "ERROR: (ResourceGroupNotFound) Resource group '$($script:ResourceGroup)' could not be found.") + } + return $script:VmListJson + } + + if ($commandLine -like 'rest *') { + $url = Get-ArgumentValue -Arguments $arguments -Name '--url' + $bodyFile = (Get-ArgumentValue -Arguments $arguments -Name '--body').Substring(1) + $vmName = ($url -split '/')[8] + $script:Associations.Add([pscustomobject]@{ + Vm = $vmName + Url = $url + Arguments = $arguments + File = $bodyFile + Bytes = [System.IO.File]::ReadAllBytes($bodyFile) + Body = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($bodyFile)) + }) + if ($script:AssociationErrors.ContainsKey($vmName)) { + $global:LASTEXITCODE = 1 + return (New-AzErrorRecord -Message $script:AssociationErrors[$vmName]) + } + return + } + + if ($commandLine -like 'vm extension list *') { + if ($null -ne $script:RawExtensionJson) { + return $script:RawExtensionJson + } + + $vmName = Get-ArgumentValue -Arguments $arguments -Name '--vm-name' + $script:ExtensionLists.Add($vmName) + $agent = Get-NextAgentState -Vm $vmName + if ($agent -is [hashtable] -and $agent.ContainsKey('Error')) { + $global:LASTEXITCODE = 1 + return (New-AzErrorRecord -Message $agent['Error']) + } + return (ConvertTo-ExtensionListJson -Vm $vmName -Agent $agent) + } + + if ($commandLine -like 'vm extension set *') { + $vmName = Get-ArgumentValue -Arguments $arguments -Name '--vm-name' + $script:Installs.Add([pscustomobject]@{ + Vm = $vmName + Agent = Get-ArgumentValue -Arguments $arguments -Name '--name' + InstanceName = Get-ArgumentValue -Arguments $arguments -Name '--extension-instance-name' + Arguments = $arguments + }) + if ($script:InstallErrors.ContainsKey($vmName)) { + $global:LASTEXITCODE = 1 + return (New-AzErrorRecord -Message $script:InstallErrors[$vmName]) + } + return + } + + throw "Unexpected az call: $commandLine" + } + Mock azd { throw "Unexpected azd call: $($args -join ' ')" } + Mock Get-AzdEnvValue { + $script:AzdEnvironments.Add($EnvironmentName) + [string]$script:AzdValues[$Key] + } + Mock Write-Host { $script:HostLines.Add([string]$Object) } + Mock Write-Warning { $script:Warnings.Add([string]$Message) } + Mock Start-Sleep { $script:Slept.Add([int]$Seconds) } + + $savedVariables = Save-AzureVariable + $setup = @{ + ResourceGroupName = $script:ResourceGroup + SubscriptionId = $script:SubscriptionId + LinuxHostDataCollectionRuleId = $script:LinuxRuleId + AvdHostDataCollectionRuleId = $script:AvdRuleId + } + } + + AfterEach { + Restore-AzureVariable -Saved $savedVariables + } + + It 'connects each Linux host to the Linux rule and each AVD session host to the AVD rule, with the matching agent' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'avd-0' -Role 'avd-host'), (New-TestVm -Name 'jump-0' -Role ''), (New-TestVm -Name 'sql-0' -Role 'database') + + Invoke-HostMonitoringSetup @setup + + $script:AzCalls[0] | Should -BeExactly "account set --subscription $($script:SubscriptionId)" + $script:AzCalls[1] | Should -BeExactly "vm list --resource-group $($script:ResourceGroup) --show-details --output json --only-show-errors" + + @($script:Associations | ForEach-Object { $_.Vm }) | Should -Be @('lnx-0', 'avd-0') + foreach ($association in $script:Associations) { + $association.Url | Should -BeExactly "$(Get-TestVmId -Name $association.Vm)/providers/Microsoft.Insights/dataCollectionRuleAssociations/linuxbroker-host-monitoring" + Get-ArgumentValue -Arguments $association.Arguments -Name '--method' | Should -BeExactly 'put' + Get-ArgumentValue -Arguments $association.Arguments -Name '--url-parameters' | Should -BeExactly 'api-version=2022-06-01' + Get-ArgumentValue -Arguments $association.Arguments -Name '--output' | Should -BeExactly 'none' + } + $script:Associations[0].Body.properties.dataCollectionRuleId | Should -BeExactly $script:LinuxRuleId + $script:Associations[1].Body.properties.dataCollectionRuleId | Should -BeExactly $script:AvdRuleId + + @($script:Installs | ForEach-Object { '{0} {1}' -f $_.Vm, $_.Agent }) | Should -Be @('lnx-0 AzureMonitorLinuxAgent', 'avd-0 AzureMonitorWindowsAgent') + foreach ($install in $script:Installs) { + Get-ArgumentValue -Arguments $install.Arguments -Name '--resource-group' | Should -BeExactly $script:ResourceGroup + Get-ArgumentValue -Arguments $install.Arguments -Name '--publisher' | Should -BeExactly 'Microsoft.Azure.Monitor' + Get-ArgumentValue -Arguments $install.Arguments -Name '--enable-auto-upgrade' | Should -BeExactly 'true' + $install.InstanceName | Should -BeExactly $install.Agent + $install.Arguments | Should -Contain '--no-wait' + $install.Arguments | Should -Not -Contain '--force-update' + $install.Arguments | Should -Not -Contain '--settings' + } + + @($script:AzCalls | Where-Object { $_ -match 'jump-0|sql-0' }).Count | Should -Be 0 + $script:Slept | Should -Be @(30) + $script:HostLines | Should -Contain "Installing the Azure Monitor agent on Linux host 'lnx-0'..." + $script:HostLines | Should -Contain "Linux host 'lnx-0' is connected, with the Azure Monitor agent installed." + $script:HostLines | Should -Contain "AVD session host 'avd-0' is connected, with the Azure Monitor agent installed." + $script:HostLines | Should -Contain 'The host monitoring is set up on 2 host(s).' + $script:Warnings.Count | Should -Be 0 + } + + It 'writes each association to a file without a byte order mark, and deletes the file' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + + Invoke-HostMonitoringSetup @setup + + $association = $script:Associations[0] + Get-ArgumentValue -Arguments $association.Arguments -Name '--body' | Should -BeExactly "@$($association.File)" + Test-Utf8Bom -Bytes $association.Bytes | Should -BeFalse + $association.Body.properties.description | Should -Not -BeNullOrEmpty + Test-Path -LiteralPath $association.File | Should -BeFalse + } + + It 'connects a host that is not running without installing the agent, and names it to run the script again for' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'lnx-1' -PowerState 'VM deallocated'), (New-TestVm -Name 'avd-0' -Role 'avd-host' -PowerState 'VM stopped') + + Invoke-HostMonitoringSetup @setup + + @($script:Associations | ForEach-Object { $_.Vm }) | Should -Be @('lnx-0', 'lnx-1', 'avd-0') + @($script:ExtensionLists | Select-Object -Unique) | Should -Be @('lnx-0') + @($script:Installs | ForEach-Object { $_.Vm }) | Should -Be @('lnx-0') + $script:Warnings | Should -Contain "Linux host 'lnx-1' is connected, but it is not running (VM deallocated), so it has no Azure Monitor agent and sends no logs yet." + $script:Warnings | Should -Contain "AVD session host 'avd-0' is connected, but it is not running (VM stopped), so it has no Azure Monitor agent and sends no logs yet." + $script:Warnings | Should -Contain 'No Azure Monitor agent yet because they are not running: lnx-1, avd-0. Once they are running, run Enable-HostMonitoring.ps1 again with -HostNames lnx-1,avd-0.' + $script:HostLines | Should -Contain 'The host monitoring is set up on 1 host(s).' + } + + It 'treats a host whose power state az does not report as running' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0' -NoPowerState) + + Invoke-HostMonitoringSetup @setup + + @($script:Installs | ForEach-Object { $_.Vm }) | Should -Be @('lnx-0') + } + + It 'leaves the agent alone on the hosts that already have it' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'avd-0' -Role 'avd-host') + Set-AgentState -Vm 'lnx-0' -State 'Succeeded' + Set-AgentState -Vm 'avd-0' -State 'Succeeded' + + Invoke-HostMonitoringSetup @setup + + @($script:Associations | ForEach-Object { $_.Vm }) | Should -Be @('lnx-0', 'avd-0') + $script:Installs.Count | Should -Be 0 + $script:Slept.Count | Should -Be 0 + $script:HostLines | Should -Contain "Linux host 'lnx-0' is connected, and already has the Azure Monitor agent." + $script:HostLines | Should -Contain "AVD session host 'avd-0' is connected, and already has the Azure Monitor agent." + $script:HostLines | Should -Contain 'The host monitoring is set up on 2 host(s).' + } + + It 'waits for an install that is instead of starting another' -TestCases @( + @{ State = 'Creating' } + @{ State = 'Updating' } + ) { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + Set-AgentState -Vm 'lnx-0' -State $State, $State, 'Succeeded' + + Invoke-HostMonitoringSetup @setup + + $script:Installs.Count | Should -Be 0 + $script:Slept | Should -Be @(30, 30) + $script:HostLines | Should -Contain "Linux host 'lnx-0' is connected, with the Azure Monitor agent installed." + } + + It 'installs the agent again, forced, where an earlier install failed' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + Set-AgentState -Vm 'lnx-0' -State 'Failed', 'Succeeded' + + Invoke-HostMonitoringSetup @setup + + $script:Installs.Count | Should -Be 1 + $script:Installs[0].InstanceName | Should -BeExactly 'AzureMonitorLinuxAgent' + $script:Installs[0].Arguments | Should -Contain '--force-update' + $script:HostLines | Should -Contain 'The host monitoring is set up on 1 host(s).' + } + + It 'recognises the agent installed under another name, and installs it again under that name' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'avd-0' -Role 'avd-host') + Set-AgentState -Vm 'lnx-0' -State @{ State = 'Succeeded'; Name = 'AMALinux' } + Set-AgentState -Vm 'avd-0' -State @{ State = 'Failed'; Name = 'AzureMonitorWindowsAgent-policy' }, 'Succeeded' + + Invoke-HostMonitoringSetup @setup + + $script:Installs.Count | Should -Be 1 + $script:Installs[0].Vm | Should -BeExactly 'avd-0' + $script:Installs[0].Agent | Should -BeExactly 'AzureMonitorWindowsAgent' + $script:Installs[0].InstanceName | Should -BeExactly 'AzureMonitorWindowsAgent-policy' + $script:Installs[0].Arguments | Should -Contain '--force-update' + $script:HostLines | Should -Contain "Linux host 'lnx-0' is connected, and already has the Azure Monitor agent." + } + + It 'reports a host whose agent fails to install, after it sets up the others' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'lnx-1') + Set-AgentState -Vm 'lnx-0' -State '', 'Failed' + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw 'The host monitoring could not be set up on: lnx-0. Run Enable-HostMonitoring.ps1 again with -HostNames lnx-0 to retry them.' + + $script:Warnings | Should -Contain "The Azure Monitor agent failed to install on Linux host 'lnx-0'. The AzureMonitorLinuxAgent extension's status on the VM says why." + $script:HostLines | Should -Contain "Linux host 'lnx-1' is connected, with the Azure Monitor agent installed." + } + + It 'stops waiting for an install at the time limit, and reports the host' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + Set-AgentState -Vm 'lnx-0' -State '', 'Creating' + + { Invoke-HostMonitoringSetup @setup -AgentTimeoutMinutes 1 } | + Should -Throw 'The host monitoring could not be set up on: lnx-0. *' + + $script:Slept | Should -Be @(30, 30) + $script:Warnings | Should -Contain "The Azure Monitor agent install on Linux host 'lnx-0' did not finish within 1 minute(s)." + } + + It 'waits 20 minutes for the installs by default' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + Set-AgentState -Vm 'lnx-0' -State '', 'Creating' + + { Invoke-HostMonitoringSetup @setup } | Should -Throw + + $script:Slept.Count | Should -Be 40 + $script:Warnings | Should -Contain "The Azure Monitor agent install on Linux host 'lnx-0' did not finish within 20 minute(s)." + } + + It 'keeps going when it cannot connect a host, and reports the host' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'lnx-1') + $script:AssociationErrors['lnx-0'] = "ERROR: (AuthorizationFailed) The client does not have authorization to perform action 'Microsoft.Insights/dataCollectionRuleAssociations/write'." + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw 'The host monitoring could not be set up on: lnx-0. Run Enable-HostMonitoring.ps1 again with -HostNames lnx-0 to retry them.' + + $script:Warnings | Should -Contain "Could not connect Linux host 'lnx-0' to the host monitoring: Could not associate the data collection rule. $($script:AssociationErrors['lnx-0'])" + Test-Path -LiteralPath $script:Associations[0].File | Should -BeFalse + @($script:ExtensionLists | Select-Object -Unique) | Should -Be @('lnx-1') + $script:HostLines | Should -Contain "Linux host 'lnx-1' is connected, with the Azure Monitor agent installed." + } + + It 'keeps going when an agent install cannot start, and reports the host' { + Set-TestVm -Vm (New-TestVm -Name 'avd-0' -Role 'avd-host'), (New-TestVm -Name 'lnx-0') + $script:InstallErrors['avd-0'] = "ERROR: (OperationNotAllowed) Operation 'PUT' is not allowed on VM 'avd-0' since the VM is deallocated or marked to be deallocated." + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw 'The host monitoring could not be set up on: avd-0. *' + + $script:Warnings | Should -Contain "Could not install the Azure Monitor agent on AVD session host 'avd-0': Could not start installing the Azure Monitor agent. $($script:InstallErrors['avd-0'])" + $script:HostLines | Should -Contain "Linux host 'lnx-0' is connected, with the Azure Monitor agent installed." + } + + It 'tries again at the next poll when it cannot read the extensions of a host it is waiting for' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + Set-AgentState -Vm 'lnx-0' -State '', @{ Error = 'ERROR: The request timed out.' }, 'Succeeded' + + Invoke-HostMonitoringSetup @setup + + $script:Slept | Should -Be @(30, 30) + $script:HostLines | Should -Contain 'The host monitoring is set up on 1 host(s).' + } + + It 'reports a host whose extensions it cannot read before the install' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0') + Set-AgentState -Vm 'lnx-0' -State @{ Error = 'ERROR: The request timed out.' } + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw 'The host monitoring could not be set up on: lnx-0. *' + + $script:Warnings | Should -Contain "Could not install the Azure Monitor agent on Linux host 'lnx-0': Could not list the extensions of 'lnx-0'. ERROR: The request timed out." + $script:Installs.Count | Should -Be 0 + } + + It 'sets up only the hosts -HostNames names, ignoring case, and warns about the names that are not hosts' { + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'lnx-1'), (New-TestVm -Name 'avd-0' -Role 'avd-host'), (New-TestVm -Name 'jump-0' -Role '') + + Invoke-HostMonitoringSetup @setup -HostNames 'LNX-1', 'avd-0', 'jump-0', 'lnx-9', ' ' + + @($script:Associations | ForEach-Object { $_.Vm }) | Should -Be @('lnx-1', 'avd-0') + @($script:Installs | ForEach-Object { $_.Vm }) | Should -Be @('lnx-1', 'avd-0') + $script:Warnings | Should -Contain "Requested host 'jump-0' is not a Linux host or AVD session host in resource group '$($script:ResourceGroup)'." + $script:Warnings | Should -Contain "Requested host 'lnx-9' is not a Linux host or AVD session host in resource group '$($script:ResourceGroup)'." + $script:Warnings.Count | Should -Be 2 + } + + It 'says so when the resource group has no hosts to connect' { + Set-TestVm -Vm (New-TestVm -Name 'jump-0' -Role '') + + Invoke-HostMonitoringSetup @setup + + $script:Associations.Count | Should -Be 0 + $script:HostLines | Should -Contain "No Linux hosts or AVD session hosts to connect to the host monitoring in resource group '$($script:ResourceGroup)'." + } + + It 'changes nothing when the deployment has no host monitoring' { + $setup.Remove('LinuxHostDataCollectionRuleId') + $setup.Remove('AvdHostDataCollectionRuleId') + $env:AZURE_ENV_NAME = 'dev' + + Invoke-HostMonitoringSetup @setup + + $script:AzCalls.Count | Should -Be 0 + @($script:AzdEnvironments | Select-Object -Unique) | Should -Be @('dev') + $script:HostLines | Should -Contain 'This deployment has no host monitoring, so there is nothing to connect: deployHostMonitoring is false, or azd provision has not run since the monitoring was added.' + } + + It 'leaves alone the hosts whose rule it does not know' { + $setup.Remove('AvdHostDataCollectionRuleId') + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'avd-0' -Role 'avd-host') + + Invoke-HostMonitoringSetup @setup + + @($script:Associations | ForEach-Object { $_.Vm }) | Should -Be @('lnx-0') + @($script:AzCalls | Where-Object { $_ -match 'avd-0' }).Count | Should -Be 0 + } + + It 'refuses as the before it changes anything' -TestCases @( + @{ Parameter = 'LinuxHostDataCollectionRuleId'; Case = 'a Log Analytics workspace'; Value = '/subscriptions/6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f/resourceGroups/rg-linuxbroker-dev/providers/Microsoft.OperationalInsights/workspaces/log-linuxbroker-dev' } + @{ Parameter = 'LinuxHostDataCollectionRuleId'; Case = 'a rule name'; Value = 'linuxbroker-dev-linux-hosts' } + @{ Parameter = 'LinuxHostDataCollectionRuleId'; Case = 'a subscription name'; Value = '/subscriptions/dev/resourceGroups/rg-linuxbroker-dev/providers/Microsoft.Insights/dataCollectionRules/linuxbroker-dev-linux-hosts' } + @{ Parameter = 'AvdHostDataCollectionRuleId'; Case = 'a path below a rule'; Value = '/subscriptions/6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f/resourceGroups/rg-linuxbroker-dev/providers/Microsoft.Insights/dataCollectionRules/linuxbroker-dev-avd-hosts/associations' } + ) { + $setup[$Parameter] = $Value + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw "'$Value' is not the resource ID of a data collection rule, such as /subscriptions//resourceGroups//providers/Microsoft.Insights/dataCollectionRules/." + + $script:AzCalls.Count | Should -Be 0 + } + + It 'reads the rules, the resource group and the subscription from the azd environment that names' -TestCases @( + @{ Variable = 'AZURE_ENV_NAME' } + @{ Variable = 'AZURE_ENVIRONMENT_NAME' } + ) { + [Environment]::SetEnvironmentVariable($Variable, 'dev', 'Process') + $script:AzdValues = @{ + resourceGroupName = $script:ResourceGroup + AZURE_SUBSCRIPTION_ID = $script:SubscriptionId + linuxHostDataCollectionRuleId = $script:LinuxRuleId + avdHostDataCollectionRuleId = $script:AvdRuleId + } + Set-TestVm -Vm (New-TestVm -Name 'lnx-0'), (New-TestVm -Name 'avd-0' -Role 'avd-host') + + Invoke-HostMonitoringSetup + + @($script:AzdEnvironments | Select-Object -Unique) | Should -Be @('dev') + $script:AzCalls[0] | Should -BeExactly "account set --subscription $($script:SubscriptionId)" + $script:AzCalls[1] | Should -BeExactly "vm list --resource-group $($script:ResourceGroup) --show-details --output json --only-show-errors" + $script:Associations[0].Body.properties.dataCollectionRuleId | Should -BeExactly $script:LinuxRuleId + $script:Associations[1].Body.properties.dataCollectionRuleId | Should -BeExactly $script:AvdRuleId + } + + It 'selects the subscription in AZURE_SUBSCRIPTION_ID when the azd environment has none' { + $setup.Remove('SubscriptionId') + $env:AZURE_SUBSCRIPTION_ID = $script:SubscriptionId + + Invoke-HostMonitoringSetup @setup + + $script:AzCalls[0] | Should -BeExactly "account set --subscription $($script:SubscriptionId)" + } + + It 'keeps the current subscription when it does not know one' { + $setup.Remove('SubscriptionId') + + Invoke-HostMonitoringSetup @setup + + $script:AzCalls[0] | Should -Match '^vm list ' + } + + It 'stops when it does not know the resource group' { + $setup.Remove('ResourceGroupName') + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw 'The host monitoring setup needs ResourceGroupName, as a parameter or an azd environment value.' + $script:AzCalls.Count | Should -Be 0 + } + + It 'stops when it cannot select the subscription' { + $script:AccountSetExitCode = 1 + + { Invoke-HostMonitoringSetup @setup } | Should -Throw "Could not select subscription '$($script:SubscriptionId)'." + $script:AzCalls.Count | Should -Be 1 + } + + It 'stops when it cannot list the VMs' { + $script:VmListExitCode = 1 + + { Invoke-HostMonitoringSetup @setup } | + Should -Throw "Could not list the VMs in resource group '$($script:ResourceGroup)'. ERROR: (ResourceGroupNotFound) Resource group '$($script:ResourceGroup)' could not be found." + $script:Associations.Count | Should -Be 0 + } + + Context 'Finding the agent among the extensions az lists' { + It 'finds the agent ' -TestCases @( + @{ Case = 'by its type, which older versions of az call virtualMachineExtensionType'; Expected = 'AMALinux Succeeded'; Json = '[{"name":"AMALinux","publisher":"Microsoft.Azure.Monitor","virtualMachineExtensionType":"AzureMonitorLinuxAgent","provisioningState":"Succeeded"}]' } + @{ Case = 'in a nested properties object, below the resource type'; Expected = 'AMALinux Creating'; Json = '[{"name":"AMALinux","type":"Microsoft.Compute/virtualMachines/extensions","properties":{"publisher":"Microsoft.Azure.Monitor","type":"AzureMonitorLinuxAgent","provisioningState":"Creating"}}]' } + @{ Case = 'by its name'; Expected = 'AzureMonitorLinuxAgent Failed'; Json = '[{"name":"AzureMonitorLinuxAgent","provisioningState":"Failed"}]' } + ) { + $script:RawExtensionJson = $Json + + $extension = Get-AgentExtension -ResourceGroupName $script:ResourceGroup -VmName 'lnx-0' -AgentName 'AzureMonitorLinuxAgent' + + '{0} {1}' -f $extension.Name, $extension.ProvisioningState | Should -BeExactly $Expected + } + + It 'does not take for the agent' -TestCases @( + @{ Case = 'an extension of the same type from another publisher'; Json = '[{"name":"imitation","publisher":"Contoso.Monitoring","typePropertiesType":"AzureMonitorLinuxAgent","provisioningState":"Succeeded"}]' } + @{ Case = 'the agent for the other operating system'; Json = '[{"name":"AzureMonitorWindowsAgent","publisher":"Microsoft.Azure.Monitor","typePropertiesType":"AzureMonitorWindowsAgent","provisioningState":"Succeeded"}]' } + @{ Case = 'an empty list'; Json = '[]' } + @{ Case = 'no output'; Json = '' } + ) { + $script:RawExtensionJson = $Json + + Get-AgentExtension -ResourceGroupName $script:ResourceGroup -VmName 'lnx-0' -AgentName 'AzureMonitorLinuxAgent' | Should -BeNullOrEmpty + } + } +} + +Describe 'The deployment outputs the script reads' { + It 'finds both rule IDs among the outputs of main.bicep, under the names it reads them by' { + $scriptText = [System.IO.File]::ReadAllText($script:MonitoringScriptPath) + $main = [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot 'deploy\bicep\main.bicep')) + $resources = [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot 'deploy\bicep\main.resources.bicep')) + + foreach ($name in 'linuxHostDataCollectionRuleId', 'avdHostDataCollectionRuleId') { + $scriptText | Should -Match ([regex]::Escape("-Key '$name'")) + $main | Should -Match ('(?m)^output {0} string = resources\.outputs\.{0}\s*$' -f $name) + $resources | Should -Match ('(?m)^output {0} string = deployHostMonitoring \? hostMonitoring!\.outputs\.{0} : ''''\s*$' -f $name) + } + } +} + +Describe 'Enable-HostMonitoring.ps1 in a PowerShell process of its own' { + BeforeAll { + # Stand-ins for az and azd, found first on PATH. The az one records each call and answers + # as az does for one running Linux host that has no agent until the install starts. + $script:FakeAz = @' +@echo off +>>"%~dp0calls.log" echo %* +if "%1"=="fail" ( + echo ERROR: The client does not have authorization to perform this action. 1>&2 + exit /b 1 +) +if "%1"=="warn" ( + echo WARNING: This command is in preview. 1>&2 + echo {"ok": true} + exit /b 0 +) +if "%1 %2"=="account set" exit /b 0 +if "%1 %2"=="vm list" ( + type "%~dp0vms.json" + exit /b 0 +) +if "%1"=="rest" exit /b 0 +if "%1 %2 %3"=="vm extension set" ( + type nul > "%~dp0installed.flag" + exit /b 0 +) +if "%1 %2 %3"=="vm extension list" ( + if exist "%~dp0installed.flag" ( + type "%~dp0extensions.json" + ) else ( + echo [] + ) + exit /b 0 +) +echo ERROR: unexpected az call 1>&2 +exit /b 1 +'@ + $script:FakeAzd = @' +@echo off +if "%3"=="missing" ( + echo ERROR: key 'missing' not found in the environment values 1>&2 + exit /b 1 +) +echo rg-linuxbroker-dev +exit /b 0 +'@ + + function New-FakeCliFolder { + $folder = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $folder | Out-Null + # cmd.exe reads batch files with CRLF line endings. + [System.IO.File]::WriteAllText((Join-Path $folder 'az.cmd'), ($script:FakeAz -replace "`r?`n", "`r`n")) + [System.IO.File]::WriteAllText((Join-Path $folder 'azd.cmd'), ($script:FakeAzd -replace "`r?`n", "`r`n")) + [System.IO.File]::WriteAllText((Join-Path $folder 'vms.json'), (ConvertTo-Json -InputObject @(New-TestVm -Name 'lnx-0') -Depth 5), $script:Utf8NoBom) + [System.IO.File]::WriteAllText((Join-Path $folder 'extensions.json'), '[{"name":"AzureMonitorLinuxAgent","publisher":"Microsoft.Azure.Monitor","type":"Microsoft.Compute/virtualMachines/extensions","typePropertiesType":"AzureMonitorLinuxAgent","provisioningState":"Succeeded"}]', $script:Utf8NoBom) + return $folder + } + + function Get-ShellPath { + param([string]$Shell) + + if ($Shell -eq 'Windows PowerShell') { + return $script:WindowsPowerShell + } + $pwsh = Get-Command pwsh -ErrorAction SilentlyContinue + if ($pwsh) { + return $pwsh.Source + } + return $null + } + } + + It 'keeps what az and azd write to stderr from stopping it, in ' -TestCases @( + @{ Shell = 'Windows PowerShell' } + @{ Shell = 'PowerShell 7' } + ) { + $shellPath = Get-ShellPath -Shell $Shell + if (-not $shellPath) { + Set-ItResult -Skipped -Because 'PowerShell 7 is not installed.' + return + } + $folder = New-FakeCliFolder + $harnessPath = Join-Path $folder 'harness.ps1' + $harness = @' +$env:PATH = __FOLDER__ + ';' + $env:PATH +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +. __SCRIPT__ +$failed = Invoke-AzCommand -Arguments @('fail') +$warned = Invoke-AzCommand -Arguments @('warn') +Write-Output ('failed={0}|{1}|{2}' -f $failed.ExitCode, $failed.Output, $failed.Error) +Write-Output ('warned={0}|{1}|{2}' -f $warned.ExitCode, $warned.Output, $warned.Error) +Write-Output ('missing=[{0}]' -f (Get-AzdEnvValue -EnvironmentName 'dev' -Key 'missing')) +Write-Output ('present=[{0}]' -f (Get-AzdEnvValue -EnvironmentName 'dev' -Key 'resourceGroupName')) +'@ + $harness = $harness.Replace('__FOLDER__', (ConvertTo-TestLiteral -Value $folder)).Replace('__SCRIPT__', (ConvertTo-TestLiteral -Value $script:MonitoringScriptPath)) + [System.IO.File]::WriteAllText($harnessPath, $harness, $script:Utf8NoBom) + + $result = Invoke-ChildProcess -FilePath $shellPath -ScriptPath $harnessPath + + $result.StdErr | Should -BeNullOrEmpty + $result.ExitCode | Should -Be 0 + @($result.StdOut.Trim() -split "`r?`n") | Should -Be @( + 'failed=1||ERROR: The client does not have authorization to perform this action.', + 'warned=0|{"ok": true}|WARNING: This command is in preview.', + 'missing=[]', + 'present=[rg-linuxbroker-dev]') + } + + It 'connects a host and installs its agent when it is run as a script in ' -TestCases @( + @{ Shell = 'Windows PowerShell' } + @{ Shell = 'PowerShell 7' } + ) { + $shellPath = Get-ShellPath -Shell $Shell + if (-not $shellPath) { + Set-ItResult -Skipped -Because 'PowerShell 7 is not installed.' + return + } + $folder = New-FakeCliFolder + $harnessPath = Join-Path $folder 'harness.ps1' + $harness = @' +$env:PATH = __FOLDER__ + ';' + $env:PATH +foreach ($name in 'AZURE_ENV_NAME', 'AZURE_ENVIRONMENT_NAME', 'AZURE_SUBSCRIPTION_ID') { + [Environment]::SetEnvironmentVariable($name, $null, 'Process') +} +function Start-Sleep { param([int]$Seconds) Write-Output "TEST: Start-Sleep $Seconds" } +try { + & __SCRIPT__ -ResourceGroupName __RESOURCE_GROUP__ -SubscriptionId __SUBSCRIPTION__ -LinuxHostDataCollectionRuleId __LINUX_RULE__ -AvdHostDataCollectionRuleId __AVD_RULE__ +} +catch { + [Console]::Error.WriteLine($_.Exception.Message) + exit 1 +} +'@ + $harness = $harness.Replace('__FOLDER__', (ConvertTo-TestLiteral -Value $folder)). + Replace('__SCRIPT__', (ConvertTo-TestLiteral -Value $script:MonitoringScriptPath)). + Replace('__RESOURCE_GROUP__', (ConvertTo-TestLiteral -Value $script:ResourceGroup)). + Replace('__SUBSCRIPTION__', (ConvertTo-TestLiteral -Value $script:SubscriptionId)). + Replace('__LINUX_RULE__', (ConvertTo-TestLiteral -Value $script:LinuxRuleId)). + Replace('__AVD_RULE__', (ConvertTo-TestLiteral -Value $script:AvdRuleId)) + [System.IO.File]::WriteAllText($harnessPath, $harness, $script:Utf8NoBom) + + $result = Invoke-ChildProcess -FilePath $shellPath -ScriptPath $harnessPath + + $result.StdErr | Should -BeNullOrEmpty + $result.ExitCode | Should -Be 0 + $result.StdOut | Should -Match ([regex]::Escape('TEST: Start-Sleep 30')) + $result.StdOut | Should -Match ([regex]::Escape('The host monitoring is set up on 1 host(s).')) + + $calls = @(Get-Content -LiteralPath (Join-Path $folder 'calls.log') | ForEach-Object { $_.Trim() }) + $calls.Count | Should -Be 6 + $calls[0] | Should -BeExactly "account set --subscription $($script:SubscriptionId)" + $calls[1] | Should -BeExactly "vm list --resource-group $($script:ResourceGroup) --show-details --output json --only-show-errors" + $association = [regex]::Match($calls[2], '^rest --method put --url (\S+) --url-parameters api-version=2022-06-01 --body "?@([^"]+?)"? --output none --only-show-errors$') + $association.Success | Should -BeTrue + $association.Groups[1].Value | Should -BeExactly "$(Get-TestVmId -Name 'lnx-0')/providers/Microsoft.Insights/dataCollectionRuleAssociations/linuxbroker-host-monitoring" + Test-Path -LiteralPath $association.Groups[2].Value | Should -BeFalse + $calls[3] | Should -BeExactly "vm extension list --resource-group $($script:ResourceGroup) --vm-name lnx-0 --output json --only-show-errors" + $calls[4] | Should -BeExactly "vm extension set --resource-group $($script:ResourceGroup) --vm-name lnx-0 --name AzureMonitorLinuxAgent --publisher Microsoft.Azure.Monitor --extension-instance-name AzureMonitorLinuxAgent --enable-auto-upgrade true --no-wait --output none --only-show-errors" + $calls[5] | Should -BeExactly $calls[3] + } +} + +Describe 'Post-Provision.ps1' { + BeforeAll { + $script:ProvisionFolder = Join-Path $TestDrive 'deploy' + New-Item -ItemType Directory -Path $script:ProvisionFolder | Out-Null + Copy-Item -LiteralPath (Join-Path $script:RepoRoot 'deploy\Post-Provision.ps1') -Destination $script:ProvisionFolder + $script:ProvisionPath = Join-Path $script:ProvisionFolder 'Post-Provision.ps1' + $script:StepLog = Join-Path $script:ProvisionFolder 'steps.log' + + # Each step records how it was run, and fails when LINUXBROKER_TEST_FAILING_STEPS names it. + $stub = @' +$step = [System.IO.Path]::GetFileNameWithoutExtension($MyInvocation.MyCommand.Path) +$arguments = @($args | ForEach-Object { [string]$_ }) +Add-Content -LiteralPath (Join-Path $PSScriptRoot 'steps.log') -Value ('{0} {1}' -f $step, ($arguments -join ' ')) +if (@([string]$env:LINUXBROKER_TEST_FAILING_STEPS -split ',') -contains $step) { + throw "$step failed on lnx-0." +} +'@ + $script:ProvisionSteps = @('Assign-FunctionAppApiRole', 'Initialize-Database', 'Build-ContainerImages', 'Assign-VmApiRoles', 'Register-LinuxHostSqlRecords', 'Enable-HostMonitoring') + foreach ($step in $script:ProvisionSteps) { + [System.IO.File]::WriteAllText((Join-Path $script:ProvisionFolder "$step.ps1"), $stub) + } + + function Get-StepLog { + if (-not (Test-Path -LiteralPath $script:StepLog)) { + return @() + } + return @(Get-Content -LiteralPath $script:StepLog) + } + } + + BeforeEach { + Remove-Item -LiteralPath $script:StepLog -ErrorAction SilentlyContinue + # Not $script: variables: in a mock that Post-Provision.ps1 calls, $script: is that + # script's scope. These are found through the scopes that called it. + $warnings = New-Object System.Collections.Generic.List[string] + Mock Write-Warning { $warnings.Add([string]$Message) } + Mock az { + if (($args -join ' ') -notlike 'account set *') { + throw "Unexpected az call: $($args -join ' ')" + } + $global:LASTEXITCODE = 0 + } + # The azd environment has no graphEndpoint. + Mock azd { $global:LASTEXITCODE = 1 } + + $savedVariables = Save-AzureVariable + $savedFailingSteps = $env:LINUXBROKER_TEST_FAILING_STEPS + $env:LINUXBROKER_TEST_FAILING_STEPS = $null + $provision = @{ + ResourceGroupName = $script:ResourceGroup + SubscriptionId = $script:SubscriptionId + EnvironmentName = 'dev' + TaskAppName = 'func-linuxbroker-dev-task' + ApiClientId = '0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' + SqlServerFqdn = 'sql-linuxbroker-dev.database.windows.net' + DatabaseName = 'linuxbroker' + SqlAdminLogin = 'sqladmin' + SqlAdminPassword = 'not-a-real-password' + AvdHostGroupId = '3f2c1b0a-9e8d-4c7b-a6f5-e4d3c2b1a098' + LinuxHostGroupId = '7a6b5c4d-3e2f-4a1b-9c8d-7e6f5a4b3c21' + } + } + + AfterEach { + Restore-AzureVariable -Saved $savedVariables + $env:LINUXBROKER_TEST_FAILING_STEPS = $savedFailingSteps + } + + It 'connects the hosts to the host monitoring last, in the resource group, subscription and environment it provisioned' { + & $script:ProvisionPath @provision + + $steps = Get-StepLog + @($steps | ForEach-Object { ($_ -split ' ')[0] }) | Should -Be $script:ProvisionSteps + $steps[-1] | Should -BeExactly "Enable-HostMonitoring -ResourceGroupName $($script:ResourceGroup) -SubscriptionId $($script:SubscriptionId) -EnvironmentName dev" + $warnings.Count | Should -Be 0 + } + + It 'warns, rather than failing the deployment, when the host monitoring cannot be fully set up' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Enable-HostMonitoring' + + { & $script:ProvisionPath @provision } | Should -Not -Throw + + $warnings | Should -Contain 'The deployment is in place, but the host monitoring is not fully set up. Enable-HostMonitoring failed on lnx-0.' + } + + It 'still stops at a step that fails before the host monitoring' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Register-LinuxHostSqlRecords' + + { & $script:ProvisionPath @provision } | Should -Throw 'Register-LinuxHostSqlRecords failed on lnx-0.' + + @(Get-StepLog | ForEach-Object { ($_ -split ' ')[0] }) | Should -Not -Contain 'Enable-HostMonitoring' + } +} diff --git a/avd_host/tests/Initialize-DeploymentEnvironment.Tests.ps1 b/avd_host/tests/Initialize-DeploymentEnvironment.Tests.ps1 new file mode 100644 index 0000000..604f161 --- /dev/null +++ b/avd_host/tests/Initialize-DeploymentEnvironment.Tests.ps1 @@ -0,0 +1,1130 @@ +# Pester 5 tests for how deploy/Initialize-DeploymentEnvironment.ps1 sets up the AVD scaling plan +# and its role, and leaves out the VM extensions of hosts that are not running, and for the Bicep +# those settings feed. From the repository root, in Windows PowerShell: +# Invoke-Pester -Path avd_host/tests + +BeforeAll { + $script:RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path + $script:InitializeScriptPath = Join-Path $script:RepoRoot 'deploy\Initialize-DeploymentEnvironment.ps1' + + # Running the script would bootstrap an azd environment, so only its functions are loaded. + $parseErrors = $null + $script:InitializeAst = [System.Management.Automation.Language.Parser]::ParseFile($script:InitializeScriptPath, [ref]$null, [ref]$parseErrors) + if (@($parseErrors).Count -gt 0) { + throw "Initialize-DeploymentEnvironment.ps1 does not parse: $($parseErrors[0].Message)" + } + + $script:FunctionDefinitions = @{} + foreach ($definition in $script:InitializeAst.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) { + $script:FunctionDefinitions[$definition.Name] = $definition + } + + foreach ($name in @( + 'Get-AzdEnvValue', 'Get-FirstNonEmptyValue', 'ConvertTo-IntParameterValue', 'Get-PropertyValue', 'Get-VmTagValue', + 'ConvertTo-TimeOfDayParameterValue', 'ConvertTo-PercentParameterValue', 'Assert-AvdScalingPlanTimeOrder', + 'ConvertTo-PositiveIntParameterValue', 'ConvertTo-EmailListParameterValue', + 'Resolve-PortalSessionStore', 'ConvertTo-PortalRedisSkuParameterValue', 'ConvertTo-PortalRedisLocationParameterValue', 'Resolve-AppServicePlanCapacity', + 'Resolve-WindowsTimeZoneId', 'Get-HostPowerStateSnapshot', 'Resolve-AvdServicePrincipalObjectId', + 'Invoke-ArmGetRequest', 'Get-AvdAutoscaleRoleState', 'Test-ActionPermitted', 'Test-CanAssignSubscriptionRole', + 'Resolve-AvdAutoscaleRolePlan')) { + if (-not $script:FunctionDefinitions.ContainsKey($name)) { + throw "Initialize-DeploymentEnvironment.ps1 has no function $name." + } + . ([scriptblock]::Create($script:FunctionDefinitions[$name].Extent.Text)) + } + + # Mock can only replace a command that exists, and the machine running the tests may not have + # the Azure CLI. + if (-not (Get-Command az -ErrorAction SilentlyContinue)) { + function script:az { } + } + + $script:SubscriptionId = '6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f' + $script:ResourceGroup = 'rg-linuxbroker-dev' + $script:AvdAppId = '9cdead84-a844-4324-93f2-b2e6bb768d07' + $script:AvdObjectId = '3f2a1b0c-9d8e-4f7a-8b6c-5d4e3f2a1b0c' + $script:RoleId = '40c5ff49-9181-41f8-ae61-143b0e78555e' + $script:SubscriptionScope = "/subscriptions/$script:SubscriptionId" + + # What az answers to a command line that matches Pattern, first match first: its output, the + # lines it writes to stderr and its exit code. + function Add-AzResponse { + param( + [Parameter(Mandatory = $true)][string]$Pattern, + $Output = $null, + [string[]]$StdErr = @(), + [int]$ExitCode = 0 + ) + + $script:AzResponses.Add(@{ Pattern = $Pattern; Output = $Output; StdErr = $StdErr; ExitCode = $ExitCode }) + } + + function Get-ArgumentValue { + param( + [string[]]$Arguments, + [string]$Name + ) + + $index = [array]::IndexOf($Arguments, $Name) + if ($index -lt 0 -or $index + 1 -ge $Arguments.Count) { + return $null + } + return $Arguments[$index + 1] + } + + function New-TestVm { + param( + [Parameter(Mandatory = $true)][string]$Name, + [hashtable]$Tags = @{ 'broker-role' = 'avd-host' }, + [string]$PowerState = 'VM running', + [switch]$NoPowerState + ) + + $vm = [ordered]@{ name = $Name; resourceGroup = $script:ResourceGroup; tags = [pscustomobject]$Tags } + if (-not $NoPowerState) { + $vm['powerState'] = $PowerState + } + return [pscustomobject]$vm + } + + function New-RoleAssignment { + param( + [string]$PrincipalId = $script:AvdObjectId, + [string]$PrincipalType = 'ServicePrincipal', + [string]$RoleDefinitionId = "$script:SubscriptionScope/providers/Microsoft.Authorization/roleDefinitions/$script:RoleId", + [string]$Scope = $script:SubscriptionScope, + [switch]$Flattened + ) + + $properties = [ordered]@{ + principalId = $PrincipalId + principalType = $PrincipalType + roleDefinitionId = $RoleDefinitionId + scope = $Scope + } + if ($Flattened) { + return $properties + } + return [ordered]@{ + id = "$Scope/providers/Microsoft.Authorization/roleAssignments/$([guid]::NewGuid())" + name = [guid]::NewGuid().ToString() + properties = $properties + } + } + + function ConvertTo-ListJson { + param( + [AllowEmptyCollection()][object[]]$Value = @(), + [string]$NextLink = '' + ) + + $body = [ordered]@{ value = @($Value) } + if ($NextLink) { + $body['nextLink'] = $NextLink + } + return ConvertTo-Json -InputObject $body -Depth 8 + } + + $script:OwnerPermission = [ordered]@{ actions = @('*'); notActions = @(); dataActions = @(); notDataActions = @() } + $script:ContributorPermission = [ordered]@{ + actions = @('*') + notActions = @('Microsoft.Authorization/*/Delete', 'Microsoft.Authorization/*/Write', 'Microsoft.Authorization/elevateAccess/Action', 'Microsoft.Blueprint/blueprintAssignments/write') + dataActions = @() + notDataActions = @() + } + $script:UserAccessAdministratorPermission = [ordered]@{ actions = @('*/read', 'Microsoft.Authorization/*', 'Microsoft.Support/*'); notActions = @() } + $script:ReaderPermission = [ordered]@{ actions = @('*/read'); notActions = @() } + $script:ConditionalPermission = [ordered]@{ + actions = @('Microsoft.Authorization/roleAssignments/write', 'Microsoft.Authorization/roleAssignments/delete', '*/read') + notActions = @() + condition = "((!(ActionMatches{'Microsoft.Authorization/roleAssignments/write'})) OR (@Request[Microsoft.Authorization/roleAssignments:RoleDefinitionId] ForAnyOfAnyValues:GuidEquals {b24988ac-6180-42a0-ab88-20f7382dd24c}))" + conditionVersion = '2.0' + } + + # What az answers while deciding on the role: whether the tenant has the Azure Virtual Desktop + # service principal, whether it holds the role, and whether the signed-in account can assign it. + function Set-RoleCheck { + param( + [ValidateSet('Assigned', 'NotAssigned', 'Unknown')][string]$RoleState = 'NotAssigned', + [ValidateSet('Yes', 'No', 'Unknown')][string]$CanAssign = 'Yes', + [switch]$ServicePrincipalMissing + ) + + if ($ServicePrincipalMissing) { + Add-AzResponse -Pattern 'ad sp list *' -Output '[]' + Add-AzResponse -Pattern 'ad sp create *' -ExitCode 1 + } + else { + Add-AzResponse -Pattern 'ad sp list *' -Output (ConvertTo-Json -InputObject @($script:AvdObjectId)) + } + + switch ($RoleState) { + 'Assigned' { Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @((New-RoleAssignment))) } + 'NotAssigned' { Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson) } + 'Unknown' { Add-AzResponse -Pattern 'rest *roleAssignments*' -StdErr @('ERROR: Bad Request') -ExitCode 1 } + } + + switch ($CanAssign) { + 'Yes' { Add-AzResponse -Pattern 'rest *permissions*' -Output (ConvertTo-ListJson -Value @($script:OwnerPermission)) } + 'No' { Add-AzResponse -Pattern 'rest *permissions*' -Output (ConvertTo-ListJson -Value @($script:ContributorPermission)) } + 'Unknown' { Add-AzResponse -Pattern 'rest *permissions*' -StdErr @('ERROR: Bad Request') -ExitCode 1 } + } + } +} + +Describe 'Initialize-DeploymentEnvironment.ps1' { + BeforeEach { + Set-StrictMode -Version Latest + $global:LASTEXITCODE = 0 + $script:AzdValues = @{} + $script:AzCalls = New-Object System.Collections.Generic.List[object] + $script:AzResponses = New-Object System.Collections.Generic.List[hashtable] + $script:Warnings = New-Object System.Collections.Generic.List[string] + + Mock Get-AzdEnvValue { [string]$script:AzdValues[$Key] } + Mock Write-Warning { $script:Warnings.Add([string]$Message) } + Mock az { + # Like az's stderr, the lines go to the error stream, which the script discards with + # 2>$null or merges into the output with 2>&1, and they never stop the script. + $ErrorActionPreference = 'Continue' + # A native command gets each item of an array argument as an argument of its own. + $arguments = [string[]]@($args | ForEach-Object { $_ } | ForEach-Object { [string]$_ }) + $commandLine = $arguments -join ' ' + $script:AzCalls.Add([pscustomobject]@{ CommandLine = $commandLine; Arguments = $arguments }) + foreach ($response in $script:AzResponses) { + if ($commandLine -like $response.Pattern) { + $global:LASTEXITCODE = $response.ExitCode + foreach ($line in $response.StdErr) { + Write-Error -Message $line -Category NotSpecified -ErrorId 'NativeCommandError' + } + return $response.Output + } + } + throw "Unexpected az call: $commandLine" + } + } + + Context 'Reading the scaling plan settings' { + It 'reads a time of day as HH:mm, from ' -TestCases @( + @{ Value = '07:00'; Expected = '07:00' } + @{ Value = '7:05'; Expected = '07:05' } + @{ Value = ' 18:30 '; Expected = '18:30' } + @{ Value = '0:00'; Expected = '00:00' } + @{ Value = '23:59'; Expected = '23:59' } + ) { + $script:AzdValues['avdScalingPlanPeakStart'] = $Value + + ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanPeakStart' -DefaultValue '09:00' | Should -BeExactly $Expected + } + + It 'uses the default time when the value is not set' { + ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanPeakStart' -DefaultValue '09:00' | Should -BeExactly '09:00' + } + + It 'refuses the time ' -TestCases @( + @{ Value = '24:00' } + @{ Value = '07:60' } + @{ Value = '7' } + @{ Value = '7:5' } + @{ Value = '07:00:00' } + @{ Value = '7am' } + @{ Value = "07:00`n" + '08:00' } + @{ Value = ('{0}{1}:00' -f [char]0xFF10, [char]0xFF17) } + ) { + $script:AzdValues['avdScalingPlanPeakStart'] = $Value + + { ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanPeakStart' -DefaultValue '09:00' } | + Should -Throw -ExpectedMessage "*'avdScalingPlanPeakStart'*HH:mm*" + } + + It 'reads a percentage from to 100' -TestCases @( + @{ Value = '0'; Minimum = 0; Expected = 0 } + @{ Value = '100'; Minimum = 0; Expected = 100 } + @{ Value = '1'; Minimum = 1; Expected = 1 } + @{ Value = ''; Minimum = 1; Expected = 60 } + ) { + $script:AzdValues['avdScalingPlanRampUpCapacityThresholdPct'] = $Value + + ConvertTo-PercentParameterValue -Key 'avdScalingPlanRampUpCapacityThresholdPct' -DefaultValue 60 -Minimum $Minimum | Should -Be $Expected + } + + It 'refuses the percentage when the minimum is ' -TestCases @( + @{ Value = '101'; Minimum = 0; Message = "*'avdScalingPlanRampUpCapacityThresholdPct'*from 0 to 100*" } + @{ Value = '-1'; Minimum = 0; Message = '*from 0 to 100*' } + @{ Value = '0'; Minimum = 1; Message = '*from 1 to 100*' } + @{ Value = 'sixty'; Minimum = 0; Message = '*must be an integer*' } + ) { + $script:AzdValues['avdScalingPlanRampUpCapacityThresholdPct'] = $Value + + { ConvertTo-PercentParameterValue -Key 'avdScalingPlanRampUpCapacityThresholdPct' -DefaultValue 60 -Minimum $Minimum } | + Should -Throw -ExpectedMessage $Message + } + + It 'accepts phase times that come in order within one day' { + $times = [ordered]@{ avdScalingPlanRampUpStart = '07:00'; avdScalingPlanPeakStart = '09:00'; avdScalingPlanRampDownStart = '18:00'; avdScalingPlanOffPeakStart = '20:00' } + + { Assert-AvdScalingPlanTimeOrder -Times $times } | Should -Not -Throw + } + + It 'refuses a phase that starts ' -TestCases @( + @{ Description = 'with the one before it'; PeakStart = '07:00' } + @{ Description = 'before the one before it'; PeakStart = '06:59' } + ) { + $times = [ordered]@{ avdScalingPlanRampUpStart = '07:00'; avdScalingPlanPeakStart = $PeakStart; avdScalingPlanRampDownStart = '18:00'; avdScalingPlanOffPeakStart = '20:00' } + + { Assert-AvdScalingPlanTimeOrder -Times $times } | + Should -Throw -ExpectedMessage "*'avdScalingPlanPeakStart' ($PeakStart) must be later than 'avdScalingPlanRampUpStart' (07:00)*" + } + + It 'refuses an off-peak time that wraps past midnight' { + $times = [ordered]@{ avdScalingPlanRampUpStart = '07:00'; avdScalingPlanPeakStart = '09:00'; avdScalingPlanRampDownStart = '18:00'; avdScalingPlanOffPeakStart = '01:00' } + + { Assert-AvdScalingPlanTimeOrder -Times $times } | Should -Throw -ExpectedMessage "*'avdScalingPlanOffPeakStart'*" + } + + It 'finds the Windows time zone as ' -TestCases @( + @{ Value = 'UTC'; Expected = 'UTC' } + @{ Value = 'utc'; Expected = 'UTC' } + @{ Value = ' eastern standard time '; Expected = 'Eastern Standard Time' } + ) { + Resolve-WindowsTimeZoneId -TimeZoneId $Value | Should -BeExactly $Expected + } + + It 'does not take the IANA name for a Windows time zone' -TestCases @( + @{ Value = 'America/New_York' } + @{ Value = 'Etc/UTC' } + @{ Value = 'Not A Zone' } + ) { + Resolve-WindowsTimeZoneId -TimeZoneId $Value | Should -BeExactly '' + } + } + + Context 'Reading the monitoring settings' { + It 'reads the alert threshold as ' -TestCases @( + @{ Value = '1'; Expected = 1 } + @{ Value = ' 25 '; Expected = 25 } + @{ Value = ''; Expected = 5 } + ) { + $script:AzdValues['alertApiErrorThreshold'] = $Value + + ConvertTo-PositiveIntParameterValue -Key 'alertApiErrorThreshold' -DefaultValue 5 | Should -Be $Expected + } + + It 'refuses the alert threshold ' -TestCases @( + @{ Value = '0'; Message = "*'alertApiErrorThreshold'*1 or more*" } + @{ Value = '-3'; Message = '*1 or more*' } + @{ Value = 'five'; Message = '*must be an integer*' } + ) { + $script:AzdValues['alertApiErrorThreshold'] = $Value + + { ConvertTo-PositiveIntParameterValue -Key 'alertApiErrorThreshold' -DefaultValue 5 } | Should -Throw -ExpectedMessage $Message + } + + It 'reads the alert email addresses as ' -TestCases @( + @{ Value = ''; Expected = '' } + @{ Value = ' , ; '; Expected = '' } + @{ Value = 'ops@contoso.com'; Expected = 'ops@contoso.com' } + @{ Value = ' ops@contoso.com ; oncall@contoso.com,'; Expected = 'ops@contoso.com,oncall@contoso.com' } + @{ Value = 'ops@contoso.com,OPS@contoso.com;a1b2c3d4.contoso.onmicrosoft.com@amer.teams.ms'; Expected = 'ops@contoso.com,a1b2c3d4.contoso.onmicrosoft.com@amer.teams.ms' } + ) { + $script:AzdValues['alertEmailAddresses'] = $Value + + ConvertTo-EmailListParameterValue -Key 'alertEmailAddresses' | Should -BeExactly $Expected + } + + It 'refuses
as an alert email address' -TestCases @( + @{ Value = 'ops'; Address = 'ops' } + @{ Value = 'ops@contoso.com, oncall@contoso'; Address = 'oncall@contoso' } + @{ Value = 'ops@contoso.com oncall@contoso.com'; Address = 'ops@contoso.com oncall@contoso.com' } + @{ Value = 'ops@@contoso.com'; Address = 'ops@@contoso.com' } + ) { + $script:AzdValues['alertEmailAddresses'] = $Value + + { ConvertTo-EmailListParameterValue -Key 'alertEmailAddresses' } | + Should -Throw -ExpectedMessage "*'alertEmailAddresses'*'$Address' is not an email address*" + } + } + + Context 'Reading the portal session and plan capacity settings' { + It 'keeps portal sessions in when portalSessionStore is in ' -TestCases @( + @{ Value = ''; Cloud = 'AzurePublic'; Expected = 'redis' } + @{ Value = ' Redis '; Cloud = 'AzureUSGovernment'; Expected = 'redis' } + @{ Value = 'filesystem'; Cloud = 'AzurePublic'; Expected = 'filesystem' } + @{ Value = 'FILESYSTEM'; Cloud = 'AzureCustom'; Expected = 'filesystem' } + ) { + $script:AzdValues['portalSessionStore'] = $Value + + Resolve-PortalSessionStore -CloudName $Cloud | Should -BeExactly $Expected + $script:Warnings | Should -BeNullOrEmpty + } + + It 'keeps portal sessions on disk in AzureCustom, with a warning, when portalSessionStore is ' -TestCases @( + @{ Value = '' } + @{ Value = 'redis' } + ) { + $script:AzdValues['portalSessionStore'] = $Value + + Resolve-PortalSessionStore -CloudName 'AzureCustom' | Should -BeExactly 'filesystem' + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike "*AzureCustom*each instance's disk*" + } + + It 'refuses the portal session store ' -TestCases @( + @{ Value = 'memcached' } + @{ Value = 'cosmos' } + ) { + $script:AzdValues['portalSessionStore'] = $Value + + { Resolve-PortalSessionStore -CloudName 'AzurePublic' } | Should -Throw -ExpectedMessage "*'portalSessionStore' must be 'redis' or 'filesystem'*'$Value'*" + } + + It 'reads the session cache size in as ' -TestCases @( + @{ Value = ''; Cloud = 'AzurePublic'; Expected = '' } + @{ Value = ' Balanced_B1 '; Cloud = 'AzurePublic'; Expected = 'Balanced_B1' } + @{ Value = 'MemoryOptimized_M10'; Cloud = 'AzurePublic'; Expected = 'MemoryOptimized_M10' } + @{ Value = ''; Cloud = 'AzureUSGovernment'; Expected = '' } + @{ Value = 'Standard_C1'; Cloud = 'AzureUSGovernment'; Expected = 'Standard_C1' } + @{ Value = 'Premium_P1'; Cloud = 'AzureUSGovernment'; Expected = 'Premium_P1' } + @{ Value = 'Basic_C0'; Cloud = 'AzureUSGovernment'; Expected = 'Basic_C0' } + ) { + $script:AzdValues['portalRedisSku'] = $Value + + ConvertTo-PortalRedisSkuParameterValue -CloudName $Cloud | Should -BeExactly $Expected + } + + It 'refuses the session cache size in ' -TestCases @( + @{ Value = 'Balanced'; Cloud = 'AzurePublic'; Service = 'Azure Managed Redis' } + @{ Value = 'balanced_b0'; Cloud = 'AzurePublic'; Service = 'Azure Managed Redis' } + @{ Value = 'Balanced_B0;x'; Cloud = 'AzurePublic'; Service = 'Azure Managed Redis' } + @{ Value = 'Balanced_B0'; Cloud = 'AzureUSGovernment'; Service = 'Azure Cache for Redis' } + @{ Value = 'Standard_P1'; Cloud = 'AzureUSGovernment'; Service = 'Azure Cache for Redis' } + @{ Value = 'Premium_P0'; Cloud = 'AzureUSGovernment'; Service = 'Azure Cache for Redis' } + @{ Value = 'standard_c1'; Cloud = 'AzureUSGovernment'; Service = 'Azure Cache for Redis' } + @{ Value = 'C1'; Cloud = 'AzureUSGovernment'; Service = 'Azure Cache for Redis' } + ) { + $script:AzdValues['portalRedisSku'] = $Value + + { ConvertTo-PortalRedisSkuParameterValue -CloudName $Cloud } | Should -Throw -ExpectedMessage "*'portalRedisSku' must be an $Service size*'$Value'*" + } + + It 'reads the session cache region as ' -TestCases @( + @{ Value = ''; Expected = '' } + @{ Value = 'eastus'; Expected = 'eastus' } + @{ Value = ' East US 2 '; Expected = 'eastus2' } + @{ Value = 'USGovVirginia'; Expected = 'usgovvirginia' } + ) { + $script:AzdValues['portalRedisLocation'] = $Value + + ConvertTo-PortalRedisLocationParameterValue | Should -BeExactly $Expected + } + + It 'refuses the session cache region ' -TestCases @( + @{ Value = 'east-us' } + @{ Value = '2eastus' } + @{ Value = 'eastus;x' } + ) { + $script:AzdValues['portalRedisLocation'] = $Value + + { ConvertTo-PortalRedisLocationParameterValue } | Should -Throw -ExpectedMessage "*'portalRedisLocation' must be an Azure region name*'$Value'*" + } + + It 'reads the plan capacity as ' -TestCases @( + @{ Value = ''; Expected = 1 } + @{ Value = '2'; Expected = 2 } + @{ Value = ' 30 '; Expected = 30 } + ) { + $script:AzdValues['appServicePlanCapacity'] = $Value + + Resolve-AppServicePlanCapacity -PortalSessionStore 'redis' -SqlDatabaseSkuName 'S1' | Should -Be $Expected + $script:Warnings | Should -BeNullOrEmpty + } + + It 'refuses the plan capacity ' -TestCases @( + @{ Value = '0'; Message = '*from 1 to 30*' } + @{ Value = '31'; Message = '*from 1 to 30*' } + @{ Value = 'two'; Message = '*must be an integer*' } + ) { + $script:AzdValues['appServicePlanCapacity'] = $Value + + { Resolve-AppServicePlanCapacity -PortalSessionStore 'redis' } | Should -Throw -ExpectedMessage $Message + } + + It 'warns that sign-ins rely on affinity when instances keep sessions on disk' -TestCases @( + @{ Capacity = '2' } + @{ Capacity = '5' } + ) { + $script:AzdValues['appServicePlanCapacity'] = $Capacity + + Resolve-AppServicePlanCapacity -PortalSessionStore 'filesystem' -SqlDatabaseSkuName 'S1' | Should -Be ([int]$Capacity) + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike '*ARR affinity*' + } + + It 'does not warn about affinity for one instance keeping sessions on disk' { + $script:AzdValues['appServicePlanCapacity'] = '1' + + Resolve-AppServicePlanCapacity -PortalSessionStore 'filesystem' -SqlDatabaseSkuName 'Basic' | Should -Be 1 + $script:Warnings | Should -BeNullOrEmpty + } + + It 'warns about the Basic database tier from instances' -TestCases @( + @{ Capacity = '3'; SqlSku = 'Basic'; Warned = $true } + @{ Capacity = '4'; SqlSku = ' basic '; Warned = $true } + @{ Capacity = '2'; SqlSku = 'Basic'; Warned = $false } + @{ Capacity = '3'; SqlSku = 'S1'; Warned = $false } + @{ Capacity = '3'; SqlSku = ''; Warned = $false } + ) { + $script:AzdValues['appServicePlanCapacity'] = $Capacity + + Resolve-AppServicePlanCapacity -PortalSessionStore 'redis' -SqlDatabaseSkuName $SqlSku | Should -Be ([int]$Capacity) + if ($Warned) { + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike '*Basic database tier runs at most 30*' + } + else { + $script:Warnings | Should -BeNullOrEmpty + } + } + } + + Context 'Listing the hosts that are not running' { + It 'lists nothing, and does not warn, when the resource group does not exist yet' { + Add-AzResponse -Pattern 'group exists *' -Output 'false' + + $snapshot = Get-HostPowerStateSnapshot -SubscriptionId $script:SubscriptionId -ResourceGroupName $script:ResourceGroup + + $snapshot.Listed | Should -BeTrue + @($snapshot.NotRunning).Count | Should -Be 0 + $snapshot.AvdScalingExclusions.Count | Should -Be 0 + $script:AzCalls.Count | Should -Be 1 + Get-ArgumentValue -Arguments $script:AzCalls[0].Arguments -Name '--name' | Should -BeExactly $script:ResourceGroup + Get-ArgumentValue -Arguments $script:AzCalls[0].Arguments -Name '--subscription' | Should -BeExactly $script:SubscriptionId + $script:Warnings.Count | Should -Be 0 + } + + It 'lists the broker hosts that are not running, and the exclusion tags on the session hosts' { + $vms = @( + (New-TestVm -Name 'lnxhost-0' -Tags @{ 'broker-role' = 'linux-host' }), + (New-TestVm -Name 'LNXHOST-1' -Tags @{ 'broker-role' = 'linux-host' } -PowerState 'VM deallocated'), + (New-TestVm -Name 'avdhost-0' -PowerState 'VM stopped'), + (New-TestVm -Name 'avdhost-1' -Tags @{ 'broker-role' = 'avd-host'; excludeFromScaling = '' }), + (New-TestVm -Name 'AvdHost-2' -Tags @{ 'broker-role' = 'avd-host'; ExcludeFromScaling = 'maintenance' } -PowerState 'VM deallocated'), + (New-TestVm -Name 'avdhost-3' -NoPowerState), + (New-TestVm -Name 'avdhost-4' -PowerState 'VM starting'), + (New-TestVm -Name 'lnxhost-2' -Tags @{ 'broker-role' = 'linux-host'; excludeFromScaling = 'true' }), + (New-TestVm -Name 'jumpbox' -Tags @{ role = 'jump' } -PowerState 'VM deallocated'), + (New-TestVm -Name 'build-agent' -Tags @{ 'broker-role' = 'build' } -PowerState 'VM deallocated') + ) + Add-AzResponse -Pattern 'group exists *' -Output 'true' + Add-AzResponse -Pattern 'vm list *' -Output (ConvertTo-Json -InputObject $vms -Depth 5) + + $snapshot = Get-HostPowerStateSnapshot -SubscriptionId $script:SubscriptionId -ResourceGroupName $script:ResourceGroup + + $snapshot.Listed | Should -BeTrue + $snapshot.NotRunning | Should -Be @('lnxhost-1', 'avdhost-0', 'avdhost-2', 'avdhost-4') + $snapshot.NotRunningDisplay | Should -Be @('LNXHOST-1 (VM deallocated)', 'avdhost-0 (VM stopped)', 'AvdHost-2 (VM deallocated)', 'avdhost-4 (VM starting)') + @($snapshot.AvdScalingExclusions.Keys) | Should -Be @('avdhost-1', 'avdhost-2') + $snapshot.AvdScalingExclusions['avdhost-1'] | Should -BeExactly '' + $snapshot.AvdScalingExclusions['avdhost-2'] | Should -BeExactly 'maintenance' + $script:Warnings.Count | Should -Be 0 + + $list = $script:AzCalls[1].Arguments + Get-ArgumentValue -Arguments $list -Name '--resource-group' | Should -BeExactly $script:ResourceGroup + Get-ArgumentValue -Arguments $list -Name '--subscription' | Should -BeExactly $script:SubscriptionId + $list | Should -Contain '--show-details' + } + + It 'lists one host that is not running as a list' { + Add-AzResponse -Pattern 'group exists *' -Output 'true' + Add-AzResponse -Pattern 'vm list *' -Output (ConvertTo-Json -InputObject @((New-TestVm -Name 'lnxhost-0' -Tags @{ 'broker-role' = 'linux-host' } -PowerState 'VM deallocated')) -Depth 5) + + $snapshot = Get-HostPowerStateSnapshot -SubscriptionId $script:SubscriptionId -ResourceGroupName $script:ResourceGroup + + , $snapshot.NotRunning | Should -BeOfType [string[]] + $snapshot.NotRunning | Should -Be @('lnxhost-0') + } + + It 'warns, and lists nothing, when ' -TestCases @( + @{ Description = 'it cannot tell whether the resource group exists'; GroupExitCode = 1; ListExitCode = 0; ListOutput = '[]'; Message = "*Could not check whether resource group 'rg-linuxbroker-dev' exists*Cannot modify extensions*" } + @{ Description = 'it cannot list the VMs'; GroupExitCode = 0; ListExitCode = 1; ListOutput = ''; Message = "*Could not list the VMs in resource group 'rg-linuxbroker-dev'*run azd provision again*" } + @{ Description = 'the VM list is not JSON'; GroupExitCode = 0; ListExitCode = 0; ListOutput = 'Traceback (most recent call last):'; Message = '*Could not list the VMs*' } + ) { + Add-AzResponse -Pattern 'group exists *' -Output 'true' -StdErr @('ERROR: something went wrong') -ExitCode $GroupExitCode + Add-AzResponse -Pattern 'vm list *' -Output $ListOutput -ExitCode $ListExitCode + + $snapshot = Get-HostPowerStateSnapshot -SubscriptionId $script:SubscriptionId -ResourceGroupName $script:ResourceGroup + + $snapshot.Listed | Should -BeFalse + @($snapshot.NotRunning).Count | Should -Be 0 + $snapshot.AvdScalingExclusions.Count | Should -Be 0 + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike $Message + } + } + + Context 'Finding the Azure Virtual Desktop service principal' { + It 'uses the configured object ID without looking it up' { + Resolve-AvdServicePrincipalObjectId -ConfiguredObjectId ' 3F2A1B0C-9D8E-4F7A-8B6C-5D4E3F2A1B0C ' -AppId $script:AvdAppId | Should -BeExactly $script:AvdObjectId + $script:AzCalls.Count | Should -Be 0 + } + + It 'refuses a configured object ID that is not a GUID' { + { Resolve-AvdServicePrincipalObjectId -ConfiguredObjectId 'Azure Virtual Desktop' -AppId $script:AvdAppId } | + Should -Throw -ExpectedMessage "*'avdServicePrincipalObjectId'*GUID*" + } + + It 'refuses an app ID that is not a GUID, and never searches by name' { + { Resolve-AvdServicePrincipalObjectId -AppId "Azure Virtual Desktop' or displayName eq 'x" } | + Should -Throw -ExpectedMessage "*'avdServicePrincipalAppId'*GUID*" + $script:AzCalls.Count | Should -Be 0 + } + + It 'finds the service principal by app ID' { + Add-AzResponse -Pattern 'ad sp list *' -Output (ConvertTo-Json -InputObject @($script:AvdObjectId)) + + Resolve-AvdServicePrincipalObjectId -AppId $script:AvdAppId | Should -BeExactly $script:AvdObjectId + + $script:AzCalls.Count | Should -Be 1 + Get-ArgumentValue -Arguments $script:AzCalls[0].Arguments -Name '--filter' | Should -BeExactly "appId eq '$script:AvdAppId'" + } + + It 'creates the service principal when the tenant has none' { + Add-AzResponse -Pattern 'ad sp list *' -Output '[]' + Add-AzResponse -Pattern 'ad sp create *' -Output $script:AvdObjectId + + Resolve-AvdServicePrincipalObjectId -AppId $script:AvdAppId | Should -BeExactly $script:AvdObjectId + + Get-ArgumentValue -Arguments $script:AzCalls[1].Arguments -Name '--id' | Should -BeExactly $script:AvdAppId + } + + It 'does not create the service principal with SkipCreate' { + Add-AzResponse -Pattern 'ad sp list *' -Output '[]' + + Resolve-AvdServicePrincipalObjectId -AppId $script:AvdAppId -SkipCreate | Should -BeExactly '' + $script:AzCalls.Count | Should -Be 1 + } + + It 'answers with nothing when ' -TestCases @( + @{ Description = 'neither the lookup nor the creation works'; ListOutput = ''; ListExitCode = 1; CreateOutput = ''; CreateExitCode = 1 } + @{ Description = 'the lookup and the creation answer with something other than a GUID'; ListOutput = '["not-a-guid"]'; ListExitCode = 0; CreateOutput = 'Insufficient privileges'; CreateExitCode = 0 } + @{ Description = 'the lookup answers with something other than JSON'; ListOutput = 'oops'; ListExitCode = 0; CreateOutput = ''; CreateExitCode = 1 } + ) { + Add-AzResponse -Pattern 'ad sp list *' -Output $ListOutput -ExitCode $ListExitCode + Add-AzResponse -Pattern 'ad sp create *' -Output $CreateOutput -ExitCode $CreateExitCode + + Resolve-AvdServicePrincipalObjectId -AppId $script:AvdAppId | Should -BeExactly '' + } + } + + Context 'Reading Azure Resource Manager through az rest' { + It 'passes the query string as URL parameters, not in the URL' { + Add-AzResponse -Pattern 'rest *' -Output '{"value":[]}' + + $result = Invoke-ArmGetRequest -Path "$script:SubscriptionScope/providers/Microsoft.Authorization/roleAssignments" -QueryParameters @('api-version=2022-04-01', "`$filter=atScope() and assignedTo('x')") + + $result.Succeeded | Should -BeTrue + @($result.Response.value).Count | Should -Be 0 + $arguments = $script:AzCalls[0].Arguments + Get-ArgumentValue -Arguments $arguments -Name '--method' | Should -BeExactly 'get' + Get-ArgumentValue -Arguments $arguments -Name '--url' | Should -BeExactly "$script:SubscriptionScope/providers/Microsoft.Authorization/roleAssignments" + $index = [array]::IndexOf($arguments, '--url-parameters') + $arguments[$index + 1] | Should -BeExactly 'api-version=2022-04-01' + $arguments[$index + 2] | Should -BeExactly "`$filter=atScope() and assignedTo('x')" + } + + It 'reports what az wrote to stderr when the request fails' { + Add-AzResponse -Pattern 'rest *' -StdErr @('ERROR: (AuthorizationFailed) The client does not have authorization.', 'Code: AuthorizationFailed') -ExitCode 1 + + $result = Invoke-ArmGetRequest -Path "$script:SubscriptionScope/providers/Microsoft.Authorization/permissions" -QueryParameters @('api-version=2022-04-01') + + $result.Succeeded | Should -BeFalse + $result.ErrorText | Should -BeLike '*(AuthorizationFailed)*Code: AuthorizationFailed' + } + + It 'treats an answer that is not a JSON object as a failure' { + Add-AzResponse -Pattern 'rest *' -Output 'null' + + (Invoke-ArmGetRequest -Path "$script:SubscriptionScope/providers/Microsoft.Authorization/permissions" -QueryParameters @('api-version=2022-04-01')).Succeeded | Should -BeFalse + } + } + + Context 'Checking whether the service principal holds the role' { + It 'asks for the assignments at and above the subscription that apply to the service principal' { + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson) + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'NotAssigned' + + $arguments = $script:AzCalls[0].Arguments + $index = [array]::IndexOf($arguments, '--url-parameters') + $arguments[$index + 1] | Should -BeExactly 'api-version=2022-04-01' + $arguments[$index + 2] | Should -BeExactly "`$filter=atScope() and assignedTo('$script:AvdObjectId')" + } + + It 'answers Assigned for the role on ' -TestCases @( + @{ Description = 'the subscription'; Scope = '/subscriptions/6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f' } + @{ Description = 'the subscription, in other letter case'; Scope = '/Subscriptions/6D1E0F3C-2B4A-4C5D-8E9F-0A1B2C3D4E5F' } + @{ Description = 'a management group'; Scope = '/providers/Microsoft.Management/managementGroups/contoso' } + @{ Description = 'the root scope'; Scope = '/' } + ) { + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @((New-RoleAssignment -Scope $Scope))) + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'Assigned' + } + + It 'answers Assigned for the role given to a group the service principal belongs to' { + $groupAssignment = New-RoleAssignment -PrincipalId ([guid]::NewGuid().ToString()) -PrincipalType 'Group' + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @($groupAssignment)) + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'Assigned' + } + + It 'answers Assigned for an assignment whose properties are not nested' { + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @((New-RoleAssignment -Flattened))) + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'Assigned' + } + + It 'answers NotAssigned when the only assignment is ' -TestCases @( + @{ Description = 'on a resource group, which autoscale does not accept'; Assignment = @{ Scope = '/subscriptions/6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f/resourceGroups/rg-linuxbroker-dev' } } + @{ Description = 'on another subscription'; Assignment = @{ Scope = '/subscriptions/00000000-1111-2222-3333-444444444444' } } + @{ Description = 'of another role'; Assignment = @{ RoleDefinitionId = '/subscriptions/6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c' } } + @{ Description = 'of a role whose ID only ends like it'; Assignment = @{ RoleDefinitionId = '/providers/Microsoft.Authorization/roleDefinitions/x40c5ff49-9181-41f8-ae61-143b0e78555e' } } + @{ Description = 'for another service principal'; Assignment = @{ PrincipalId = '11111111-2222-3333-4444-555555555555' } } + ) { + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @((New-RoleAssignment @Assignment))) + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'NotAssigned' + } + + It 'answers Unknown when ' -TestCases @( + @{ Description = 'the request fails'; Output = $null; ExitCode = 1 } + @{ Description = 'more pages might hold the role'; Output = 'next-page'; ExitCode = 0 } + ) { + $body = $null + if ($Output -eq 'next-page') { + $body = ConvertTo-ListJson -Value @((New-RoleAssignment -Scope '/subscriptions/6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f/resourceGroups/rg')) -NextLink 'https://management.azure.com/next' + } + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output $body -StdErr @('ERROR: Bad Request') -ExitCode $ExitCode + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'Unknown' + } + + It 'still answers Assigned when more pages follow a match' { + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @((New-RoleAssignment)) -NextLink 'https://management.azure.com/next') + + Get-AvdAutoscaleRoleState -PrincipalObjectId $script:AvdObjectId -SubscriptionId $script:SubscriptionId | Should -BeExactly 'Assigned' + } + } + + Context 'Checking whether the signed-in account can assign roles' { + It 'matches the action against as ' -TestCases @( + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('*'); Expected = $true } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('Microsoft.Authorization/*'); Expected = $true } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('Microsoft.Authorization/*/Write'); Expected = $true } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('microsoft.authorization/roleassignments/WRITE'); Expected = $true } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('*/read', 'Microsoft.Support/*'); Expected = $false } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('Microsoft.Authorization/roleAssignments/writ.'); Expected = $false } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @('Microsoft.Authorization/roleAssignments/write/extra'); Expected = $false } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @($null, '', ' '); Expected = $false } + @{ Action = 'Microsoft.Authorization/roleAssignments/write'; Patterns = @(); Expected = $false } + ) { + Test-ActionPermitted -Action $Action -Patterns $Patterns | Should -Be $Expected + } + + It 'answers for an account that is ' -TestCases @( + @{ Description = 'an Owner'; Permissions = @('Owner'); Expected = 'Yes' } + @{ Description = 'a User Access Administrator'; Permissions = @('UserAccessAdministrator'); Expected = 'Yes' } + @{ Description = 'a Contributor and a User Access Administrator'; Permissions = @('Contributor', 'UserAccessAdministrator'); Expected = 'Yes' } + @{ Description = 'a Contributor'; Permissions = @('Contributor'); Expected = 'No' } + @{ Description = 'a Reader'; Permissions = @('Reader'); Expected = 'No' } + @{ Description = 'a Role Based Access Control Administrator limited by a condition'; Permissions = @('Conditional'); Expected = 'Unknown' } + @{ Description = 'limited by a condition, and an Owner as well'; Permissions = @('Conditional', 'Owner'); Expected = 'Yes' } + ) { + $lookup = @{ + Owner = $script:OwnerPermission + UserAccessAdministrator = $script:UserAccessAdministratorPermission + Contributor = $script:ContributorPermission + Reader = $script:ReaderPermission + Conditional = $script:ConditionalPermission + } + $values = @($Permissions | ForEach-Object { $lookup[$_] }) + Add-AzResponse -Pattern 'rest *permissions*' -Output (ConvertTo-ListJson -Value $values) + + Test-CanAssignSubscriptionRole -SubscriptionId $script:SubscriptionId | Should -BeExactly $Expected + + Get-ArgumentValue -Arguments $script:AzCalls[0].Arguments -Name '--url' | Should -BeExactly "$script:SubscriptionScope/providers/Microsoft.Authorization/permissions" + } + + It 'answers No when the account may not even read its permissions' { + Add-AzResponse -Pattern 'rest *permissions*' -StdErr @("ERROR: (AuthorizationFailed) The client 'user@contoso.com' does not have authorization to perform action 'Microsoft.Authorization/permissions/read'.") -ExitCode 1 + + Test-CanAssignSubscriptionRole -SubscriptionId $script:SubscriptionId | Should -BeExactly 'No' + } + + It 'answers Unknown when ' -TestCases @( + @{ Description = 'the request fails for another reason'; Output = $null; StdErr = @('ERROR: HTTPSConnectionPool: Max retries exceeded'); ExitCode = 1 } + @{ Description = 'more pages might grant it'; Output = 'next-page'; StdErr = @(); ExitCode = 0 } + ) { + $body = $null + if ($Output -eq 'next-page') { + $body = ConvertTo-ListJson -Value @($script:ReaderPermission) -NextLink 'https://management.azure.com/next' + } + Add-AzResponse -Pattern 'rest *permissions*' -Output $body -StdErr $StdErr -ExitCode $ExitCode + + Test-CanAssignSubscriptionRole -SubscriptionId $script:SubscriptionId | Should -BeExactly 'Unknown' + } + } + + Context 'Deciding whether the deployment assigns the role and the scaling plan' { + BeforeEach { + $plan = @{ + AvdSessionHostsDeployed = $true + ScalingPlanRequested = $true + StartVmOnConnect = $true + AssignRoleRequested = $true + ConfiguredObjectId = '' + AppId = $script:AvdAppId + SubscriptionId = $script:SubscriptionId + } + $ownerCommand = "az role assignment create --assignee-object-id $script:AvdObjectId --assignee-principal-type ServicePrincipal --role $script:RoleId --scope /subscriptions/$script:SubscriptionId" + } + + It 'checks nothing when ' -TestCases @( + @{ Description = 'no AVD session hosts are deployed'; Settings = @{ AvdSessionHostsDeployed = $false } } + @{ Description = 'neither the scaling plan nor Start VM on Connect is on'; Settings = @{ ScalingPlanRequested = $false; StartVmOnConnect = $false } } + ) { + foreach ($key in $Settings.Keys) { + $plan[$key] = $Settings[$key] + } + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ServicePrincipalObjectId | Should -BeExactly '' + $result.ScalingPlanEnabled | Should -Be $plan.ScalingPlanRequested + $script:AzCalls.Count | Should -Be 0 + $script:Warnings.Count | Should -Be 0 + } + + It 'assigns the role, quietly, when it is missing and the account can assign it' { + Set-RoleCheck -RoleState 'NotAssigned' -CanAssign 'Yes' + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeTrue + $result.ScalingPlanEnabled | Should -BeTrue + $result.ServicePrincipalObjectId | Should -BeExactly $script:AvdObjectId + $result.Summary | Should -BeLike '*gives the Azure Virtual Desktop service principal Desktop Virtualization Power On Off Contributor*' + $script:Warnings.Count | Should -Be 0 + } + + It 'leaves out the role assignment, and keeps the plan, when the role is already assigned' { + Set-RoleCheck -RoleState 'Assigned' + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeTrue + $result.ServicePrincipalObjectId | Should -BeExactly $script:AvdObjectId + @($script:AzCalls | Where-Object { $_.CommandLine -like 'rest *permissions*' }).Count | Should -Be 0 + $script:Warnings.Count | Should -Be 0 + } + + It 'leaves out the role and the plan assignment, with the command an Owner can run, when the account cannot assign roles' { + Set-RoleCheck -RoleState 'NotAssigned' -CanAssign 'No' + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeFalse + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike "*cannot assign roles on subscription '$script:SubscriptionId'*scaling plan is deployed but assigned to no host pool*Start VM on Connect cannot start session hosts*run azd provision again*assignAvdAutoscaleRole to false*" + $script:Warnings[0] | Should -BeLike "*Command: $ownerCommand" + } + + It 'names only when the account cannot assign roles' -TestCases @( + @{ Description = 'Start VM on Connect'; ScalingPlanRequested = $false; StartVmOnConnect = $true; Present = '*Start VM on Connect cannot start session hosts*'; Absent = '*scaling plan is deployed*' } + @{ Description = 'the scaling plan'; ScalingPlanRequested = $true; StartVmOnConnect = $false; Present = '*scaling plan is deployed but assigned to no host pool*'; Absent = '*Start VM on Connect*' } + ) { + $plan.ScalingPlanRequested = $ScalingPlanRequested + $plan.StartVmOnConnect = $StartVmOnConnect + Set-RoleCheck -RoleState 'NotAssigned' -CanAssign 'No' + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeFalse + $script:Warnings[0] | Should -BeLike $Present + $script:Warnings[0] | Should -Not -BeLike $Absent + } + + It 'assigns the role, and says how to recover, when ' -TestCases @( + @{ Description = 'it cannot tell whether the role is assigned'; RoleState = 'Unknown'; CanAssign = 'Yes' } + @{ Description = 'it cannot tell whether the account can assign it'; RoleState = 'NotAssigned'; CanAssign = 'Unknown' } + @{ Description = 'it can tell neither'; RoleState = 'Unknown'; CanAssign = 'Unknown' } + ) { + Set-RoleCheck -RoleState $RoleState -CanAssign $CanAssign + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeTrue + $result.ScalingPlanEnabled | Should -BeTrue + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike '*RoleAssignmentExists*set assignAvdAutoscaleRole to false*AuthorizationFailed*' + $script:Warnings[0] | Should -BeLike "*Command: $ownerCommand" + } + + It 'leaves out the role and the plan assignment when the service principal cannot be found or created' { + Set-RoleCheck -ServicePrincipalMissing + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeFalse + $result.ServicePrincipalObjectId | Should -BeExactly '' + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike "*app ID $script:AvdAppId*Set avdServicePrincipalObjectId*assignAvdAutoscaleRole to false*" + @($script:AzCalls | Where-Object { $_.CommandLine -like 'rest *' }).Count | Should -Be 0 + } + + It 'uses the configured object ID' { + $plan.ConfiguredObjectId = $script:AvdObjectId + Add-AzResponse -Pattern 'rest *roleAssignments*' -Output (ConvertTo-ListJson -Value @((New-RoleAssignment))) + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.ServicePrincipalObjectId | Should -BeExactly $script:AvdObjectId + @($script:AzCalls | Where-Object { $_.CommandLine -like 'ad *' }).Count | Should -Be 0 + } + + Context 'with assignAvdAutoscaleRole set to false' { + BeforeEach { + $plan.AssignRoleRequested = $false + } + + It 'keeps the plan and warns when the role is missing' { + Set-RoleCheck -RoleState 'NotAssigned' + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeTrue + $result.ServicePrincipalObjectId | Should -BeExactly $script:AvdObjectId + $result.Summary | Should -BeLike '*assignAvdAutoscaleRole is false*' + $script:Warnings.Count | Should -Be 1 + $script:Warnings[0] | Should -BeLike '*Azure refuses to assign the scaling plan to the host pool, which fails the deployment*Start VM on Connect cannot start session hosts*' + $script:Warnings[0] | Should -BeLike "*command: $ownerCommand" + @($script:AzCalls | Where-Object { $_.CommandLine -like 'rest *permissions*' }).Count | Should -Be 0 + } + + It 'does not warn when the role is assigned, or when it cannot tell' -TestCases @( + @{ RoleState = 'Assigned' } + @{ RoleState = 'Unknown' } + ) { + Set-RoleCheck -RoleState $RoleState + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeTrue + $script:Warnings.Count | Should -Be 0 + } + + It 'neither creates the service principal nor warns when the tenant has none' { + Set-RoleCheck -ServicePrincipalMissing + + $result = Resolve-AvdAutoscaleRolePlan @plan + + $result.AssignRole | Should -BeFalse + $result.ScalingPlanEnabled | Should -BeTrue + $result.ServicePrincipalObjectId | Should -BeExactly '' + @($script:AzCalls | Where-Object { $_.CommandLine -like 'ad sp create *' }).Count | Should -Be 0 + $script:Warnings.Count | Should -Be 0 + } + } + } +} + +Describe 'The deployment settings and the Bicep they feed' { + BeforeAll { + # Git can check the files out with CRLF line endings, which $ in a multiline pattern does + # not match before. + function Read-RepositoryText { + param([Parameter(Mandatory = $true)][string]$RelativePath) + + return [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot $RelativePath)).Replace("`r`n", "`n") + } + + $script:ScriptText = Read-RepositoryText -RelativePath 'deploy\Initialize-DeploymentEnvironment.ps1' + $script:MainBicep = Read-RepositoryText -RelativePath 'deploy\bicep\main.bicep' + $script:ResourcesBicep = Read-RepositoryText -RelativePath 'deploy\bicep\main.resources.bicep' + $script:AvdBicep = Read-RepositoryText -RelativePath 'deploy\bicep\modules\AVD\main.bicep' + $script:LinuxBicep = Read-RepositoryText -RelativePath 'deploy\bicep\modules\Linux\main.bicep' + $script:ScalingPlanBicep = Read-RepositoryText -RelativePath 'deploy\bicep\modules\AVD\scaling-plan.bicep' + $script:MainBicepParameters = @([regex]::Matches($script:MainBicep, '(?m)^param\s+(\w+)\s') | ForEach-Object { $_.Groups[1].Value }) + $script:ScriptParameters = @([regex]::Matches($script:ScriptText, "-ParameterName '(\w+)'") | ForEach-Object { $_.Groups[1].Value }) + } + + It 'writes only parameters main.bicep declares, each once' { + $script:ScriptParameters.Count | Should -BeGreaterThan 40 + foreach ($name in $script:ScriptParameters) { + $script:MainBicepParameters | Should -Contain $name + } + @($script:ScriptParameters | Group-Object | Where-Object { $_.Count -gt 1 } | ForEach-Object { $_.Name }) | Should -Be @() + } + + It 'writes every autoscale parameter' { + foreach ($name in @( + 'avdScalingPlanEnabled', 'avdStartVmOnConnect', 'avdScalingPlanTimeZone', 'avdScalingPlanRampUpStart', 'avdScalingPlanPeakStart', + 'avdScalingPlanRampDownStart', 'avdScalingPlanOffPeakStart', 'avdScalingPlanRampUpMinimumHostsPct', + 'avdScalingPlanRampUpCapacityThresholdPct', 'avdScalingPlanRampDownMinimumHostsPct', 'avdScalingPlanRampDownCapacityThresholdPct', + 'avdScalingPlanWeekendMinimumHostsPct', 'avdScalingExclusions', 'hostNamesNotRunning', 'avdServicePrincipalObjectId', + 'assignAvdAutoscaleRole')) { + $script:ScriptParameters | Should -Contain $name + $script:MainBicepParameters | Should -Contain $name + } + } + + It 'writes every monitoring parameter and passes it on to the monitoring module' { + $monitoringBicep = Read-RepositoryText -RelativePath 'deploy\bicep\modules\core\host-monitoring.bicep' + foreach ($name in @('deployHostMonitoring', 'alertEmailAddresses', 'alertCheckoutRefusalThreshold', 'alertApiErrorThreshold', 'alertNfsLatencyThresholdMs')) { + $script:ScriptParameters | Should -Contain $name + $script:MainBicepParameters | Should -Contain $name + $script:MainBicep | Should -Match "(?m)^\s+${name}: ${name}$" + $script:ResourcesBicep | Should -Match "(?m)^param ${name} " + } + + $script:ResourcesBicep | Should -Match ([regex]::Escape("module hostMonitoring 'modules/core/host-monitoring.bicep' = if (deployHostMonitoring) {")) + foreach ($name in @('alertEmailAddresses', 'alertCheckoutRefusalThreshold', 'alertApiErrorThreshold', 'alertNfsLatencyThresholdMs')) { + $script:ResourcesBicep | Should -Match "(?m)^\s+${name}: ${name}$" + $monitoringBicep | Should -Match "(?m)^param ${name} " + } + } + + It 'writes the plan capacity and portal session parameters and passes them on' { + foreach ($name in @('appServicePlanCapacity', 'portalSessionStore', 'portalRedisSku', 'portalRedisLocation')) { + $script:ScriptParameters | Should -Contain $name + $script:MainBicepParameters | Should -Contain $name + $script:MainBicep | Should -Match "(?m)^\s+${name}: ${name}$" + $script:ResourcesBicep | Should -Match "(?m)^param ${name} " + } + + $script:ResourcesBicep | Should -Match '(?m)^\s+capacity: appServicePlanCapacity$' + $script:ResourcesBicep | Should -Match '(?m)^\s+skuName: portalRedisSku$' + $script:ResourcesBicep | Should -Match ([regex]::Escape('cacheLocation: empty(portalRedisLocation) ? location : portalRedisLocation')) + (Read-RepositoryText -RelativePath 'deploy\bicep\modules\core\app-service-plan.bicep') | Should -Match '(?m)^\s+capacity: capacity$' + } + + It 'puts the session cache in its own region but its private endpoint in the virtual network''s' { + $redisBicep = Read-RepositoryText -RelativePath 'deploy\bicep\modules\core\redis-session-store.bicep' + $redisBicep | Should -Match '(?m)^param cacheLocation string = location$' + foreach ($resource in @('managedRedis', 'cache')) { + $block = [regex]::Match($redisBicep, "(?s)\nresource $resource '[^']+' = if \([^)]*\) \{.*?\n\}").Value + $block | Should -Match '(?m)^ location: cacheLocation$' + } + $endpoint = [regex]::Match($redisBicep, "(?s)\nresource privateEndpoint '[^']+' = \{.*?\n\}").Value + $endpoint | Should -Match '(?m)^ location: location$' + } + + It 'allows the plan capacities the script allows' { + foreach ($text in @($script:MainBicep, $script:ResourcesBicep, (Read-RepositoryText -RelativePath 'deploy\bicep\modules\core\app-service-plan.bicep'))) { + $text | Should -Match '(?m)^@minValue\(1\)\n@maxValue\(30\)\nparam (appServicePlanCapacity|capacity) int = 1$' + } + $script:FunctionDefinitions['Resolve-AppServicePlanCapacity'].Extent.Text | Should -Match ([regex]::Escape('$capacity -lt 1 -or $capacity -gt 30')) + } + + It 'puts the portal on the virtual network and turns its affinity off only when its sessions are in Redis' { + $portal = [regex]::Match($script:ResourcesBicep, "(?s)\nmodule frontendApp 'modules/apps/container-web-app\.bicep' = \{.*?\n\}").Value + $portal | Should -Match '(?m)^\s+appSettings: union\(frontendSettings, portalSessionSettings\)$' + $portal | Should -Match ([regex]::Escape("virtualNetworkSubnetId: usePortalRedis ? networking.outputs.appSubnetId : ''")) + $portal | Should -Match '(?m)^\s+clientAffinityEnabled: !usePortalRedis$' + + $api = [regex]::Match($script:ResourcesBicep, "(?s)\nmodule apiApp 'modules/apps/container-web-app\.bicep' = \{.*?\n\}").Value + $api | Should -Match '(?m)^\s+clientAffinityEnabled: false$' + } + + It 'names the same session settings the portal reads' { + $sessionStore = Read-RepositoryText -RelativePath 'front_end\session_store.py' + $settings = [regex]::Match($script:ResourcesBicep, '(?s)var portalSessionSettings = .*?\n\}\n').Value + $names = @([regex]::Matches($settings, '(?m)^\s+([A-Z_]+):') | ForEach-Object { $_.Groups[1].Value } | Sort-Object -Unique) + $names | Should -Be @('REDIS_ENTRA_RESOURCE', 'REDIS_HOST', 'REDIS_PORT', 'SESSION_BACKEND') + foreach ($name in $names) { + $sessionStore | Should -Match "'$name'" + } + } + + It 'lists in the example parameters file only parameters main.bicep declares' { + $example = ConvertFrom-Json -InputObject (Read-RepositoryText -RelativePath 'deploy\bicep\main.parameters.example.json') + $names = @($example.parameters.PSObject.Properties | ForEach-Object { $_.Name }) + $names | Should -Contain 'avdScalingPlanEnabled' + $names | Should -Contain 'hostNamesNotRunning' + foreach ($name in $names) { + $script:MainBicepParameters | Should -Contain $name + } + } + + It 'leaves out every VM extension of a host that is not running, in the module' -TestCases @( + @{ Module = 'AVD'; Expected = 3 } + @{ Module = 'Linux'; Expected = 1 } + ) { + $text = $script:AvdBicep + if ($Module -eq 'Linux') { + $text = $script:LinuxBicep + } + + $text | Should -Match ([regex]::Escape('var skipExtensionVmNamesLower = [for vmName in skipExtensionVmNames: toLower(vmName)]')) + $lines = $text -split "`r?`n" + $extensions = 0 + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -match "^resource\s+\w+\s+'Microsoft\.Compute/virtualMachines/extensions@") { + $extensions++ + $lines[$i + 1] | Should -Match ([regex]::Escape('for (name, i) in vmNames: if (!contains(skipExtensionVmNamesLower, toLower(name))) {')) + } + } + $extensions | Should -Be $Expected + } + + It 'passes the hosts that are not running and the exclusion tags to the host modules' { + $script:MainBicep | Should -Match '(?m)^\s+hostNamesNotRunning: hostNamesNotRunning$' + $script:MainBicep | Should -Match '(?m)^\s+avdScalingExclusions: avdScalingExclusions$' + $script:MainBicep | Should -Match '(?m)^\s+avdStartVmOnConnect: avdStartVmOnConnect$' + [regex]::Matches($script:ResourcesBicep, '(?m)^\s+skipExtensionVmNames: hostNamesNotRunning$').Count | Should -Be 2 + $script:ResourcesBicep | Should -Match '(?m)^\s+scalingExclusions: avdScalingExclusions$' + $script:ResourcesBicep | Should -Match '(?m)^\s+startVmOnConnect: avdStartVmOnConnect$' + } + + It 'writes back the tag the scaling plan excludes session hosts by' { + $script:ScalingPlanBicep | Should -Match "(?m)^param exclusionTag string = 'excludeFromScaling'$" + $script:AvdBicep | Should -Match ([regex]::Escape('tags: contains(scalingExclusions, toLower(name))')) + $script:AvdBicep | Should -Match ([regex]::Escape('excludeFromScaling: scalingExclusions[toLower(name)]')) + $script:FunctionDefinitions['Get-HostPowerStateSnapshot'].Extent.Text | Should -Match ([regex]::Escape("-Name 'excludeFromScaling'")) + } + + It 'turns Start VM on Connect on or off in both places that write the host pool' { + $script:AvdBicep | Should -Match '(?m)^\s+startVMOnConnect: startVmOnConnect$' + [regex]::Matches($script:AvdBicep, '(?m)^\s+startVMOnConnect: startVmOnConnect$').Count | Should -Be 2 + $tokenBicep = Read-RepositoryText -RelativePath 'deploy\bicep\modules\AVD\token.bicep' + $tokenBicep | Should -Match '(?m)^\s+startVMOnConnect: startVMOnConnect$' + } + + It 'assigns the scaling plan only after the role and the host pool' { + $match = [regex]::Match($script:MainBicep, "(?s)module avdScalingPlan 'modules/AVD/scaling-plan\.bicep'.*?dependsOn:\s*\[(?[^\]]*)\]") + $match.Success | Should -BeTrue + $dependencies = @($match.Groups['deps'].Value -split '\s+' | Where-Object { $_ }) + $dependencies | Should -Contain 'resources' + $dependencies | Should -Contain 'avdAutoscaleRole' + } + + It 'checks for the same role main.bicep assigns, for the same service principal' { + $bicepRole = [regex]::Match($script:MainBicep, "var avdPowerOnOffContributorRoleGuid = '([0-9a-f-]{36})'").Groups[1].Value + $bicepRole | Should -BeExactly $script:RoleId + $script:ScriptText | Should -Match ([regex]::Escape("`$avdAutoscaleRoleDefinitionId = '$bicepRole'")) + foreach ($name in @('Get-AvdAutoscaleRoleState', 'Resolve-AvdAutoscaleRolePlan')) { + $parameter = $script:FunctionDefinitions[$name].Body.ParamBlock.Parameters | Where-Object { $_.Name.VariablePath.UserPath -eq 'RoleDefinitionId' } + $parameter.DefaultValue.Value | Should -BeExactly $bicepRole + } + + $script:ScriptText | Should -Match ([regex]::Escape("`$avdServicePrincipalAppId = '$script:AvdAppId'")) + $script:MainBicep | Should -Match ([regex]::Escape("app ID $script:AvdAppId")) + } +} diff --git a/avd_host/tests/PSScriptAnalyzerSettings.psd1 b/avd_host/tests/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 0000000..811ec66 --- /dev/null +++ b/avd_host/tests/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,13 @@ +# PSScriptAnalyzer settings for the AVD session host script and the script that updates it, +# used by the avd-scripts job in .github/workflows/front-end-tests.yml. +@{ + Severity = @('Error', 'Warning') + ExcludeRules = @( + # The flagged New-* functions only build objects in memory; there is nothing for -WhatIf + # to skip. + 'PSUseShouldProcessForStateChangingFunctions', + # Update-AvdHostBrokerScript.ps1 reports its progress to the operator who runs it, as the + # other deployment scripts do. + 'PSAvoidUsingWriteHost' + ) +} diff --git a/avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 b/avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 new file mode 100644 index 0000000..d0524c9 --- /dev/null +++ b/avd_host/tests/Update-AvdHostBrokerScript.Tests.ps1 @@ -0,0 +1,1009 @@ +# Pester 5 tests for deploy/Update-AvdHostBrokerScript.ps1, which replaces Connect-LinuxBroker.ps1 +# on the existing AVD session hosts through Run Command, and for how +# deploy/Migrate-ExistingEnvironment.ps1 runs it. From the repository root, in Windows PowerShell: +# Invoke-Pester -Path avd_host/tests + +BeforeAll { + $script:RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path + $script:UpdateScriptPath = Join-Path $script:RepoRoot 'deploy\Update-AvdHostBrokerScript.ps1' + $script:BrokerScriptPath = Join-Path $script:RepoRoot 'avd_host\broker\Connect-LinuxBroker.ps1' + . $script:UpdateScriptPath + + # Mock can only replace a command that exists, and the machine running the tests may not have + # the Azure CLIs. + if (-not (Get-Command az -ErrorAction SilentlyContinue)) { + function script:az { } + } + if (-not (Get-Command azd -ErrorAction SilentlyContinue)) { + function script:azd { } + } + + $script:ResourceGroup = 'rg-linuxbroker-dev' + $script:SubscriptionId = '6d1e0f3c-2b4a-4c5d-8e9f-0a1b2c3d4e5f' + $script:ApiUrl = 'https://app-linuxbroker-api.azurewebsites.net/api' + $script:ClientId = '0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' + $script:DefaultSourceUrl = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main/avd_host/broker/Connect-LinuxBroker.ps1' + $script:BrokerScriptVersion = [regex]::Match([System.IO.File]::ReadAllText($script:BrokerScriptPath), '(?m)^\$ScriptVersion = ''([^'']+)''').Groups[1].Value + $script:WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + $script:Utf8NoBom = New-Object System.Text.UTF8Encoding -ArgumentList $false + $script:AzureVariables = @('AZURE_ENV_NAME', 'AZURE_ENVIRONMENT_NAME', 'AZURE_SUBSCRIPTION_ID') + + function New-TestVm { + param( + [Parameter(Mandatory = $true)] + [string]$Name, + + $Tags = @{ 'broker-role' = 'avd-host' }, + + [string]$PowerState = 'VM running', + + [switch]$NoPowerState + ) + + $vm = [ordered]@{ name = $Name; resourceGroup = $script:ResourceGroup; tags = $Tags } + if (-not $NoPowerState) { + $vm['powerState'] = $PowerState + } + return [pscustomobject]$vm + } + + function Set-TestVm { + param([object[]]$Vm) + + $script:VmListJson = ConvertTo-Json -InputObject @($Vm) -Depth 5 + } + + function ConvertTo-RunCommandJson { + param( + [AllowEmptyString()] + [string]$StdOut = '', + + [AllowEmptyString()] + [string]$StdErr = '' + ) + + return ConvertTo-Json -Depth 5 -InputObject ([ordered]@{ + value = @( + [ordered]@{ code = 'ComponentStatus/StdOut/succeeded'; displayStatus = 'Provisioning succeeded'; level = 'Info'; message = $StdOut }, + [ordered]@{ code = 'ComponentStatus/StdErr/succeeded'; displayStatus = 'Provisioning succeeded'; level = 'Info'; message = $StdErr } + ) + }) + } + + # Queues what Run Command answers on a VM, one entry per call: @{ StdOut; StdErr } for a run, + # @{ Error } for az failing, or @{ Raw } for output that is not JSON. A VM with nothing queued + # reports a completed update. + function Add-RunCommandOutcome { + param( + [Parameter(Mandatory = $true)] + [string]$Vm, + + [Parameter(Mandatory = $true)] + [hashtable[]]$Outcome + ) + + if (-not $script:RunCommandOutcomes.ContainsKey($Vm)) { + $script:RunCommandOutcomes[$Vm] = New-Object System.Collections.Queue + } + foreach ($entry in $Outcome) { + $script:RunCommandOutcomes[$Vm].Enqueue($entry) + } + } + + function Get-ArgumentValue { + param( + [string[]]$Arguments, + + [string]$Name + ) + + $index = [array]::IndexOf($Arguments, $Name) + if ($index -lt 0 -or $index + 1 -ge $Arguments.Count) { + return $null + } + return $Arguments[$index + 1] + } + + # The value a top-level assignment in the session host's update script gives a variable. + function Get-PayloadValue { + param( + [Parameter(Mandatory = $true)] + [string]$Payload, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + $parseErrors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseInput($Payload, [ref]$null, [ref]$parseErrors) + $assignment = $ast.Find({ + param($node) + $node -is [System.Management.Automation.Language.AssignmentStatementAst] -and + $node.Left -is [System.Management.Automation.Language.VariableExpressionAst] -and + $node.Left.VariablePath.UserPath -eq $Name + }, $false) + if ($null -eq $assignment) { + return $null + } + return $assignment.Right.Expression.Value + } + + function Test-Utf8Bom { + param([byte[]]$Bytes) + + return ($Bytes.Length -ge 3 -and $Bytes[0] -eq 0xEF -and $Bytes[1] -eq 0xBB -and $Bytes[2] -eq 0xBF) + } + + function Invoke-ChildProcess { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + + [Parameter(Mandatory = $true)] + [string]$ScriptPath + ) + + $folder = Split-Path -Parent $ScriptPath + $stdoutPath = Join-Path $folder 'stdout.txt' + $stderrPath = Join-Path $folder 'stderr.txt' + $process = Start-Process -FilePath $FilePath ` + -ArgumentList @('-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-File', ('"{0}"' -f $ScriptPath)) ` + -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -Wait -PassThru -NoNewWindow + + return [pscustomobject]@{ + ExitCode = $process.ExitCode + StdOut = [System.IO.File]::ReadAllText($stdoutPath) + StdErr = [System.IO.File]::ReadAllText($stderrPath) + } + } + + # Runs the script Run Command would run on a session host, in Windows PowerShell as Run Command + # does, with the destination in the test drive. The stubs keep it from registering an event + # source on the machine running the tests and from waiting between attempts, and a failure is + # reported on one line, as Run Command returns it. + function Invoke-HostUpdate { + param( + [Parameter(Mandatory = $true)] + [string]$SourceUrl, + + [Parameter(Mandatory = $true)] + [string]$Destination + ) + + $folder = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $folder | Out-Null + $payloadPath = Join-Path $folder 'payload.ps1' + $harnessPath = Join-Path $folder 'harness.ps1' + + $payload = Get-AvdHostUpdateScript -SourceUrl $SourceUrl -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId -Destination $Destination + [System.IO.File]::WriteAllText($payloadPath, $payload, $script:Utf8NoBom) + $harness = @' +function New-EventLog { param($LogName, $Source) Write-Output "TEST: New-EventLog $LogName $Source" } +function Start-Sleep { param($Seconds) Write-Output "TEST: Start-Sleep $Seconds" } +try { + . __PAYLOAD__ +} +catch { + [Console]::Error.WriteLine($_.Exception.Message) + exit 1 +} +'@ + [System.IO.File]::WriteAllText($harnessPath, $harness.Replace('__PAYLOAD__', (ConvertTo-PowerShellLiteral -Value $payloadPath)), $script:Utf8NoBom) + + return Invoke-ChildProcess -FilePath $script:WindowsPowerShell -ScriptPath $harnessPath + } + + function New-InstalledScript { + param([string]$Content = "# Connect-LinuxBroker.ps1 from an earlier release`r`n") + + $destination = Join-Path $TestDrive ('{0}\Connect-LinuxBroker.ps1' -f [guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path (Split-Path -Parent $destination) | Out-Null + [System.IO.File]::WriteAllText($destination, $Content) + return $destination + } + + function Get-FolderEntry { + param([string]$Path) + + return @(Get-ChildItem -LiteralPath (Split-Path -Parent $Path) -Force | ForEach-Object { $_.Name }) + } + + function Save-AzureVariable { + $saved = @{} + foreach ($name in $script:AzureVariables) { + $saved[$name] = [Environment]::GetEnvironmentVariable($name, 'Process') + [Environment]::SetEnvironmentVariable($name, $null, 'Process') + } + return $saved + } + + function Restore-AzureVariable { + param([hashtable]$Saved) + + foreach ($name in $Saved.Keys) { + [Environment]::SetEnvironmentVariable($name, $Saved[$name], 'Process') + } + } +} + +Describe 'ConvertTo-PowerShellLiteral' { + It 'doubles single quotes, including the curly ones PowerShell also ends a string at' { + ConvertTo-PowerShellLiteral -Value "it's" | Should -BeExactly "'it''s'" + $curly = [string][char]0x2019 + ConvertTo-PowerShellLiteral -Value "it$($curly)s" | Should -BeExactly "'it$curly$($curly)s'" + ConvertTo-PowerShellLiteral -Value '' | Should -BeExactly "''" + } +} + +Describe 'Test-SafeHttpsUrl' { + It 'accepts ' -TestCases @( + @{ Value = 'https://app-linuxbroker-api.azurewebsites.net/api' } + @{ Value = 'https://app-linuxbroker-api.azurewebsites.us:8443/api' } + @{ Value = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main' } + @{ Value = 'https://mirror.contoso.us/linux%20broker/~main/' } + @{ Value = 'https://10.0.0.4' } + ) { + Test-SafeHttpsUrl -Value $Value | Should -BeTrue + } + + It 'refuses a URL that ' -TestCases @( + @{ Description = 'is empty'; Value = '' } + @{ Description = 'is not https'; Value = 'http://app/api' } + @{ Description = 'names no host'; Value = 'https:///api' } + @{ Description = 'holds a double quote'; Value = 'https://app/api"' } + @{ Description = 'holds a single quote'; Value = "https://app/it's" } + @{ Description = 'holds a typographic double quote'; Value = ('https://app/api{0}' -f [char]0x201D) } + @{ Description = 'holds a typographic single quote'; Value = ('https://app/it{0}s' -f [char]0x2019) } + @{ Description = 'holds a letter outside ASCII'; Value = ('https://app/{0}/api' -f [char]0x00D1) } + @{ Description = 'holds the Kelvin sign, which a match without regard to case takes for a k'; Value = ('https://app/{0}/api' -f [char]0x212A) } + @{ Description = 'holds a subexpression'; Value = 'https://app/$(Get-Process)/api' } + @{ Description = 'holds a backtick'; Value = 'https://app/`api' } + @{ Description = 'holds a space'; Value = 'https://app /api' } + @{ Description = 'ends with a line break'; Value = "https://app/api`n" } + @{ Description = 'has a query string'; Value = 'https://app/api?code=1' } + @{ Description = 'has a fragment'; Value = 'https://app/api#top' } + @{ Description = 'names a user'; Value = 'https://user@app/api' } + @{ Description = 'has a port that is not a number'; Value = 'https://app:port/api' } + @{ Description = 'has a port above 65535'; Value = 'https://app:99999/api' } + ) { + Test-SafeHttpsUrl -Value $Value | Should -BeFalse + } +} + +Describe 'Get-AzdEnvValue' { + BeforeEach { + Set-StrictMode -Version Latest + $script:AzdCalls = New-Object System.Collections.Generic.List[string] + $script:AzdExitCode = 0 + $script:AzdOutput = @('rg-linuxbroker-dev', '') + Mock azd { + $script:AzdCalls.Add(($args -join ' ')) + $global:LASTEXITCODE = $script:AzdExitCode + $script:AzdOutput + } + } + + It 'reads a value from the azd environment' { + Get-AzdEnvValue -EnvironmentName 'linuxbroker-dev' -Key 'resourceGroupName' | Should -BeExactly 'rg-linuxbroker-dev' + $script:AzdCalls | Should -Be @('env get-value resourceGroupName --environment linuxbroker-dev') + } + + It 'answers with nothing when azd has no such value' { + $script:AzdExitCode = 1 + $script:AzdOutput = 'ERROR: key ''apiUrl'' not found in the environment values' + + Get-AzdEnvValue -EnvironmentName 'linuxbroker-dev' -Key 'apiUrl' | Should -BeExactly '' + } + + It 'does not ask azd without an environment' { + Get-AzdEnvValue -EnvironmentName '' -Key 'apiUrl' | Should -BeExactly '' + $script:AzdCalls.Count | Should -Be 0 + } +} + +Describe 'Get-AvdHostUpdateScript' { + BeforeEach { + Set-StrictMode -Version Latest + } + + It 'fills in exactly, as a string PowerShell 5.1 parses' -TestCases @( + @{ Description = 'ordinary values'; Value = 'https://app-linuxbroker-api.azurewebsites.net/api' } + @{ Description = 'a single quote'; Value = "https://example.com/it's" } + @{ Description = 'a curly quote'; Value = ('https://example.com/it{0}s' -f [char]0x2019) } + @{ Description = 'what a double-quoted string would expand'; Value = 'https://example.com/$(Get-Process)/`t/$env:COMPUTERNAME"' } + ) { + $payload = Get-AvdHostUpdateScript -SourceUrl $Value -ApiBaseUrl $Value -ApiClientId $Value -Destination $Value + + $parseErrors = $null + [void][System.Management.Automation.Language.Parser]::ParseInput($payload, [ref]$null, [ref]$parseErrors) + @($parseErrors).Count | Should -Be 0 + foreach ($name in 'sourceUrl', 'apiBaseUrl', 'apiClientId', 'destination') { + Get-PayloadValue -Payload $payload -Name $name | Should -BeExactly $Value + } + $payload | Should -Not -Match '__(SOURCE_URL|API_BASE_URL|API_CLIENT_ID|DESTINATION)__' + } + + It 'fills in every value in one pass, so a value that holds a token keeps it' { + $sourceUrl = 'https://example.com/__DESTINATION__/__API_CLIENT_ID__/Connect-LinuxBroker.ps1' + + $payload = Get-AvdHostUpdateScript -SourceUrl $sourceUrl -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId + + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly $sourceUrl + Get-PayloadValue -Payload $payload -Name 'apiClientId' | Should -BeExactly $script:ClientId + Get-PayloadValue -Payload $payload -Name 'destination' | Should -BeExactly 'C:\Temp\Connect-LinuxBroker.ps1' + } + + It 'installs the script where Configure-AVD-Host.ps1 does and the RemoteApp runs it from' { + $default = Get-PayloadValue -Payload (Get-AvdHostUpdateScript -SourceUrl $script:DefaultSourceUrl -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId) -Name 'destination' + + $bicep = [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot 'deploy\bicep\modules\AVD\main.bicep')) + $remoteAppPath = [regex]::Match($bicep, 'commandLineArguments: ''[^'']*-File ([^\s''$]+)').Groups[1].Value.Replace('\\', '\') + $remoteAppPath | Should -BeExactly $default + $configure = [System.IO.File]::ReadAllText((Join-Path $script:RepoRoot 'custom_script_extensions\Configure-AVD-Host.ps1')) + $configure | Should -Match ([regex]::Escape('$folderPath = "C:\Temp"')) + $configure | Should -Match ([regex]::Escape('$outputPath = "$folderPath\Connect-LinuxBroker.ps1"')) + } +} + +Describe 'Updating the session hosts through Run Command' { + BeforeEach { + Set-StrictMode -Version Latest + $global:LASTEXITCODE = 0 + $script:AzCalls = New-Object System.Collections.Generic.List[string] + $script:RunCommands = New-Object System.Collections.Generic.List[object] + $script:RunCommandOutcomes = @{} + $script:AccountSetExitCode = 0 + $script:VmListExitCode = 0 + $script:VmListJson = '[]' + $script:HostLines = New-Object System.Collections.Generic.List[string] + $script:Warnings = New-Object System.Collections.Generic.List[string] + $script:Slept = New-Object System.Collections.Generic.List[int] + $script:AzdValues = @{} + $script:AzdEnvironments = New-Object System.Collections.Generic.List[string] + + Mock az { + $arguments = [string[]]@($args | ForEach-Object { [string]$_ }) + $commandLine = $arguments -join ' ' + $script:AzCalls.Add($commandLine) + $global:LASTEXITCODE = 0 + + if ($commandLine -like 'account set *') { + $global:LASTEXITCODE = $script:AccountSetExitCode + return + } + + if ($commandLine -like 'vm list *') { + $global:LASTEXITCODE = $script:VmListExitCode + if ($script:VmListExitCode -ne 0) { + return + } + return $script:VmListJson + } + + if ($commandLine -like 'vm run-command invoke *') { + $vmName = Get-ArgumentValue -Arguments $arguments -Name '--name' + $scriptFile = (Get-ArgumentValue -Arguments $arguments -Name '--scripts').Substring(1) + $script:RunCommands.Add([pscustomobject]@{ + Vm = $vmName + ResourceGroup = Get-ArgumentValue -Arguments $arguments -Name '--resource-group' + Arguments = $arguments + File = $scriptFile + Script = [System.IO.File]::ReadAllText($scriptFile) + Bytes = [System.IO.File]::ReadAllBytes($scriptFile) + }) + + $outcome = @{ StdOut = "Updated Connect-LinuxBroker.ps1 on $vmName from 1.0.0 to 2.0.0." } + if ($script:RunCommandOutcomes.ContainsKey($vmName) -and $script:RunCommandOutcomes[$vmName].Count -gt 0) { + $outcome = $script:RunCommandOutcomes[$vmName].Dequeue() + } + + if ($outcome.ContainsKey('Error')) { + $global:LASTEXITCODE = 1 + return (New-Object System.Management.Automation.ErrorRecord -ArgumentList @( + (New-Object System.Exception -ArgumentList $outcome['Error']), + 'NativeCommandError', + [System.Management.Automation.ErrorCategory]::NotSpecified, + $null)) + } + if ($outcome.ContainsKey('Raw')) { + return $outcome['Raw'] + } + return (ConvertTo-RunCommandJson -StdOut ([string]$outcome['StdOut']) -StdErr ([string]$outcome['StdErr'])) + } + + throw "Unexpected az call: $commandLine" + } + Mock azd { throw "Unexpected azd call: $($args -join ' ')" } + Mock Get-AzdEnvValue { + $script:AzdEnvironments.Add($EnvironmentName) + [string]$script:AzdValues[$Key] + } + Mock Write-Host { $script:HostLines.Add([string]$Object) } + Mock Write-Warning { $script:Warnings.Add([string]$Message) } + Mock Start-Sleep { $script:Slept.Add([int]$Seconds) } + + $savedVariables = Save-AzureVariable + $update = @{ + ResourceGroupName = $script:ResourceGroup + ApiBaseUrl = $script:ApiUrl + ApiClientId = $script:ClientId + SubscriptionId = $script:SubscriptionId + } + } + + AfterEach { + Restore-AzureVariable -Saved $savedVariables + } + + Context 'Invoke-RunCommandWithRetry' { + BeforeEach { + $testScript = "Write-Output 'first'`r`nWrite-Output 'second'`r`n" + } + + It 'returns what the script wrote to standard output and standard error' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ StdOut = "first`nsecond`n"; StdErr = 'WARNING: The CredentialManager module is not installed.' } + + $result = Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript + + $result.StdOut | Should -BeExactly "first`nsecond" + $result.StdErr | Should -BeExactly 'WARNING: The CredentialManager module is not installed.' + $script:RunCommands.Count | Should -Be 1 + $script:Slept.Count | Should -Be 0 + } + + It 'hands az the script as a file, without a byte order mark, and deletes the file afterwards' { + Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript | Out-Null + + $call = $script:RunCommands[0] + $call.Script | Should -BeExactly $testScript + Test-Utf8Bom -Bytes $call.Bytes | Should -BeFalse + Test-Path -LiteralPath $call.File | Should -BeFalse + Get-ArgumentValue -Arguments $call.Arguments -Name '--command-id' | Should -BeExactly 'RunPowerShellScript' + $call.ResourceGroup | Should -BeExactly $script:ResourceGroup + } + + It 'retries a Run Command that fails, waiting longer each time' { + $conflict = 'ERROR: (Conflict) Run command extension execution is in progress. Please wait for completion before invoking a run command.' + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Error = $conflict }, @{ Error = $conflict }, @{ StdOut = 'done' } + + $result = Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript + + $result.StdOut | Should -BeExactly 'done' + $script:RunCommands.Count | Should -Be 3 + $script:Slept | Should -Be @(5, 10) + $script:Warnings | Should -Contain "Run Command failed on 'avd-0' attempt 1 of 3. Retrying in 5 seconds." + $script:Warnings | Should -Contain $conflict + } + + It 'gives up after the last attempt, reporting the last error, and deletes the file' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Error = 'ERROR: first' }, @{ Error = 'ERROR: second' }, @{ Error = 'ERROR: third' } + + { Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript } | + Should -Throw "Run Command failed on 'avd-0'. Last error: ERROR: third" + + $script:RunCommands.Count | Should -Be 3 + $script:Slept | Should -Be @(5, 10) + Test-Path -LiteralPath $script:RunCommands[0].File | Should -BeFalse + } + + It 'waits at most 30 seconds between attempts' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Error = 'ERROR' }, @{ Error = 'ERROR' }, @{ Error = 'ERROR' }, @{ Error = 'ERROR' }, @{ StdOut = 'done' } + + Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript -MaxAttempts 5 -InitialDelaySeconds 20 | Out-Null + + $script:Slept | Should -Be @(20, 30, 30, 30) + } + + It 'refuses output that is not JSON' { + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ Raw = 'This command is in preview and under development.' } + + { Invoke-RunCommandWithRetry -ResourceGroupName $script:ResourceGroup -VmName 'avd-0' -Script $testScript } | + Should -Throw "Run Command on 'avd-0' returned output that is not JSON*" + $script:RunCommands.Count | Should -Be 1 + } + } + + Context 'Invoke-AvdHostBrokerScriptUpdate' { + It 'updates every running AVD session host in the resource group and no other VM' { + Set-TestVm -Vm @( + (New-TestVm -Name 'avd-0'), + (New-TestVm -Name 'lnx-0' -Tags @{ 'broker-role' = 'linux-host' }), + (New-TestVm -Name 'jumpbox' -Tags @{ environment = 'dev' }), + (New-TestVm -Name 'untagged' -Tags $null), + (New-TestVm -Name 'avd-1'), + (New-TestVm -Name 'avd-2' -NoPowerState) + ) + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-0', 'avd-1', 'avd-2') + @($script:RunCommands | ForEach-Object { $_.ResourceGroup } | Select-Object -Unique) | Should -Be @($script:ResourceGroup) + $script:AzCalls[0] | Should -BeExactly "account set --subscription $script:SubscriptionId" + $script:AzCalls[1] | Should -BeExactly "vm list --resource-group $script:ResourceGroup --show-details --output json --only-show-errors" + $script:HostLines | Should -Contain 'Connect-LinuxBroker.ps1 is up to date on 3 AVD session host(s).' + $script:Warnings.Count | Should -Be 0 + } + + It 'runs the same update on every session host, filled in with this environment''s values' { + Set-TestVm -Vm @((New-TestVm -Name 'avd-0'), (New-TestVm -Name 'avd-1')) + + Invoke-AvdHostBrokerScriptUpdate @update + + $payload = $script:RunCommands[0].Script + $script:RunCommands[1].Script | Should -BeExactly $payload + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly $script:DefaultSourceUrl + Get-PayloadValue -Payload $payload -Name 'apiBaseUrl' | Should -BeExactly $script:ApiUrl + Get-PayloadValue -Payload $payload -Name 'apiClientId' | Should -BeExactly $script:ClientId + Get-PayloadValue -Payload $payload -Name 'destination' | Should -BeExactly 'C:\Temp\Connect-LinuxBroker.ps1' + } + + It 'downloads Connect-LinuxBroker.ps1 from ScriptSourceRoot and drops a trailing slash from the API URL' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.ScriptSourceRoot = 'https://mirror.contoso.com/linuxbroker/' + $update.ApiBaseUrl = " $script:ApiUrl/ " + $update.ApiClientId = " $script:ClientId " + + Invoke-AvdHostBrokerScriptUpdate @update + + $payload = $script:RunCommands[0].Script + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly 'https://mirror.contoso.com/linuxbroker/avd_host/broker/Connect-LinuxBroker.ps1' + Get-PayloadValue -Payload $payload -Name 'apiBaseUrl' | Should -BeExactly $script:ApiUrl + Get-PayloadValue -Payload $payload -Name 'apiClientId' | Should -BeExactly $script:ClientId + } + + It 'skips the session hosts that are not running and names them at the end' { + Set-TestVm -Vm @( + (New-TestVm -Name 'avd-0'), + (New-TestVm -Name 'avd-1' -PowerState 'VM deallocated'), + (New-TestVm -Name 'avd-2' -PowerState 'VM stopped') + ) + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-0') + $script:Warnings | Should -Contain "Skipping AVD session host 'avd-1' because it is not running (VM deallocated). Start it and run this script again with -AvdHostNames avd-1." + $script:Warnings | Should -Contain 'Not updated because they are not running: avd-1, avd-2.' + $script:HostLines | Should -Contain 'Connect-LinuxBroker.ps1 is up to date on 1 AVD session host(s).' + } + + It 'tries every session host and then names the ones where the update failed' { + Set-TestVm -Vm @((New-TestVm -Name 'avd-0'), (New-TestVm -Name 'avd-1'), (New-TestVm -Name 'avd-2')) + $downloadFailure = "Could not download $script:DefaultSourceUrl after 3 attempts: The remote server returned an error: (404) Not Found." + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ StdOut = ''; StdErr = $downloadFailure } + $agentFailure = 'ERROR: (VMAgentStatusCommunicationError) The VM agent is not reporting.' + Add-RunCommandOutcome -Vm 'avd-1' -Outcome @{ Error = $agentFailure }, @{ Error = $agentFailure }, @{ Error = $agentFailure } + + { Invoke-AvdHostBrokerScriptUpdate @update } | + Should -Throw 'The Connect-LinuxBroker.ps1 update failed on: avd-0, avd-1. Rerun with -AvdHostNames to retry them.' + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-0', 'avd-1', 'avd-1', 'avd-1', 'avd-2') + $script:Warnings | Should -Contain "The update did not run to completion on 'avd-0'. $downloadFailure" + $script:Warnings | Should -Contain "Run Command failed on 'avd-1'. Last error: $agentFailure" + @($script:HostLines | Where-Object { $_ -like '*is up to date*' }).Count | Should -Be 0 + } + + It 'treats a run that did not reach its last line as failed, even without an error' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + Add-RunCommandOutcome -Vm 'avd-0' -Outcome @{ StdOut = 'WARNING: The LinuxBrokerScript event source could not be created.' } + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw 'The Connect-LinuxBroker.ps1 update failed on: avd-0.*' + $script:Warnings | Should -Contain "The update did not run to completion on 'avd-0'." + } + + It 'reads the values it is not given from the azd environment' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $script:AzdValues = @{ + resourceGroupName = $script:ResourceGroup + AZURE_SUBSCRIPTION_ID = $script:SubscriptionId + apiUrl = $script:ApiUrl + apiClientId = $script:ClientId + } + + Invoke-AvdHostBrokerScriptUpdate -EnvironmentName 'linuxbroker-dev' + + @($script:AzdEnvironments | Select-Object -Unique) | Should -Be @('linuxbroker-dev') + $script:AzCalls | Should -Contain "account set --subscription $script:SubscriptionId" + $script:AzCalls | Should -Contain "vm list --resource-group $script:ResourceGroup --show-details --output json --only-show-errors" + Get-PayloadValue -Payload $script:RunCommands[0].Script -Name 'apiBaseUrl' | Should -BeExactly $script:ApiUrl + Get-PayloadValue -Payload $script:RunCommands[0].Script -Name 'apiClientId' | Should -BeExactly $script:ClientId + } + + It 'uses the azd environment in when none is given' -TestCases @( + @{ Variable = 'AZURE_ENV_NAME' } + @{ Variable = 'AZURE_ENVIRONMENT_NAME' } + ) { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + [Environment]::SetEnvironmentVariable($Variable, 'linuxbroker-hook', 'Process') + $update.Remove('ResourceGroupName') + $script:AzdValues = @{ resourceGroupName = $script:ResourceGroup } + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:AzdEnvironments | Select-Object -Unique) | Should -Be @('linuxbroker-hook') + $script:RunCommands.Count | Should -Be 1 + } + + It 'selects the subscription in AZURE_SUBSCRIPTION_ID when azd has none' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.Remove('SubscriptionId') + $env:AZURE_SUBSCRIPTION_ID = $script:SubscriptionId + + Invoke-AvdHostBrokerScriptUpdate @update + + $script:AzCalls | Should -Contain "account set --subscription $script:SubscriptionId" + } + + It 'keeps the current subscription when none is known' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.Remove('SubscriptionId') + + Invoke-AvdHostBrokerScriptUpdate @update + + @($script:AzCalls | Where-Object { $_ -like 'account set*' }).Count | Should -Be 0 + $script:RunCommands.Count | Should -Be 1 + } + + It 'stops before it calls Azure when is not known' -TestCases @( + @{ Missing = 'ResourceGroupName' } + @{ Missing = 'ApiBaseUrl' } + @{ Missing = 'ApiClientId' } + ) { + $update.Remove($Missing) + + { Invoke-AvdHostBrokerScriptUpdate @update } | + Should -Throw 'The AVD session host update needs ResourceGroupName, ApiBaseUrl and ApiClientId, as parameters or azd environment values.' + $script:AzCalls.Count | Should -Be 0 + } + + It 'refuses an API URL that ' -TestCases @( + @{ Description = 'is not https'; Url = 'http://app-linuxbroker-api.azurewebsites.net/api' } + @{ Description = 'holds a double quote'; Url = 'https://app/api";Remove-Item("C:\Temp");"' } + @{ Description = 'holds a typographic double quote'; Url = ('https://app/api{0};Remove-Item("C:\Temp");{0}' -f [char]0x201D) } + @{ Description = 'holds a letter outside ASCII'; Url = ('https://app/{0}/api' -f [char]0x00D1) } + @{ Description = 'holds a single quote'; Url = "https://app/api'" } + @{ Description = 'holds a subexpression'; Url = 'https://app/$(Get-Process)/api' } + @{ Description = 'holds a backtick'; Url = 'https://app/`api' } + @{ Description = 'holds a space'; Url = 'https://app /api' } + ) { + $update.ApiBaseUrl = $Url + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "ApiBaseUrl must be the broker API's https:// URL, such as https:///api, not *" + $script:AzCalls.Count | Should -Be 0 + } + + It 'refuses a ScriptSourceRoot that ' -TestCases @( + @{ Description = 'is not https'; Root = 'http://mirror.contoso.com/linuxbroker' } + @{ Description = 'holds a single quote'; Root = "https://mirror.contoso.com/it's" } + @{ Description = 'holds a typographic single quote'; Root = ('https://mirror.contoso.com/it{0}s' -f [char]0x2019) } + @{ Description = 'holds a letter outside ASCII'; Root = ('https://mirror.contoso.com/{0}' -f [char]0x00D1) } + @{ Description = 'is empty'; Root = '' } + ) { + $update.ScriptSourceRoot = $Root + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw 'ScriptSourceRoot must be the https:// URL of the repository or of a mirror of it, such as https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main, not *' + $script:AzCalls.Count | Should -Be 0 + } + + It 'accepts URLs with a port, percent-encoding and a tilde' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + $update.ScriptSourceRoot = 'https://mirror.contoso.us:8443/linux%20broker/~main' + $update.ApiBaseUrl = 'https://app-linuxbroker-api.azurewebsites.us:8443/api' + + Invoke-AvdHostBrokerScriptUpdate @update + + $payload = $script:RunCommands[0].Script + Get-PayloadValue -Payload $payload -Name 'sourceUrl' | Should -BeExactly 'https://mirror.contoso.us:8443/linux%20broker/~main/avd_host/broker/Connect-LinuxBroker.ps1' + Get-PayloadValue -Payload $payload -Name 'apiBaseUrl' | Should -BeExactly 'https://app-linuxbroker-api.azurewebsites.us:8443/api' + } + + It 'refuses a client ID that is not a GUID in its usual form: ' -TestCases @( + @{ ClientId = 'api://0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' } + @{ ClientId = '{0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21}' } + @{ ClientId = '0b5a9c3e6f1d4a2b9c8e7d6f5e4a3b21' } + @{ ClientId = 'linuxbroker-api' } + ) { + $update.ApiClientId = $ClientId + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "ApiClientId must be the broker API app registration's client ID, a GUID, not *" + $script:AzCalls.Count | Should -Be 0 + } + + It 'updates only the session hosts named with -AvdHostNames, in any case, and warns about the rest' { + Set-TestVm -Vm @( + (New-TestVm -Name 'avd-0'), + (New-TestVm -Name 'avd-1'), + (New-TestVm -Name 'lnx-0' -Tags @{ 'broker-role' = 'linux-host' }) + ) + + Invoke-AvdHostBrokerScriptUpdate @update -AvdHostNames 'AVD-1', ' avd-9 ', 'lnx-0', '' + + @($script:RunCommands | ForEach-Object { $_.Vm }) | Should -Be @('avd-1') + $script:Warnings | Should -Contain "Requested AVD session host 'avd-9' was not found in resource group '$script:ResourceGroup'." + $script:Warnings | Should -Contain "Requested AVD session host 'lnx-0' was not found in resource group '$script:ResourceGroup'." + $script:Warnings.Count | Should -Be 2 + } + + It 'does nothing when no session host has a requested name' { + Set-TestVm -Vm @(New-TestVm -Name 'avd-0') + + Invoke-AvdHostBrokerScriptUpdate @update -AvdHostNames 'avd-9' + + $script:RunCommands.Count | Should -Be 0 + $script:HostLines | Should -Contain 'No AVD session hosts matched the requested update scope.' + } + + It 'does nothing in a resource group with ' -TestCases @( + @{ Description = 'no VMs'; Linux = $false } + @{ Description = 'only Linux hosts'; Linux = $true } + ) { + if ($Linux) { + Set-TestVm -Vm @(New-TestVm -Name 'lnx-0' -Tags @{ 'broker-role' = 'linux-host' }) + } + + Invoke-AvdHostBrokerScriptUpdate @update + + $script:RunCommands.Count | Should -Be 0 + $script:HostLines | Should -Contain "No AVD session host VMs found in resource group '$script:ResourceGroup'." + } + + It 'stops when the VMs cannot be listed' { + $script:VmListExitCode = 1 + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "Could not list the VMs in resource group '$script:ResourceGroup'." + $script:RunCommands.Count | Should -Be 0 + } + + It 'stops when the subscription cannot be selected' { + $script:AccountSetExitCode = 1 + + { Invoke-AvdHostBrokerScriptUpdate @update } | Should -Throw "Could not select subscription '$script:SubscriptionId'." + @($script:AzCalls | Where-Object { $_ -like 'vm *' }).Count | Should -Be 0 + } + } +} + +Describe 'The update on a session host' { + BeforeAll { + $script:SourcePath = Join-Path $TestDrive 'source\Connect-LinuxBroker.ps1' + New-Item -ItemType Directory -Path (Split-Path -Parent $script:SourcePath) | Out-Null + Copy-Item -LiteralPath $script:BrokerScriptPath -Destination $script:SourcePath + $script:SourceUrl = ([Uri]$script:SourcePath).AbsoluteUri + $script:ExpectedScript = [System.IO.File]::ReadAllText($script:BrokerScriptPath).Replace('https://your_linuxbroker_api_base_url/api', $script:ApiUrl).Replace('your_linuxbroker_api_client_id', $script:ClientId) + } + + It 'installs the current script where there is none, with the API URL and client ID filled in' { + $destination = Join-Path $TestDrive ('{0}\Temp\Connect-LinuxBroker.ps1' -f [guid]::NewGuid().ToString('N')) + + $result = Invoke-HostUpdate -SourceUrl $script:SourceUrl -Destination $destination + + $result.StdErr | Should -BeNullOrEmpty + $result.ExitCode | Should -Be 0 + $script:BrokerScriptVersion | Should -Not -BeNullOrEmpty + $result.StdOut | Should -Match ([regex]::Escape("Updated Connect-LinuxBroker.ps1 on $env:COMPUTERNAME from no script to $script:BrokerScriptVersion.")) + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $script:ExpectedScript + $script:ExpectedScript | Should -Match ([regex]::Escape("`$apiBaseUrl = `"$script:ApiUrl`"")) + $script:ExpectedScript | Should -Match ([regex]::Escape("api://$script:ClientId")) + Test-Utf8Bom -Bytes ([System.IO.File]::ReadAllBytes($destination)) | Should -BeFalse + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } + + It 'replaces in place' -TestCases @( + @{ Description = 'a script from before versions'; Content = "# Connect-LinuxBroker.ps1`r`n`$apiBaseUrl = `"https://app-linuxbroker-api.azurewebsites.net/api`"`r`n"; Previous = 'a script without a version' } + @{ Description = 'an earlier version'; Content = "`$ScriptVersion = '1.9.0'`r`n"; Previous = '1.9.0' } + ) { + $destination = New-InstalledScript -Content $Content + + $result = Invoke-HostUpdate -SourceUrl $script:SourceUrl -Destination $destination + + $result.StdErr | Should -BeNullOrEmpty + $result.ExitCode | Should -Be 0 + $result.StdOut | Should -Match ([regex]::Escape("from $Previous to $script:BrokerScriptVersion.")) + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $script:ExpectedScript + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } + + It 'leaves the installed script alone when the download fails' { + $destination = New-InstalledScript + $before = [System.IO.File]::ReadAllText($destination) + $missingUrl = ([Uri](Join-Path $TestDrive 'missing\Connect-LinuxBroker.ps1')).AbsoluteUri + + $result = Invoke-HostUpdate -SourceUrl $missingUrl -Destination $destination + + $result.ExitCode | Should -Be 1 + $result.StdErr | Should -Match ('Could not download ' + [regex]::Escape($missingUrl) + ' after 3 attempts') + $result.StdOut | Should -Match 'TEST: Start-Sleep 2' + $result.StdOut | Should -Match 'TEST: Start-Sleep 4' + $result.StdOut | Should -Not -Match 'Updated Connect-LinuxBroker.ps1' + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $before + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } + + It 'refuses and leaves the installed script alone' -TestCases @( + @{ + Description = 'a file that is not a Connect-LinuxBroker.ps1' + Content = "Write-Output 'hello'`r`n" + Message = 'is not a Connect-LinuxBroker.ps1 that this update can install.' + } + @{ + Description = 'a script without the placeholders to fill in' + Content = "`$ScriptVersion = '9.9.9'`r`n`$apiBaseUrl = `"https://app-linuxbroker-api.azurewebsites.net/api`"`r`n" + Message = 'is not a Connect-LinuxBroker.ps1 that this update can install.' + } + @{ + Description = 'a script that does not parse' + Content = "`$ScriptVersion = '9.9.9'`r`n`$apiBaseUrl = `"https://your_linuxbroker_api_base_url/api`"`r`n`$apiAppIdUri = `"api://your_linuxbroker_api_client_id`"`r`nfunction {`r`n" + Message = 'Connect-LinuxBroker.ps1 9.9.9 does not parse once the API URL and client ID are filled in' + } + @{ + Description = 'a script with characters outside ASCII' + Content = "`$ScriptVersion = '9.9.9'`r`n`$apiBaseUrl = `"https://your_linuxbroker_api_base_url/api`"`r`n`$apiAppIdUri = `"api://your_linuxbroker_api_client_id`"`r`n# Caf$([char]0x00E9)`r`n" + Message = 'Connect-LinuxBroker.ps1 9.9.9 holds characters outside ASCII, which Windows PowerShell could misread.' + } + ) { + $sourcePath = Join-Path $TestDrive ('{0}.ps1' -f [guid]::NewGuid().ToString('N')) + [System.IO.File]::WriteAllText($sourcePath, $Content) + $destination = New-InstalledScript + $before = [System.IO.File]::ReadAllText($destination) + + $result = Invoke-HostUpdate -SourceUrl ([Uri]$sourcePath).AbsoluteUri -Destination $destination + + $result.ExitCode | Should -Be 1 + $result.StdErr | Should -Match ([regex]::Escape($Message)) + $result.StdOut | Should -Not -Match 'Updated Connect-LinuxBroker.ps1' + [System.IO.File]::ReadAllText($destination) | Should -BeExactly $before + Get-FolderEntry -Path $destination | Should -Be @('Connect-LinuxBroker.ps1') + } +} + +Describe 'Update-AvdHostBrokerScript.ps1' { + It 'runs the update with its parameters when it is run as a script in PowerShell 7' -Skip:(-not (Get-Command pwsh -ErrorAction SilentlyContinue)) { + $folder = Join-Path $TestDrive ([guid]::NewGuid().ToString('N')) + New-Item -ItemType Directory -Path $folder | Out-Null + $harnessPath = Join-Path $folder 'harness.ps1' + $harness = @' +try { + & __SCRIPT__ -ResourceGroupName 'rg-linuxbroker-dev' -ApiBaseUrl 'http://app-linuxbroker-api.azurewebsites.net/api' -ApiClientId '0b5a9c3e-6f1d-4a2b-9c8e-7d6f5e4a3b21' +} +catch { + [Console]::Error.WriteLine($_.Exception.Message) + exit 1 +} +'@ + [System.IO.File]::WriteAllText($harnessPath, $harness.Replace('__SCRIPT__', (ConvertTo-PowerShellLiteral -Value $script:UpdateScriptPath)), $script:Utf8NoBom) + + $result = Invoke-ChildProcess -FilePath (Get-Command pwsh).Source -ScriptPath $harnessPath + + $result.ExitCode | Should -Be 1 + $result.StdErr.Trim() | Should -BeExactly "ApiBaseUrl must be the broker API's https:// URL, such as https:///api, not 'http://app-linuxbroker-api.azurewebsites.net/api'." + } +} + +Describe 'Migrate-ExistingEnvironment.ps1' { + BeforeAll { + $script:MigrationFolder = Join-Path $TestDrive 'deploy' + New-Item -ItemType Directory -Path $script:MigrationFolder | Out-Null + Copy-Item -LiteralPath (Join-Path $script:RepoRoot 'deploy\Migrate-ExistingEnvironment.ps1') -Destination $script:MigrationFolder + $script:MigrationPath = Join-Path $script:MigrationFolder 'Migrate-ExistingEnvironment.ps1' + $script:StepLog = Join-Path $script:MigrationFolder 'steps.log' + + # Each step records how it was run, with an array argument as a,b, and fails when + # LINUXBROKER_TEST_FAILING_STEPS names it. + $stub = @' +$step = [System.IO.Path]::GetFileNameWithoutExtension($MyInvocation.MyCommand.Path) +$arguments = @($args | ForEach-Object { if ($_ -is [array]) { $_ -join ',' } else { [string]$_ } }) +Add-Content -LiteralPath (Join-Path $PSScriptRoot 'steps.log') -Value ('{0} {1}' -f $step, ($arguments -join ' ')) +if (@([string]$env:LINUXBROKER_TEST_FAILING_STEPS -split ',') -contains $step) { + throw "$step failed on host-1." +} +'@ + foreach ($step in 'Post-Provision', 'Migrate-LinuxHostReleaseAgent', 'Update-AvdHostBrokerScript') { + [System.IO.File]::WriteAllText((Join-Path $script:MigrationFolder "$step.ps1"), $stub) + } + + function Get-StepLog { + if (-not (Test-Path -LiteralPath $script:StepLog)) { + return @() + } + return @(Get-Content -LiteralPath $script:StepLog) + } + + function Get-StepName { + return @(Get-StepLog | ForEach-Object { ($_ -split ' ')[0] }) + } + } + + BeforeEach { + Remove-Item -LiteralPath $script:StepLog -ErrorAction SilentlyContinue + # Not $script: variables: in a mock that Migrate-ExistingEnvironment.ps1 calls, $script: is + # that script's scope. These are found through the scopes that called it. + $hostLines = New-Object System.Collections.Generic.List[string] + $warnings = New-Object System.Collections.Generic.List[string] + Mock Write-Host { $hostLines.Add([string]$Object) } + Mock Write-Warning { $warnings.Add([string]$Message) } + Mock az { throw "Unexpected az call: $($args -join ' ')" } + Mock azd { throw "Unexpected azd call: $($args -join ' ')" } + + $savedVariables = Save-AzureVariable + $savedFailingSteps = $env:LINUXBROKER_TEST_FAILING_STEPS + $env:LINUXBROKER_TEST_FAILING_STEPS = $null + $migration = @{ + ResourceGroupName = $script:ResourceGroup + ApiBaseUrl = $script:ApiUrl + ApiClientId = $script:ClientId + } + } + + AfterEach { + Restore-AzureVariable -Saved $savedVariables + $env:LINUXBROKER_TEST_FAILING_STEPS = $savedFailingSteps + } + + It 'updates the AVD session hosts after the Linux hosts, with the same values' { + & $script:MigrationPath @migration -LinuxHostNames 'lnx-0' -AvdHostNames 'avd-0', 'avd-1' + + Get-StepName | Should -Be @('Post-Provision', 'Migrate-LinuxHostReleaseAgent', 'Update-AvdHostBrokerScript') + $steps = Get-StepLog + $steps[1] | Should -Match ([regex]::Escape('-LinuxHostNames lnx-0')) + $steps[2] | Should -Match ([regex]::Escape("-ResourceGroupName $script:ResourceGroup -ApiBaseUrl $script:ApiUrl -ApiClientId $script:ClientId")) + $steps[2] | Should -Match ([regex]::Escape('-ScriptSourceRoot https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main')) + $steps[2] | Should -Match ([regex]::Escape('-AvdHostNames avd-0,avd-1')) + $hostLines | Should -Contain 'Existing environment migration completed.' + } + + It 'still updates the AVD session hosts when the Linux host migration fails, and then fails' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Migrate-LinuxHostReleaseAgent' + + { & $script:MigrationPath @migration } | + Should -Throw 'Existing environment migration failed in the Linux host migration. The warnings above name the hosts to retry.' + + Get-StepName | Should -Be @('Post-Provision', 'Migrate-LinuxHostReleaseAgent', 'Update-AvdHostBrokerScript') + $warnings | Should -Contain 'Migrate-LinuxHostReleaseAgent failed on host-1.' + $hostLines | Should -Not -Contain 'Existing environment migration completed.' + } + + It 'names both host updates when both fail' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Migrate-LinuxHostReleaseAgent,Update-AvdHostBrokerScript' + + { & $script:MigrationPath @migration } | + Should -Throw 'Existing environment migration failed in the Linux host migration and the AVD session host update. The warnings above name the hosts to retry.' + $warnings | Should -Contain 'Update-AvdHostBrokerScript failed on host-1.' + } + + It 'stops at a failed post-provision step, before it updates any host' { + $env:LINUXBROKER_TEST_FAILING_STEPS = 'Post-Provision' + + { & $script:MigrationPath @migration } | Should -Throw 'Post-Provision failed on host-1.' + Get-StepName | Should -Be @('Post-Provision') + } + + It 'leaves the AVD session hosts alone with -SkipAvdHostScriptUpdate' { + & $script:MigrationPath @migration -SkipAvdHostScriptUpdate + + Get-StepName | Should -Be @('Post-Provision', 'Migrate-LinuxHostReleaseAgent') + } + + It 'updates only the AVD session hosts with -SkipLinuxHostReleaseAgentMigration' { + & $script:MigrationPath @migration -SkipPostProvision -SkipLinuxHostReleaseAgentMigration + + Get-StepName | Should -Be @('Update-AvdHostBrokerScript') + } + + It 'stops before it updates any host when is not known' -TestCases @( + @{ Missing = 'ResourceGroupName' } + @{ Missing = 'ApiBaseUrl' } + @{ Missing = 'ApiClientId' } + ) { + $migration.Remove($Missing) + + { & $script:MigrationPath @migration } | + Should -Throw 'Updating the Linux hosts and the AVD session hosts requires ResourceGroupName, ApiBaseUrl, and ApiClientId.' + Get-StepName | Should -Be @('Post-Provision') + } + + It 'needs none of them when both host updates are skipped' { + & $script:MigrationPath -SkipLinuxHostReleaseAgentMigration -SkipAvdHostScriptUpdate + + Get-StepName | Should -Be @('Post-Provision') + $hostLines | Should -Contain 'Existing environment migration completed.' + } +} diff --git a/custom_script_extensions/Configure-RHEL8-Host.sh b/custom_script_extensions/Configure-RHEL8-Host.sh index 434dca6..008691d 100644 --- a/custom_script_extensions/Configure-RHEL8-Host.sh +++ b/custom_script_extensions/Configure-RHEL8-Host.sh @@ -42,6 +42,8 @@ patch_host_script_url="$script_source_root/linux_host/patch-host.sh" patch_host_script="/usr/local/bin/patch-host.sh" xrdp_startwm_script_url="$script_source_root/linux_host/xrdp-startwm.sh" xrdp_startwm_script="/usr/local/bin/xrdp-startwm.sh" +install_host_config_script_url="$script_source_root/linux_host/install-host-config.sh" +install_host_config_script="/usr/local/bin/install-host-config.sh" arch=$( /bin/arch ) @@ -219,6 +221,9 @@ sudo wget -O "$patch_host_script" "$patch_host_script_url" echo "Downloading xrdp-startwm.sh..." sudo wget -O "$xrdp_startwm_script" "$xrdp_startwm_script_url" +echo "Downloading install-host-config.sh..." +sudo wget -O "$install_host_config_script" "$install_host_config_script_url" + echo "Setting execute permissions for downloaded scripts..." sudo chmod +x "$SCRIPT_PATH" sudo chmod +x "$output_directory/xrdp-who-xorg.sh" @@ -229,6 +234,7 @@ sudo chmod +x "$apply_settings_script" sudo chmod +x "$session_control_script" sudo chmod +x "$patch_host_script" sudo chmod +x "$xrdp_startwm_script" +sudo chmod +x "$install_host_config_script" echo "Downloaded scripts are now executable." # xrdp starts every session through xrdp-startwm.sh, which starts the desktop named here. For @@ -248,6 +254,13 @@ if ! sudo "$xrdp_startwm_script" --install; then exit 1 fi +# Log rotation, the NFS read-ahead and the users' local caches. The host works without them, +# so a failure is reported and the bootstrap goes on. +echo "Installing the Linux Broker host configuration..." +if ! sudo "$install_host_config_script"; then + echo "WARNING: install-host-config.sh did not complete. The host works without it; run it again as root." +fi + echo "Creating log and user details files..." sudo mkdir -p "$state_directory" sudo touch "$LOG_FILE" "$CURRENT_USERS_DETAILS" "$PREVIOUS_USERS_FILE" "$DISCONNECTED_USERS_FILE" diff --git a/custom_script_extensions/Configure-RHEL9-Host.sh b/custom_script_extensions/Configure-RHEL9-Host.sh index 1fd3423..6ab1e62 100644 --- a/custom_script_extensions/Configure-RHEL9-Host.sh +++ b/custom_script_extensions/Configure-RHEL9-Host.sh @@ -43,6 +43,8 @@ patch_host_script_url="$script_source_root/linux_host/patch-host.sh" patch_host_script="/usr/local/bin/patch-host.sh" xrdp_startwm_script_url="$script_source_root/linux_host/xrdp-startwm.sh" xrdp_startwm_script="/usr/local/bin/xrdp-startwm.sh" +install_host_config_script_url="$script_source_root/linux_host/install-host-config.sh" +install_host_config_script="/usr/local/bin/install-host-config.sh" arch=$( /bin/arch ) @@ -253,6 +255,9 @@ sudo wget -O "$patch_host_script" "$patch_host_script_url" echo "Downloading xrdp-startwm.sh..." sudo wget -O "$xrdp_startwm_script" "$xrdp_startwm_script_url" +echo "Downloading install-host-config.sh..." +sudo wget -O "$install_host_config_script" "$install_host_config_script_url" + echo "Setting execute permissions for downloaded scripts..." sudo chmod +x "$SCRIPT_PATH" sudo chmod +x "$output_directory/xrdp-who-xorg.sh" @@ -263,6 +268,7 @@ sudo chmod +x "$apply_settings_script" sudo chmod +x "$session_control_script" sudo chmod +x "$patch_host_script" sudo chmod +x "$xrdp_startwm_script" +sudo chmod +x "$install_host_config_script" echo "Downloaded scripts are now executable." # xrdp starts every session through xrdp-startwm.sh, which starts the desktop named here. For @@ -282,6 +288,13 @@ if ! sudo "$xrdp_startwm_script" --install; then exit 1 fi +# Log rotation, the NFS read-ahead and the users' local caches. The host works without them, +# so a failure is reported and the bootstrap goes on. +echo "Installing the Linux Broker host configuration..." +if ! sudo "$install_host_config_script"; then + echo "WARNING: install-host-config.sh did not complete. The host works without it; run it again as root." +fi + echo "Creating log and user details files..." sudo mkdir -p "$state_directory" sudo touch "$LOG_FILE" "$CURRENT_USERS_DETAILS" "$PREVIOUS_USERS_FILE" "$DISCONNECTED_USERS_FILE" diff --git a/custom_script_extensions/Configure-Ubuntu24_desktop-Host.sh b/custom_script_extensions/Configure-Ubuntu24_desktop-Host.sh index eca0d07..f38fcf7 100644 --- a/custom_script_extensions/Configure-Ubuntu24_desktop-Host.sh +++ b/custom_script_extensions/Configure-Ubuntu24_desktop-Host.sh @@ -46,6 +46,8 @@ patch_host_script_url="$script_source_root/linux_host/patch-host.sh" patch_host_script="/usr/local/bin/patch-host.sh" xrdp_startwm_script_url="$script_source_root/linux_host/xrdp-startwm.sh" xrdp_startwm_script="/usr/local/bin/xrdp-startwm.sh" +install_host_config_script_url="$script_source_root/linux_host/install-host-config.sh" +install_host_config_script="/usr/local/bin/install-host-config.sh" # Disable the screen saver and screen lock on this host. Enabled by default because a # locked greeter inside an xrdp session often cannot be unlocked after a reconnect, which @@ -271,6 +273,9 @@ wget -O "$patch_host_script" "$patch_host_script_url" echo "Downloading xrdp-startwm.sh..." wget -O "$xrdp_startwm_script" "$xrdp_startwm_script_url" +echo "Downloading install-host-config.sh..." +wget -O "$install_host_config_script" "$install_host_config_script_url" + echo "Setting execute permissions for downloaded scripts..." chmod +x "$SCRIPT_PATH" chmod +x "$output_directory/xrdp-who-xorg.sh" @@ -281,6 +286,7 @@ chmod +x "$apply_settings_script" chmod +x "$session_control_script" chmod +x "$patch_host_script" chmod +x "$xrdp_startwm_script" +chmod +x "$install_host_config_script" echo "Downloaded scripts are now executable." # xrdp starts every session through xrdp-startwm.sh, which starts the desktop named here. @@ -299,6 +305,13 @@ if ! "$xrdp_startwm_script" --install; then exit 1 fi +# Log rotation, the NFS read-ahead and the users' local caches. The host works without them, +# so a failure is reported and the bootstrap goes on. +echo "Installing the Linux Broker host configuration..." +if ! "$install_host_config_script"; then + echo "WARNING: install-host-config.sh did not complete. The host works without it; run it again as root." +fi + # A restart, rather than the reload --install asks for, so xrdp also joins ssl-cert. echo "Enabling and restarting xrdp..." systemctl enable xrdp diff --git a/deploy/DEPLOYMENT.md b/deploy/DEPLOYMENT.md index 41841b3..8be14fd 100644 --- a/deploy/DEPLOYMENT.md +++ b/deploy/DEPLOYMENT.md @@ -44,7 +44,7 @@ For upgrade scenarios, keep one more distinction clear: You need enough access to do all of the following: - Create resource groups and deploy Azure resources. -- Create Azure role assignments. +- Create Azure role assignments. When AVD session hosts are deployed, one of them is on the subscription, which needs **Owner** or **User Access Administrator** on the subscription. Without that, the deployment still succeeds, but the AVD scaling plan is left unassigned until an administrator assigns the role; see [AVD Autoscale](#avd-autoscale). - Create or update Microsoft Entra app registrations. - Create or update service principals. - Create or update Entra security groups. @@ -64,8 +64,11 @@ At a high level, the deployment provisions and configures the following: - App Service plan, storage account, Application Insights, Log Analytics, and networking. - A private DNS zone, `linuxbroker.internal`, linked to the virtual network with auto-registration, so the broker reaches each Linux host as `.linuxbroker.internal`. It is skipped when you supply `domainName`. - A premium Azure Files NFS share for Linux home directories, reachable only through a private endpoint. It is skipped when you supply `nfsShare`, set `deployNfsShare` to `false`, or deploy no Linux hosts. +- A Redis cache for the portal's sign-in sessions, reachable only through a private endpoint: Azure Managed Redis in the public cloud, and Azure Cache for Redis in Azure Government. It is skipped when `portalSessionStore` is `filesystem` or `azureCloudName` is `AzureCustom`. See [Portal And API Scale-Out](#portal-and-api-scale-out). - Optional Linux hosts and optional AVD hosts, depending on azd environment settings. - When AVD hosts are deployed, a RemoteApp application group that publishes **Linux Desktop**, which runs `Connect-LinuxBroker.ps1` on the session host to check out a Linux host and open an RDP session to it. +- When AVD hosts are deployed, an Azure Virtual Desktop scaling plan that starts and stops the session hosts on a schedule, and Start VM on Connect on the host pool. See [AVD Autoscale](#avd-autoscale). +- Unless `deployHostMonitoring` is `false`, data collection rules that send the hosts' logs to Log Analytics, a workbook that shows the state of the Linux host fleet, and alerts. See [Host Monitoring](#host-monitoring). - Two Entra app registrations: frontend and API. - Two Entra security groups for VM authorization: AVD hosts and Linux hosts. - When AVD hosts are deployed and `avdUsersGroupId` is not set, a third Entra security group for AVD users, with the deploying user added as a member. @@ -79,9 +82,11 @@ The deployment model now follows these runtime rules: - The keyring vault holds one secret per user, `keyring-`, which the API creates at the user's first checkout. The template adds none. - Frontend and API auth secrets are stored in app settings, not in Key Vault. - Linux hosts are registered into SQL during `postprovision`. AVD hosts are not. -- The API and function apps are integrated with the virtual network's app subnet, so the API reaches Linux hosts on their private IP addresses for SSH and the portal's connectivity test. +- The API and function apps are integrated with the virtual network's app subnet, so the API reaches Linux hosts on their private IP addresses for SSH and the portal's connectivity test. The portal is integrated with the same subnet while its sessions are in Redis, to reach the cache. +- The portal's managed identity holds the session cache's `default` access policy on Azure Managed Redis, or `Data Contributor` on Azure Cache for Redis, and signs in to the cache with Microsoft Entra ID. The cache's access keys are turned off. - The API managed identity holds **Desktop Virtualization Power On Off Contributor** on the VM resource group, which lets it start and stop hosts for the portal and scaling rules without broader write access. Stopping a host powers it off without deallocating it, so a stopped host still accrues compute charges. - Members of the AVD users group hold **Desktop Virtualization User** on the RemoteApp application group and **Virtual Machine User Login** on each session host. Both are required: the first publishes **Linux Desktop** to the user, and the second lets the user sign in to the Microsoft Entra joined session host. +- The Azure Virtual Desktop service principal holds **Desktop Virtualization Power On Off Contributor** on the subscription, which autoscale and Start VM on Connect need. Autoscale does not work with the role on a resource group, and the role lets Azure Virtual Desktop start and stop any session host in the subscription, not only this deployment's. ## Required And Common azd Environment Values @@ -94,6 +99,10 @@ The checked-in [bicep/main.parameters.example.json](bicep/main.parameters.exampl - `appName`: base name for generated resources. - `AZURE_LOCATION`: deployment region. - `appServicePlanSku`: App Service plan SKU. The deployment baseline defaults to Premium v3 `P2mv3` for 4 vCPUs and 32 GB memory. +- `appServicePlanCapacity`: instances of the App Service plan, from 1 to 30. Defaults to `1`. The portal, the API and the task function run on every instance. See [More than one instance](#more-than-one-instance). +- `portalSessionStore`: `redis` or `filesystem`. Defaults to `redis`, or to `filesystem` when `azureCloudName` is `AzureCustom`. Where the portal keeps sign-in sessions: in a Redis cache that every portal instance shares, or on each instance's disk. See [Portal And API Scale-Out](#portal-and-api-scale-out). +- `portalRedisSku`: size of that cache. Leave empty, the default, for `Balanced_B0` (Azure Managed Redis) in the public cloud or `Standard_C1` (Azure Cache for Redis) in Azure Government. See [Size and cost](#size-and-cost). +- `portalRedisLocation`: region of that cache, such as `eastus`. Leave empty, the default, for `AZURE_LOCATION`. Set it where that region does not offer the cache or has no capacity for it. See [Size and cost](#size-and-cost). - `allowedClientIp`: your public client IP for SQL bootstrap from the local machine. - `deployLinuxHosts`: `true` or `false`. - `deployAvdHosts`: `true` or `false`. @@ -109,11 +118,21 @@ The checked-in [bicep/main.parameters.example.json](bicep/main.parameters.exampl - `domainName`: DNS suffix the broker appends to Linux host names when it connects over SSH. Leave empty to use the deployment's private DNS zone, `linuxbroker.internal`. If you set it, you are responsible for DNS records that resolve `.` from the API's virtual network. - `nfsShare`: an existing NFS share, in `:/` form, to mount for Linux home directories. Leave empty to have the deployment provision one. - `deployNfsShare`: `true` or `false`. Provisions a premium Azure Files NFS share when `nfsShare` is empty. Defaults to `true`. -- `nfsShareQuotaGiB`: provisioned size of that share in GiB. Premium shares have a 100 GiB minimum, and cost is based on the provisioned size. Defaults to `100`. +- `nfsShareQuotaGiB`: provisioned size of that share in GiB. Premium shares have a 100 GiB minimum, and the size sets the share's IOPS, throughput and cost as well as its capacity. Defaults to `100`. See [Sizing the share](#sizing-the-share). - `avdUsersGroupId`: object ID of an existing Entra group whose members can launch **Linux Desktop**. Leave empty to have `preprovision` create `--avd-users-sg` and add you to it. +- `avdLinuxDesktopFullScreen`: `true` or `false`. Defaults to `true`, which opens the Linux desktop full screen. `false` opens it in a window on one monitor. See [Linux Desktop Display](#linux-desktop-display). +- `avdLinuxDesktopMultiMonitor`: `true` or `false`. Defaults to `true`, which spreads a full-screen Linux desktop across every monitor of the user's AVD session. `false` keeps it on one monitor. See [Linux Desktop Display](#linux-desktop-display). +- `avdScalingPlanEnabled`: `true` or `false`. Defaults to `true`, which assigns the deployment's scaling plan to the AVD host pool, so autoscale starts and stops the session hosts. `false` deploys the plan assigned to no host pool and leaves the session hosts as they are. See [AVD Autoscale](#avd-autoscale). +- `avdStartVmOnConnect`: `true` or `false`. Defaults to `true`, which starts a stopped AVD session host for a user who opens **Linux Desktop** when no running session host can take the session. +- `avdScalingPlanTimeZone`: Windows time zone ID the scaling plan's times are in, such as `Eastern Standard Time`. Defaults to `UTC`. The schedule itself comes from `avdScalingPlanRampUpStart`, `avdScalingPlanPeakStart`, `avdScalingPlanRampDownStart`, `avdScalingPlanOffPeakStart` and the five `avdScalingPlan...Pct` values; see [AVD Autoscale](#avd-autoscale). +- `assignAvdAutoscaleRole`: `true` or `false`. Defaults to `true`, which lets the deployment give the Azure Virtual Desktop service principal **Desktop Virtualization Power On Off Contributor** on the subscription when it does not hold it. Set it to `false` when an administrator manages that role. +- `avdServicePrincipalObjectId`: object ID of the Azure Virtual Desktop service principal in your tenant. Leave empty to have `preprovision` find it by its app ID, `avdServicePrincipalAppId`, which defaults to `9cdead84-a844-4324-93f2-b2e6bb768d07`. +- `deployHostMonitoring`: `true` or `false`. Defaults to `true`, or to `false` when `azureCloudName` is `AzureCustom`. Collects the hosts' logs in Log Analytics and adds the fleet workbook and the alerts. See [Host Monitoring](#host-monitoring). +- `alertEmailAddresses`: email addresses that receive the alerts, separated by commas or semicolons. For a Teams channel, use the channel's email address. Leave empty, the default, to deploy the alerts without notifications. +- `alertCheckoutRefusalThreshold`, `alertApiErrorThreshold` and `alertNfsLatencyThresholdMs`: the refused checkouts and the API `5xx` responses in 15 minutes, and the average latency of the home directory share in milliseconds, that raise an alert. They default to `1`, `5` and `50`. See [Alerts](#alerts). - `vmHostResourceGroup`: override if managed VMs live in a different resource group. - `brokerReaderGroupId`, `brokerOperatorGroupId`, `brokerAdminGroupId`: optional object IDs of Entra groups to assign the Broker API's `Reader`, `Operator` and `FullAccess` app roles to. Assigning an app role to a group needs Microsoft Entra ID P1 or P2; without it, assign the roles to users in **Enterprise applications**. See [Portal roles](#portal-roles). -- `sqlDatabaseSkuName`: Azure SQL Database SKU, for example `Basic`, `S1` or `GP_S_Gen5_1`. Defaults to `Basic`, which suits small pools. Use `S1` or higher when many hosts and portal users call the broker at once; each API worker process opens at most `DB_MAX_CONCURRENCY` connections (6 by default). +- `sqlDatabaseSkuName`: Azure SQL Database SKU, for example `Basic`, `S1` or `GP_S_Gen5_1`. Defaults to `Basic`, which suits small pools. Use `S1` or higher when many hosts and portal users call the broker at once, or when `appServicePlanCapacity` is 3 or more: each instance's API runs two worker processes, each process opens at most `DB_MAX_CONCURRENCY` connections (6 by default), and the Basic tier runs at most 30 requests at once. - `allowLegacyScopeAccess`: `true` or `false`. Defaults to `false`. When `true`, any portal user holding the `access_as_user` scope is treated as `FullAccess`, as in releases before role enforcement. Use it only while you assign roles during an upgrade. ### Values that are usually auto-generated @@ -205,6 +224,8 @@ The mirror must preserve the repository layout, because the bootstrap scripts ap Confirm before deploying to a non-commercial cloud that the region offers Azure Virtual Desktop, the App Service Premium v3 `P2mv3` SKU, and the Linux and Windows VM images referenced by the deployment. Availability differs between clouds, and a missing SKU or image surfaces as a provisioning failure rather than a validation error. +The portal's session cache needs Azure Managed Redis in the public cloud, which not every region offers, and Azure Cache for Redis in Azure Government. See [Size and cost](#size-and-cost) for how to check, and what to do where the region lacks it. + ### SSH key values for Linux hosts The broker requires SSH key-based access to Linux hosts. The relevant azd environment values are: @@ -325,6 +346,498 @@ Expect `uint32 0`, `false`, and `true` on GNOME, `0`, `false`, and `true` on MAT twice on Xfce. If `gsettings` still reports the distribution defaults, check that `/etc/dconf/profile/user` contains `system-db:local` and rerun `sudo dconf update`. +## Linux Desktop Display + +**Linux Desktop** runs [Connect-LinuxBroker.ps1](../avd_host/broker/Connect-LinuxBroker.ps1) on the +AVD session host, and the script starts Remote Desktop Connection (`mstsc.exe`) to the user's Linux +host. From version 2.0.0 of the script, the Linux desktop opens full screen across every monitor, +with `mstsc /v:
/f /multimon`. Earlier scripts ran `mstsc /v:
`, which left full +screen and monitors to the user's own Remote Desktop Connection settings. + +Two azd environment values change this for every user: + +| Value | Set to `false`, it | +| --- | --- | +| `avdLinuxDesktopFullScreen` | opens the desktop in a window on one monitor, whatever `avdLinuxDesktopMultiMonitor` says | +| `avdLinuxDesktopMultiMonitor` | keeps a full-screen desktop on one monitor | + +```powershell +azd env set avdLinuxDesktopMultiMonitor false +azd provision +``` + +Provisioning passes a `false` value to the script as `-FullScreen Off` or `-MultiMonitor Off` on the +RemoteApp's command line. The RemoteApp requires that command line, and every `azd provision` sets +it again, so change these values rather than the command line in the Azure portal. + +Before you set either value to `false`, update every session host to script 2.0.0 or later, +including the stopped ones, which [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1) +skips. An older script refuses the argument, and **Linux Desktop** then closes on that session host +without connecting. While both values are `true`, the command line is the one earlier releases +used, which every script accepts. + +The Linux desktop can use only the monitors that the user's AVD session has. An AVD session uses +every monitor of the user's device, if the user's client supports several, unless the host pool's +RDP properties set `use multimon:i:0`, which this deployment does not. + +### Why the script writes no connection file + +Since the April 2026 Windows security update, Remote Desktop Connection warns about every `.rdp` file +that no trusted publisher signed, and turns off every redirection the file asks for, the clipboard +included, until the user turns each one back on. The warning applies only to connections started +from a file, so the script passes switches to `mstsc.exe` instead of writing one. See +[Understanding security warnings when opening Remote Desktop (RDP) files](https://learn.microsoft.com/windows-server/remote/remote-desktop-services/remotepc/understanding-security-warnings). + +Do not apply Microsoft's +[high-security configuration](https://learn.microsoft.com/windows-server/remote/remote-desktop-services/remotepc/manage-rdp-file-security-settings-with-group-policy#high-security-environments) +for RDP files to the session hosts. With **Allow .rdp files from valid publishers and user's +default .rdp settings** disabled, Remote Desktop Connection opens only files signed by a trusted +publisher, and refuses the connection the script starts as well. Disabling **Allow .rdp files from unknown +publishers** on its own does no harm. Connection files signed by a publisher the session hosts +trust, which would allow the high-security configuration, are in the +[security hardening backlog](../docs/ROADMAP.md#security-hardening-backlog). + +## AVD Autoscale + +The AVD session hosts only run Remote Desktop Connection to the Linux hosts, so they need to run +only while users need them. With the session hosts, the deployment creates an Azure Virtual Desktop +[scaling plan](https://learn.microsoft.com/azure/virtual-desktop/autoscale-scenarios), +`-scaling-plan`, that starts and stops them on a schedule, and turns on +[Start VM on Connect](https://learn.microsoft.com/azure/virtual-desktop/start-virtual-machine-connect) +for the host pool. + +The plan has one schedule for Monday to Friday and one for Saturday and Sunday, with the same times, +in `avdScalingPlanTimeZone`: + +| Phase | Starts at | Session hosts kept on | Starts another session host above | New sessions go to | +| --- | --- | --- | --- | --- | +| Ramp-up | `avdScalingPlanRampUpStart`, 07:00 | `avdScalingPlanRampUpMinimumHostsPct`, 20% | `avdScalingPlanRampUpCapacityThresholdPct`, 60% | the session host with the fewest sessions | +| Peak | `avdScalingPlanPeakStart`, 09:00 | as ramp-up | as ramp-up | the session host with the fewest sessions | +| Ramp-down | `avdScalingPlanRampDownStart`, 18:00 | `avdScalingPlanRampDownMinimumHostsPct`, 10% | `avdScalingPlanRampDownCapacityThresholdPct`, 90% | the busiest session host with room | +| Off-peak | `avdScalingPlanOffPeakStart`, 20:00 | as ramp-down | as ramp-down | the busiest session host with room | + +- **Session hosts kept on** is a share of all the session hosts, rounded up, so with up to five + session hosts, 20% and 10% both keep one on. On Saturday and Sunday, + `avdScalingPlanWeekendMinimumHostsPct`, 0% by default, takes the place of both. +- Autoscale starts another session host when the sessions fill more than the threshold's share of + what the running session hosts can hold, `avdMaxSessionLimit` each. During peak, ramp-down and + off-peak, it also stops session hosts that have no sessions, as long as the others stay under the + threshold. It stops none during ramp-up. Its load balancing replaces the host pool's own. +- Users are never signed out. A session host stops only once it has no sessions, disconnected ones + included, so a user who closes their Remote Desktop client without signing out keeps their session + host, and their Linux session, running. +- The times are `HH:mm` on a 24-hour clock and must come in the order of the table within one day. + Each share is a whole number from 0 to 100, and each threshold from 1 to 100. `preprovision` + refuses other values. + +`avdScalingPlanTimeZone` is a Windows time zone ID, such as `Eastern Standard Time`, not an IANA +name such as `America/New_York`, and `preprovision` warns when it does not know the value. It is +separate from the time zone of the broker's own scaling schedule for the Linux hosts, which an +administrator sets on the portal's **Scaling** page, so change them together. + +With the weekend share at 0%, every session host can stop. The first user to open **Linux Desktop** +then waits while Start VM on Connect starts a session host, which their Remote Desktop client +reports, and if no Linux host is ready either, waits again inside the session host while the broker +starts one. So the first user can wait through two starts, of several minutes in all. Keep a +weekend share above 0% if that is too long. For a pooled host pool, Start VM on Connect starts a +session host only when none is running, and another only when the running ones reach their session +limit; autoscale starts the rest. + +To keep autoscale away from a session host, for example while you maintain it, tag it +`excludeFromScaling`, with any value. Autoscale then never starts or stops it, and leaves its drain +mode alone, which it otherwise overrides in a pooled host pool. A deployment replaces a VM's tags, +so `preprovision` records the tag and `azd provision` writes it back. + +```powershell +az vm update -g -n --set tags.excludeFromScaling=maintenance +az vm update -g -n --remove tags.excludeFromScaling +``` + +The plan uses power-management autoscale, which starts and stops existing session hosts and works in +Azure Government. Dynamic autoscale, which also creates and deletes session hosts, is not available +there, and the deployment does not use it. + +### The autoscale role + +Autoscale and Start VM on Connect start and stop the session hosts as the Azure Virtual Desktop +service principal, which must hold **Desktop Virtualization Power On Off Contributor** on the +subscription: autoscale does not work with the role on a resource group or a VM. The role lets +Azure Virtual Desktop start and stop any session host in the subscription, and assigning it needs +**Owner** or **User Access Administrator** on the subscription. + +`preprovision` finds the service principal by its app ID, `avdServicePrincipalAppId`, never by its +name, and, unless `assignAvdAutoscaleRole` is `false`, creates it in the tenant if it is missing. It +then checks whether the service principal holds the role on the subscription already, directly, +through a group or from a management group, and whether your account can assign roles there: + +| `preprovision` finds that | The deployment | +| --- | --- | +| The role is assigned | assigns nothing | +| The role is not assigned, and you can assign it | assigns it | +| The role is not assigned, and you cannot assign it | deploys the scaling plan assigned to no host pool, and `preprovision` prints the command for an Owner or User Access Administrator to run. Start VM on Connect stays on, but starts nothing until the role is assigned. | +| The service principal cannot be found or created | does the same, and `preprovision` asks you to set `avdServicePrincipalObjectId` | +| It cannot tell | assigns the role, and `preprovision` warns what a failure would mean | + +Once an administrator has assigned the role, run `azd provision` again, and it assigns the plan to +the host pool. + +- `assignAvdAutoscaleRole=false` stops the deployment from assigning the role, for when an + administrator manages it, for example through a custom role, which the check does not recognize. + The plan is then assigned as `avdScalingPlanEnabled` says, and `preprovision` only warns when the + service principal holds no **Desktop Virtualization Power On Off Contributor** assignment on the + subscription. +- `avdScalingPlanEnabled=false` keeps the plan but assigns it to no host pool, so autoscale leaves + the session hosts alone. Set it when the host pool already has a scaling plan, because a host pool + can have only one and the deployment would fail, or when something else scales the session hosts, + because autoscale must not be combined with another scaling tool. +- `avdStartVmOnConnect=false` turns off Start VM on Connect. With the plan off too, the deployment + assigns no role. Start VM on Connect on its own needs only **Desktop Virtualization Power On + Contributor**, on any scope that contains the session hosts: to use it on the resource group + instead of the subscription role, assign it to the service principal yourself and set + `assignAvdAutoscaleRole` to `false`. +- `avdServicePrincipalObjectId` skips the lookup. `avdServicePrincipalAppId` defaults to + `9cdead84-a844-4324-93f2-b2e6bb768d07`, the app ID Microsoft documents for Azure Virtual Desktop; + set it only for a cloud where the service principal has another. + +## Home Directory Share + +Linux hosts keep each broker user's home directory on an NFS share, so a user's files follow them +from host to host. At checkout, `create-user.sh` mounts the user's directory on the share over +`/home/`, and the host keeps it mounted until the broker returns the host. Unless `nfsShare` +names a share of your own, the deployment creates a premium Azure Files NFS share of +`nfsShareQuotaGiB` GiB, 100 by default. + +### Sizing the share + +The share uses the provisioned v1 model, where the size you provision sets how fast the share is +as well as how much it holds: + +| Provisioned size (GiB) | Baseline IOPS | Burst IOPS | Throughput (MiB/s) | +| ---: | ---: | ---: | ---: | +| 100 | 3,100 | 10,000 | 110 | +| 256 | 3,256 | 10,000 | 127 | +| 512 | 3,512 | 10,000 | 152 | +| 1,024 | 4,024 | 10,000 | 203 | +| 2,048 | 5,048 | 10,000 | 305 | +| 5,120 | 8,120 | 15,360 | 613 | +| 10,240 | 13,240 | 30,720 | 1,125 | + +Baseline IOPS are 3,000 plus one for each GiB, burst IOPS are three for each GiB but at least +10,000, and throughput is 100 MiB/s plus about 0.1 MiB/s for each GiB, as +[Microsoft documents](https://learn.microsoft.com/azure/storage/files/understanding-billing#provisioned-v1-model). +A share gathers credits while it runs below its baseline, and with a full set it can run at its +burst IOPS for an hour. + +- **Size for capacity first.** The share keeps the profile of every user who has ever signed in, + not only of those signed in now, so it needs room for all of them: the number of users times a + typical profile, and room to grow. `df -h ~` in a session shows how full the share is. +- **IOPS follow how many users are signed in at once,** and what they run. With caches on the + local disk, as described below, a session's traffic to the share is mostly the files its user + opens and saves, and an idle session moves at most a few hundred bytes a minute. Signing in and + starting applications cost the most, so many users who sign in together, such as at the start + of a shift, make the peak. Burst credits are there for peaks like that. +- **Start at 100 GiB and watch the share.** In the storage account's **Metrics**, choose the + **File** metric namespace and **Transactions**, and split it by **Response type**. + `SuccessWithThrottling`, the `SuccessWithShare...Throttling` types and the + `ClientShare...ThrottlingError` types mean the share held requests back because they went over + its IOPS or throughput. **Success E2E Latency** shows how long requests took, and **Success + Server Latency** how much of that the share itself took. If the share throttles, make it larger. +- **Change the size with azd:** `azd env set nfsShareQuotaGiB `, then `azd provision`. A + larger size takes effect within minutes, and the hosts don't need to mount the share again. A + share can be made smaller only 24 hours after it was last made larger. A size changed anywhere + else, such as in the Azure portal, lasts only until the next `azd provision` sets it back. + +Microsoft now recommends the provisioned v2 model for new shares. It provisions IOPS and throughput +separately from capacity, so a share that holds little but serves many users at once costs less. +The deployment still creates a provisioned v1 share, and a storage account can't move from one +model to the other, so moving an environment to provisioned v2 takes a new share and a copy of the +profiles. + +### How hosts use the share + +Every Linux host runs `install-host-config.sh` from its bootstrap, and again whenever +[Migrate-LinuxHostReleaseAgent.ps1](Migrate-LinuxHostReleaseAgent.ps1) runs. It writes only files +of its own, and only when they differ. A host works without them, so when the script fails the +bootstrap reports it and goes on. + +- **Read-ahead.** On an NFS mount, Linux reads ahead 128 KiB at a time. Microsoft + [recommends 15 MiB](https://learn.microsoft.com/azure/storage/files/nfs-performance) for Azure + Files NFS shares, which the script sets on NFS mounts that already exist and, for later ones, + with the udev rule Microsoft publishes, in `/etc/udev/rules.d/99-nfs.rules`. The rule applies to + every NFS mount on the host, not only to the home directory share. It takes the place of a rule + of the same name in `/usr/lib/udev/rules.d`, which some distributions install with their NFS + packages. A rule an administrator wrote at that path is left alone, as long as it doesn't carry + the script's marker line. The rule differs from Microsoft's in one character, `$$4` where + Microsoft writes `$4`: udev reads `$$` as a literal `$`, so awk still gets `$4`, and systemd 255 + and later no longer report the rule as invalid each time udev loads its rules. +- **Caches on the local disk.** At checkout, `create-user.sh` creates + `/var/cache/linuxbroker/users/`, which only the user can read, and the session launcher + points `XDG_CACHE_HOME` at it, as `/etc/profile.d/linuxbroker-cache.sh` does in login shells. + Applications that follow the XDG base directory specification, such as the desktops' own + components and most browsers, then keep their caches there instead of in `~/.cache` on the + share. Returning the host deletes the cache, and a restart empties the directory. Only root can + create entries in `/var/cache/linuxbroker/users`, so no user can create another user's cache + ahead of them, as they could in a directory every user can write to, such as `/var/tmp`. +- **Log rotation.** `/etc/logrotate.d/linuxbroker` rotates the broker's logs in `/var/log` every + week, or sooner once one passes 50 MB, and keeps four, compressed. `linuxbroker-patch.log` is + left out, because `patch-host.sh` trims it itself. +- **The old mount point.** `create-user.sh` mounts the share on `/nfs_profiles` while it creates a + home directory. The script removes the previous mount point, `/awipsprofiles`, when it is empty + and nothing is mounted on it. + +Some caches still go to the share: + +- The Firefox snap on Ubuntu keeps its cache in `~/snap/firefox/common/.cache`, because a snap + sets its own `XDG_CACHE_HOME`. +- Services that the user's systemd instance starts before the desktop hands it its environment + use `~/.cache`. +- Applications that keep caches in their own directories, such as Visual Studio Code under + `~/.config/Code`, ignore `XDG_CACHE_HOME`. + +Caches that sessions left in `~/.cache` before the host was updated stay on the share and can be +deleted; an application rebuilds a cache it needs. + +## Host Monitoring + +With `deployHostMonitoring`, which is `true` unless `azureCloudName` is `AzureCustom`, the +deployment collects the hosts' logs in the Log Analytics workspace that Application Insights writes +to, adds a workbook that shows the state of the fleet, and adds alerts for the problems that keep +users from a Linux desktop. With `` for `-`, it creates: + +- the `LinuxBrokerHost_CL` table in the workspace, for the broker's logs from the Linux hosts +- two data collection rules, `dcr--linux` for the Linux hosts and `dcr--avd` for + the AVD session hosts +- the **Linux Broker fleet (``)** workbook +- five log search alert rules and, when the deployment creates the NFS share, two metric alert + rules on the share, all named `alert--...` +- the action group `ag-`, when `alertEmailAddresses` is set + +A data collection rule collects nothing until a VM is associated with it and runs the Azure +Monitor agent. Azure refuses to add an extension to a VM that is not running, and scaling, start on +demand and autoscale all stop hosts, so the deployment doesn't install the agent itself; +[Enable-HostMonitoring.ps1](Enable-HostMonitoring.ps1) does, from `postprovision`. + +Set `deployHostMonitoring` to `false` to leave all of this out, for example in a cloud that lacks +data collection rules, workbooks or log search alerts, which is why an `AzureCustom` cloud has to +opt in. Setting it to `false` later removes nothing that an earlier deployment created. + +### What the hosts send + +| From | Table | What | +| --- | --- | --- | +| Linux hosts | `LinuxBrokerHost_CL` | The records of `release-session.log`, `release-session-watcher.log`, `linuxbroker-host-settings.log`, `createuser.log`, `linuxbroker-session-control.log`, `linuxbroker-patch.log`, `xrdp.log` and `xrdp-sesman.log` in `/var/log`, in `RawData`, with the file in `FilePath` and the host in `Computer` | +| Linux hosts | `Syslog` | The messages of the `auth` and `authpriv` facilities from `info` up, of `kern` and `user` from `notice` up, and of every other facility from `warning` up | +| AVD session hosts | `Event` | The events that `Connect-LinuxBroker.ps1` writes to the Application log under the `LinuxBrokerScript` source, and the critical, error and warning events in the Remote Desktop client's `Microsoft-Windows-TerminalServices-RDPClient/Operational` log | +| The broker API | `AppTraces` | A fleet snapshot every five minutes, described below | + +- Each line of the broker's logs starts with a `YYYY-MM-DD HH:MM:SS` timestamp, which starts a + record, so a message of several lines arrives as one record. xrdp starts its lines in a format + the agent doesn't recognize, so each line of its logs arrives on its own. +- The kernel reports an NFS server that stops answering at `notice`, which is why `kern` is + collected from there. The agent takes syslog messages from rsyslog or syslog-ng, so a host that + runs neither sends no syslog. +- The logs of the release agent and its watcher name the users signed in to the host, so only + root can read them, and the `syslog` group, where the host has one. Version 1.45 of the Azure + Monitor agent reads the logs as root, and installing it creates the `syslog` group on RHEL 9. +- Log Analytics charges for what it ingests. Each sign-in, `sudo` command and SSH connection, the + broker's own included, adds a few `auth` lines, and each broker action a few log lines, so the + volume follows how busy the hosts are. Each alert rule also has a small monthly charge. + +The broker's own host heartbeats, which **Fleet health** shows, go to its database rather than to +Log Analytics, so at the end of each scaling run, every five minutes, the API logs a +`fleet snapshot` trace with the figures of the `GetFleetSnapshot` procedure as custom dimensions: +`ReadyHosts`, the hosts a new user could be given at once; `PoweredOn`, `Serviceable` and `InUse`, +counted as scaling counts them; `Waiting` users and `Booting` hosts; the hosts whose heartbeat is +older than `StaleAfterSeconds` (`StaleHeartbeats`), that report the home directory share +unreachable (`NfsUnreachable`) or that report no xrdp (`XrdpInactive`); `Draining`, `Maintenance` +and `TotalHosts`; the scaling minimum and maximum in effect, `EffectiveMinVMs` and `MaxVMs`; and +`StartOnDemandEnabled`. Until SQL script `157` is applied, the API logs no snapshot, and each API +process logs once that `GetFleetSnapshot` is not deployed yet. + +### Connecting the hosts + +`postprovision` runs [Enable-HostMonitoring.ps1](Enable-HostMonitoring.ps1) after every other step, +so every `azd up` and `azd provision` connects the hosts, new ones included. The script: + +1. Associates each VM tagged `broker-role=linux-host` with the Linux rule, and each VM tagged + `broker-role=avd-host` with the AVD rule, under the name `linuxbroker-host-monitoring`. An + association works whether the VM is running or not. +2. Installs the Azure Monitor agent, `AzureMonitorLinuxAgent` or `AzureMonitorWindowsAgent`, with + automatic upgrades, on each running VM that doesn't have it, and installs it again, under its + existing name, where an earlier install failed. The agent authenticates with the VM's + system-assigned managed identity. +3. Waits for the installs, for up to 20 minutes, which `-AgentTimeoutMinutes` changes. + +A host that isn't running keeps its association but gets no agent, so it sends nothing, and the +script names it. A host keeps the agent once it has it, so start each host the script named once +and run the script for it: + +```powershell +Set-Location .\deploy +.\Enable-HostMonitoring.ps1 -EnvironmentName -HostNames , +``` + +Scaling leaves a Linux host running for at least 10 minutes after it starts, which is usually long +enough, so run the script as soon as the host is up. While the scaling plan is assigned, autoscale +can stop an AVD session host you started before its agent is installed, so tag the session host +`excludeFromScaling` before you start it and remove the tag afterwards; see +[AVD Autoscale](#avd-autoscale). An install that failed is repaired the next time the script runs. + +The script tries every host, reports the ones it could not set up together, and exits with an +error naming them, which `postprovision` reports as a warning, because the rest of the deployment +is in place by then. Running it again is safe. It reads the rules, the resource group and the +subscription from the azd environment unless you pass them, and it finds the hosts by their +`broker-role` tag in that resource group, so hosts elsewhere need `-ResourceGroupName` and the tag. +Without `-HostNames`, it goes through every host. + +The agent sends over HTTPS. If you restrict the hosts' outbound traffic, allow the `AzureMonitor` +and `AzureResourceManager` service tags on port 443, which the agent +[needs](https://learn.microsoft.com/azure/azure-monitor/agents/azure-monitor-agent-network-configuration). + +### The workbook + +Open **Linux Broker fleet (``)** from **Workbooks** in the Log Analytics workspace, or from +the resource group. For the time range you choose, it shows: + +- the latest fleet snapshot, and the ready, in-use, waiting, booting and powered-on hosts over time +- checkouts by outcome: assigned, told to wait while a host starts, refused, or failed +- how long successful checkouts took, at the 50th and 95th percentile +- releases and returns, and whether they succeeded +- hosts started on demand, and starts that failed +- the host settings that a host could not apply +- the errors and warnings in each Linux host's broker logs, and its syslog errors +- NFS home directory share failures: mounts that failed, and an NFS server that stopped answering +- errors and warnings by AVD session host + +### Alerts + +The alerts are evaluated every five minutes over the last 15 minutes, and resolve themselves once +the problem is gone. + +| Alert | Severity | Fires when | +| --- | --- | --- | +| No ready hosts | 1 | Every fleet snapshot in the window showed no ready Linux host while the scaling minimum is above 0 or users are waiting. A pool scaled to zero that nobody is waiting for raises nothing. | +| No fleet snapshot | 2 | The window has no fleet snapshot: the task function, the API or the database may be down, or SQL script `157` isn't applied. | +| Checkouts refused | 2 | The API refused at least `alertCheckoutRefusalThreshold` checkouts with `409`, 1 by default. | +| Unhealthy hosts | 2 | Every fleet snapshot in the window showed a Linux host with a stale heartbeat, an unreachable home directory share or no xrdp. | +| API errors | 2 | The API answered at least `alertApiErrorThreshold` requests with a `5xx` status, 5 by default. | +| NFS throttling | 2 | Azure Files throttled a request to the home directory share. See [Sizing the share](#sizing-the-share). | +| NFS latency | 3 | Requests to the home directory share took more than `alertNfsLatencyThresholdMs` milliseconds on average, 50 by default. | + +- Because the fleet alerts need every snapshot in the window to show the problem, a host that is + briefly busy or starting raises nothing. +- The NFS alerts exist only when the deployment creates the share. A share of your own, named in + `nfsShare`, isn't watched. +- Without `alertEmailAddresses`, the alerts appear only under **Alerts** in Azure Monitor. Set it to + one or more addresses, separated by commas or semicolons, to get them by email. For a Teams + channel, use the channel's email address. + +## Portal And API Scale-Out + +The portal, the API and the task function run on one App Service plan, and every instance of the +plan runs all three. `appServicePlanCapacity` sets how many instances the plan has, 1 by default. +The portal keeps each administrator's sign-in, with the Entra ID token it calls the API with, in a +session on the server, so the browser holds only an opaque cookie. On more than one instance, every +instance has to find that session, so with `portalSessionStore` set to `redis`, the default, the +deployment keeps the sessions in a cache that every portal instance shares: + +| `azureCloudName` | Session cache | Default size | Private DNS zone | +| --- | --- | --- | --- | +| `AzurePublic` | Azure Managed Redis | `Balanced_B0` | `privatelink.redis.azure.net` | +| `AzureUSGovernment` | Azure Cache for Redis | `Standard_C1` | `privatelink.redis.cache.usgovcloudapi.net` | +| `AzureCustom` | None: the portal keeps sessions on each instance's disk | | | + +The clouds differ because new customers can no longer create Azure Cache for Redis in the public +cloud, and Azure Managed Redis isn't offered in Azure Government. Azure Cache for Redis retires on +30 September 2028, and Azure Government lets new caches be created until then. + +### Portal sessions + +- The cache, `redis--`, takes no traffic from the internet and has its access + keys turned off. The portal reaches it through the private endpoint `pe-redis--` + in the private endpoint subnet, so the deployment integrates the portal with the virtual network's + app subnet, as it does the API and the task function, and links the private DNS zone to the + virtual network. +- The portal signs in to the cache as its system-assigned managed identity, over TLS, on port 10000 + for Azure Managed Redis or 6380 for Azure Cache for Redis. The deployment gives the identity the + database's `default` access policy on Azure Managed Redis, or `Data Contributor` on Azure Cache for + Redis. +- A session ends 12 hours after the administrator's last request, and the cache deletes it then. + The `SESSION_LIFETIME_HOURS` app setting on the portal, from 1 to 720, changes that, but + `azd provision` replaces the portal's app settings, so set it again after each provision. The + session cookie is `Secure`, `HttpOnly` and `SameSite=Lax`. +- When the cache can't be reached, the portal answers `503` to the requests that need a session and + says that sign-in is unavailable for now. `/health` doesn't use the session, so App Service keeps + the instances, and the portal recovers by itself when the cache answers again. See + [The portal answers 503 and says it cannot reach its session store](#the-portal-answers-503-and-says-it-cannot-reach-its-session-store). +- Moving the sessions to the cache signs every administrator out once. The `azd provision` that + creates the cache restarts the portal with the new settings before the portal's identity gets its + access policy, which can then take a few more minutes to take effect, so the portal can answer + `503` for those minutes. +- With `portalSessionStore` set to `filesystem`, the portal keeps sessions on each instance's disk, + and the deployment doesn't integrate it with the virtual network. That disk doesn't survive a + restart, so each restart signs out the administrators on that instance. The deployment turns on + ARR affinity for the portal, which sends each browser back to the instance it signed in on. + Changing `portalSessionStore` to `filesystem` later deletes nothing that an earlier deployment + created. + +### Size and cost + +The cache is billed for every hour it exists, whether or not anyone signs in, and so is its private +endpoint. Sessions are small, so the default sizes hold far more sessions than a deployment has +administrators: + +- `Balanced_B0` is the smallest size of Azure Managed Redis. The deployment keeps its high + availability on, with a replica on a second node, which Microsoft recommends for everything but + dev/test. +- `Standard_C1` is the smallest size of Azure Cache for Redis that Microsoft recommends outside + dev/test. `Standard_C0` costs less but shares a CPU core with other caches, and the `Basic` tier + has no replica and no SLA. + +Set `portalRedisSku` to use another size, for example where the region doesn't offer the default. +In the public cloud, name an Azure Managed Redis size, such as `Balanced_B1` or `Balanced_B3`. In +Azure Government, name an Azure Cache for Redis size, written `_`: +`Basic_C0` to `Basic_C6`, `Standard_C0` to `Standard_C6`, or `Premium_P1` to `Premium_P5`. + +Azure Managed Redis isn't offered in every region, and a region that offers it can run out of room +for new caches, which Azure reports as `InsufficientCapacity`. This lists the regions that offer it: + +```powershell +az provider show --namespace Microsoft.Cache --query "resourceTypes[?resourceType=='redisEnterprise'].locations | [0]" --output tsv +``` + +Where the deployment's region doesn't offer it or has no room, set `portalRedisLocation` to a nearby +region that does, such as `eastus` for a deployment in `eastus2`. Only the cache moves: its private +endpoint stays in the virtual network, so the portal still reaches it privately, and each request +to it crosses between the regions, which adds a few milliseconds and is billed as data transfer. +Otherwise set `portalSessionStore` to `filesystem`, and keep `appServicePlanCapacity` at 1 or rely +on ARR affinity. + +### More than one instance + +More instances add capacity, and let one instance keep answering while App Service maintains +another. Each instance runs the portal, the API and the task function and is billed as a plan +instance, so check the following before you raise `appServicePlanCapacity`: + +- **The database.** Each instance's API runs two worker processes, and each runs at most + `DB_MAX_CONCURRENCY` database requests at once, 6 by default, so each instance adds 12. The `Basic` + tier runs at most 30 at once, so with 3 or more instances, set `sqlDatabaseSkuName` to `S1` or + higher. `preprovision` warns when the Basic tier is too small. +- **ARR affinity.** The deployment turns ARR affinity off for the API, whose callers keep nothing on + an instance, and for the portal while its sessions are in the cache, so requests spread across the + instances. With `portalSessionStore` set to `filesystem`, it stays on for the portal, and + `preprovision` warns that sign-ins then depend on it. +- **The task function stays on the plan.** Its timers run once per schedule, however many instances + there are: the Functions host lets only one instance run a timer, through a lease in the function + app's storage account. Each run is a few calls to the API, and the busiest timers run once a + minute, so it doesn't compete with the API. A plan of its own, or Flex Consumption, would add cost + and a second virtual network integration for no gain. +- **Database connections.** Each API worker process keeps the connections it has finished with and + reuses them, instead of signing in to the database for each request. On App Service, each closed + connection also holds one of the instance's outbound ports for four minutes. `DB_POOL_ENABLED`, + `DB_POOL_IDLE_SECONDS` and `DB_POOL_MAX_LIFETIME_SECONDS` on the API tune the pool; see + [Database Access](../api/README.md#database-access). + ## Quick Start From the repository root: @@ -373,6 +886,8 @@ It currently does all of the following: - Creates or reuses frontend and API client secrets. - Generates or reuses Linux host SSH keys. - When Linux hosts are deployed with `linuxHostOsVersion=rocky-9`, accepts the Azure Marketplace terms of the Rocky Linux 9 image in the deployment subscription unless they are accepted already. +- Lists the Linux hosts and AVD session hosts already in the resource group that are not running, so the deployment leaves out their VM extensions, which Azure refuses to change on a VM that is not running, and records the `excludeFromScaling` tag of each AVD session host, which the deployment writes back. See [A deployment failed with `Cannot modify extensions in the VM when the VM is not running`](#a-deployment-failed-with-cannot-modify-extensions-in-the-vm-when-the-vm-is-not-running). +- Checks the AVD scaling plan's times and shares, and, when AVD session hosts are deployed, finds the Azure Virtual Desktop service principal and decides whether the deployment gives it **Desktop Virtualization Power On Off Contributor** on the subscription. See [The autoscale role](#the-autoscale-role). - Writes resolved values back into the azd environment in both uppercase and camelCase forms expected by the deployment. The API app registration is also configured with the Graph application permissions the API uses to validate host and group membership. @@ -393,6 +908,9 @@ Important deployment characteristics: - The API's `NFS_SHARE` setting points at the provisioned Azure Files share unless `nfsShare` is set. The storage account disables public network access and shared key access, and it allows non-HTTPS traffic because NFS does not use HTTPS; the private endpoint is the only path to it. - RHEL, Rocky Linux and AlmaLinux hosts use Generation 2 images so they can run with Trusted Launch. - The AVD host pool prefers RemoteApp and sets RDP properties that enable Microsoft Entra single sign-on to the Microsoft Entra joined session hosts. +- The AVD host pool has Start VM on Connect on, and a scaling plan starts and stops its session hosts on a schedule. See [AVD Autoscale](#avd-autoscale). +- Unless `deployHostMonitoring` is `false`, the host monitoring's data collection rules, workbook and alerts are created, but no host gets the Azure Monitor agent until `postprovision`. See [Host Monitoring](#host-monitoring). +- Hosts that were not running when `preprovision` listed them keep their VM extensions as they were. Changes that the extensions carry, such as the API URL, `scriptSourceRoot`, `linuxHostDesktop` and `linuxHostDisableScreenLock`, reach such a host the next time `azd provision` runs while it is running. - Linux hosts run the desktop that `linuxHostDesktop` names, GNOME by default, with the screen saver and screen lock disabled unless `linuxHostDisableScreenLock` is `false`. See [Linux Host Screen Lock](#linux-host-screen-lock). - Key Vault stores `db-password` and `linux-host`. - The API app receives Key Vault Secrets User access so it can read those secrets at runtime. @@ -409,6 +927,7 @@ It currently runs, in order: 3. [Build-ContainerImages.ps1](Build-ContainerImages.ps1) 4. [Assign-VmApiRoles.ps1](Assign-VmApiRoles.ps1) 5. [Register-LinuxHostSqlRecords.ps1](Register-LinuxHostSqlRecords.ps1) +6. [Enable-HostMonitoring.ps1](Enable-HostMonitoring.ps1) That means `postprovision` does all of the following: @@ -419,6 +938,7 @@ That means `postprovision` does all of the following: - Restarts the API app, then the function app, then the frontend app, so nothing starts ahead of the API endpoints it calls. - Adds AVD and Linux VM managed identities to the corresponding Entra groups, retrying while new identities replicate, and fails the hook if a membership still cannot be confirmed. - Registers Linux hosts into `dbo.VirtualMachines` through `dbo.RegisterLinuxHostVm`. +- Connects the Linux hosts and AVD session hosts to the host monitoring and installs the Azure Monitor agent on the running ones. A host it cannot set up is a warning that names the host, not a failure of the hook. See [Connecting the hosts](#connecting-the-hosts). ### Front End Build Requirements @@ -430,7 +950,7 @@ If the build environment cannot reach `registry.npmjs.org`, point npm at an inte ## Migration For Existing Deployments -Use the migration flow when you already have a deployed customer environment and want to roll forward the current application, SQL, and Linux-host release-agent changes without treating that as part of the normal `azd up` lifecycle. +Use the migration flow when you already have a deployed customer environment and want to roll forward the current application, SQL, Linux host release-agent and AVD session host script changes without treating that as part of the normal `azd up` lifecycle. The migration entrypoint is [Migrate-ExistingEnvironment.ps1](Migrate-ExistingEnvironment.ps1). @@ -439,10 +959,13 @@ That script intentionally stays separate from `azd up`: - `azd up` continues to express the desired greenfield deployment for new environments. - [Migrate-ExistingEnvironment.ps1](Migrate-ExistingEnvironment.ps1) is the supported in-place process for existing environments. -By default, the migration script does two things: +By default, the migration script does three things: -1. Runs [Post-Provision.ps1](Post-Provision.ps1) so the existing environment gets the latest container images, SQL scripts, role assignments, VM group sync, and Linux host SQL registration. +1. Runs [Post-Provision.ps1](Post-Provision.ps1) so the existing environment gets the latest container images, SQL scripts, role assignments, VM group sync, and Linux host SQL registration, and its hosts are connected to the host monitoring. 2. Runs [Migrate-LinuxHostReleaseAgent.ps1](Migrate-LinuxHostReleaseAgent.ps1) so existing Linux hosts get the current release-agent files, one-minute reconciliation timer, and `systemd-logind` watcher. +3. Runs [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1) so existing AVD session hosts get the current `Connect-LinuxBroker.ps1`, the script the **Linux Desktop** RemoteApp runs. + +A failed post-provision step stops the migration. The two host steps run even when the other fails, and the migration then fails, naming the steps that did; the warnings before it name the hosts to retry. Example full migration: @@ -460,6 +983,17 @@ Set-Location .\deploy -LinuxHostNames lnxhost-01,lnxhost-02 ``` +`-LinuxHostNames` limits only the Linux host step. `-AvdHostNames` limits the AVD session host step in the same way, and `-SkipLinuxHostReleaseAgentMigration` and `-SkipAvdHostScriptUpdate` leave out a step. Example update of only selected AVD session hosts: + +```powershell +Set-Location .\deploy +.\Migrate-ExistingEnvironment.ps1 ` + -EnvironmentName ` + -SkipPostProvision ` + -SkipLinuxHostReleaseAgentMigration ` + -AvdHostNames , +``` + Example host-only migration when you do not want to rerun the post-provision steps: ```powershell @@ -480,6 +1014,8 @@ Set-Location .\deploy The host migration step updates only the release-agent-related files and services on existing Linux VMs. It does not reprovision infrastructure, replace the VM image, rerun the full Linux custom script extension, or attempt to reconcile every manual drift in an older environment. +The AVD session host step changes only `C:\Temp\Connect-LinuxBroker.ps1`, through Run Command, on the VMs in the resource group tagged `broker-role=avd-host`. On each running session host it downloads the script from `ScriptSourceRoot`, fills in the API URL and client ID as `Configure-AVD-Host.ps1` does, checks that the result is a broker script in plain ASCII that parses, and replaces the installed script in one step. Because the API URL and `ScriptSourceRoot` are written into PowerShell code on the session host, the update refuses either one unless it is an `https://` URL made only of ASCII letters, digits and the characters `.`, `-`, `_`, `~`, `%` and `/`, with an optional port. A download or check that fails leaves the installed script as it was, and sessions already open keep the script they started with. Session hosts that are not running are skipped and named at the end; update each one with `-AvdHostNames` once it is started. While the scaling plan is assigned, autoscale can stop a session host you started before the update reaches it, so tag the session host `excludeFromScaling` before you start it and remove the tag once it is updated; see [AVD Autoscale](#avd-autoscale). To update only the session hosts, you can also run [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1) on its own, with the same `-EnvironmentName`, `-ScriptSourceRoot` and `-AvdHostNames` parameters. + The migration also rewrites `/etc/sudoers.d/avdadmin`. Older hosts were provisioned with a broad allowlist that included `cat`, `rm`, `chmod`, `chown`, `cp`, `mount`, and `umount`. The current policy grants only `userdel`, `groupadd`, `usermod`, `chpasswd`, `/usr/local/bin/create-user.sh`, `/usr/local/bin/manage-lease.sh`, `/usr/local/bin/apply-host-settings.sh`, `/usr/local/bin/session-control.sh`, and `/usr/local/bin/patch-host.sh`; all privileged file work now happens inside those root-owned scripts. The generated policy is validated with `visudo -c` and moved into place only if it passes. `apply-host-settings.sh` is the only way the broker API can change host configuration. It accepts a JSON settings document on stdin and nothing on argv, rejects unknown keys, and clamps every value to a supported range before writing anything, so a bad value cannot strand the fleet. @@ -617,6 +1153,38 @@ This release changes which Linux distributions and desktops the deployment offer Every layer tolerates the others being one release behind during the rollout, and the database does not change. Hosts on agent 1.1.0 ignore the key the new API sends and keep working, flagged **Agent outdated**. A 1.2.0 host that meets the previous API, or an API without `KEYRING_VAULT_URL`, gets no key and starts the desktop as before. The previous API drops the launcher's version from the heartbeat. +## Upgrading To Start On Demand + +This release lets a checkout that finds no ready host start one, so a pool can scale to zero when nobody needs it (item 4.1 of the [roadmap](../docs/ROADMAP.md)), opens the Linux desktop full screen across every monitor (item 4.8), starts and stops the AVD session hosts on a schedule (item 4.5), moves users' caches off the home directory share (item 4.6), collects the hosts' logs in Log Analytics, with a workbook and alerts for the fleet (item 4.7), and keeps the portal's sign-ins in Redis, so the portal and the API can run on more than one instance (item 4.9). The schedule adds a scaling plan and a role assignment on the subscription, the monitoring adds data collection rules, a workbook and alerts, and the portal's sessions add a Redis cache, so the release needs `azd provision`, the AVD session hosts need the new `Connect-LinuxBroker.ps1`, and the Linux hosts need agent 1.3.0. + +- **Start on demand.** When a checkout finds no ready host, the API starts a stopped one for the user and answers `202` with how long to wait, instead of refusing. On the AVD session host, **Linux Desktop** shows *Your Linux desktop is starting…* with **Cancel**, asks again when the broker says to, and opens the desktop as soon as the host is reachable, for up to 10 minutes. The start is recorded as a scaling start is, in the scaling activity log as `Start On Demand` and in the audit log as `vm.start_on_demand`, and uses the API's existing **Desktop Virtualization Power On Off Contributor** role. It never takes the pool past the active rule's or window's `MaxVMs`, so a user who arrives when that many hosts are on and none is free is refused as before. Start on demand is on after the upgrade; an administrator can turn it off, and set how many hosts may start at once for waiting users (2 by default, up to 20), in the **Start on demand** card on the **Scaling** page. +- **Scale to zero.** While start on demand is on, the default rule and schedule windows may keep a minimum of 0 hosts, so idle hosts stop, for example overnight, and the first user to arrive waits a minute or two for one to start. Scaling counts waiting users as demand, and a scale-down keeps a host for each of them. One user on the only host running makes the pool full, so the next scaling run starts `ScaleUpIncrement` more hosts, as it does for any full pool, and a scale-down stops them only once utilization falls to the scale-down ratio. A minimum of 0 cannot be saved while start on demand is off. If it is turned off afterwards, scaling keeps one host on for those rules and windows, and the card says so. +- **Connect-LinuxBroker.ps1 2.0.0.** Besides waiting for a host to start, the script tries a request again after no answer, a `408`, a `429` or a `5xx`, until three fail in a row, where the previous script tried three times at once. It requests a new token once when the broker refuses one, and tells the user in plain words what went wrong. Each attempt is logged under the **LinuxBrokerScript** source in the session host's Application event log. Session hosts keep the script they were deployed with, so the migration's new third step, [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1), replaces it on every running session host. An older script treats `202` as a failure: its user is told that no Linux host is available while a host starts for them, and gets that host at their next try once it is up. The script now reports its version with each checkout, and the **Start on demand** card shows, for the session hosts that asked for a host in the last seven days, how many run each version, and names those whose script cannot wait. +- **Waits on the dashboard.** Every time a user is told to wait, the checkout records a `Starting` event. The dashboard's checkout card adds how many users waited, how many of them got a host, the median and 95th percentile wait, and how many are waiting now, and the capacity chart shows the users who waited. **Attention** no longer reports that no host is ready for a pool scaled to zero that nobody is waiting on. +- **Full screen across every monitor.** The new script opens the Linux desktop full screen across every monitor of the user's AVD session, where the previous one left that to the user's own Remote Desktop Connection settings, so users with more than one monitor see the desktop on all of them. The new `avdLinuxDesktopFullScreen` and `avdLinuxDesktopMultiMonitor` values open it in a window or on one monitor instead; set them only after every session host runs the new script. See [Linux Desktop Display](#linux-desktop-display). +- **The AVD session hosts start and stop on a schedule.** `azd provision` creates a scaling plan, assigns it to the host pool, turns on Start VM on Connect and gives the Azure Virtual Desktop service principal **Desktop Virtualization Power On Off Contributor** on the subscription, which needs Owner or User Access Administrator there. The plan takes effect at once. In UTC, unless you set `avdScalingPlanTimeZone`, it keeps at least 20% of the session hosts on from 07:00 on weekdays, 10% from 18:00 and none on Saturday and Sunday, rounded up, and it stops a session host only once the session host has no sessions. Review the schedule before you provision, and set `avdScalingPlanEnabled` to `false` if the host pool already has a scaling plan or the session hosts should stay as they are. See [AVD Autoscale](#avd-autoscale). +- **`azd provision` leaves stopped hosts' extensions alone.** Azure refuses to change a VM extension on a VM that is not running, and scaling, start on demand and autoscale all stop hosts, so `preprovision` lists the hosts that are not running and the deployment leaves out their extensions. It also keeps any `excludeFromScaling` tag on the session hosts, which a deployment would otherwise remove. +- **Caches stay on the local disk.** Each broker user's cache moves from `~/.cache`, in the home directory on the NFS share, to `/var/cache/linuxbroker/users/` on the host's own disk. `create-user.sh` creates it at checkout, `manage-lease.sh` deletes it when the broker returns the host, and the session launcher points `XDG_CACHE_HOME` at it, as the new `/etc/profile.d/linuxbroker-cache.sh` does in login shells. A cache no longer follows its user to the next host, so applications rebuild their caches the first time they start on each host. Caches that sessions left in `~/.cache` stay on the share and can be deleted. See [How hosts use the share](#how-hosts-use-the-share). +- **NFS read-ahead of 15 MiB.** Linux hosts now read ahead 15 MiB at a time on NFS mounts, instead of 128 KiB, as Microsoft recommends for Azure Files NFS shares. This applies to every NFS mount on a host, not only to the home directory share. +- **The broker's logs rotate.** The logs of the release agent, its watcher, `create-user.sh`, the host settings and `session-control.sh` in `/var/log` grew without limit until now. They now rotate every week, or sooner once one passes 50 MB, and each keeps four compressed copies. +- **`install-host-config.sh`.** The three bootstraps and the host migration install `/usr/local/bin/install-host-config.sh` and run it as root. It writes the read-ahead rule, the log rotation, the cache directory with the file that empties it at every restart, and the profile script, and it removes the unused `/awipsprofiles`, because `create-user.sh` now mounts the share on `/nfs_profiles` while it creates a home directory. `azd provision` doesn't run the bootstrap again on existing hosts, so they get the script from the migration; a host the migration skips because it is stopped keeps its caches in the home directory until it is migrated. The script can be run again at any time. See [Sizing the share](#sizing-the-share) for how the share's size sets its IOPS. +- **Host monitoring.** `azd provision` creates the data collection rules, the **Linux Broker fleet** workbook and the alerts, and its `postprovision` step then connects the hosts and installs the Azure Monitor agent on the running ones, so their logs arrive in Log Analytics. Once SQL script `157` is applied, the API logs a fleet snapshot every five minutes, which the workbook's fleet charts and three of the alerts use. The alerts notify nobody until you set `alertEmailAddresses`. Set `deployHostMonitoring` to `false` before you provision to leave the monitoring out. See [Host Monitoring](#host-monitoring). +- **Log timestamps and permissions.** Each line of `createuser.log` now starts with the date and time, as the broker's other logs do. The logs of the release agent and its watcher name the users signed in to the host, so only root could read them; now the `syslog` group can too, where the host has one, in case the Azure Monitor agent reads them as that account. Version 1.45 of the agent reads them as root. +- **Host agent 1.3.0.** Every script in `linux_host/` declares 1.3.0 and the API expects it, so **Fleet health** flags every host as **Agent outdated** until [Migrate-LinuxHostReleaseAgent.ps1](Migrate-LinuxHostReleaseAgent.ps1) from this release has updated it. 1.3.0 carries `install-host-config.sh`, the cache that `create-user.sh` creates and `manage-lease.sh` deletes, the share mounted on `/nfs_profiles`, the timestamps in `createuser.log` and the release agent's log permissions, and its heartbeat also reports the version of `install-host-config.sh`. Sessions in progress keep running during the migration, and each user's cache moves to the host's disk from their next checkout. Hosts that are not running are skipped and named at the end; migrate each one with `-LinuxHostNames` once it is started, because until then it keeps its old agent. +- **Portal sessions move to Redis.** `azd provision` creates the portal's session cache, Azure Managed Redis in the public cloud and Azure Cache for Redis in Azure Government, with a private endpoint, integrates the portal with the virtual network's app subnet, and points the portal at the cache, which signs every administrator out once. The deployment gives the portal's identity its access policy after the portal restarts with the new settings, so the portal can answer `503` for a few minutes. The cache is billed by the hour; set `portalSessionStore` to `filesystem` before you provision to keep the sessions on the portal's disk, as before. The session cookie is now also `Secure` and `SameSite=Lax`. See [Portal And API Scale-Out](#portal-and-api-scale-out). +- **More than one instance.** The new `appServicePlanCapacity` sets how many instances the App Service plan runs, still 1 by default. ARR affinity is now off for the API, and for the portal while its sessions are in Redis. Each instance's API adds 12 database requests at once, so the Basic database tier suits at most 2 instances. The API now also reuses its database connections, which needs no change. See [More than one instance](#more-than-one-instance). + +### Recommended order + +1. Run [Migrate-ExistingEnvironment.ps1](Migrate-ExistingEnvironment.ps1). It applies SQL scripts `144`–`157` before the new images start, restarts the apps in the order API, task, front end, migrates the Linux hosts, and then updates `Connect-LinuxBroker.ps1` on the running AVD session hosts. Running it before `azd provision` updates the session hosts before autoscale starts stopping them. Its post-provision step reports that the deployment has no host monitoring yet, which `azd provision` adds next. +2. Start any AVD session host the migration skipped because it was not running, and update it with `-SkipPostProvision -SkipLinuxHostReleaseAgentMigration -AvdHostNames `. Likewise start any Linux host it skipped and migrate it with `-SkipPostProvision -SkipAvdHostScriptUpdate -LinuxHostNames `, then open **Fleet health** and confirm that no powered-on host is flagged **Agent outdated**. +3. Review the [AVD autoscale](#avd-autoscale) values, the [host monitoring](#host-monitoring) values, such as `alertEmailAddresses`, and the [portal session](#portal-and-api-scale-out) values, then run `azd provision`. It creates the scaling plan, the role assignment, the host monitoring and the portal's session cache and turns on Start VM on Connect, and its `postprovision` step builds the images, restarts the apps again and connects the hosts to the host monitoring. If `preprovision` warns that it cannot assign the role, have an Owner or User Access Administrator run the command it prints, then run `azd provision` again. If the deployment fails with `InsufficientCapacity` on the Redis cache, see [A deployment failed with `InsufficientCapacity` on the Redis cache](#a-deployment-failed-with-insufficientcapacity-on-the-redis-cache). Administrators then sign in to the portal again; if it answers `503`, see [The portal answers 503 and says it cannot reach its session store](#the-portal-answers-503-and-says-it-cannot-reach-its-session-store). +4. For each host that `postprovision` named because it was not running, start it and run [Enable-HostMonitoring.ps1](Enable-HostMonitoring.ps1) for it, as [Connecting the hosts](#connecting-the-hosts) describes. Within about 15 minutes, the workbook should show a fleet snapshot, and `Heartbeat | where Category == "Azure Monitor Agent"` in the workspace should list each host that has the agent. +5. With a test user, open **Linux Desktop** while no host is free, for example with every idle host stopped, and confirm that the wait window appears and the desktop opens once the host is up. In a terminal in that session, `echo $XDG_CACHE_HOME` should print `/var/cache/linuxbroker/users/`. +6. Before you set any minimum to 0, confirm that the **Start on demand** card names no session host whose script cannot wait. The card goes by each session host's latest checkout in the last seven days, so a session host updated since then is listed until its next checkout. + +Every layer tolerates the others being one release behind during the rollout. Against a database without the new procedures, the new API refuses a checkout that finds no host with `409`, as before, and logs it once, and the portal says that the broker does not support start on demand yet. Without `GetFleetSnapshot`, it logs no fleet snapshot, and neither does the previous API build, so the **No fleet snapshot** alert fires until both SQL script `157` and the new API are in place. The previous API build keeps working against the new database: it never answers `202`, so nobody waits, and it refuses a minimum of 0 as before. Session hosts that still run an older script behave as described above. The Linux host changes don't depend on the API: a migrated host keeps caches on its own disk with either API, and a host that is not migrated yet keeps them in the home directory. A host that is not migrated yet gets the agent too, and sends its logs, which the agent reads as root, but its `createuser.log` lines have no timestamp, so each line arrives as a record of its own. The new portal image keeps sessions on its disk, as before, until `azd provision` gives it the cache's settings. The previous portal image ignores those settings and keeps sessions on its disk, which works on one instance, so raise `appServicePlanCapacity` only once the new images run. + ## Manual Steps After `azd up` ### Admin consent @@ -671,6 +1239,8 @@ Verify that the expected resources exist in the target resource group: - the `linuxbroker.internal` private DNS zone with an A record for each Linux host, unless `domainName` was supplied - the NFS storage account, its `home` share, and its private endpoint, unless `nfsShare` was supplied or `deployNfsShare` is `false` - for AVD, the host pool, the desktop and RemoteApp application groups, the workspace, and the **Linux Desktop** application +- unless `deployHostMonitoring` is `false`, the data collection rules `dcr---linux` and `dcr---avd`, the **Linux Broker fleet** workbook, the `alert---...` alert rules, and, when `alertEmailAddresses` is set, the action group +- unless `portalSessionStore` is `filesystem` or `azureCloudName` is `AzureCustom`, the portal's session cache `redis--`, its private endpoint, and the `privatelink.redis.azure.net` private DNS zone, or `privatelink.redis.cache.usgovcloudapi.net` in Azure Government ### Key Vault @@ -720,9 +1290,11 @@ Confirm that: - the frontend and API apps restarted after the ACR builds - the function app restarted after the image build - frontend sign-in works after admin consent is granted +- with the session cache, the portal's app settings have `SESSION_BACKEND` set to `redis` and `REDIS_HOST` set to the cache's host name, and its **Networking** page shows virtual network integration with the `snet-appsvc` subnet - the portal's connectivity test succeeds for each Linux host, which confirms DNS resolution and SSH from the API - a user in the AVD users group can open **Linux Desktop** and land on a Linux desktop, and their home directory is on the NFS share (`df -h ~` on the Linux host) - AVD checkout and Linux host release operations work end to end +- with the host monitoring, each running host appears in `Heartbeat | where Category == "Azure Monitor Agent"` in the Log Analytics workspace, and the **Linux Broker fleet** workbook shows a fleet snapshot from the last 15 minutes ## Rerunning Parts Of The Deployment @@ -742,7 +1314,16 @@ Set-Location .\deploy .\Post-Provision.ps1 -EnvironmentName ``` -That reruns the image build, SQL bootstrap, function-role assignment, VM group sync, and Linux host SQL registration. +That reruns the image build, SQL bootstrap, function-role assignment, VM group sync, Linux host SQL registration, and the host monitoring setup. + +### Connect hosts to the host monitoring + +```powershell +Set-Location .\deploy +.\Enable-HostMonitoring.ps1 -EnvironmentName +``` + +That connects every Linux host and AVD session host and installs the Azure Monitor agent on the running ones that lack it. Add `-HostNames` to limit it to some hosts. See [Connecting the hosts](#connecting-the-hosts). ## Troubleshooting @@ -790,9 +1371,27 @@ Users cannot sign in to a session host that is not joined, even though the `AADL A device object left over from an earlier deployment that used the same VM name blocks the join. In Microsoft Entra ID, find the devices with the session host's name, confirm from the Azure resource ID on the device that its VM no longer exists, and delete that device. The host retries the join on its own, typically within 15 minutes. +### Linux Desktop closes without connecting after the display values changed + +A session host whose `Connect-LinuxBroker.ps1` is older than 2.0.0 refuses the `-FullScreen` and `-MultiMonitor` arguments that `azd provision` adds to the RemoteApp's command line when `avdLinuxDesktopFullScreen` or `avdLinuxDesktopMultiMonitor` is `false`. PowerShell stops before the script runs, so **Linux Desktop** closes at once and nothing is logged under the **LinuxBrokerScript** source. Update the script on that session host with [Update-AvdHostBrokerScript.ps1](Update-AvdHostBrokerScript.ps1) and `-AvdHostNames `, or set both values back to `true` and run `azd provision` again. See [Linux Desktop Display](#linux-desktop-display). + ### Linux Desktop opens but reports that no Linux host is available -`Connect-LinuxBroker.ps1` shows this when checkout does not return a host. Check the **LinuxBrokerScript** source in the session host's Application event log. A `403` from the API means the session host's managed identity is not yet in the AVD host group; rerun `azd hooks run postprovision`. Otherwise, confirm in the portal that at least one Linux host is available. +`Connect-LinuxBroker.ps1` shows this when the broker answers `409`: no host is ready and the broker cannot start one, because start on demand is off, the active rule or window already has `MaxVMs` hosts on, or no stopped host is free to start. Hosts in maintenance, draining or waiting for cleanup are never started. The session host's Application event log has the broker's answer to each attempt under the **LinuxBrokerScript** source, and the portal's scaling activity log shows each start on demand. Otherwise, confirm in the portal that at least one Linux host is available. + +A session host whose script is older than 2.0.0 shows the same message while a host starts for its user, after logging `Failed to retrieve VM information from API response.` three times. The **Start on demand** card on the **Scaling** page names such session hosts; update them as described in [Migration For Existing Deployments](#migration-for-existing-deployments). + +### Linux Desktop says the Linux desktop is taking longer than usual to start + +The user waited `MaxWaitSeconds` (10 minutes by default) and no host became ready for them. Usually the host the broker started did not become reachable on SSH: find it in the portal's scaling activity log under the `Start On Demand` action, then check its power state and reachability. The broker stops counting a host as starting 10 minutes after it started, so the next request can start another, if a stopped host is free and `MaxVMs` allows. A start that Azure refuses is logged by the API, and the user is then refused with `409` instead of waiting. + +### Linux Desktop reports that the Linux Broker could not authenticate this session host + +`Connect-LinuxBroker.ps1` shows this when the session host cannot get a token from its managed identity (`Failed to obtain access token` in the **LinuxBrokerScript** event log) or the broker refuses it (`The broker refused this session host (403)`). A `403` means the session host's managed identity is not yet in the AVD host group; rerun `azd hooks run postprovision`. The script renews a token the broker refuses with `401` once before it gives up. + +### Start VM on Connect does not start a session host + +Start VM on Connect needs the Azure Virtual Desktop service principal to hold **Desktop Virtualization Power On Off Contributor** or **Desktop Virtualization Power On Contributor** on a scope that contains the session hosts. `preprovision` warns when the deployment cannot assign it, and a new assignment can take a few minutes to take effect. Also confirm that `avdStartVmOnConnect` is `true` and that the host pool's **Properties** in the Azure portal show **Start VM on Connect** turned on. In a pooled host pool, Start VM on Connect starts a session host only when none is running, and another only when the running ones reach their session limit. ### The portal shows "No access" after signing in @@ -852,6 +1451,60 @@ azd env set avdVmSize Standard_D8as_v4 `avdVmSize` accepts only the sizes listed in `main.bicep`, and both host types use Trusted Launch, so pick a Gen2-capable size. +### A deployment failed with `Cannot modify extensions in the VM when the VM is not running` + +A host was stopped or deallocated when the deployment tried to update its VM extensions. `preprovision` lists the hosts that are not running so that the deployment leaves out their extensions, so either the host stopped after `preprovision` listed it, which the broker's scaling, start on demand and autoscale can all do at any time, or `preprovision` could not list the VMs and warned that the deployment would update the extensions of every host. Run `azd provision` again, which lists the hosts again. + +### A deployment failed with `AuthorizationFailed` on a role assignment on the subscription + +The deployment tried to give the Azure Virtual Desktop service principal **Desktop Virtualization Power On Off Contributor** on the subscription, and your account cannot assign roles there. `preprovision` could not check this beforehand and warned about it. Have an Owner or User Access Administrator run the command from that warning, which looks like this: + +```powershell +az role assignment create --assignee-object-id --assignee-principal-type ServicePrincipal --role 40c5ff49-9181-41f8-ae61-143b0e78555e --scope /subscriptions/ +``` + +`40c5ff49-9181-41f8-ae61-143b0e78555e` is **Desktop Virtualization Power On Off Contributor**, and `az ad sp list --filter "appId eq '9cdead84-a844-4324-93f2-b2e6bb768d07'" --query "[0].id" --output tsv` prints the object ID of the service principal. Then set `assignAvdAutoscaleRole` to `false`, because the check that could not tell may not tell next time either, and run `azd provision` again. To do without autoscale instead, set `avdScalingPlanEnabled` and `avdStartVmOnConnect` to `false`. See [The autoscale role](#the-autoscale-role). + +### A deployment failed with `RoleAssignmentExists` + +The Azure Virtual Desktop service principal already holds **Desktop Virtualization Power On Off Contributor** on the subscription through an assignment that `preprovision` could not see, so the deployment tried to assign it again. Set `assignAvdAutoscaleRole` to `false` and run `azd provision` again. + +### The AVD scaling plan is not assigned to the host pool + +The deployment leaves the plan assigned to no host pool when `avdScalingPlanEnabled` is `false`, and when `preprovision` finds that the Azure Virtual Desktop service principal cannot get its role, because the service principal cannot be found or created, or because it does not hold the role and your account cannot assign it. `preprovision` says which in a warning. Once the role is assigned, run `azd provision` again. See [The autoscale role](#the-autoscale-role). + +If the `avdScalingPlan` deployment fails with `BadRequest` and `unable to access host pool`, Azure Virtual Desktop could not use its role on the host pool. The deployment assigns the plan last, but a role assigned in the same deployment can still take a few more minutes to take effect, so wait and run `azd provision` again. With `assignAvdAutoscaleRole` set to `false`, also confirm that the service principal holds the role on the subscription. + +A host pool can have only one scaling plan. If the host pool is assigned to another plan, remove it from that plan in the Azure portal, or set `avdScalingPlanEnabled` to `false` to keep the other plan. + +### `preprovision` warns that `avdScalingPlanTimeZone` is not a Windows time zone ID + +`preprovision` looks the value up among the Windows time zone IDs that the machine it runs on knows. Use the Windows ID, such as `Eastern Standard Time` for `America/New_York`. On Windows, `Get-TimeZone -ListAvailable` lists the IDs in its `Id` column, and in PowerShell 7 on any platform the first line below converts an IANA name: + +```powershell +$id = $null; [System.TimeZoneInfo]::TryConvertIanaIdToWindowsId('America/New_York', [ref]$id) | Out-Null; $id +azd env set avdScalingPlanTimeZone "Eastern Standard Time" +``` + +`preprovision` passes a value it does not know to the deployment unchanged, and Azure refuses the scaling plan if Azure does not know it either. + +### A deployment failed with `InsufficientCapacity` on the Redis cache + +Azure had no room for the portal's session cache, at that size, in that region, and leaves the +cache in a failed state. `azd provision` again tries the same cache again, which works once the +region has room. To use another size or region instead, first delete the failed cache, because +Azure refuses to resize a cache that failed to create, and a cache can't change region: + +```powershell +az resource delete --resource-group --name --resource-type Microsoft.Cache/redisEnterprise +``` + +In Azure Government, the type of the cache is `Microsoft.Cache/redis`. + +Then set `portalRedisSku` or `portalRedisLocation`, as described in [Size and cost](#size-and-cost), +and run `azd provision` again. Everything else in the failed deployment is in place, except the +changes to the portal app and the AVD scaling plan, which wait for the cache. + ### Home directories are not on the NFS share Linux hosts mount the share when the broker first creates a user. Confirm that `NFS_SHARE` is set on the API app, that `.file.core.windows.net` resolves to a private IP address from the Linux host, and that the storage account's private endpoint is approved. @@ -860,6 +1513,22 @@ If the share is reachable but `df -h ~` inside a session shows the local disk, c Current hosts keep the home mounted while the host holds the user's lease, which lasts from checkout until the broker returns the host. At return, `manage-lease.sh` unmounts the home before the broker runs `userdel -r`, so only the empty local mount point is removed and the profile stays on the share. The API also refuses to run `userdel -r` while the home is still mounted, and logs `home directory is still mounted` instead. If a checkout fails after the host has written the lease, the API runs the same cleanup before it puts the host back in the pool. +### Sessions are slow on the home directory share + +First check whether the share holds requests back, as described in [Sizing the share](#sizing-the-share), and make it larger if it does. Then check the host, and a session on it: + +```bash +# The read-ahead of each NFS mount on the host, in KiB. It should be 15360. +awk 'NR > 1 { print $4 }' /proc/fs/nfsfs/volumes | while read -r device; do + echo "$device $(cat "/sys/class/bdi/$device/read_ahead_kb")" +done +# In a terminal in the session. It should print /var/cache/linuxbroker/users/. +echo "$XDG_CACHE_HOME" +``` + +- A read-ahead of 128 means the host has not run `install-host-config.sh`, or `/etc/udev/rules.d/99-nfs.rules` is a rule of your own, which the script leaves alone. Run `sudo /usr/local/bin/install-host-config.sh`, which reports each step, or migrate the host with [Migrate-LinuxHostReleaseAgent.ps1](Migrate-LinuxHostReleaseAgent.ps1). +- An empty `XDG_CACHE_HOME` means the session keeps its caches in `~/.cache` on the share. Either the host was migrated after the session started, or `create-user.sh` could not create the cache and said why in `/var/log/createuser.log`, in a line that ends `so the cache of stays in the home directory.` + ### A RHEL session is stuck on a lock screen that will not accept the password The GNOME lock screen inside an xrdp session often cannot be unlocked after a reconnect. Confirm the screen lock configuration actually applied on the host using the commands in [Linux Host Screen Lock](#linux-host-screen-lock). The most common cause is a missing `system-db:local` line in `/etc/dconf/profile/user`, which makes GNOME ignore the settings even though the files under `/etc/dconf/db/local.d/` are present. @@ -897,6 +1566,50 @@ Check `/var/log/release-session.log` on the host once the session has been idle The bootstrap fails deliberately if the host settings cannot be applied, so the problem is visible instead of silently leaving the lock screen enabled. Check that `scriptSourceRoot` is reachable from the host so `apply-host-settings.sh` can be downloaded, or set `linuxHostDisableScreenLock` to `false` to seed the profile with the lock screen left enabled. +### A host's logs do not appear in Log Analytics + +First check whether the host's Azure Monitor agent reaches the workspace: + +```kusto +Heartbeat +| where Category == "Azure Monitor Agent" and Computer startswith "" +| summarize LastHeartbeat = max(TimeGenerated) by Computer +``` + +- **No heartbeat** means the host has no working agent. While the host is running, run [Enable-HostMonitoring.ps1](Enable-HostMonitoring.ps1) with `-HostNames `: it associates the host again, installs the agent, or installs it again where the install failed, and says why when it cannot. If the agent is installed and still sends nothing, check that the host can reach the `AzureMonitor` and `AzureResourceManager` service tags on port 443 and that the VM has its system-assigned managed identity. +- **A heartbeat but no `Syslog` rows** from a Linux host means that rsyslog or syslog-ng doesn't pass messages to the agent. See Microsoft's [troubleshooting guide for Linux](https://learn.microsoft.com/azure/azure-monitor/agents/azure-monitor-agent-troubleshoot-linux-vm). +- **A heartbeat but no rows from a log file** in `LinuxBrokerHost_CL`: the agent sends a file's lines as they are added, so a file that gains none sends nothing. Otherwise check the file's owner and mode: + + ```bash + ls -l /var/log/release-session.log /var/log/release-session-watcher.log + getent group syslog + ``` + + At each run, the release agent and its watcher give their logs to the `syslog` group with mode `-rw-r-----` where the group exists, and keep them `-rw-------` where it doesn't. Logs that are still `-rw-------` although the group exists mean an agent older than 1.3.0; migrate the host with [Migrate-LinuxHostReleaseAgent.ps1](Migrate-LinuxHostReleaseAgent.ps1). + +### The No fleet snapshot alert fires + +The API logs a fleet snapshot at the end of each scaling run that succeeds, so the alert means that none has succeeded for 15 minutes, or that the API cannot read the snapshot: + +- In **Application Insights**, search the API's traces for `GetFleetSnapshot is not deployed yet`. The API logs it once per process until SQL script `157` is applied. Rerun `Post-Provision.ps1`, which applies the SQL scripts, or run [Initialize-Database.ps1](Initialize-Database.ps1). +- Search them for `Could not read the fleet snapshot`, which the API logs with the database's error. +- Check that the function app runs and that the scaling activity log gains an entry every five minutes. When it doesn't, the task function cannot reach the API, or the scaling run fails; the API's failed requests to `/api/scaling/trigger` say why. + +### The portal answers 503 and says it cannot reach its session store + +The portal answers `503` when it cannot read or write a session in its Redis cache. It keeps answering `/health`, so App Service keeps its instances, and it recovers by itself once the cache answers. The portal's log, in its log stream and in Application Insights, gives the cause after `The session store could not be reached`: + +- **Right after the `azd provision` that created the cache**, the portal's identity may not have its access policy yet. The deployment assigns it after the portal restarts with the new settings, and it can take a few more minutes to take effect. Wait, and try again. +- **`AuthenticationError` or `NoPermissionError`** means that the cache refused the portal's identity. Check that the portal app's managed identity holds the `default` access policy on Azure Managed Redis, listed on the cache's **Authentication** page, or `Data Contributor` on Azure Cache for Redis, listed on its **Data Access Configuration** page, with Microsoft Entra authentication on under **Authentication**. `azd provision` assigns it again. +- **`TimeoutError` or `ConnectionError`** means that the portal cannot reach the cache. Check that the cache is running, that the portal app's **Networking** page shows virtual network integration with `snet-appsvc`, that the private DNS zone, `privatelink.redis.azure.net` or `privatelink.redis.cache.usgovcloudapi.net`, has an A record for the cache and a link to the virtual network, and that `REDIS_HOST` and `REDIS_PORT` in the portal's app settings match the cache's host name and port: 10000 for Azure Managed Redis, 6380 for Azure Cache for Redis. +- **`RequestTokenErr` or `TokenRenewalErr`** means that the portal's managed identity could not get a token for the cache. Check that the portal app has its system-assigned identity turned on. In Azure Government, the deployment sets `REDIS_ENTRA_RESOURCE` to `acca5fbb-b7e4-4009-81f1-37e38fd66d78`, the application ID that Azure Cache for Redis issues tokens for. + +To bring sign-in back while you investigate, set `portalSessionStore` to `filesystem`, with `appServicePlanCapacity` at 1, and run `azd provision`. + +### Every administrator was signed out of the portal + +This happens once, when an upgrade moves the sessions from the portal's disk to the Redis cache, or back. Otherwise, a session ends 12 hours after its last request, or after `SESSION_LIFETIME_HOURS`. With `portalSessionStore` set to `filesystem`, each restart of a portal instance also signs out its administrators, for example when `azd provision` or a new image restarts the portal. Sessions in the cache survive the portal's restarts, but not the loss of the cache's data, for example when the cache is deleted and created again, or when a `Basic` cache, which has no replica, restarts for maintenance. + ## Related Files - [azure.yaml](azure.yaml) @@ -905,5 +1618,9 @@ The bootstrap fails deliberately if the host settings cannot be applied, so the - [Build-ContainerImages.ps1](Build-ContainerImages.ps1) - [Initialize-Database.ps1](Initialize-Database.ps1) - [Register-LinuxHostSqlRecords.ps1](Register-LinuxHostSqlRecords.ps1) +- [Enable-HostMonitoring.ps1](Enable-HostMonitoring.ps1) - [bicep/main.bicep](bicep/main.bicep) +- [bicep/modules/core/host-monitoring.bicep](bicep/modules/core/host-monitoring.bicep) +- [bicep/modules/core/redis-session-store.bicep](bicep/modules/core/redis-session-store.bicep) +- [bicep/modules/core/redis-session-access.bicep](bicep/modules/core/redis-session-access.bicep) - [../sql_queries/README.md](../sql_queries/README.md) \ No newline at end of file diff --git a/deploy/Enable-HostMonitoring.ps1 b/deploy/Enable-HostMonitoring.ps1 new file mode 100644 index 0000000..4913628 --- /dev/null +++ b/deploy/Enable-HostMonitoring.ps1 @@ -0,0 +1,522 @@ +<# +.SYNOPSIS + Connects the Linux hosts and the AVD session hosts to the host monitoring. + +.DESCRIPTION + With deployHostMonitoring on, the deployment creates two data collection rules: one that + collects the broker's logs, the xrdp logs and syslog from the Linux hosts, and one that + collects the Connect-LinuxBroker.ps1 events and Remote Desktop client problems from the AVD + session hosts. The deployment cannot install the Azure Monitor agent itself, because Azure + refuses to add an extension to a VM that is not running, and scaling powers hosts off. + + This script associates every VM tagged broker-role=linux-host with the Linux rule and every + VM tagged broker-role=avd-host with the AVD rule, which works whether the VM is running or + not. It then installs the Azure Monitor agent, with automatic upgrades, on each running VM + that does not have it yet, and waits for the installs to finish. + + A VM that is not running keeps its association but gets no agent, so it sends nothing: start + it and run this script again, with -HostNames to limit the run to it. Every host is + attempted, and the ones that failed are reported together at the end, with a non-zero exit + code. Running the script again is safe. + + Values that are not given are read from the azd environment. When the deployment has no host + monitoring, the script says so and changes nothing. + +.EXAMPLE + .\Enable-HostMonitoring.ps1 -EnvironmentName + +.EXAMPLE + .\Enable-HostMonitoring.ps1 -EnvironmentName -HostNames , +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $false)] + [string]$SubscriptionId, + + [Parameter(Mandatory = $false)] + [string]$EnvironmentName, + + [Parameter(Mandatory = $false)] + [string]$LinuxHostDataCollectionRuleId, + + [Parameter(Mandatory = $false)] + [string]$AvdHostDataCollectionRuleId, + + [Parameter(Mandatory = $false)] + [string[]]$HostNames, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 120)] + [int]$AgentTimeoutMinutes = 20 +) + +# One association per VM under a fixed name, so running the script again replaces it. +$AssociationName = 'linuxbroker-host-monitoring' +$AgentPollSeconds = 30 + +# Runs az, keeping what it writes to standard output apart from its error lines. The error lines +# must not stop the script, whatever the caller's preference. +function Invoke-AzCommand { + param( + [Parameter(Mandatory = $true)] + [string[]]$Arguments + ) + + $callerErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $output = @(az @Arguments 2>&1) + $exitCode = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $callerErrorActionPreference + } + + $standardOutput = @($output | Where-Object { $_ -isnot [System.Management.Automation.ErrorRecord] } | ForEach-Object { [string]$_ }) + $errorOutput = @($output | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] } | ForEach-Object { $_.ToString() }) + return [pscustomobject]@{ + ExitCode = $exitCode + Output = ($standardOutput -join "`n").Trim() + Error = ($errorOutput -join ' ').Trim() + } +} + +function Get-AzdEnvValue { + param( + [AllowEmptyString()] + [string]$EnvironmentName, + + [Parameter(Mandatory = $true)] + [string]$Key + ) + + if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { + return '' + } + + # azd reports a value it does not have on stderr, which must not stop the script. + $callerErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $value = azd env get-value $Key --environment $EnvironmentName 2>$null + $exitCode = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $callerErrorActionPreference + } + + if ($exitCode -ne 0) { + return '' + } + + return ($value | Out-String).Trim() +} + +# A property's value, or $null when the object or the property is missing, which StrictMode +# would otherwise refuse. +function Get-PropertyValue { + param( + $InputObject, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + if ($null -eq $InputObject) { + return $null + } + + $property = $InputObject.PSObject.Properties[$Name] + if ($null -eq $property) { + return $null + } + + return $property.Value +} + +function Get-VmTagValue { + param( + [Parameter(Mandatory = $true)] + $Vm, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + return [string](Get-PropertyValue -InputObject (Get-PropertyValue -InputObject $Vm -Name 'tags') -Name $Name) +} + +function Test-DataCollectionRuleId { + param( + [AllowEmptyString()] + [string]$Value + ) + + return $Value -match '^/subscriptions/[0-9a-fA-F-]{36}/resourceGroups/[^/\s]+/providers/Microsoft\.Insights/dataCollectionRules/[^/\s]+\z' +} + +function Set-DataCollectionRuleAssociation { + param( + [Parameter(Mandatory = $true)] + [string]$VmId, + + [Parameter(Mandatory = $true)] + [string]$RuleId + ) + + $body = ConvertTo-Json -Compress -InputObject ([ordered]@{ + properties = [ordered]@{ + description = 'Linux Broker host monitoring, from deploy/Enable-HostMonitoring.ps1.' + dataCollectionRuleId = $RuleId + } + }) + + # The body reaches az as @file: on Windows az runs through az.cmd, where cmd.exe would take + # the quotes in inline JSON for its own. + $bodyFile = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath ('linuxbroker-dcra-{0}.json' -f [guid]::NewGuid().ToString('N')) + [System.IO.File]::WriteAllText($bodyFile, $body, (New-Object System.Text.UTF8Encoding -ArgumentList $false)) + try { + # A URL without a host is a resource ID on the current cloud's Resource Manager. The API + # version goes in --url-parameters, because az.cmd would hand an unquoted & to cmd.exe. + $result = Invoke-AzCommand -Arguments @( + 'rest', '--method', 'put', + '--url', "$VmId/providers/Microsoft.Insights/dataCollectionRuleAssociations/$AssociationName", + '--url-parameters', 'api-version=2022-06-01', + '--body', "@$bodyFile", + '--output', 'none', '--only-show-errors') + } + finally { + Remove-Item -LiteralPath $bodyFile -Force -ErrorAction SilentlyContinue + } + + if ($result.ExitCode -ne 0) { + throw ("Could not associate the data collection rule. {0}" -f $result.Error).Trim() + } +} + +# An extension property, which az lists flattened; a nested properties object is read as well. +# NestedNames are read only from the nested object, where 'type' is the extension's type rather +# than the resource type it is at the top level. +function Get-ExtensionPropertyValue { + param( + [Parameter(Mandatory = $true)] + $Extension, + + [Parameter(Mandatory = $true)] + [string[]]$Names, + + [string[]]$NestedNames = @() + ) + + $properties = Get-PropertyValue -InputObject $Extension -Name 'properties' + $candidates = @($Names | ForEach-Object { [pscustomobject]@{ Source = $Extension; Name = $_ } }) + + @(($Names + $NestedNames) | ForEach-Object { [pscustomobject]@{ Source = $properties; Name = $_ } }) + foreach ($candidate in $candidates) { + $value = [string](Get-PropertyValue -InputObject $candidate.Source -Name $candidate.Name) + if ($value.Length -gt 0) { + return $value + } + } + + return '' +} + +# The agent's extension on the VM, as its instance name and provisioning state, or $null when the +# VM has none. Azure allows one extension of each type on a VM, and the portal or a policy may +# have installed the agent under another name, so the type counts as well as the name. +function Get-AgentExtension { + param( + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string]$VmName, + + [Parameter(Mandatory = $true)] + [string]$AgentName + ) + + $result = Invoke-AzCommand -Arguments @('vm', 'extension', 'list', '--resource-group', $ResourceGroupName, '--vm-name', $VmName, '--output', 'json', '--only-show-errors') + if ($result.ExitCode -ne 0) { + throw ("Could not list the extensions of '{0}'. {1}" -f $VmName, $result.Error).Trim() + } + + if ([string]::IsNullOrWhiteSpace($result.Output)) { + return $null + } + + foreach ($extension in @(ConvertFrom-Json -InputObject $result.Output | ForEach-Object { $_ })) { + $name = [string](Get-PropertyValue -InputObject $extension -Name 'name') + # Older versions of az call the type virtualMachineExtensionType. + $type = Get-ExtensionPropertyValue -Extension $extension -Names @('typePropertiesType', 'virtualMachineExtensionType') -NestedNames @('type') + $publisher = Get-ExtensionPropertyValue -Extension $extension -Names @('publisher') + if ($name -eq $AgentName -or ($type -eq $AgentName -and $publisher -eq 'Microsoft.Azure.Monitor')) { + return [pscustomobject]@{ + Name = $name + ProvisioningState = Get-ExtensionPropertyValue -Extension $extension -Names @('provisioningState') + } + } + } + + return $null +} + +function Start-AgentInstall { + param( + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string]$VmName, + + [Parameter(Mandatory = $true)] + [string]$AgentName, + + [Parameter(Mandatory = $true)] + [string]$InstanceName, + + # An extension that failed runs again only when forced, as its settings are unchanged. + [switch]$Reinstall + ) + + # The agent authenticates with the VM's system-assigned identity, which needs no settings. + $arguments = @( + 'vm', 'extension', 'set', + '--resource-group', $ResourceGroupName, + '--vm-name', $VmName, + '--name', $AgentName, + '--publisher', 'Microsoft.Azure.Monitor', + '--extension-instance-name', $InstanceName, + '--enable-auto-upgrade', 'true') + if ($Reinstall) { + $arguments += '--force-update' + } + + $result = Invoke-AzCommand -Arguments ($arguments + @('--no-wait', '--output', 'none', '--only-show-errors')) + if ($result.ExitCode -ne 0) { + throw ("Could not start installing the Azure Monitor agent. {0}" -f $result.Error).Trim() + } +} + +function Invoke-HostMonitoringSetup { + param( + [string]$ResourceGroupName, + [string]$SubscriptionId, + [string]$EnvironmentName, + [string]$LinuxHostDataCollectionRuleId, + [string]$AvdHostDataCollectionRuleId, + [string[]]$HostNames, + [ValidateRange(1, 120)] + [int]$AgentTimeoutMinutes = 20 + ) + + if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { + $EnvironmentName = if (-not [string]::IsNullOrWhiteSpace($env:AZURE_ENV_NAME)) { + $env:AZURE_ENV_NAME + } + elseif (-not [string]::IsNullOrWhiteSpace($env:AZURE_ENVIRONMENT_NAME)) { + $env:AZURE_ENVIRONMENT_NAME + } + else { + '' + } + } + + if ([string]::IsNullOrWhiteSpace($LinuxHostDataCollectionRuleId)) { + $LinuxHostDataCollectionRuleId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'linuxHostDataCollectionRuleId' + } + + if ([string]::IsNullOrWhiteSpace($AvdHostDataCollectionRuleId)) { + $AvdHostDataCollectionRuleId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'avdHostDataCollectionRuleId' + } + + $LinuxHostDataCollectionRuleId = ([string]$LinuxHostDataCollectionRuleId).Trim() + $AvdHostDataCollectionRuleId = ([string]$AvdHostDataCollectionRuleId).Trim() + if ($LinuxHostDataCollectionRuleId.Length -eq 0 -and $AvdHostDataCollectionRuleId.Length -eq 0) { + Write-Host 'This deployment has no host monitoring, so there is nothing to connect: deployHostMonitoring is false, or azd provision has not run since the monitoring was added.' + return + } + + foreach ($ruleId in @($LinuxHostDataCollectionRuleId, $AvdHostDataCollectionRuleId)) { + if ($ruleId.Length -gt 0 -and -not (Test-DataCollectionRuleId -Value $ruleId)) { + throw "'$ruleId' is not the resource ID of a data collection rule, such as /subscriptions//resourceGroups//providers/Microsoft.Insights/dataCollectionRules/." + } + } + + if ([string]::IsNullOrWhiteSpace($ResourceGroupName)) { + $ResourceGroupName = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'resourceGroupName' + } + + if ([string]::IsNullOrWhiteSpace($ResourceGroupName)) { + throw 'The host monitoring setup needs ResourceGroupName, as a parameter or an azd environment value.' + } + + if ([string]::IsNullOrWhiteSpace($SubscriptionId)) { + $SubscriptionId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'AZURE_SUBSCRIPTION_ID' + } + + if ([string]::IsNullOrWhiteSpace($SubscriptionId)) { + $SubscriptionId = $env:AZURE_SUBSCRIPTION_ID + } + + if (-not [string]::IsNullOrWhiteSpace($SubscriptionId)) { + $result = Invoke-AzCommand -Arguments @('account', 'set', '--subscription', $SubscriptionId) + if ($result.ExitCode -ne 0) { + throw "Could not select subscription '$SubscriptionId'." + } + } + + $roles = @{ + 'linux-host' = [pscustomobject]@{ Label = 'Linux host'; RuleId = $LinuxHostDataCollectionRuleId; Agent = 'AzureMonitorLinuxAgent' } + 'avd-host' = [pscustomobject]@{ Label = 'AVD session host'; RuleId = $AvdHostDataCollectionRuleId; Agent = 'AzureMonitorWindowsAgent' } + } + + $result = Invoke-AzCommand -Arguments @('vm', 'list', '--resource-group', $ResourceGroupName, '--show-details', '--output', 'json', '--only-show-errors') + if ($result.ExitCode -ne 0) { + throw "Could not list the VMs in resource group '$ResourceGroupName'. $($result.Error)".Trim() + } + + $vms = @() + if (-not [string]::IsNullOrWhiteSpace($result.Output)) { + $vms = @(ConvertFrom-Json -InputObject $result.Output | ForEach-Object { $_ }) + } + + $hosts = @($vms | Where-Object { + $vmRole = Get-VmTagValue -Vm $_ -Name 'broker-role' + $roles.ContainsKey($vmRole) -and $roles[$vmRole].RuleId.Length -gt 0 + }) + + if ($HostNames -and $HostNames.Count -gt 0) { + $requestedNames = New-Object 'System.Collections.Generic.HashSet[string]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($name in $HostNames) { + if (-not [string]::IsNullOrWhiteSpace($name)) { + [void]$requestedNames.Add($name.Trim()) + } + } + + $filteredHosts = @($hosts | Where-Object { $requestedNames.Contains([string]$_.name) }) + foreach ($name in $requestedNames) { + if (-not ($filteredHosts | Where-Object { $_.name -eq $name })) { + Write-Warning "Requested host '$name' is not a Linux host or AVD session host in resource group '$ResourceGroupName'." + } + } + + $hosts = $filteredHosts + } + + if ($hosts.Count -eq 0) { + Write-Host "No Linux hosts or AVD session hosts to connect to the host monitoring in resource group '$ResourceGroupName'." + return + } + + # One host that is off or failing must not leave the rest unmonitored, so every one is + # attempted and the failures are reported together at the end. + $connected = New-Object System.Collections.Generic.List[string] + $notRunning = New-Object System.Collections.Generic.List[string] + $failures = New-Object System.Collections.Generic.List[string] + $installing = New-Object System.Collections.Generic.List[object] + + foreach ($vm in $hosts) { + $name = [string]$vm.name + $role = $roles[(Get-VmTagValue -Vm $vm -Name 'broker-role')] + try { + Set-DataCollectionRuleAssociation -VmId ([string]$vm.id) -RuleId $role.RuleId + } + catch { + Write-Warning "Could not connect $($role.Label) '$name' to the host monitoring: $($_.Exception.Message)" + $failures.Add($name) + continue + } + + $powerState = [string](Get-PropertyValue -InputObject $vm -Name 'powerState') + if (-not [string]::IsNullOrWhiteSpace($powerState) -and $powerState -notmatch 'running') { + Write-Warning "$($role.Label) '$name' is connected, but it is not running ($powerState), so it has no Azure Monitor agent and sends no logs yet." + $notRunning.Add($name) + continue + } + + try { + $extension = Get-AgentExtension -ResourceGroupName $ResourceGroupName -VmName $name -AgentName $role.Agent + $state = if ($null -ne $extension) { $extension.ProvisioningState } else { '' } + if ($state -eq 'Succeeded') { + Write-Host "$($role.Label) '$name' is connected, and already has the Azure Monitor agent." + $connected.Add($name) + continue + } + + # An install that is under way is waited for rather than started again. + if ($state -ne 'Creating' -and $state -ne 'Updating') { + $instanceName = if ($null -ne $extension -and $extension.Name.Length -gt 0) { $extension.Name } else { $role.Agent } + Write-Host "Installing the Azure Monitor agent on $($role.Label) '$name'..." + Start-AgentInstall -ResourceGroupName $ResourceGroupName -VmName $name -AgentName $role.Agent -InstanceName $instanceName -Reinstall:($null -ne $extension) + } + + $installing.Add([pscustomobject]@{ Name = $name; Role = $role }) + } + catch { + Write-Warning "Could not install the Azure Monitor agent on $($role.Label) '$name': $($_.Exception.Message)" + $failures.Add($name) + } + } + + # The installs run side by side. A poll that cannot read a VM's extensions is retried at the + # next one. + $pollsLeft = [int][Math]::Ceiling(($AgentTimeoutMinutes * 60) / $AgentPollSeconds) + while ($installing.Count -gt 0 -and $pollsLeft -gt 0) { + $pollsLeft-- + Start-Sleep -Seconds $AgentPollSeconds + foreach ($install in $installing.ToArray()) { + try { + $extension = Get-AgentExtension -ResourceGroupName $ResourceGroupName -VmName $install.Name -AgentName $install.Role.Agent + } + catch { + continue + } + + $state = if ($null -ne $extension) { $extension.ProvisioningState } else { '' } + if ($state -eq 'Succeeded') { + Write-Host "$($install.Role.Label) '$($install.Name)' is connected, with the Azure Monitor agent installed." + $connected.Add($install.Name) + [void]$installing.Remove($install) + } + elseif ($state -eq 'Failed') { + Write-Warning "The Azure Monitor agent failed to install on $($install.Role.Label) '$($install.Name)'. The $($install.Role.Agent) extension's status on the VM says why." + $failures.Add($install.Name) + [void]$installing.Remove($install) + } + } + } + + foreach ($install in $installing) { + Write-Warning "The Azure Monitor agent install on $($install.Role.Label) '$($install.Name)' did not finish within $AgentTimeoutMinutes minute(s)." + $failures.Add($install.Name) + } + + if ($notRunning.Count -gt 0) { + Write-Warning "No Azure Monitor agent yet because they are not running: $($notRunning -join ', '). Once they are running, run Enable-HostMonitoring.ps1 again with -HostNames $($notRunning -join ',')." + } + + if ($failures.Count -gt 0) { + throw "The host monitoring could not be set up on: $($failures -join ', '). Run Enable-HostMonitoring.ps1 again with -HostNames $($failures -join ',') to retry them." + } + + Write-Host "The host monitoring is set up on $($connected.Count) host(s)." +} + +# Dot-sourcing the script, as its tests do, defines the functions without changing anything. +if ($MyInvocation.InvocationName -ne '.') { + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + + Invoke-HostMonitoringSetup ` + -ResourceGroupName $ResourceGroupName ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -LinuxHostDataCollectionRuleId $LinuxHostDataCollectionRuleId ` + -AvdHostDataCollectionRuleId $AvdHostDataCollectionRuleId ` + -HostNames $HostNames ` + -AgentTimeoutMinutes $AgentTimeoutMinutes +} diff --git a/deploy/Initialize-DeploymentEnvironment.ps1 b/deploy/Initialize-DeploymentEnvironment.ps1 index 3d35ef7..ed6748e 100644 --- a/deploy/Initialize-DeploymentEnvironment.ps1 +++ b/deploy/Initialize-DeploymentEnvironment.ps1 @@ -59,6 +59,11 @@ if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { $graphAppId = '00000003-0000-0000-c000-000000000000' $defaultAccessAppRoleId = '00000000-0000-0000-0000-000000000000' +# The Azure Virtual Desktop first-party app, whose service principal starts and stops session +# hosts for the scaling plan and Start VM on Connect. +$avdServicePrincipalAppId = '9cdead84-a844-4324-93f2-b2e6bb768d07' +# Desktop Virtualization Power On Off Contributor, which main.bicep assigns to that service principal. +$avdAutoscaleRoleDefinitionId = '40c5ff49-9181-41f8-ae61-143b0e78555e' $frontendGraphDelegatedPermissions = @( @{ name = 'User.Read'; id = 'e1fe6dd8-ba31-4d61-89e7-88639da4683d' } @{ name = 'profile'; id = '14dad69e-099b-42c9-810b-d002981feec1' } @@ -815,6 +820,646 @@ function Ensure-EntraRdpAuthentication { } } +# A property's value, or $null when the object or the property is missing. The lookup ignores +# case, as Azure does with tag names. +function Get-PropertyValue { + param( + $InputObject, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + if ($null -eq $InputObject) { + return $null + } + + $property = $InputObject.PSObject.Properties[$Name] + if ($null -eq $property) { + return $null + } + + return $property.Value +} + +function Get-VmTagValue { + param( + [Parameter(Mandatory = $true)] + $Vm, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + return [string](Get-PropertyValue -InputObject (Get-PropertyValue -InputObject $Vm -Name 'tags') -Name $Name) +} + +# A time of day for the AVD scaling plan, as HH:mm on a 24-hour clock. +function ConvertTo-TimeOfDayParameterValue { + param( + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][string]$DefaultValue + ) + + $value = Get-AzdEnvValue -Key $Key + if ([string]::IsNullOrWhiteSpace($value)) { + $value = $DefaultValue + } + + $match = [regex]::Match($value.Trim(), '^([01]?[0-9]|2[0-3]):([0-5][0-9])\z') + if (-not $match.Success) { + throw "azd environment value '$Key' must be a time of day as HH:mm on a 24-hour clock, such as 07:00, but was '$value'." + } + + return ('{0:D2}:{1}' -f [int]$match.Groups[1].Value, $match.Groups[2].Value) +} + +function ConvertTo-PercentParameterValue { + param( + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][int]$DefaultValue, + [Parameter(Mandatory = $false)][int]$Minimum = 0 + ) + + $value = ConvertTo-IntParameterValue -Key $Key -DefaultValue $DefaultValue + if ($value -lt $Minimum -or $value -gt 100) { + throw "azd environment value '$Key' must be a whole number from $Minimum to 100, but was '$value'." + } + + return $value +} + +function ConvertTo-PositiveIntParameterValue { + param( + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][int]$DefaultValue + ) + + $value = ConvertTo-IntParameterValue -Key $Key -DefaultValue $DefaultValue + if ($value -lt 1) { + throw "azd environment value '$Key' must be a whole number of 1 or more, but was '$value'." + } + + return $value +} + +# Email addresses separated by commas or semicolons, returned comma-separated without repeats. +function ConvertTo-EmailListParameterValue { + param([Parameter(Mandatory = $true)][string]$Key) + + $value = Get-AzdEnvValue -Key $Key + if ([string]::IsNullOrWhiteSpace($value)) { + return '' + } + + $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + $addresses = New-Object System.Collections.Generic.List[string] + foreach ($entry in ($value -split '[,;]')) { + $address = $entry.Trim() + if ($address.Length -eq 0) { + continue + } + + if ($address -notmatch '^[^@\s]+@[^@\s]+\.[^@\s]+\z') { + throw "azd environment value '$Key' must list email addresses separated by commas or semicolons, but '$address' is not an email address." + } + + if ($seen.Add($address)) { + $addresses.Add($address) + } + } + + return ($addresses -join ',') +} + +# Where the portal keeps sign-in sessions. AzureCustom has no known Redis service, private DNS +# zone or Entra resource, so its portal keeps them on disk. +function Resolve-PortalSessionStore { + param([Parameter(Mandatory = $true)][string]$CloudName) + + $value = Get-AzdEnvValue -Key 'portalSessionStore' + if ([string]::IsNullOrWhiteSpace($value)) { + $value = 'redis' + } + + $store = $value.Trim().ToLowerInvariant() + if ($store -notin @('redis', 'filesystem')) { + throw "azd environment value 'portalSessionStore' must be 'redis' or 'filesystem', but was '$value'." + } + + if ($store -eq 'redis' -and $CloudName -eq 'AzureCustom') { + Write-Warning "portalSessionStore is 'redis', but the AzureCustom cloud has no Redis profile, so the portal keeps sessions on each instance's disk." + return 'filesystem' + } + + return $store +} + +# Azure Managed Redis sizes look like Balanced_B0; Azure Cache for Redis sizes like Standard_C1. +function ConvertTo-PortalRedisSkuParameterValue { + param([Parameter(Mandatory = $true)][string]$CloudName) + + $value = Get-AzdEnvValue -Key 'portalRedisSku' + if ([string]::IsNullOrWhiteSpace($value)) { + return '' + } + + $sku = $value.Trim() + if ($CloudName -eq 'AzureUSGovernment') { + # The Bicep reads the tier, family and capacity out of the name. + if ($sku -cnotmatch '^(Basic_C[0-6]|Standard_C[0-6]|Premium_P[1-5])\z') { + throw "azd environment value 'portalRedisSku' must be an Azure Cache for Redis size, such as Standard_C1, Standard_C2 or Premium_P1, but was '$value'." + } + } + elseif ($sku -cnotmatch '^[A-Z][A-Za-z]*_[A-Z]+[0-9]+\z') { + throw "azd environment value 'portalRedisSku' must be an Azure Managed Redis size, such as Balanced_B0 or Balanced_B1, but was '$value'." + } + + return $sku +} + +# The region of the portal's session cache, when it cannot be the deployment's, for example where +# Azure reports InsufficientCapacity for the cache there. Written as az lists regions, such as +# eastus, so a display name like 'East US' is folded into that form. +function ConvertTo-PortalRedisLocationParameterValue { + $value = Get-AzdEnvValue -Key 'portalRedisLocation' + if ([string]::IsNullOrWhiteSpace($value)) { + return '' + } + + $location = ($value -replace '\s', '').ToLowerInvariant() + if ($location -notmatch '^[a-z][a-z0-9]*\z') { + throw "azd environment value 'portalRedisLocation' must be an Azure region name, such as eastus or centralus, but was '$value'." + } + + return $location +} + +# Instances of the App Service plan, which the portal, the API and the task function all run on. +function Resolve-AppServicePlanCapacity { + param( + [Parameter(Mandatory = $true)][string]$PortalSessionStore, + [Parameter(Mandatory = $false)][AllowEmptyString()][string]$SqlDatabaseSkuName = '' + ) + + $capacity = ConvertTo-IntParameterValue -Key 'appServicePlanCapacity' -DefaultValue 1 + if ($capacity -lt 1 -or $capacity -gt 30) { + throw "azd environment value 'appServicePlanCapacity' must be a whole number from 1 to 30, but was '$capacity'." + } + + if ($capacity -gt 1 -and $PortalSessionStore -eq 'filesystem') { + Write-Warning "appServicePlanCapacity is $capacity, but the portal keeps sessions on each instance's disk. Sign-ins then rely on ARR affinity, and anyone whose instance restarts or is removed must sign in again." + } + + # Each instance's API runs two worker processes, each running up to DB_MAX_CONCURRENCY (6) + # database requests at once. + if ($SqlDatabaseSkuName.Trim() -eq 'Basic' -and $capacity * 12 -gt 30) { + Write-Warning "appServicePlanCapacity is $capacity, and each instance's API can run 12 database requests at once, but the Basic database tier runs at most 30. Set sqlDatabaseSkuName to S1 or higher." + } + + return $capacity +} + +# The scaling plan's phases start in order within one day: ramp-up, peak, ramp-down, off-peak. +function Assert-AvdScalingPlanTimeOrder { + param([Parameter(Mandatory = $true)][System.Collections.IDictionary]$Times) + + $previousKey = $null + $previousMinutes = -1 + foreach ($key in @($Times.Keys)) { + $parts = ([string]$Times[$key]).Split(':') + $minutes = ([int]$parts[0] * 60) + [int]$parts[1] + if ($null -ne $previousKey -and $minutes -le $previousMinutes) { + throw "azd environment value '$key' ($($Times[$key])) must be later than '$previousKey' ($($Times[$previousKey])). The scaling plan's ramp-up, peak, ramp-down and off-peak times must come in that order within one day." + } + + $previousKey = $key + $previousMinutes = $minutes + } +} + +# The canonical form of a Windows time zone ID, or '' when this machine does not know it. It does +# not use FindSystemTimeZoneById, which also accepts IANA names on .NET 6 and later. +function Resolve-WindowsTimeZoneId { + param([Parameter(Mandatory = $true)][string]$TimeZoneId) + + $id = $TimeZoneId.Trim() + if ([System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT) { + foreach ($zone in [System.TimeZoneInfo]::GetSystemTimeZones()) { + if ([string]::Equals($zone.Id, $id, [System.StringComparison]::OrdinalIgnoreCase)) { + return $zone.Id + } + } + + return '' + } + + # Elsewhere the system time zones have IANA IDs, so ask ICU whether it knows the Windows ID. + $ianaId = $null + if ($null -ne [System.TimeZoneInfo].GetMethod('TryConvertWindowsIdToIanaId', [type[]]@([string], [string].MakeByRefType())) -and + [System.TimeZoneInfo]::TryConvertWindowsIdToIanaId($id, [ref]$ianaId)) { + return $id + } + + return '' +} + +# The broker's Linux hosts and AVD session hosts that exist but are not running, and the +# excludeFromScaling tags on the session hosts. Azure refuses to change an extension on a VM that +# is not running, and a deployment replaces the tags of every VM it declares. +function Get-HostPowerStateSnapshot { + param( + [Parameter(Mandatory = $true)][string]$SubscriptionId, + [Parameter(Mandatory = $true)][string]$ResourceGroupName + ) + + $snapshot = @{ + Listed = $false + NotRunning = @() + NotRunningDisplay = @() + AvdScalingExclusions = [ordered]@{} + } + $unlistedWarning = "so this deployment updates the VM extensions of every host. If one is not running, the deployment fails with 'Cannot modify extensions in the VM when the VM is not running'; start that host, or run azd provision again." + + $groupExists = (@(az group exists --name $ResourceGroupName --subscription $SubscriptionId --only-show-errors 2>$null) -join '').Trim() + if ($LASTEXITCODE -ne 0) { + Write-Warning "Could not check whether resource group '$ResourceGroupName' exists, $unlistedWarning" + return $snapshot + } + + if ($groupExists -ne 'true') { + $snapshot.Listed = $true + return $snapshot + } + + $vmJson = (@(az vm list --resource-group $ResourceGroupName --subscription $SubscriptionId --show-details --output json --only-show-errors 2>$null) -join "`n") + $vms = $null + if ($LASTEXITCODE -eq 0) { + try { + $vms = @(ConvertFrom-Json -InputObject $vmJson | ForEach-Object { $_ }) + } + catch { + $vms = $null + } + } + + if ($null -eq $vms) { + Write-Warning "Could not list the VMs in resource group '$ResourceGroupName', $unlistedWarning" + return $snapshot + } + + $notRunning = New-Object System.Collections.Generic.List[string] + $notRunningDisplay = New-Object System.Collections.Generic.List[string] + foreach ($vm in $vms) { + if ($null -eq $vm) { + continue + } + + $role = Get-VmTagValue -Vm $vm -Name 'broker-role' + if ($role -ne 'linux-host' -and $role -ne 'avd-host') { + continue + } + + $name = [string](Get-PropertyValue -InputObject $vm -Name 'name') + if ([string]::IsNullOrWhiteSpace($name)) { + continue + } + + # An empty power state means Azure could not report one, so the host is treated as running. + $powerState = [string](Get-PropertyValue -InputObject $vm -Name 'powerState') + if (-not [string]::IsNullOrWhiteSpace($powerState) -and $powerState -notmatch 'running') { + $notRunning.Add($name.ToLowerInvariant()) + $notRunningDisplay.Add("$name ($powerState)") + } + + # Autoscale leaves alone a session host that carries the tag, whatever its value. + if ($role -eq 'avd-host') { + $exclusion = Get-PropertyValue -InputObject (Get-PropertyValue -InputObject $vm -Name 'tags') -Name 'excludeFromScaling' + if ($null -ne $exclusion) { + $snapshot.AvdScalingExclusions[$name.ToLowerInvariant()] = [string]$exclusion + } + } + } + + $snapshot.Listed = $true + $snapshot.NotRunning = $notRunning.ToArray() + $snapshot.NotRunningDisplay = $notRunningDisplay.ToArray() + return $snapshot +} + +# The Azure Virtual Desktop service principal's object ID in this tenant, or '' when it cannot be +# found or created. It is looked up by app ID only, never by display name, so that a lookalike app +# registration can never be given a role on the subscription. +function Resolve-AvdServicePrincipalObjectId { + param( + [AllowEmptyString()][string]$ConfiguredObjectId = '', + [Parameter(Mandatory = $true)][string]$AppId, + [switch]$SkipCreate + ) + + $parsed = [guid]::Empty + if (-not [string]::IsNullOrWhiteSpace($ConfiguredObjectId)) { + if (-not [guid]::TryParseExact($ConfiguredObjectId.Trim(), 'D', [ref]$parsed)) { + throw "azd environment value 'avdServicePrincipalObjectId' must be the object ID of the Azure Virtual Desktop service principal, a GUID, but was '$ConfiguredObjectId'." + } + + return $parsed.ToString() + } + + if (-not [guid]::TryParseExact($AppId.Trim(), 'D', [ref]$parsed)) { + throw "azd environment value 'avdServicePrincipalAppId' must be the app ID of the Azure Virtual Desktop service principal, a GUID, but was '$AppId'." + } + + $appIdValue = $parsed.ToString() + $listJson = (@(az ad sp list --filter "appId eq '$appIdValue'" --query '[].id' --output json --only-show-errors 2>$null) -join "`n") + if ($LASTEXITCODE -eq 0) { + $ids = @() + try { + $ids = @(ConvertFrom-Json -InputObject $listJson | ForEach-Object { [string]$_ }) + } + catch { + $ids = @() + } + + foreach ($id in $ids) { + if ([guid]::TryParseExact($id, 'D', [ref]$parsed)) { + return $parsed.ToString() + } + } + } + + if ($SkipCreate) { + return '' + } + + $createdId = (@(az ad sp create --id $appIdValue --query id --output tsv --only-show-errors 2>$null) -join '').Trim() + if ($LASTEXITCODE -eq 0 -and [guid]::TryParseExact($createdId, 'D', [ref]$parsed)) { + return $parsed.ToString() + } + + return '' +} + +# A GET on Azure Resource Manager through az rest. The query string goes in --url-parameters, +# because az.cmd would hand an unquoted & in the URL to cmd.exe. +function Invoke-ArmGetRequest { + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][string[]]$QueryParameters + ) + + # Merged stderr lines must not stop the script, whatever the caller's preference. + $callerErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { + $output = az rest --method get --url $Path --url-parameters $QueryParameters --output json --only-show-errors 2>&1 + $succeeded = $LASTEXITCODE -eq 0 + } + finally { + $ErrorActionPreference = $callerErrorActionPreference + } + + $standardOutput = New-Object System.Collections.Generic.List[string] + $errorOutput = New-Object System.Collections.Generic.List[string] + foreach ($line in @($output)) { + if ($line -is [System.Management.Automation.ErrorRecord]) { + $errorOutput.Add([string]$line) + } + elseif ($null -ne $line) { + $standardOutput.Add([string]$line) + } + } + + $response = $null + if ($succeeded) { + try { + $response = ConvertFrom-Json -InputObject ($standardOutput -join "`n") + } + catch { + $response = $null + } + + if ($null -eq $response) { + $succeeded = $false + $errorOutput.Add('The response was not a JSON object.') + } + } + + return @{ + Succeeded = $succeeded + Response = $response + ErrorText = ($errorOutput -join "`n") + } +} + +# Whether the principal holds the role on the whole subscription, directly or from a management +# group: Assigned, NotAssigned or Unknown. Autoscale is not satisfied by the role on a resource +# group. +function Get-AvdAutoscaleRoleState { + param( + [Parameter(Mandatory = $true)][string]$PrincipalObjectId, + [Parameter(Mandatory = $true)][string]$SubscriptionId, + [string]$RoleDefinitionId = '40c5ff49-9181-41f8-ae61-143b0e78555e' + ) + + $subscriptionScope = "/subscriptions/$SubscriptionId" + # atScope() returns the assignments on the subscription and the scopes above it, and + # assignedTo() the principal's own and those of the groups it belongs to. + $request = Invoke-ArmGetRequest -Path "$subscriptionScope/providers/Microsoft.Authorization/roleAssignments" -QueryParameters @( + 'api-version=2022-04-01', + ('$filter=atScope() and assignedTo(''{0}'')' -f $PrincipalObjectId) + ) + if (-not $request.Succeeded) { + return 'Unknown' + } + + foreach ($assignment in @(Get-PropertyValue -InputObject $request.Response -Name 'value')) { + $properties = Get-PropertyValue -InputObject $assignment -Name 'properties' + if ($null -eq $properties) { + $properties = $assignment + } + + $principalId = [string](Get-PropertyValue -InputObject $properties -Name 'principalId') + $principalType = [string](Get-PropertyValue -InputObject $properties -Name 'principalType') + $roleDefinition = [string](Get-PropertyValue -InputObject $properties -Name 'roleDefinitionId') + $scope = [string](Get-PropertyValue -InputObject $properties -Name 'scope') + if (-not [string]::Equals($principalId, $PrincipalObjectId, [System.StringComparison]::OrdinalIgnoreCase) -and $principalType -ne 'Group') { + continue + } + + if (-not $roleDefinition.EndsWith("/$RoleDefinitionId", [System.StringComparison]::OrdinalIgnoreCase)) { + continue + } + + if ($scope -eq '/' -or + [string]::Equals($scope.TrimEnd('/'), $subscriptionScope, [System.StringComparison]::OrdinalIgnoreCase) -or + $scope.StartsWith('/providers/Microsoft.Management/managementGroups/', [System.StringComparison]::OrdinalIgnoreCase)) { + return 'Assigned' + } + } + + if (-not [string]::IsNullOrWhiteSpace([string](Get-PropertyValue -InputObject $request.Response -Name 'nextLink'))) { + return 'Unknown' + } + + return 'NotAssigned' +} + +# Whether an Azure RBAC action matches one of a role's action patterns, in which * is a wildcard. +function Test-ActionPermitted { + param( + [Parameter(Mandatory = $true)][string]$Action, + [AllowNull()][AllowEmptyCollection()][object[]]$Patterns = @() + ) + + foreach ($pattern in @($Patterns)) { + $text = [string]$pattern + if ([string]::IsNullOrWhiteSpace($text)) { + continue + } + + $expression = '^' + [regex]::Escape($text.Trim()).Replace('\*', '.*') + '\z' + if ([regex]::IsMatch($Action, $expression, [System.Text.RegularExpressions.RegexOptions]::IgnoreCase)) { + return $true + } + } + + return $false +} + +# Whether the signed-in account can create role assignments on the whole subscription: Yes, No or +# Unknown. It reads the account's effective permissions there, which do not show deny assignments, +# so the deployment can still be refused. +function Test-CanAssignSubscriptionRole { + param([Parameter(Mandatory = $true)][string]$SubscriptionId) + + $action = 'Microsoft.Authorization/roleAssignments/write' + $request = Invoke-ArmGetRequest -Path "/subscriptions/$SubscriptionId/providers/Microsoft.Authorization/permissions" -QueryParameters @('api-version=2022-04-01') + if (-not $request.Succeeded) { + if ($request.ErrorText -match 'AuthorizationFailed|Forbidden') { + return 'No' + } + + return 'Unknown' + } + + $conditional = $false + foreach ($permission in @(Get-PropertyValue -InputObject $request.Response -Name 'value')) { + if ($null -eq $permission) { + continue + } + + $actions = @(Get-PropertyValue -InputObject $permission -Name 'actions') + $notActions = @(Get-PropertyValue -InputObject $permission -Name 'notActions') + if ((Test-ActionPermitted -Action $action -Patterns $actions) -and -not (Test-ActionPermitted -Action $action -Patterns $notActions)) { + # A condition, such as one that limits which roles may be assigned, may not allow this one. + if ([string]::IsNullOrWhiteSpace([string](Get-PropertyValue -InputObject $permission -Name 'condition'))) { + return 'Yes' + } + + $conditional = $true + } + } + + if ($conditional -or -not [string]::IsNullOrWhiteSpace([string](Get-PropertyValue -InputObject $request.Response -Name 'nextLink'))) { + return 'Unknown' + } + + return 'No' +} + +# Whether the deployment gives the Azure Virtual Desktop service principal Desktop Virtualization +# Power On Off Contributor on the subscription, and whether it assigns the scaling plan to the +# host pool, which Azure refuses while the service principal lacks that role. An assignment that +# is certain to be refused is left out with a warning, so that it does not fail the deployment. +function Resolve-AvdAutoscaleRolePlan { + param( + [Parameter(Mandatory = $true)][bool]$AvdSessionHostsDeployed, + [Parameter(Mandatory = $true)][bool]$ScalingPlanRequested, + [Parameter(Mandatory = $true)][bool]$StartVmOnConnect, + [Parameter(Mandatory = $true)][bool]$AssignRoleRequested, + [AllowEmptyString()][string]$ConfiguredObjectId = '', + [Parameter(Mandatory = $true)][string]$AppId, + [Parameter(Mandatory = $true)][string]$SubscriptionId, + [string]$RoleDefinitionId = '40c5ff49-9181-41f8-ae61-143b0e78555e' + ) + + $plan = @{ + ServicePrincipalObjectId = '' + AssignRole = $false + ScalingPlanEnabled = $ScalingPlanRequested + Summary = '' + } + + if (-not $AvdSessionHostsDeployed) { + $plan.Summary = 'no AVD session hosts are deployed.' + return $plan + } + + if (-not $ScalingPlanRequested -and -not $StartVmOnConnect) { + $plan.Summary = 'neither the scaling plan nor Start VM on Connect needs a role.' + return $plan + } + + $roleName = 'Desktop Virtualization Power On Off Contributor' + $startVmImpact = 'Start VM on Connect cannot start session hosts' + $leftOutImpact = New-Object System.Collections.Generic.List[string] + $keptImpact = New-Object System.Collections.Generic.List[string] + if ($ScalingPlanRequested) { + $leftOutImpact.Add('the scaling plan is deployed but assigned to no host pool, because Azure refuses the assignment without the role') + $keptImpact.Add('Azure refuses to assign the scaling plan to the host pool, which fails the deployment') + } + if ($StartVmOnConnect) { + $leftOutImpact.Add($startVmImpact) + $keptImpact.Add($startVmImpact) + } + + $objectId = Resolve-AvdServicePrincipalObjectId -ConfiguredObjectId $ConfiguredObjectId -AppId $AppId -SkipCreate:(-not $AssignRoleRequested) + $plan.ServicePrincipalObjectId = $objectId + $ownerCommand = "az role assignment create --assignee-object-id $objectId --assignee-principal-type ServicePrincipal --role $RoleDefinitionId --scope /subscriptions/$SubscriptionId" + + if (-not $AssignRoleRequested) { + # The administrator manages the role, possibly through a custom role this check cannot see, + # so the scaling plan stays as requested. + if (-not [string]::IsNullOrWhiteSpace($objectId) -and + (Get-AvdAutoscaleRoleState -PrincipalObjectId $objectId -SubscriptionId $SubscriptionId -RoleDefinitionId $RoleDefinitionId) -eq 'NotAssigned') { + Write-Warning "assignAvdAutoscaleRole is false, and the Azure Virtual Desktop service principal does not hold $roleName on subscription '$SubscriptionId'. Unless it has the same permissions through another role, $($keptImpact -join ', and '). An Owner or User Access Administrator can assign the role with this command: $ownerCommand" + } + + $plan.Summary = 'the role is managed outside this deployment (assignAvdAutoscaleRole is false).' + return $plan + } + + if ([string]::IsNullOrWhiteSpace($objectId)) { + $plan.ScalingPlanEnabled = $false + Write-Warning "Could not find or create the service principal of Azure Virtual Desktop (app ID $AppId) in this tenant, so the deployment cannot give it $roleName on subscription '$SubscriptionId'. Until it holds the role, $($leftOutImpact -join ', and '). Set avdServicePrincipalObjectId to the object ID of the service principal and run azd provision again, or, if it already holds the role, set assignAvdAutoscaleRole to false." + $plan.Summary = 'the Azure Virtual Desktop service principal was not found, so no role is assigned.' + return $plan + } + + $roleState = Get-AvdAutoscaleRoleState -PrincipalObjectId $objectId -SubscriptionId $SubscriptionId -RoleDefinitionId $RoleDefinitionId + if ($roleState -eq 'Assigned') { + $plan.Summary = "the Azure Virtual Desktop service principal already holds $roleName on the subscription." + return $plan + } + + $canAssign = Test-CanAssignSubscriptionRole -SubscriptionId $SubscriptionId + if ($canAssign -eq 'No') { + $plan.ScalingPlanEnabled = $false + Write-Warning "Your account cannot assign roles on subscription '$SubscriptionId', so the deployment cannot give the Azure Virtual Desktop service principal $roleName there. Until it holds the role, $($leftOutImpact -join ', and '). Have an Owner or User Access Administrator run the command below, then run azd provision again. If the service principal has the same permissions through another role, set assignAvdAutoscaleRole to false instead. Command: $ownerCommand" + $plan.Summary = 'the role is not assigned, and your account cannot assign it.' + return $plan + } + + $plan.AssignRole = $true + if ($roleState -eq 'Unknown' -or $canAssign -eq 'Unknown') { + Write-Warning "Could not confirm whether the Azure Virtual Desktop service principal already holds $roleName on subscription '$SubscriptionId', or whether your account can assign it, so the deployment assigns it. If the deployment fails with RoleAssignmentExists, the role is already assigned: set assignAvdAutoscaleRole to false. If it fails with AuthorizationFailed, have an Owner or User Access Administrator run the command below, then set assignAvdAutoscaleRole to false. Command: $ownerCommand" + } + + $plan.Summary = "the deployment gives the Azure Virtual Desktop service principal $roleName on the subscription." + return $plan +} + function Ensure-FrontendApplication { param( [Parameter(Mandatory = $true)][hashtable]$CloudContext, @@ -1090,17 +1735,43 @@ Ensure-DefaultEnvValue -Key 'linuxHostDesktop' -ValueFactory { 'gnome' } | Out-N Ensure-DefaultEnvValue -Key 'linuxHostVmSize' -ValueFactory { 'Standard_D2s_v5' } | Out-Null Ensure-DefaultEnvValue -Key 'avdVmSize' -ValueFactory { 'Standard_D8s_v5' } | Out-Null Ensure-DefaultEnvValue -Key 'avdMaxSessionLimit' -ValueFactory { '5' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdLinuxDesktopFullScreen' -ValueFactory { 'true' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdLinuxDesktopMultiMonitor' -ValueFactory { 'true' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanEnabled' -ValueFactory { 'true' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdStartVmOnConnect' -ValueFactory { 'true' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanTimeZone' -ValueFactory { 'UTC' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanRampUpStart' -ValueFactory { '07:00' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanPeakStart' -ValueFactory { '09:00' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanRampDownStart' -ValueFactory { '18:00' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanOffPeakStart' -ValueFactory { '20:00' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanRampUpMinimumHostsPct' -ValueFactory { '20' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanRampUpCapacityThresholdPct' -ValueFactory { '60' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanRampDownMinimumHostsPct' -ValueFactory { '10' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanRampDownCapacityThresholdPct' -ValueFactory { '90' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdScalingPlanWeekendMinimumHostsPct' -ValueFactory { '0' } | Out-Null +Ensure-DefaultEnvValue -Key 'assignAvdAutoscaleRole' -ValueFactory { 'true' } | Out-Null +Ensure-DefaultEnvValue -Key 'avdServicePrincipalAppId' -ValueFactory { $avdServicePrincipalAppId } | Out-Null Ensure-DefaultEnvValue -Key 'vmSubscriptionId' -ValueFactory { $subscription.id } | Out-Null Ensure-DefaultEnvValue -Key 'sqlAdminLogin' -ValueFactory { 'brokeradmin' } | Out-Null Ensure-DefaultEnvValue -Key 'sqlDatabaseName' -ValueFactory { 'LinuxBroker' } | Out-Null Ensure-DefaultEnvValue -Key 'sqlDatabaseSkuName' -ValueFactory { 'Basic' } | Out-Null Ensure-DefaultEnvValue -Key 'allowLegacyScopeAccess' -ValueFactory { 'false' } | Out-Null Ensure-DefaultEnvValue -Key 'appServicePlanSku' -ValueFactory { 'P2mv3' } | Out-Null +Ensure-DefaultEnvValue -Key 'appServicePlanCapacity' -ValueFactory { '1' } | Out-Null +Ensure-DefaultEnvValue -Key 'portalSessionStore' -ValueFactory { if ($cloudContext.Name -eq 'AzureCustom') { 'filesystem' } else { 'redis' } } | Out-Null +Ensure-DefaultEnvValue -Key 'portalRedisSku' -ValueFactory { '' } | Out-Null +Ensure-DefaultEnvValue -Key 'portalRedisLocation' -ValueFactory { '' } | Out-Null Ensure-DefaultEnvValue -Key 'linuxHostAdminLoginName' -ValueFactory { 'avdadmin' } | Out-Null Ensure-DefaultEnvValue -Key 'domainName' -ValueFactory { '' } | Out-Null Ensure-DefaultEnvValue -Key 'nfsShare' -ValueFactory { '' } | Out-Null Ensure-DefaultEnvValue -Key 'deployNfsShare' -ValueFactory { 'true' } | Out-Null Ensure-DefaultEnvValue -Key 'nfsShareQuotaGiB' -ValueFactory { '100' } | Out-Null +# An air-gapped cloud may lack the resource types or API versions the monitoring uses, so it opts in. +Ensure-DefaultEnvValue -Key 'deployHostMonitoring' -ValueFactory { if ($cloudContext.Name -eq 'AzureCustom') { 'false' } else { 'true' } } | Out-Null +Ensure-DefaultEnvValue -Key 'alertEmailAddresses' -ValueFactory { '' } | Out-Null +Ensure-DefaultEnvValue -Key 'alertCheckoutRefusalThreshold' -ValueFactory { '1' } | Out-Null +Ensure-DefaultEnvValue -Key 'alertApiErrorThreshold' -ValueFactory { '5' } | Out-Null +Ensure-DefaultEnvValue -Key 'alertNfsLatencyThresholdMs' -ValueFactory { '50' } | Out-Null Ensure-DefaultEnvValue -Key 'avdUsersGroupId' -ValueFactory { '' } | Out-Null Ensure-DefaultEnvValue -Key 'vmHostResourceGroup' -ValueFactory { '' } | Out-Null Ensure-DefaultEnvValue -Key 'scriptSourceRoot' -ValueFactory { 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main' } | Out-Null @@ -1131,14 +1802,15 @@ if ($deployLinuxHostsValue -eq 'true') { [void](Ensure-LinuxHostSshKeys) } +# The subscription azd provisions, where the hosts and the scaling plan deploy; not vmSubscriptionId. +$deploymentSubscriptionId = Get-FirstNonEmptyValue -Values @( + (Get-AzdEnvValue -Key 'AZURE_SUBSCRIPTION_ID'), + $env:AZURE_SUBSCRIPTION_ID, + $subscription.id +) + if ($deployLinuxHostsValue -eq 'true' -and (ConvertTo-IntParameterValue -Key 'linuxHostCount') -gt 0) { - # The Linux hosts deploy to the subscription azd provisions, not to vmSubscriptionId. - $linuxHostSubscriptionId = Get-FirstNonEmptyValue -Values @( - (Get-AzdEnvValue -Key 'AZURE_SUBSCRIPTION_ID'), - $env:AZURE_SUBSCRIPTION_ID, - $subscription.id - ) - Ensure-LinuxHostImageTerms -OsVersion (Get-AzdEnvValue -Key 'linuxHostOsVersion') -SubscriptionId $linuxHostSubscriptionId + Ensure-LinuxHostImageTerms -OsVersion (Get-AzdEnvValue -Key 'linuxHostOsVersion') -SubscriptionId $deploymentSubscriptionId } $apiApp = Ensure-ApiApplication -CloudContext $cloudContext -DisplayName $apiAppDisplayName @@ -1234,6 +1906,107 @@ else { Write-Host "AVD users group: $avdUsersGroupSummary" Write-Host 'Automatic admin consent was attempted for the configured application permissions. If consent was not granted, complete it manually in Microsoft Entra ID.' +# Azure refuses to change an extension on a VM that is not running, and autoscale and the broker's +# own scaling leave hosts deallocated. So the deployment leaves out the extensions of the hosts that +# are not running, and writes back the excludeFromScaling tags it would otherwise remove. This is +# the resource group main.bicep deploys to, since resourceGroupName is not set here. +$deploymentResourceGroupName = "rg-$(Get-RequiredAzdEnvValue -Key 'appName')-$EnvironmentName" +$hostNamesNotRunning = @() +$avdScalingExclusions = [ordered]@{} +if ((ConvertTo-BoolParameterValue -Key 'deployLinuxHosts') -or (ConvertTo-BoolParameterValue -Key 'deployAvdHosts')) { + $hostSnapshot = Get-HostPowerStateSnapshot -SubscriptionId $deploymentSubscriptionId -ResourceGroupName $deploymentResourceGroupName + $hostNamesNotRunning = @($hostSnapshot.NotRunning) + $avdScalingExclusions = $hostSnapshot.AvdScalingExclusions + if ($hostNamesNotRunning.Count -gt 0) { + Write-Warning "These hosts are not running, so this deployment leaves their VM extensions as they are: $($hostSnapshot.NotRunningDisplay -join ', '). Changes that the extensions carry, such as the API URL, scriptSourceRoot, linuxHostDesktop and linuxHostDisableScreenLock, reach a host only once it is running and azd provision runs again. If a host stops while this deployment runs, the deployment can still fail; run azd provision again." + } + + if ($avdScalingExclusions.Count -gt 0) { + Write-Host "Keeping the excludeFromScaling tag on AVD session hosts: $(@($avdScalingExclusions.Keys) -join ', ')." + } +} + +# The AVD scaling plan and the role that lets Azure Virtual Desktop start and stop session hosts. +$avdScalingPlanTimes = [ordered]@{ + avdScalingPlanRampUpStart = ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanRampUpStart' -DefaultValue '07:00' + avdScalingPlanPeakStart = ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanPeakStart' -DefaultValue '09:00' + avdScalingPlanRampDownStart = ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanRampDownStart' -DefaultValue '18:00' + avdScalingPlanOffPeakStart = ConvertTo-TimeOfDayParameterValue -Key 'avdScalingPlanOffPeakStart' -DefaultValue '20:00' +} +Assert-AvdScalingPlanTimeOrder -Times $avdScalingPlanTimes +$avdScalingPlanRampUpMinimumHostsPct = ConvertTo-PercentParameterValue -Key 'avdScalingPlanRampUpMinimumHostsPct' -DefaultValue 20 +$avdScalingPlanRampUpCapacityThresholdPct = ConvertTo-PercentParameterValue -Key 'avdScalingPlanRampUpCapacityThresholdPct' -DefaultValue 60 -Minimum 1 +$avdScalingPlanRampDownMinimumHostsPct = ConvertTo-PercentParameterValue -Key 'avdScalingPlanRampDownMinimumHostsPct' -DefaultValue 10 +$avdScalingPlanRampDownCapacityThresholdPct = ConvertTo-PercentParameterValue -Key 'avdScalingPlanRampDownCapacityThresholdPct' -DefaultValue 90 -Minimum 1 +$avdScalingPlanWeekendMinimumHostsPct = ConvertTo-PercentParameterValue -Key 'avdScalingPlanWeekendMinimumHostsPct' -DefaultValue 0 +$avdStartVmOnConnect = ConvertTo-BoolParameterValue -Key 'avdStartVmOnConnect' -DefaultValue $true +$avdSessionHostsDeployed = (ConvertTo-BoolParameterValue -Key 'deployAvdHosts') -and + ((ConvertTo-IntParameterValue -Key 'avdSessionHostCount') -gt 0) -and + (-not [string]::IsNullOrWhiteSpace((Get-AzdEnvValue -Key 'avdHostPoolName'))) + +$avdScalingPlanTimeZone = Get-FirstNonEmptyValue -Values @((Get-AzdEnvValue -Key 'avdScalingPlanTimeZone'), 'UTC') +$knownTimeZoneId = Resolve-WindowsTimeZoneId -TimeZoneId $avdScalingPlanTimeZone +if (-not [string]::IsNullOrWhiteSpace($knownTimeZoneId)) { + $avdScalingPlanTimeZone = $knownTimeZoneId +} +elseif ($avdSessionHostsDeployed) { + $timeZoneHint = '' + if ($avdScalingPlanTimeZone.Contains('/')) { + $timeZoneHint = ' It looks like an IANA time zone name; use the Windows ID instead, such as Eastern Standard Time for America/New_York.' + } + + Write-Warning "avdScalingPlanTimeZone '$avdScalingPlanTimeZone' is not a Windows time zone ID this machine knows, so Azure may refuse the scaling plan.$timeZoneHint" +} + +$avdAutoscaleArguments = @{ + AvdSessionHostsDeployed = $avdSessionHostsDeployed + ScalingPlanRequested = (ConvertTo-BoolParameterValue -Key 'avdScalingPlanEnabled' -DefaultValue $true) + StartVmOnConnect = $avdStartVmOnConnect + AssignRoleRequested = (ConvertTo-BoolParameterValue -Key 'assignAvdAutoscaleRole' -DefaultValue $true) + ConfiguredObjectId = (Get-AzdEnvValue -Key 'avdServicePrincipalObjectId') + AppId = (Get-FirstNonEmptyValue -Values @((Get-AzdEnvValue -Key 'avdServicePrincipalAppId'), $avdServicePrincipalAppId)) + SubscriptionId = $deploymentSubscriptionId + RoleDefinitionId = $avdAutoscaleRoleDefinitionId +} +$avdAutoscale = Resolve-AvdAutoscaleRolePlan @avdAutoscaleArguments +if ($avdSessionHostsDeployed) { + $avdScalingPlanState = "assigned to the host pool, in time zone $avdScalingPlanTimeZone" + if (-not $avdAutoscale.ScalingPlanEnabled) { + $avdScalingPlanState = 'deployed but assigned to no host pool' + } + + $avdStartVmOnConnectState = 'off' + if ($avdStartVmOnConnect) { + $avdStartVmOnConnectState = 'on' + } + + Write-Host "AVD autoscale: scaling plan $avdScalingPlanState; Start VM on Connect $avdStartVmOnConnectState; $($avdAutoscale.Summary)" +} + +$portalSessionStore = Resolve-PortalSessionStore -CloudName $cloudContext.Name +$portalRedisSku = '' +$portalRedisLocation = '' +if ($portalSessionStore -eq 'redis') { + $portalRedisSku = ConvertTo-PortalRedisSkuParameterValue -CloudName $cloudContext.Name + $portalRedisLocation = ConvertTo-PortalRedisLocationParameterValue +} +$appServicePlanCapacity = Resolve-AppServicePlanCapacity -PortalSessionStore $portalSessionStore -SqlDatabaseSkuName (Get-AzdEnvValue -Key 'sqlDatabaseSkuName') + +$portalSessionStoreState = "on each instance's disk" +if ($portalSessionStore -eq 'redis') { + $portalSessionStoreState = 'in Azure Managed Redis' + if ($cloudContext.Name -eq 'AzureUSGovernment') { + $portalSessionStoreState = 'in Azure Cache for Redis' + } + if ($portalRedisSku) { + $portalSessionStoreState += " ($portalRedisSku)" + } + if ($portalRedisLocation) { + $portalSessionStoreState += " in $portalRedisLocation" + } +} +Write-Host "Portal sessions: kept $portalSessionStoreState. App Service plan instances: $appServicePlanCapacity." + # Generate the Bicep parameters file with the real values so azd passes them # to the ARM deployment. azd collects parameters BEFORE running preprovision, # so env values set above would not be picked up through ${...} references or @@ -1269,6 +2042,10 @@ Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterNa Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'vmSubscriptionId' -Value (Get-RequiredAzdEnvValue -Key 'vmSubscriptionId') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'allowedClientIp' -Value (Get-AzdEnvValue -Key 'allowedClientIp') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'appServicePlanSku' -Value (Get-RequiredAzdEnvValue -Key 'appServicePlanSku') +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'appServicePlanCapacity' -Value $appServicePlanCapacity +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'portalSessionStore' -Value $portalSessionStore +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'portalRedisSku' -Value $portalRedisSku +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'portalRedisLocation' -Value $portalRedisLocation Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'deployLinuxHosts' -Value (ConvertTo-BoolParameterValue -Key 'deployLinuxHosts') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'deployAvdHosts' -Value (ConvertTo-BoolParameterValue -Key 'deployAvdHosts') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'linuxHostVmNamePrefix' -Value (Get-RequiredAzdEnvValue -Key 'linuxHostVmNamePrefix') @@ -1281,6 +2058,24 @@ Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterNa Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdHostPoolName' -Value (Get-RequiredAzdEnvValue -Key 'avdHostPoolName') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdSessionHostCount' -Value (ConvertTo-IntParameterValue -Key 'avdSessionHostCount') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdMaxSessionLimit' -Value (ConvertTo-IntParameterValue -Key 'avdMaxSessionLimit' -DefaultValue 5) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdLinuxDesktopFullScreen' -Value (ConvertTo-BoolParameterValue -Key 'avdLinuxDesktopFullScreen' -DefaultValue $true) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdLinuxDesktopMultiMonitor' -Value (ConvertTo-BoolParameterValue -Key 'avdLinuxDesktopMultiMonitor' -DefaultValue $true) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanEnabled' -Value ([bool]$avdAutoscale.ScalingPlanEnabled) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdStartVmOnConnect' -Value $avdStartVmOnConnect +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanTimeZone' -Value $avdScalingPlanTimeZone +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanRampUpStart' -Value $avdScalingPlanTimes['avdScalingPlanRampUpStart'] +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanPeakStart' -Value $avdScalingPlanTimes['avdScalingPlanPeakStart'] +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanRampDownStart' -Value $avdScalingPlanTimes['avdScalingPlanRampDownStart'] +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanOffPeakStart' -Value $avdScalingPlanTimes['avdScalingPlanOffPeakStart'] +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanRampUpMinimumHostsPct' -Value $avdScalingPlanRampUpMinimumHostsPct +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanRampUpCapacityThresholdPct' -Value $avdScalingPlanRampUpCapacityThresholdPct +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanRampDownMinimumHostsPct' -Value $avdScalingPlanRampDownMinimumHostsPct +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanRampDownCapacityThresholdPct' -Value $avdScalingPlanRampDownCapacityThresholdPct +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingPlanWeekendMinimumHostsPct' -Value $avdScalingPlanWeekendMinimumHostsPct +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdScalingExclusions' -Value $avdScalingExclusions +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'hostNamesNotRunning' -Value @($hostNamesNotRunning) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdServicePrincipalObjectId' -Value $avdAutoscale.ServicePrincipalObjectId +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'assignAvdAutoscaleRole' -Value ([bool]$avdAutoscale.AssignRole) Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdVmNamePrefix' -Value (Get-RequiredAzdEnvValue -Key 'avdVmNamePrefix') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'avdVmSize' -Value (Get-RequiredAzdEnvValue -Key 'avdVmSize') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'azureCloudName' -Value (Get-RequiredAzdEnvValue -Key 'azureCloudName') @@ -1289,6 +2084,11 @@ Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterNa Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'stsIssuerHost' -Value (Get-RequiredAzdEnvValue -Key 'stsIssuerHost') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'appServiceDomain' -Value (Get-RequiredAzdEnvValue -Key 'appServiceDomain') Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'scriptSourceRoot' -Value (Get-RequiredAzdEnvValue -Key 'scriptSourceRoot') +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'deployHostMonitoring' -Value (ConvertTo-BoolParameterValue -Key 'deployHostMonitoring' -DefaultValue $true) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'alertEmailAddresses' -Value (ConvertTo-EmailListParameterValue -Key 'alertEmailAddresses') +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'alertCheckoutRefusalThreshold' -Value (ConvertTo-PositiveIntParameterValue -Key 'alertCheckoutRefusalThreshold' -DefaultValue 1) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'alertApiErrorThreshold' -Value (ConvertTo-PositiveIntParameterValue -Key 'alertApiErrorThreshold' -DefaultValue 5) +Add-BicepParameterValue -ParameterCollection $bicepParameterEntries -ParameterName 'alertNfsLatencyThresholdMs' -Value (ConvertTo-PositiveIntParameterValue -Key 'alertNfsLatencyThresholdMs' -DefaultValue 50) $bicepParameters = [ordered]@{ '$schema' = 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' diff --git a/deploy/Migrate-ExistingEnvironment.ps1 b/deploy/Migrate-ExistingEnvironment.ps1 index 58d926c..a4772e5 100644 --- a/deploy/Migrate-ExistingEnvironment.ps1 +++ b/deploy/Migrate-ExistingEnvironment.ps1 @@ -45,6 +45,9 @@ param( [Parameter(Mandatory = $false)] [string[]]$LinuxHostNames, + [Parameter(Mandatory = $false)] + [string[]]$AvdHostNames, + [Parameter(Mandatory = $false)] [ValidateRange(1, 300)] [int]$WatcherDebounceSeconds = 10, @@ -57,7 +60,10 @@ param( [switch]$SkipPostProvision, [Parameter(Mandatory = $false)] - [switch]$SkipLinuxHostReleaseAgentMigration + [switch]$SkipLinuxHostReleaseAgentMigration, + + [Parameter(Mandatory = $false)] + [switch]$SkipAvdHostScriptUpdate ) Set-StrictMode -Version Latest @@ -132,23 +138,58 @@ if (-not $SkipPostProvision) { -LinuxHostGroupId $LinuxHostGroupId } +if ((-not $SkipLinuxHostReleaseAgentMigration -or -not $SkipAvdHostScriptUpdate) -and + ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhiteSpace($ApiBaseUrl) -or [string]::IsNullOrWhiteSpace($ApiClientId))) { + throw 'Updating the Linux hosts and the AVD session hosts requires ResourceGroupName, ApiBaseUrl, and ApiClientId.' +} + +# Neither host update stops the other, so a Linux host that fails does not leave the AVD session +# hosts on a Connect-LinuxBroker.ps1 that cannot wait for a host to start. Both scripts name the +# hosts to retry, and the failures end the migration together. +$failedSteps = New-Object System.Collections.Generic.List[string] + if (-not $SkipLinuxHostReleaseAgentMigration) { - if ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhiteSpace($ApiBaseUrl) -or [string]::IsNullOrWhiteSpace($ApiClientId)) { - throw 'Linux host release-agent migration requires ResourceGroupName, ApiBaseUrl, and ApiClientId.' + Write-Host 'Migrating existing Linux hosts to the current release-agent layout.' + + try { + & "$PSScriptRoot/Migrate-LinuxHostReleaseAgent.ps1" ` + -ResourceGroupName $ResourceGroupName ` + -ApiBaseUrl $ApiBaseUrl ` + -ApiClientId $ApiClientId ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -ScriptSourceRoot $ScriptSourceRoot ` + -LinuxHostNames $LinuxHostNames ` + -WatcherDebounceSeconds $WatcherDebounceSeconds ` + -WatcherSettleSeconds $WatcherSettleSeconds + } + catch { + Write-Warning $_.Exception.Message + $failedSteps.Add('the Linux host migration') } +} - Write-Host 'Migrating existing Linux hosts to the current release-agent layout.' +if (-not $SkipAvdHostScriptUpdate) { + Write-Host 'Updating Connect-LinuxBroker.ps1 on the existing AVD session hosts.' + + try { + & "$PSScriptRoot/Update-AvdHostBrokerScript.ps1" ` + -ResourceGroupName $ResourceGroupName ` + -ApiBaseUrl $ApiBaseUrl ` + -ApiClientId $ApiClientId ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -ScriptSourceRoot $ScriptSourceRoot ` + -AvdHostNames $AvdHostNames + } + catch { + Write-Warning $_.Exception.Message + $failedSteps.Add('the AVD session host update') + } +} - & "$PSScriptRoot/Migrate-LinuxHostReleaseAgent.ps1" ` - -ResourceGroupName $ResourceGroupName ` - -ApiBaseUrl $ApiBaseUrl ` - -ApiClientId $ApiClientId ` - -SubscriptionId $SubscriptionId ` - -EnvironmentName $EnvironmentName ` - -ScriptSourceRoot $ScriptSourceRoot ` - -LinuxHostNames $LinuxHostNames ` - -WatcherDebounceSeconds $WatcherDebounceSeconds ` - -WatcherSettleSeconds $WatcherSettleSeconds +if ($failedSteps.Count -gt 0) { + throw "Existing environment migration failed in $($failedSteps -join ' and '). The warnings above name the hosts to retry." } Write-Host 'Existing environment migration completed.' \ No newline at end of file diff --git a/deploy/Migrate-LinuxHostReleaseAgent.ps1 b/deploy/Migrate-LinuxHostReleaseAgent.ps1 index 189ac0c..78eb5dc 100644 --- a/deploy/Migrate-LinuxHostReleaseAgent.ps1 +++ b/deploy/Migrate-LinuxHostReleaseAgent.ps1 @@ -219,6 +219,7 @@ apply_settings_script="$output_directory/apply-host-settings.sh" session_control_script="$output_directory/session-control.sh" patch_host_script="$output_directory/patch-host.sh" xrdp_startwm_script="$output_directory/xrdp-startwm.sh" +install_host_config_script="$output_directory/install-host-config.sh" release_service_name='linuxbroker-release-session.service' release_timer_name='linuxbroker-release-session.timer' watcher_service_name='linuxbroker-release-session-watcher.service' @@ -393,6 +394,7 @@ apply_settings_script_url="$script_source_root/linux_host/apply-host-settings.sh session_control_script_url="$script_source_root/linux_host/session-control.sh" patch_host_script_url="$script_source_root/linux_host/patch-host.sh" xrdp_startwm_script_url="$script_source_root/linux_host/xrdp-startwm.sh" +install_host_config_script_url="$script_source_root/linux_host/install-host-config.sh" mkdir -p "$output_directory" "$state_directory" "$state_directory/leases" @@ -405,8 +407,9 @@ download_file "$apply_settings_script_url" "$apply_settings_script" download_file "$session_control_script_url" "$session_control_script" download_file "$patch_host_script_url" "$patch_host_script" download_file "$xrdp_startwm_script_url" "$xrdp_startwm_script" +download_file "$install_host_config_script_url" "$install_host_config_script" -chmod +x "$release_script" "$xorg_script" "$watcher_script" "$create_user_script" "$manage_lease_script" "$apply_settings_script" "$session_control_script" "$patch_host_script" "$xrdp_startwm_script" +chmod +x "$release_script" "$xorg_script" "$watcher_script" "$create_user_script" "$manage_lease_script" "$apply_settings_script" "$session_control_script" "$patch_host_script" "$xrdp_startwm_script" "$install_host_config_script" sed -i "s|YOUR_LINUX_BROKER_API_CLIENT_ID|$api_client_id|g" "$release_script" sed -i "s|YOUR_LINUX_BROKER_API_BASE_URL|$api_base_url|g" "$release_script" @@ -537,6 +540,13 @@ if [ "$launcher_status" -ne 0 ] && [ "$launcher_status" -ne 3 ]; then echo "WARNING: xrdp-startwm.sh --install failed with exit code $launcher_status." fi +# Log rotation, the NFS read-ahead and the users' local caches. The host works without them. +config_status=0 +"$install_host_config_script" || config_status=$? +if [ "$config_status" -ne 0 ]; then + echo "WARNING: install-host-config.sh failed with exit code $config_status." +fi + systemctl disable --now "$watcher_service_name" >/dev/null 2>&1 || true systemctl disable --now "$release_timer_name" >/dev/null 2>&1 || true systemctl disable --now "$release_service_name" >/dev/null 2>&1 || true diff --git a/deploy/Post-Provision.ps1 b/deploy/Post-Provision.ps1 index 1bcd99a..e7bf7ef 100644 --- a/deploy/Post-Provision.ps1 +++ b/deploy/Post-Provision.ps1 @@ -158,3 +158,17 @@ if ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhite -DatabaseName $DatabaseName ` -SqlAdminLogin $SqlAdminLogin ` -SqlAdminPassword $SqlAdminPassword + +# The deployment cannot install the Azure Monitor agent, because Azure refuses extensions on VMs +# that scaling has powered off, so the hosts are connected to the host monitoring here. The rest +# of the deployment is in place by now, so a host that cannot be set up is a warning, which names +# the hosts to run the script again for. +try { + & "$PSScriptRoot/Enable-HostMonitoring.ps1" ` + -ResourceGroupName $ResourceGroupName ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName +} +catch { + Write-Warning "The deployment is in place, but the host monitoring is not fully set up. $($_.Exception.Message)" +} diff --git a/deploy/Update-AvdHostBrokerScript.ps1 b/deploy/Update-AvdHostBrokerScript.ps1 new file mode 100644 index 0000000..76ec450 --- /dev/null +++ b/deploy/Update-AvdHostBrokerScript.ps1 @@ -0,0 +1,556 @@ +<# +.SYNOPSIS + Replaces Connect-LinuxBroker.ps1 on the existing AVD session hosts with the current one. + +.DESCRIPTION + Configure-AVD-Host.ps1 installs Connect-LinuxBroker.ps1 only when a session host is + provisioned, so session hosts deployed by an earlier release keep the script they were + deployed with. A script older than 2.0.0 cannot wait while the broker starts a Linux host for + its user, and shows an error instead. + + Through Run Command, this script downloads Connect-LinuxBroker.ps1 from ScriptSourceRoot onto + every running AVD session host in the resource group (VMs tagged broker-role=avd-host), fills + in the broker API's URL and client ID as Configure-AVD-Host.ps1 does, and replaces + C:\Temp\Connect-LinuxBroker.ps1 in one step, so a user who connects meanwhile runs either the + old script or the new one. + + Session hosts that are not running are skipped and named at the end: update each one with + -AvdHostNames once it is started. Every running session host is attempted, and the ones where + the update failed are reported together at the end, with a non-zero exit code. + + Values that are not given are read from the azd environment. ScriptSourceRoot and the API URL + are written into PowerShell code, so both must be https:// URLs made only of ASCII letters, + digits and the characters . - _ ~ % and /, with an optional port. + +.EXAMPLE + .\Update-AvdHostBrokerScript.ps1 -EnvironmentName + +.EXAMPLE + .\Update-AvdHostBrokerScript.ps1 -EnvironmentName -AvdHostNames +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $false)] + [string]$ApiBaseUrl, + + [Parameter(Mandatory = $false)] + [string]$ApiClientId, + + [Parameter(Mandatory = $false)] + [string]$SubscriptionId, + + [Parameter(Mandatory = $false)] + [string]$EnvironmentName, + + [Parameter(Mandatory = $false)] + [string]$ScriptSourceRoot = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main', + + [Parameter(Mandatory = $false)] + [string[]]$AvdHostNames +) + +$CompletionMarker = 'Updated Connect-LinuxBroker.ps1 on' + +function Get-AzdEnvValue { + param( + [AllowEmptyString()] + [string]$EnvironmentName, + + [Parameter(Mandatory = $true)] + [string]$Key + ) + + if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { + return '' + } + + $value = azd env get-value $Key --environment $EnvironmentName 2>$null + if ($LASTEXITCODE -ne 0) { + return '' + } + + return ($value | Out-String).Trim() +} + +# A property's value, or $null when the object or the property is missing, which StrictMode +# would otherwise refuse. +function Get-PropertyValue { + param( + $InputObject, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + if ($null -eq $InputObject) { + return $null + } + + $property = $InputObject.PSObject.Properties[$Name] + if ($null -eq $property) { + return $null + } + + return $property.Value +} + +function Get-VmTagValue { + param( + [Parameter(Mandatory = $true)] + $Vm, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + return [string](Get-PropertyValue -InputObject (Get-PropertyValue -InputObject $Vm -Name 'tags') -Name $Name) +} + +function ConvertTo-PowerShellLiteral { + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [string]$Value + ) + + return "'" + [System.Management.Automation.Language.CodeGeneration]::EscapeSingleQuotedStringContent($Value) + "'" +} + +# Whether a URL can be written into PowerShell code without ending the string it is in. Only plain +# ASCII URL characters pass: PowerShell also ends strings at typographic quotes, and Windows +# PowerShell reads a script without a byte order mark in the ANSI code page, where some accented +# letters turn into quotes. The match is case-sensitive, because matching without regard to case +# lets the Kelvin sign through as a k. +function Test-SafeHttpsUrl { + param( + [AllowEmptyString()] + [string]$Value + ) + + if ($Value -cnotmatch '^https://[A-Za-z0-9._-]+(:[0-9]{1,5})?(/[A-Za-z0-9._~%/-]*)?\z') { + return $false + } + + # The pattern lets through a port above 65535, which a URI cannot have. + $uri = $null + return [Uri]::TryCreate($Value, [UriKind]::Absolute, [ref]$uri) +} + +# The script Run Command runs on each session host, in Windows PowerShell 5.1 as SYSTEM. +function Get-AvdHostUpdateScript { + param( + [Parameter(Mandatory = $true)] + [string]$SourceUrl, + + [Parameter(Mandatory = $true)] + [string]$ApiBaseUrl, + + [Parameter(Mandatory = $true)] + [string]$ApiClientId, + + [Parameter(Mandatory = $false)] + [string]$Destination = 'C:\Temp\Connect-LinuxBroker.ps1' + ) + + $template = @' +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +$sourceUrl = __SOURCE_URL__ +$apiBaseUrl = __API_BASE_URL__ +$apiClientId = __API_CLIENT_ID__ +$destination = __DESTINATION__ + +$baseUrlPlaceholder = 'https://your_linuxbroker_api_base_url/api' +$clientIdPlaceholder = 'your_linuxbroker_api_client_id' + +function Get-BrokerScriptVersion { + param([string]$Text) + + $match = [regex]::Match($Text, '(?m)^\$ScriptVersion = ''([^'']+)''') + if ($match.Success) { + return $match.Groups[1].Value + } + return $null +} + +$folder = Split-Path -Parent $destination +if (-not (Test-Path -LiteralPath $folder)) { + New-Item -ItemType Directory -Path $folder -Force | Out-Null +} + +$previous = 'no script' +if (Test-Path -LiteralPath $destination) { + $previous = Get-BrokerScriptVersion ([System.IO.File]::ReadAllText($destination)) + if (-not $previous) { + $previous = 'a script without a version' + } +} + +$staged = '{0}.{1}.download' -f $destination, [guid]::NewGuid().ToString('N') +try { + $attempt = 0 + while ($true) { + $attempt++ + try { + Invoke-WebRequest -Uri $sourceUrl -OutFile $staged -UseBasicParsing + break + } + catch { + if ($attempt -ge 3) { + throw "Could not download $sourceUrl after $attempt attempts: $($_.Exception.Message)" + } + Start-Sleep -Seconds (2 * $attempt) + } + } + + $text = [System.IO.File]::ReadAllText($staged) + $version = Get-BrokerScriptVersion $text + if (-not $version -or -not $text.Contains($baseUrlPlaceholder) -or -not $text.Contains($clientIdPlaceholder)) { + throw "$sourceUrl is not a Connect-LinuxBroker.ps1 that this update can install." + } + + $text = $text.Replace($baseUrlPlaceholder, $apiBaseUrl).Replace($clientIdPlaceholder, $apiClientId) + + # The script is written without a byte order mark, which Windows PowerShell reads in the ANSI + # code page, so any character outside ASCII could read differently, even as a quote. + if ($text -cmatch '[^\x00-\x7F]') { + throw "Connect-LinuxBroker.ps1 $version holds characters outside ASCII, which Windows PowerShell could misread." + } + + $parseErrors = $null + [void][System.Management.Automation.Language.Parser]::ParseInput($text, [ref]$null, [ref]$parseErrors) + if ($parseErrors) { + throw "Connect-LinuxBroker.ps1 $version does not parse once the API URL and client ID are filled in: $($parseErrors[0].Message)" + } + + # Without a byte order mark, as Configure-AVD-Host.ps1 writes it. + [System.IO.File]::WriteAllText($staged, $text, (New-Object System.Text.UTF8Encoding -ArgumentList $false)) + + # In one step, so a user who connects meanwhile runs either the old script or the new one. + $attempt = 0 + while ($true) { + $attempt++ + try { + if (Test-Path -LiteralPath $destination) { + [System.IO.File]::Replace($staged, $destination, [NullString]::Value) + } + else { + [System.IO.File]::Move($staged, $destination) + } + break + } + catch { + if ($attempt -ge 5) { + throw "Could not replace ${destination}: $($_.Exception.Message)" + } + Start-Sleep -Seconds 1 + } + } +} +finally { + Remove-Item -LiteralPath $staged -Force -ErrorAction SilentlyContinue +} + +Unblock-File -LiteralPath $destination +if ((Get-BrokerScriptVersion ([System.IO.File]::ReadAllText($destination))) -ne $version) { + throw "$destination does not hold Connect-LinuxBroker.ps1 $version after the update." +} + +# Configure-AVD-Host.ps1 creates the event source the script logs to, and installs the module it +# saves the Linux host's credentials with. +try { + if (-not [System.Diagnostics.EventLog]::SourceExists('LinuxBrokerScript')) { + New-EventLog -LogName Application -Source 'LinuxBrokerScript' + } +} +catch { + Write-Output "WARNING: The LinuxBrokerScript event source could not be created, so Connect-LinuxBroker.ps1 cannot log: $($_.Exception.Message)" +} + +if (-not (Get-Module -ListAvailable -Name CredentialManager)) { + Write-Output 'WARNING: The CredentialManager module is not installed, so Connect-LinuxBroker.ps1 cannot save the Linux host credentials. Install it with Install-Module CredentialManager -Scope AllUsers.' +} + +Write-Output ('Updated Connect-LinuxBroker.ps1 on {0} from {1} to {2}.' -f $env:COMPUTERNAME, $previous, $version) +'@ + + $values = @{ + SOURCE_URL = ConvertTo-PowerShellLiteral -Value $SourceUrl + API_BASE_URL = ConvertTo-PowerShellLiteral -Value $ApiBaseUrl + API_CLIENT_ID = ConvertTo-PowerShellLiteral -Value $ApiClientId + DESTINATION = ConvertTo-PowerShellLiteral -Value $Destination + } + + # In one pass, so a value that happens to contain a token is not replaced again. + return [regex]::Replace($template, '__(SOURCE_URL|API_BASE_URL|API_CLIENT_ID|DESTINATION)__', { + param($match) + $values[$match.Groups[1].Value] + }) +} + +# Returns what the script wrote to standard output and standard error on the session host. +function Invoke-RunCommandWithRetry { + param( + [Parameter(Mandatory = $true)] + [string]$ResourceGroupName, + + [Parameter(Mandatory = $true)] + [string]$VmName, + + [Parameter(Mandatory = $true)] + [string]$Script, + + [Parameter(Mandatory = $false)] + [int]$MaxAttempts = 3, + + [Parameter(Mandatory = $false)] + [int]$InitialDelaySeconds = 5 + ) + + $lastError = '' + + # The script reaches az as @file. Passed inline, on Windows it goes through az.cmd, where + # cmd.exe ends the command at the first newline, and the session host runs only the first line. + $scriptFile = Join-Path ([System.IO.Path]::GetTempPath()) ('linuxbroker-avd-update-{0}.ps1' -f [guid]::NewGuid().ToString('N')) + [System.IO.File]::WriteAllText($scriptFile, $Script, (New-Object System.Text.UTF8Encoding -ArgumentList $false)) + + try { + for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) { + $output = @(az vm run-command invoke ` + --resource-group $ResourceGroupName ` + --name $VmName ` + --command-id RunPowerShellScript ` + --scripts "@$scriptFile" ` + --output json ` + --only-show-errors 2>&1) + $exitCode = $LASTEXITCODE + $errorText = (@($output | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] } | ForEach-Object { $_.ToString() }) -join [Environment]::NewLine).Trim() + $json = (@($output | Where-Object { $_ -isnot [System.Management.Automation.ErrorRecord] } | ForEach-Object { [string]$_ }) -join "`n").Trim() + + if ($exitCode -eq 0) { + try { + $result = ConvertFrom-Json -InputObject $json + } + catch { + throw "Run Command on '$VmName' returned output that is not JSON: $json" + } + + $standardOutput = New-Object System.Collections.Generic.List[string] + $standardError = New-Object System.Collections.Generic.List[string] + foreach ($status in @(Get-PropertyValue -InputObject $result -Name 'value')) { + $message = [string](Get-PropertyValue -InputObject $status -Name 'message') + if ([string](Get-PropertyValue -InputObject $status -Name 'code') -like '*StdErr*') { + $standardError.Add($message) + } + else { + $standardOutput.Add($message) + } + } + + return [pscustomobject]@{ + StdOut = ($standardOutput -join "`n").Trim() + StdErr = ($standardError -join "`n").Trim() + } + } + + $lastError = $errorText + if ($attempt -ge $MaxAttempts) { + break + } + + $delaySeconds = [Math]::Min($InitialDelaySeconds * [Math]::Pow(2, $attempt - 1), 30) + Write-Warning "Run Command failed on '$VmName' attempt $attempt of $MaxAttempts. Retrying in $([int]$delaySeconds) seconds." + if (-not [string]::IsNullOrWhiteSpace($lastError)) { + Write-Warning $lastError + } + + Start-Sleep -Seconds ([int]$delaySeconds) + } + } + finally { + Remove-Item -LiteralPath $scriptFile -Force -ErrorAction SilentlyContinue + } + + if ([string]::IsNullOrWhiteSpace($lastError)) { + throw "Run Command failed on '$VmName'." + } + + throw "Run Command failed on '$VmName'. Last error: $lastError" +} + +function Invoke-AvdHostBrokerScriptUpdate { + param( + [string]$ResourceGroupName, + [string]$ApiBaseUrl, + [string]$ApiClientId, + [string]$SubscriptionId, + [string]$EnvironmentName, + [string]$ScriptSourceRoot = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main', + [string[]]$AvdHostNames + ) + + if ([string]::IsNullOrWhiteSpace($EnvironmentName)) { + $EnvironmentName = if (-not [string]::IsNullOrWhiteSpace($env:AZURE_ENV_NAME)) { + $env:AZURE_ENV_NAME + } + elseif (-not [string]::IsNullOrWhiteSpace($env:AZURE_ENVIRONMENT_NAME)) { + $env:AZURE_ENVIRONMENT_NAME + } + else { + '' + } + } + + if ([string]::IsNullOrWhiteSpace($ResourceGroupName)) { + $ResourceGroupName = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'resourceGroupName' + } + + if ([string]::IsNullOrWhiteSpace($SubscriptionId)) { + $SubscriptionId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'AZURE_SUBSCRIPTION_ID' + } + + if ([string]::IsNullOrWhiteSpace($SubscriptionId)) { + $SubscriptionId = $env:AZURE_SUBSCRIPTION_ID + } + + if ([string]::IsNullOrWhiteSpace($ApiBaseUrl)) { + $ApiBaseUrl = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'apiUrl' + } + + if ([string]::IsNullOrWhiteSpace($ApiClientId)) { + $ApiClientId = Get-AzdEnvValue -EnvironmentName $EnvironmentName -Key 'apiClientId' + } + + if ([string]::IsNullOrWhiteSpace($ResourceGroupName) -or [string]::IsNullOrWhiteSpace($ApiBaseUrl) -or [string]::IsNullOrWhiteSpace($ApiClientId)) { + throw 'The AVD session host update needs ResourceGroupName, ApiBaseUrl and ApiClientId, as parameters or azd environment values.' + } + + # The API URL and client ID end up inside double-quoted strings in Connect-LinuxBroker.ps1, and + # the source URL in the script Run Command runs as SYSTEM. + $ApiBaseUrl = $ApiBaseUrl.Trim().TrimEnd('/') + if (-not (Test-SafeHttpsUrl -Value $ApiBaseUrl)) { + throw "ApiBaseUrl must be the broker API's https:// URL, such as https:///api, not '$ApiBaseUrl'." + } + + $ApiClientId = $ApiClientId.Trim() + $parsedClientId = [guid]::Empty + if (-not [guid]::TryParseExact($ApiClientId, 'D', [ref]$parsedClientId)) { + throw "ApiClientId must be the broker API app registration's client ID, a GUID, not '$ApiClientId'." + } + + $ScriptSourceRoot = ([string]$ScriptSourceRoot).Trim().TrimEnd('/') + if (-not (Test-SafeHttpsUrl -Value $ScriptSourceRoot)) { + throw "ScriptSourceRoot must be the https:// URL of the repository or of a mirror of it, such as https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main, not '$ScriptSourceRoot'." + } + + $sourceUrl = "$ScriptSourceRoot/avd_host/broker/Connect-LinuxBroker.ps1" + + if (-not [string]::IsNullOrWhiteSpace($SubscriptionId)) { + az account set --subscription $SubscriptionId | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Could not select subscription '$SubscriptionId'." + } + } + + $vmJson = (@(az vm list --resource-group $ResourceGroupName --show-details --output json --only-show-errors) -join "`n") + if ($LASTEXITCODE -ne 0) { + throw "Could not list the VMs in resource group '$ResourceGroupName'." + } + + $avdHosts = @(ConvertFrom-Json -InputObject $vmJson | ForEach-Object { $_ } | + Where-Object { (Get-VmTagValue -Vm $_ -Name 'broker-role') -eq 'avd-host' }) + + if ($avdHosts.Count -eq 0) { + Write-Host "No AVD session host VMs found in resource group '$ResourceGroupName'." + return + } + + if ($AvdHostNames -and $AvdHostNames.Count -gt 0) { + $requestedNames = New-Object 'System.Collections.Generic.HashSet[string]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($name in $AvdHostNames) { + if (-not [string]::IsNullOrWhiteSpace($name)) { + [void]$requestedNames.Add($name.Trim()) + } + } + + $filteredHosts = @($avdHosts | Where-Object { $requestedNames.Contains([string]$_.name) }) + foreach ($name in $requestedNames) { + if (-not ($filteredHosts | Where-Object { $_.name -eq $name })) { + Write-Warning "Requested AVD session host '$name' was not found in resource group '$ResourceGroupName'." + } + } + + $avdHosts = $filteredHosts + } + + if ($avdHosts.Count -eq 0) { + Write-Host 'No AVD session hosts matched the requested update scope.' + return + } + + $remoteScript = Get-AvdHostUpdateScript -SourceUrl $sourceUrl -ApiBaseUrl $ApiBaseUrl -ApiClientId $ApiClientId + + # One session host that is off or failing must not leave the rest on the old script, so every + # one is attempted and the failures are reported together at the end. + $updated = New-Object System.Collections.Generic.List[string] + $failures = New-Object System.Collections.Generic.List[string] + $skipped = New-Object System.Collections.Generic.List[string] + + foreach ($avdHost in $avdHosts) { + $name = [string]$avdHost.name + $powerState = [string](Get-PropertyValue -InputObject $avdHost -Name 'powerState') + if (-not [string]::IsNullOrWhiteSpace($powerState) -and $powerState -notmatch 'running') { + Write-Warning "Skipping AVD session host '$name' because it is not running ($powerState). Start it and run this script again with -AvdHostNames $name." + $skipped.Add($name) + continue + } + + Write-Host "Updating Connect-LinuxBroker.ps1 on AVD session host '$name'..." + try { + $output = Invoke-RunCommandWithRetry -ResourceGroupName $ResourceGroupName -VmName $name -Script $remoteScript + if (-not [string]::IsNullOrWhiteSpace($output.StdOut)) { + Write-Host $output.StdOut + } + # Run Command reports success whatever the script did, so the script's own closing + # line is the only evidence that it ran to the end. + if ($output.StdOut -notmatch [regex]::Escape($CompletionMarker)) { + $detail = if ([string]::IsNullOrWhiteSpace($output.StdErr)) { '' } else { " $($output.StdErr)" } + throw "The update did not run to completion on '$name'.$detail" + } + $updated.Add($name) + } + catch { + Write-Warning $_.Exception.Message + $failures.Add($name) + } + } + + if ($skipped.Count -gt 0) { + Write-Warning "Not updated because they are not running: $($skipped -join ', ')." + } + + if ($failures.Count -gt 0) { + throw "The Connect-LinuxBroker.ps1 update failed on: $($failures -join ', '). Rerun with -AvdHostNames to retry them." + } + + Write-Host "Connect-LinuxBroker.ps1 is up to date on $($updated.Count) AVD session host(s)." +} + +# Dot-sourcing the script, as its tests do, defines the functions without running the update. +if ($MyInvocation.InvocationName -ne '.') { + Set-StrictMode -Version Latest + $ErrorActionPreference = 'Stop' + + Invoke-AvdHostBrokerScriptUpdate ` + -ResourceGroupName $ResourceGroupName ` + -ApiBaseUrl $ApiBaseUrl ` + -ApiClientId $ApiClientId ` + -SubscriptionId $SubscriptionId ` + -EnvironmentName $EnvironmentName ` + -ScriptSourceRoot $ScriptSourceRoot ` + -AvdHostNames $AvdHostNames +} diff --git a/deploy/bicep/main.bicep b/deploy/bicep/main.bicep index 9d33ae4..b84049b 100644 --- a/deploy/bicep/main.bicep +++ b/deploy/bicep/main.bicep @@ -117,6 +117,24 @@ param allowedClientIp string = '' @description('App Service plan SKU name.') param appServicePlanSku string = 'P2mv3' +@description('Instances of the App Service plan. The portal, the API and the task function run on every instance, so each instance adds API connections to the database: check the database tier allows them.') +@minValue(1) +@maxValue(30) +param appServicePlanCapacity int = 1 + +@description('Where the portal keeps sign-in sessions. redis keeps them in a cache every portal instance shares, reached over a private endpoint: Azure Managed Redis in the public cloud, Azure Cache for Redis in Azure Government. filesystem keeps them on each instance, which works on one instance, or on more with ARR affinity. AzureCustom always uses filesystem.') +@allowed([ + 'redis' + 'filesystem' +]) +param portalSessionStore string = 'redis' + +@description('Size of the portal session cache. Leave empty for the default: Balanced_B0 for Azure Managed Redis, Standard_C1 for Azure Cache for Redis.') +param portalRedisSku string = '' + +@description('Region of the portal session cache, such as eastus. Leave empty, the default, for the deployment\'s region. Set it where that region does not offer the cache or has no capacity for it: the portal reaches a cache in another region through the same private endpoint in the virtual network.') +param portalRedisLocation string = '' + @description('Deploy Linux broker host VMs.') param deployLinuxHosts bool = false @@ -172,6 +190,70 @@ param avdSessionHostCount int = 0 @description('Maximum number of sessions per AVD session host.') param avdMaxSessionLimit int = 5 +@description('Opens the Linux desktop full screen. False opens it in a window on one monitor. Turn it off only once every AVD session host runs Connect-LinuxBroker.ps1 2.0.0 or later (deploy/Update-AvdHostBrokerScript.ps1): older scripts refuse the argument and the Linux Desktop app fails to open.') +param avdLinuxDesktopFullScreen bool = true + +@description('Spreads a full-screen Linux desktop across every monitor. False keeps it on one monitor. Turn it off only once every AVD session host runs Connect-LinuxBroker.ps1 2.0.0 or later.') +param avdLinuxDesktopMultiMonitor bool = true + +@description('Starts and stops the AVD session hosts on a schedule through an Azure Virtual Desktop scaling plan. The plan is always deployed with the session hosts; false leaves it assigned to no host pool, so the hosts stay as they are and you can assign a plan of your own instead. Azure refuses to assign the plan unless the Azure Virtual Desktop service principal holds Desktop Virtualization Power On Off Contributor on the subscription.') +param avdScalingPlanEnabled bool = true + +@description('Starts a deallocated AVD session host when a user opens the Linux Desktop app and no running session host can take the session. Needs the same role as the scaling plan.') +param avdStartVmOnConnect bool = true + +@description('Windows time zone ID the scaling plan\'s times are in, such as UTC or Eastern Standard Time. It is separate from the broker\'s own scaling time zone, which the portal sets.') +param avdScalingPlanTimeZone string = 'UTC' + +@description('Start of ramp-up, as HH:mm in avdScalingPlanTimeZone. The weekday and weekend schedules use the same times.') +param avdScalingPlanRampUpStart string = '07:00' + +@description('Start of peak hours, as HH:mm. Must come after ramp-up.') +param avdScalingPlanPeakStart string = '09:00' + +@description('Start of ramp-down, as HH:mm. Must come after peak.') +param avdScalingPlanRampDownStart string = '18:00' + +@description('Start of off-peak hours, as HH:mm. Must come after ramp-down.') +param avdScalingPlanOffPeakStart string = '20:00' + +@description('Share of the AVD session hosts kept running from ramp-up until ramp-down on weekdays, rounded up to whole hosts.') +@minValue(0) +@maxValue(100) +param avdScalingPlanRampUpMinimumHostsPct int = 20 + +@description('Share of the running session hosts\' capacity in use that makes autoscale start another one during ramp-up and peak hours.') +@minValue(1) +@maxValue(100) +param avdScalingPlanRampUpCapacityThresholdPct int = 60 + +@description('Share of the AVD session hosts kept running from ramp-down until the next ramp-up on weekdays, rounded up to whole hosts.') +@minValue(0) +@maxValue(100) +param avdScalingPlanRampDownMinimumHostsPct int = 10 + +@description('Share of the running session hosts\' capacity in use that makes autoscale start another one during ramp-down and off-peak hours.') +@minValue(1) +@maxValue(100) +param avdScalingPlanRampDownCapacityThresholdPct int = 90 + +@description('Share of the AVD session hosts kept running on Saturday and Sunday. 0 lets them all stop, and Start VM on Connect starts one for the first user.') +@minValue(0) +@maxValue(100) +param avdScalingPlanWeekendMinimumHostsPct int = 0 + +@description('Generated by the preprovision hook: lowercase AVD session host names mapped to the value of the excludeFromScaling tag each carries, so the deployment writes the tag back.') +param avdScalingExclusions object = {} + +@description('Generated by the preprovision hook: the Linux hosts and AVD session hosts that are not running, whose VM extensions this deployment leaves out because Azure refuses to change an extension on a VM that is not running.') +param hostNamesNotRunning array = [] + +@description('Object ID of the Azure Virtual Desktop service principal (app ID 9cdead84-a844-4324-93f2-b2e6bb768d07) in this tenant. The preprovision hook finds it. Without it, the role below is not assigned.') +param avdServicePrincipalObjectId string = '' + +@description('Assigns Desktop Virtualization Power On Off Contributor on the subscription to the Azure Virtual Desktop service principal, for the scaling plan and Start VM on Connect. The preprovision hook passes false when the role is already assigned, or when the signed-in account cannot assign roles on the subscription.') +param assignAvdAutoscaleRole bool = true + @description('AVD session host VM name prefix.') param avdVmNamePrefix string = 'avdhost' @@ -189,8 +271,29 @@ param avdVmNamePrefix string = 'avdhost' @description('AVD session host VM size.') param avdVmSize string = 'Standard_D8s_v5' +@description('Deploy the host monitoring: log collection from the Linux hosts and AVD session hosts, the fleet workbook, and the alerts. Set to false in a cloud that lacks data collection rules, workbooks or log search alerts. Enable-HostMonitoring.ps1 installs the Azure Monitor agent after provisioning.') +param deployHostMonitoring bool = true + +@description('Email addresses that receive the alerts, separated by commas or semicolons. For a Teams channel, use the channel\'s email address. Leave empty to deploy the alerts without notifications.') +param alertEmailAddresses string = '' + +@description('Refused checkouts (HTTP 409) in 15 minutes that raise an alert.') +@minValue(1) +param alertCheckoutRefusalThreshold int = 1 + +@description('API responses with a 5xx status in 15 minutes that raise an alert.') +@minValue(1) +param alertApiErrorThreshold int = 5 + +@description('Average end-to-end latency of the provisioned NFS share over 15 minutes, in milliseconds, that raises an alert.') +@minValue(1) +param alertNfsLatencyThresholdMs int = 50 + @description('Resource group used for deployment.') var effectiveResourceGroupName = empty(resourceGroupName) ? 'rg-${appName}-${environmentName}' : resourceGroupName +var deployAvdSessionHosts = deployAvdHosts && avdSessionHostCount > 0 && !empty(avdHostPoolName) +// Desktop Virtualization Power On Off Contributor. +var avdPowerOnOffContributorRoleGuid = '40c5ff49-9181-41f8-ae61-143b0e78555e' resource rg 'Microsoft.Resources/resourceGroups@2024-03-01' = { name: effectiveResourceGroupName @@ -236,6 +339,10 @@ module resources 'main.resources.bicep' = { scriptSourceRoot: scriptSourceRoot allowedClientIp: allowedClientIp appServicePlanSku: appServicePlanSku + appServicePlanCapacity: appServicePlanCapacity + portalSessionStore: portalSessionStore + portalRedisSku: portalRedisSku + portalRedisLocation: portalRedisLocation deployLinuxHosts: deployLinuxHosts deployAvdHosts: deployAvdHosts linuxHostVmNamePrefix: linuxHostVmNamePrefix @@ -249,9 +356,61 @@ module resources 'main.resources.bicep' = { avdHostPoolName: avdHostPoolName avdSessionHostCount: avdSessionHostCount avdMaxSessionLimit: avdMaxSessionLimit + avdLinuxDesktopFullScreen: avdLinuxDesktopFullScreen + avdLinuxDesktopMultiMonitor: avdLinuxDesktopMultiMonitor + avdStartVmOnConnect: avdStartVmOnConnect + avdScalingExclusions: avdScalingExclusions + hostNamesNotRunning: hostNamesNotRunning avdVmNamePrefix: avdVmNamePrefix avdVmSize: avdVmSize + deployHostMonitoring: deployHostMonitoring + alertEmailAddresses: alertEmailAddresses + alertCheckoutRefusalThreshold: alertCheckoutRefusalThreshold + alertApiErrorThreshold: alertApiErrorThreshold + alertNfsLatencyThresholdMs: alertNfsLatencyThresholdMs + } +} + +// Autoscale only works with the role on the whole subscription, and Start VM on Connect is +// covered by it too. +resource avdAutoscaleRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (deployAvdSessionHosts && (avdScalingPlanEnabled || avdStartVmOnConnect) && assignAvdAutoscaleRole && !empty(avdServicePrincipalObjectId)) { + name: guid(subscription().id, avdServicePrincipalObjectId, avdPowerOnOffContributorRoleGuid) + properties: { + principalId: avdServicePrincipalObjectId + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', avdPowerOnOffContributorRoleGuid) + description: 'Lets Azure Virtual Desktop start and stop the Linux Broker AVD session hosts, for the scaling plan and Start VM on Connect.' + } +} + +// Deployed last, so that a role assigned above has had the rest of the deployment to take effect +// before Azure checks it while assigning the plan to the host pool. +module avdScalingPlan 'modules/AVD/scaling-plan.bicep' = if (deployAvdSessionHosts) { + name: 'avdScalingPlan' + scope: rg + params: { + location: location + tags: union(tags, { + 'broker-role': 'avd-host' + }) + name: take('${avdHostPoolName}-scaling-plan', 64) + hostPoolName: avdHostPoolName + enabled: avdScalingPlanEnabled + timeZone: avdScalingPlanTimeZone + rampUpStartTime: avdScalingPlanRampUpStart + peakStartTime: avdScalingPlanPeakStart + rampDownStartTime: avdScalingPlanRampDownStart + offPeakStartTime: avdScalingPlanOffPeakStart + rampUpMinimumHostsPct: avdScalingPlanRampUpMinimumHostsPct + rampUpCapacityThresholdPct: avdScalingPlanRampUpCapacityThresholdPct + rampDownMinimumHostsPct: avdScalingPlanRampDownMinimumHostsPct + rampDownCapacityThresholdPct: avdScalingPlanRampDownCapacityThresholdPct + weekendMinimumHostsPct: avdScalingPlanWeekendMinimumHostsPct } + dependsOn: [ + resources + avdAutoscaleRole + ] } output resourceGroupName string = rg.name @@ -268,3 +427,7 @@ output sqlDatabaseName string = resources.outputs.sqlDatabaseName output virtualNetworkName string = resources.outputs.virtualNetworkName output linuxHostDomainName string = resources.outputs.linuxHostDomainName output nfsSharePath string = resources.outputs.nfsSharePath +output linuxHostDataCollectionRuleId string = resources.outputs.linuxHostDataCollectionRuleId +output avdHostDataCollectionRuleId string = resources.outputs.avdHostDataCollectionRuleId +output portalSessionStore string = resources.outputs.portalSessionStore +output portalRedisName string = resources.outputs.portalRedisName diff --git a/deploy/bicep/main.json b/deploy/bicep/main.json index c487dfc..50126c7 100644 --- a/deploy/bicep/main.json +++ b/deploy/bicep/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "9307409657661009644" + "templateHash": "18348097897885014198" } }, "parameters": { @@ -257,6 +257,40 @@ "description": "App Service plan SKU name." } }, + "appServicePlanCapacity": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "maxValue": 30, + "metadata": { + "description": "Instances of the App Service plan. The portal, the API and the task function run on every instance, so each instance adds API connections to the database: check the database tier allows them." + } + }, + "portalSessionStore": { + "type": "string", + "defaultValue": "redis", + "allowedValues": [ + "redis", + "filesystem" + ], + "metadata": { + "description": "Where the portal keeps sign-in sessions. redis keeps them in a cache every portal instance shares, reached over a private endpoint: Azure Managed Redis in the public cloud, Azure Cache for Redis in Azure Government. filesystem keeps them on each instance, which works on one instance, or on more with ARR affinity. AzureCustom always uses filesystem." + } + }, + "portalRedisSku": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Size of the portal session cache. Leave empty for the default: Balanced_B0 for Azure Managed Redis, Standard_C1 for Azure Cache for Redis." + } + }, + "portalRedisLocation": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Region of the portal session cache, such as eastus. Leave empty, the default, for the deployment's region. Set it where that region does not offer the cache or has no capacity for it: the portal reaches a cache in another region through the same private endpoint in the virtual network." + } + }, "deployLinuxHosts": { "type": "bool", "defaultValue": false, @@ -364,6 +398,142 @@ "description": "Maximum number of sessions per AVD session host." } }, + "avdLinuxDesktopFullScreen": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Opens the Linux desktop full screen. False opens it in a window on one monitor. Turn it off only once every AVD session host runs Connect-LinuxBroker.ps1 2.0.0 or later (deploy/Update-AvdHostBrokerScript.ps1): older scripts refuse the argument and the Linux Desktop app fails to open." + } + }, + "avdLinuxDesktopMultiMonitor": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Spreads a full-screen Linux desktop across every monitor. False keeps it on one monitor. Turn it off only once every AVD session host runs Connect-LinuxBroker.ps1 2.0.0 or later." + } + }, + "avdScalingPlanEnabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Starts and stops the AVD session hosts on a schedule through an Azure Virtual Desktop scaling plan. The plan is always deployed with the session hosts; false leaves it assigned to no host pool, so the hosts stay as they are and you can assign a plan of your own instead. Azure refuses to assign the plan unless the Azure Virtual Desktop service principal holds Desktop Virtualization Power On Off Contributor on the subscription." + } + }, + "avdStartVmOnConnect": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Starts a deallocated AVD session host when a user opens the Linux Desktop app and no running session host can take the session. Needs the same role as the scaling plan." + } + }, + "avdScalingPlanTimeZone": { + "type": "string", + "defaultValue": "UTC", + "metadata": { + "description": "Windows time zone ID the scaling plan's times are in, such as UTC or Eastern Standard Time. It is separate from the broker's own scaling time zone, which the portal sets." + } + }, + "avdScalingPlanRampUpStart": { + "type": "string", + "defaultValue": "07:00", + "metadata": { + "description": "Start of ramp-up, as HH:mm in avdScalingPlanTimeZone. The weekday and weekend schedules use the same times." + } + }, + "avdScalingPlanPeakStart": { + "type": "string", + "defaultValue": "09:00", + "metadata": { + "description": "Start of peak hours, as HH:mm. Must come after ramp-up." + } + }, + "avdScalingPlanRampDownStart": { + "type": "string", + "defaultValue": "18:00", + "metadata": { + "description": "Start of ramp-down, as HH:mm. Must come after peak." + } + }, + "avdScalingPlanOffPeakStart": { + "type": "string", + "defaultValue": "20:00", + "metadata": { + "description": "Start of off-peak hours, as HH:mm. Must come after ramp-down." + } + }, + "avdScalingPlanRampUpMinimumHostsPct": { + "type": "int", + "defaultValue": 20, + "minValue": 0, + "maxValue": 100, + "metadata": { + "description": "Share of the AVD session hosts kept running from ramp-up until ramp-down on weekdays, rounded up to whole hosts." + } + }, + "avdScalingPlanRampUpCapacityThresholdPct": { + "type": "int", + "defaultValue": 60, + "minValue": 1, + "maxValue": 100, + "metadata": { + "description": "Share of the running session hosts' capacity in use that makes autoscale start another one during ramp-up and peak hours." + } + }, + "avdScalingPlanRampDownMinimumHostsPct": { + "type": "int", + "defaultValue": 10, + "minValue": 0, + "maxValue": 100, + "metadata": { + "description": "Share of the AVD session hosts kept running from ramp-down until the next ramp-up on weekdays, rounded up to whole hosts." + } + }, + "avdScalingPlanRampDownCapacityThresholdPct": { + "type": "int", + "defaultValue": 90, + "minValue": 1, + "maxValue": 100, + "metadata": { + "description": "Share of the running session hosts' capacity in use that makes autoscale start another one during ramp-down and off-peak hours." + } + }, + "avdScalingPlanWeekendMinimumHostsPct": { + "type": "int", + "defaultValue": 0, + "minValue": 0, + "maxValue": 100, + "metadata": { + "description": "Share of the AVD session hosts kept running on Saturday and Sunday. 0 lets them all stop, and Start VM on Connect starts one for the first user." + } + }, + "avdScalingExclusions": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Generated by the preprovision hook: lowercase AVD session host names mapped to the value of the excludeFromScaling tag each carries, so the deployment writes the tag back." + } + }, + "hostNamesNotRunning": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Generated by the preprovision hook: the Linux hosts and AVD session hosts that are not running, whose VM extensions this deployment leaves out because Azure refuses to change an extension on a VM that is not running." + } + }, + "avdServicePrincipalObjectId": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Object ID of the Azure Virtual Desktop service principal (app ID 9cdead84-a844-4324-93f2-b2e6bb768d07) in this tenant. The preprovision hook finds it. Without it, the role below is not assigned." + } + }, + "assignAvdAutoscaleRole": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Assigns Desktop Virtualization Power On Off Contributor on the subscription to the Azure Virtual Desktop service principal, for the scaling plan and Start VM on Connect. The preprovision hook passes false when the role is already assigned, or when the signed-in account cannot assign roles on the subscription." + } + }, "avdVmNamePrefix": { "type": "string", "defaultValue": "avdhost", @@ -388,10 +558,50 @@ "metadata": { "description": "AVD session host VM size." } + }, + "deployHostMonitoring": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Deploy the host monitoring: log collection from the Linux hosts and AVD session hosts, the fleet workbook, and the alerts. Set to false in a cloud that lacks data collection rules, workbooks or log search alerts. Enable-HostMonitoring.ps1 installs the Azure Monitor agent after provisioning." + } + }, + "alertEmailAddresses": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Email addresses that receive the alerts, separated by commas or semicolons. For a Teams channel, use the channel's email address. Leave empty to deploy the alerts without notifications." + } + }, + "alertCheckoutRefusalThreshold": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "metadata": { + "description": "Refused checkouts (HTTP 409) in 15 minutes that raise an alert." + } + }, + "alertApiErrorThreshold": { + "type": "int", + "defaultValue": 5, + "minValue": 1, + "metadata": { + "description": "API responses with a 5xx status in 15 minutes that raise an alert." + } + }, + "alertNfsLatencyThresholdMs": { + "type": "int", + "defaultValue": 50, + "minValue": 1, + "metadata": { + "description": "Average end-to-end latency of the provisioned NFS share over 15 minutes, in milliseconds, that raises an alert." + } } }, "variables": { - "effectiveResourceGroupName": "[if(empty(parameters('resourceGroupName')), format('rg-{0}-{1}', parameters('appName'), parameters('environmentName')), parameters('resourceGroupName'))]" + "effectiveResourceGroupName": "[if(empty(parameters('resourceGroupName')), format('rg-{0}-{1}', parameters('appName'), parameters('environmentName')), parameters('resourceGroupName'))]", + "deployAvdSessionHosts": "[and(and(parameters('deployAvdHosts'), greater(parameters('avdSessionHostCount'), 0)), not(empty(parameters('avdHostPoolName'))))]", + "avdPowerOnOffContributorRoleGuid": "40c5ff49-9181-41f8-ae61-143b0e78555e" }, "resources": [ { @@ -401,6 +611,18 @@ "location": "[parameters('location')]", "tags": "[parameters('tags')]" }, + { + "condition": "[and(and(and(variables('deployAvdSessionHosts'), or(parameters('avdScalingPlanEnabled'), parameters('avdStartVmOnConnect'))), parameters('assignAvdAutoscaleRole')), not(empty(parameters('avdServicePrincipalObjectId'))))]", + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(subscription().id, parameters('avdServicePrincipalObjectId'), variables('avdPowerOnOffContributorRoleGuid'))]", + "properties": { + "principalId": "[parameters('avdServicePrincipalObjectId')]", + "principalType": "ServicePrincipal", + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('avdPowerOnOffContributorRoleGuid'))]", + "description": "Lets Azure Virtual Desktop start and stop the Linux Broker AVD session hosts, for the scaling plan and Start VM on Connect." + } + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2025-04-01", @@ -514,6 +736,18 @@ "appServicePlanSku": { "value": "[parameters('appServicePlanSku')]" }, + "appServicePlanCapacity": { + "value": "[parameters('appServicePlanCapacity')]" + }, + "portalSessionStore": { + "value": "[parameters('portalSessionStore')]" + }, + "portalRedisSku": { + "value": "[parameters('portalRedisSku')]" + }, + "portalRedisLocation": { + "value": "[parameters('portalRedisLocation')]" + }, "deployLinuxHosts": { "value": "[parameters('deployLinuxHosts')]" }, @@ -553,11 +787,41 @@ "avdMaxSessionLimit": { "value": "[parameters('avdMaxSessionLimit')]" }, + "avdLinuxDesktopFullScreen": { + "value": "[parameters('avdLinuxDesktopFullScreen')]" + }, + "avdLinuxDesktopMultiMonitor": { + "value": "[parameters('avdLinuxDesktopMultiMonitor')]" + }, + "avdStartVmOnConnect": { + "value": "[parameters('avdStartVmOnConnect')]" + }, + "avdScalingExclusions": { + "value": "[parameters('avdScalingExclusions')]" + }, + "hostNamesNotRunning": { + "value": "[parameters('hostNamesNotRunning')]" + }, "avdVmNamePrefix": { "value": "[parameters('avdVmNamePrefix')]" }, "avdVmSize": { "value": "[parameters('avdVmSize')]" + }, + "deployHostMonitoring": { + "value": "[parameters('deployHostMonitoring')]" + }, + "alertEmailAddresses": { + "value": "[parameters('alertEmailAddresses')]" + }, + "alertCheckoutRefusalThreshold": { + "value": "[parameters('alertCheckoutRefusalThreshold')]" + }, + "alertApiErrorThreshold": { + "value": "[parameters('alertApiErrorThreshold')]" + }, + "alertNfsLatencyThresholdMs": { + "value": "[parameters('alertNfsLatencyThresholdMs')]" } }, "template": { @@ -567,7 +831,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "7015886803379737196" + "templateHash": "11506867499235198244" } }, "parameters": { @@ -738,6 +1002,40 @@ "type": "string", "defaultValue": "P2mv3" }, + "appServicePlanCapacity": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "maxValue": 30, + "metadata": { + "description": "Instances of the App Service plan. The portal, the API and the task function run on every instance." + } + }, + "portalSessionStore": { + "type": "string", + "defaultValue": "redis", + "allowedValues": [ + "redis", + "filesystem" + ], + "metadata": { + "description": "Where the portal keeps sign-in sessions: redis, a cache every portal instance shares, or filesystem, each instance's own disk. AzureCustom always uses filesystem." + } + }, + "portalRedisSku": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Size of the portal session cache. Leave empty for the default." + } + }, + "portalRedisLocation": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Region of the portal session cache. Leave empty for the deployment's region." + } + }, "deployLinuxHosts": { "type": "bool", "defaultValue": false @@ -812,6 +1110,41 @@ "type": "int", "defaultValue": 5 }, + "avdLinuxDesktopFullScreen": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Opens the Linux desktop full screen." + } + }, + "avdLinuxDesktopMultiMonitor": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Spreads a full-screen Linux desktop across every monitor." + } + }, + "avdStartVmOnConnect": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Starts a deallocated AVD session host when a user connects and no running one can take the session." + } + }, + "avdScalingExclusions": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Lowercase AVD session host names mapped to their excludeFromScaling tag values, which the deployment writes back." + } + }, + "hostNamesNotRunning": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Linux hosts and AVD session hosts whose VM extensions are left out because the VMs are not running." + } + }, "avdVmNamePrefix": { "type": "string", "defaultValue": "avdhost" @@ -830,6 +1163,35 @@ "Standard_F16s_v2", "Standard_D16as_v4" ] + }, + "deployHostMonitoring": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Deploy host log collection, the fleet workbook and the alerts." + } + }, + "alertEmailAddresses": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Email addresses that receive the alerts, separated by commas or semicolons." + } + }, + "alertCheckoutRefusalThreshold": { + "type": "int", + "defaultValue": 1, + "minValue": 1 + }, + "alertApiErrorThreshold": { + "type": "int", + "defaultValue": 5, + "minValue": 1 + }, + "alertNfsLatencyThresholdMs": { + "type": "int", + "defaultValue": 50, + "minValue": 1 } }, "variables": { @@ -868,24 +1230,35 @@ "nfsStorageAccountName": "[take(format('nfs{0}{1}', variables('sanitizedApp'), variables('suffix')), 24)]", "nfsShareName": "home", "effectiveNfsShare": "[if(variables('provisionNfsShare'), format('{0}.file.{1}:/{2}/{3}', variables('nfsStorageAccountName'), environment().suffixes.storage, variables('nfsStorageAccountName'), variables('nfsShareName')), parameters('nfsShare'))]", + "portalRedisName": "[format('redis-{0}-{1}', take(format('{0}{1}', variables('sanitizedApp'), variables('sanitizedEnv')), 40), variables('suffix'))]", "cloudProfiles": { "AzurePublic": { "graphEndpoint": "https://graph.microsoft.com", "stsIssuerHost": "https://sts.windows.net", - "appServiceDomain": "azurewebsites.net" + "appServiceDomain": "azurewebsites.net", + "portalRedisKind": "managed", + "portalRedisPrivateDnsZoneName": "privatelink.redis.azure.net", + "portalRedisEntraResource": "" }, "AzureUSGovernment": { "graphEndpoint": "https://graph.microsoft.us", "stsIssuerHost": "https://sts.windows.net", - "appServiceDomain": "azurewebsites.us" + "appServiceDomain": "azurewebsites.us", + "portalRedisKind": "cache", + "portalRedisPrivateDnsZoneName": "privatelink.redis.cache.usgovcloudapi.net", + "portalRedisEntraResource": "acca5fbb-b7e4-4009-81f1-37e38fd66d78" }, "AzureCustom": { "graphEndpoint": "", "stsIssuerHost": "", - "appServiceDomain": "" + "appServiceDomain": "", + "portalRedisKind": "", + "portalRedisPrivateDnsZoneName": "", + "portalRedisEntraResource": "" } }, "cloudProfile": "[variables('cloudProfiles')[parameters('azureCloudName')]]", + "usePortalRedis": "[and(equals(parameters('portalSessionStore'), 'redis'), not(empty(variables('cloudProfile').portalRedisKind)))]", "resolvedAuthorityHost": "[if(empty(parameters('azureAuthorityHost')), environment().authentication.loginEndpoint, parameters('azureAuthorityHost'))]", "resolvedGraphEndpoint": "[if(empty(parameters('graphEndpoint')), variables('cloudProfile').graphEndpoint, parameters('graphEndpoint'))]", "resolvedStsIssuerHost": "[if(empty(parameters('stsIssuerHost')), variables('cloudProfile').stsIssuerHost, parameters('stsIssuerHost'))]", @@ -1551,7 +1924,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "10067415946987550221" + "templateHash": "5500961998070303674" } }, "parameters": { @@ -1616,15 +1989,20 @@ "logAnalyticsWorkspaceId": { "type": "string", "value": "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]" + }, + "logAnalyticsWorkspaceName": { + "type": "string", + "value": "[parameters('logAnalyticsWorkspaceName')]" } } } } }, { + "condition": "[parameters('deployHostMonitoring')]", "type": "Microsoft.Resources/deployments", "apiVersion": "2025-04-01", - "name": "containerRegistry", + "name": "hostMonitoring", "properties": { "expressionEvaluationOptions": { "scope": "inner" @@ -1637,8 +2015,27 @@ "tags": { "value": "[parameters('tags')]" }, - "containerRegistryName": { - "value": "[variables('containerRegistryName')]" + "namePrefix": { + "value": "[format('{0}-{1}', parameters('appName'), parameters('environmentName'))]" + }, + "logAnalyticsWorkspaceName": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'observability'), '2025-04-01').outputs.logAnalyticsWorkspaceName.value]" + }, + "apiAppName": { + "value": "[variables('apiAppName')]" + }, + "nfsStorageAccountName": "[if(variables('provisionNfsShare'), createObject('value', variables('nfsStorageAccountName')), createObject('value', ''))]", + "alertEmailAddresses": { + "value": "[parameters('alertEmailAddresses')]" + }, + "alertCheckoutRefusalThreshold": { + "value": "[parameters('alertCheckoutRefusalThreshold')]" + }, + "alertApiErrorThreshold": { + "value": "[parameters('alertApiErrorThreshold')]" + }, + "alertNfsLatencyThresholdMs": { + "value": "[parameters('alertNfsLatencyThresholdMs')]" } }, "template": { @@ -1648,56 +2045,811 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "3588414506722776755" + "templateHash": "4193567527631524882" } }, "parameters": { "location": { "type": "string", - "defaultValue": "[resourceGroup().location]" + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location of the data collection rules, the workbook and the log alerts." + } }, "tags": { "type": "object", "defaultValue": {} }, - "containerRegistryName": { - "type": "string" - } - }, - "resources": [ - { - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2023-07-01", - "name": "[parameters('containerRegistryName')]", - "location": "[parameters('location')]", - "tags": "[parameters('tags')]", - "sku": { - "name": "Basic" - }, - "properties": { - "adminUserEnabled": false, - "publicNetworkAccess": "Enabled" + "namePrefix": { + "type": "string", + "metadata": { + "description": "Prefix of the resource names, -." } - } - ], - "outputs": { - "id": { + }, + "logAnalyticsWorkspaceName": { "type": "string", - "value": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]" + "metadata": { + "description": "Log Analytics workspace that Application Insights writes to. The host logs go there too." + } }, - "loginServer": { + "apiAppName": { "type": "string", - "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName')), '2023-07-01').loginServer]" + "metadata": { + "description": "Name of the broker API web app, which is the cloud role name of its telemetry." + } }, - "name": { + "nfsStorageAccountName": { "type": "string", - "value": "[parameters('containerRegistryName')]" - } - } - } - } - }, - { + "defaultValue": "", + "metadata": { + "description": "Storage account of the NFS share the deployment provisions, or empty when it provisions none." + } + }, + "alertEmailAddresses": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Email addresses that receive the alerts, separated by commas or semicolons. Leave empty to deploy the alerts without notifications." + } + }, + "alertCheckoutRefusalThreshold": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "metadata": { + "description": "Refused checkouts (HTTP 409) in 15 minutes that raise an alert." + } + }, + "alertApiErrorThreshold": { + "type": "int", + "defaultValue": 5, + "minValue": 1, + "metadata": { + "description": "API responses with a 5xx status in 15 minutes that raise an alert." + } + }, + "alertNfsLatencyThresholdMs": { + "type": "int", + "defaultValue": 50, + "minValue": 1, + "metadata": { + "description": "Average end-to-end latency of the NFS share over 15 minutes, in milliseconds, that raises an alert." + } + } + }, + "variables": { + "hostLogTableName": "LinuxBrokerHost_CL", + "hostLogStreamName": "[format('Custom-{0}', variables('hostLogTableName'))]", + "linuxHostLogFiles": [ + "/var/log/release-session.log", + "/var/log/release-session-watcher.log", + "/var/log/linuxbroker-host-settings.log", + "/var/log/createuser.log", + "/var/log/linuxbroker-session-control.log", + "/var/log/linuxbroker-patch.log", + "/var/log/xrdp.log", + "/var/log/xrdp-sesman.log" + ], + "alertEmailList": "[filter(map(split(replace(parameters('alertEmailAddresses'), ';', ','), ','), lambda('address', trim(lambdaVariables('address')))), lambda('address', not(empty(lambdaVariables('address')))))]", + "alertActionGroupIds": "[if(empty(variables('alertEmailList')), createArray(), createArray(resourceId('Microsoft.Insights/actionGroups', format('ag-{0}', parameters('namePrefix')))))]", + "monitorNfsShare": "[not(empty(parameters('nfsStorageAccountName')))]", + "nfsFileServiceId": "[resourceId('Microsoft.Storage/storageAccounts/fileServices', if(variables('monitorNfsShare'), parameters('nfsStorageAccountName'), 'none'), 'default')]", + "fleetSnapshots": [ + "AppTraces", + "[format('| where AppRoleName == \"{0}\" and Message == \"fleet snapshot\"', parameters('apiAppName'))]" + ], + "apiRequests": [ + "AppRequests", + "[format('| where AppRoleName == \"{0}\"', parameters('apiAppName'))]" + ], + "requestPath": "| extend Path = coalesce(tostring(parse_url(Url).Path), tostring(split(Name, \" \")[-1]))", + "workbookBin": "bin(TimeGenerated, max_of(5m, {TimeRange:grain}))", + "hostLogLevel": "| extend Level = case(RawData has_cs \"ERROR\", \"Error\", RawData has_cs \"WARNING\" or RawData has_cs \"WARN\", \"Warning\", RawData has_any (\"failed\", \"cannot\", \"unable\") or RawData has \"could not\", \"Error\", \"\")", + "latestSnapshotQuery": "[join(concat(variables('fleetSnapshots'), createArray('| top 1 by TimeGenerated desc', '| project TimeGenerated, Ready = toint(Properties.ReadyHosts), InUse = toint(Properties.InUse), Waiting = toint(Properties.Waiting), Booting = toint(Properties.Booting), PoweredOn = toint(Properties.PoweredOn), Total = toint(Properties.TotalHosts), MinVMs = toint(Properties.EffectiveMinVMs), MaxVMs = toint(Properties.MaxVMs), StaleHeartbeats = toint(Properties.StaleHeartbeats), NfsUnreachable = toint(Properties.NfsUnreachable), XrdpInactive = toint(Properties.XrdpInactive), Draining = toint(Properties.Draining), Maintenance = toint(Properties.Maintenance), StartOnDemand = iff(toint(Properties.StartOnDemandEnabled) == 1, \"On\", \"Off\")')), '\n')]", + "fleetTrendQuery": "[join(concat(variables('fleetSnapshots'), createArray(format('| summarize Ready = min(toint(Properties.ReadyHosts)), InUse = max(toint(Properties.InUse)), Waiting = max(toint(Properties.Waiting)), Booting = max(toint(Properties.Booting)), PoweredOn = max(toint(Properties.PoweredOn)) by {0}', variables('workbookBin')), '| order by TimeGenerated asc')), '\n')]", + "checkoutOutcomeQuery": "[join(concat(variables('apiRequests'), createArray(variables('requestPath'), '| where Path == \"/api/vms/checkout\"', '| extend Outcome = case(ResultCode == \"200\", \"Assigned (200)\", ResultCode == \"202\", \"Starting a host (202)\", ResultCode == \"409\", \"Refused (409)\", toint(ResultCode) >= 500, \"Failed (5xx)\", strcat(\"Other (\", ResultCode, \")\"))', format('| summarize Checkouts = count() by Outcome, {0}', variables('workbookBin')), '| order by TimeGenerated asc')), '\n')]", + "checkoutLatencyQuery": "[join(concat(variables('apiRequests'), createArray('| where ResultCode == \"200\"', variables('requestPath'), '| where Path == \"/api/vms/checkout\"', format('| summarize P50 = percentile(DurationMs, 50), P95 = percentile(DurationMs, 95) by {0}', variables('workbookBin')), '| order by TimeGenerated asc')), '\n')]", + "releaseQuery": "[join(concat(variables('apiRequests'), createArray(variables('requestPath'), '| extend Action = case(Path matches regex @\"^/api/vms/[^/]+/release$\", \"Release\", Path matches regex @\"^/api/vms/[^/]+/return$\", \"Return\", \"\")', '| where isnotempty(Action)', '| extend Series = strcat(Action, iff(toint(ResultCode) < 400, \" succeeded\", strcat(\" failed (\", ResultCode, \")\")))', format('| summarize Requests = count() by Series, {0}', variables('workbookBin')), '| order by TimeGenerated asc')), '\n')]", + "startOnDemandQuery": "[join(createArray('union', ' (AppTraces', format(' | where AppRoleName == \"{0}\" and tostring(Properties.audit_action) == \"vm.start_on_demand\"', parameters('apiAppName')), ' | extend Series = iff(tostring(Properties.audit_outcome) == \"success\", \"Hosts started\", \"Starts that failed\")),', ' (AppRequests', format(' | where AppRoleName == \"{0}\" and ResultCode == \"202\"', parameters('apiAppName')), format(' {0}', variables('requestPath')), ' | where Path == \"/api/vms/checkout\"', ' | extend Series = \"Checkouts told to wait (202)\")', format('| summarize Events = count() by Series, {0}', variables('workbookBin')), '| order by TimeGenerated asc'), '\n')]", + "settingsFailureQuery": "[join(createArray('LinuxBrokerHost_CL', '| where FilePath endswith \"linuxbroker-host-settings.log\"', '| where RawData has_cs \"ERROR\" or RawData has_cs \"WARNING\"', '| project TimeGenerated, Computer, Level = iff(RawData has_cs \"ERROR\", \"Error\", \"Warning\"), Message = RawData', '| order by TimeGenerated desc', '| take 200'), '\n')]", + "hostErrorQuery": "[join(createArray('union isfuzzy=true', ' (LinuxBrokerHost_CL', format(' {0}', variables('hostLogLevel')), ' | where isnotempty(Level)', ' | extend Source = extract(@\"([^/]+)$\", 1, FilePath), Message = RawData),', ' (Syslog', ' | where SeverityLevel in (\"emerg\", \"alert\", \"crit\", \"err\")', ' | extend Level = \"Error\", Source = iff(isempty(ProcessName), strcat(\"syslog \", Facility), strcat(\"syslog \", Facility, \" \", ProcessName)), Message = SyslogMessage)', '| summarize Events = count(), arg_max(TimeGenerated, Message) by Computer, Source, Level', '| project Computer, Source, Level, Events, LastSeen = TimeGenerated, LatestMessage = Message', '| order by Events desc'), '\n')]", + "nfsFailureQuery": "[join(createArray('union isfuzzy=true', ' (LinuxBrokerHost_CL', ' | where FilePath endswith \"createuser.log\" and RawData has \"Failed to mount NFS share\"', ' | extend Source = \"create-user.sh\"),', ' (LinuxBrokerHost_CL', ' | where FilePath endswith \"linuxbroker-session-control.log\" and RawData has \"Could not mount the profile share\"', ' | extend Source = \"session-control.sh\"),', ' (Syslog', ' | where Facility == \"kern\" and SyslogMessage has \"nfs\" and (SyslogMessage has \"not responding\" or SyslogMessage has \"timed out\")', ' | extend Source = \"kernel\", RawData = SyslogMessage)', '| project TimeGenerated, Computer, Source, Message = RawData', '| order by TimeGenerated desc', '| take 500'), '\n')]", + "avdEventQuery": "[join(createArray('Event', '| where Source == \"LinuxBrokerScript\" or EventLog == \"Microsoft-Windows-TerminalServices-RDPClient/Operational\"', '| where EventLevelName in (\"Critical\", \"Error\", \"Warning\")', '| summarize Events = count(), arg_max(TimeGenerated, RenderedDescription) by Computer, Source, Level = EventLevelName', '| project Computer, Source, Level, Events, LastSeen = TimeGenerated, LatestMessage = RenderedDescription', '| order by Events desc'), '\n')]", + "noReadyHostsAlertQuery": "[join(concat(variables('fleetSnapshots'), createArray('| extend Ready = toint(Properties.ReadyHosts), MinVMs = toint(Properties.EffectiveMinVMs), Waiting = toint(Properties.Waiting)', '| summarize Snapshots = count(), Short = countif(Ready == 0 and (MinVMs > 0 or Waiting > 0)), MostWaiting = max(Waiting)', '| where Snapshots > 0 and Short == Snapshots')), '\n')]", + "unhealthyHostsAlertQuery": "[join(concat(variables('fleetSnapshots'), createArray('| extend Stale = toint(Properties.StaleHeartbeats), Nfs = toint(Properties.NfsUnreachable), Xrdp = toint(Properties.XrdpInactive)', '| summarize Snapshots = count(), Lowest = min(Stale + Nfs + Xrdp), StaleHeartbeats = max(Stale), NfsUnreachable = max(Nfs), XrdpInactive = max(Xrdp)', '| where Snapshots > 0 and Lowest > 0')), '\n')]", + "refusedCheckoutAlertQuery": "[join(concat(variables('apiRequests'), createArray('| where ResultCode == \"409\"', variables('requestPath'), '| where Path == \"/api/vms/checkout\"')), '\n')]", + "apiErrorAlertQuery": "[join(concat(variables('apiRequests'), createArray('| where toint(ResultCode) >= 500')), '\n')]", + "logAlerts": [ + { + "name": "no-ready-hosts", + "displayName": "[format('Linux Broker: no ready hosts ({0})', parameters('namePrefix'))]", + "description": "For 15 minutes every fleet snapshot showed no ready Linux host while the scaling minimum is above 0 or users are waiting. Check the Linux hosts and the scaling settings.", + "severity": 1, + "query": "[variables('noReadyHostsAlertQuery')]", + "operator": "GreaterThan", + "threshold": 0 + }, + { + "name": "fleet-snapshot-missing", + "displayName": "[format('Linux Broker: no fleet snapshot ({0})', parameters('namePrefix'))]", + "description": "The scaling run has logged no fleet snapshot for 15 minutes. The task function, the API or the database may be down, or SQL script 157 is not applied yet.", + "severity": 2, + "query": "[join(variables('fleetSnapshots'), '\n')]", + "operator": "LessThan", + "threshold": 1 + }, + { + "name": "checkouts-refused", + "displayName": "[format('Linux Broker: checkouts refused ({0})', parameters('namePrefix'))]", + "description": "The broker refused checkouts (HTTP 409) in the last 15 minutes, so users could not get a Linux desktop.", + "severity": 2, + "query": "[variables('refusedCheckoutAlertQuery')]", + "operator": "GreaterThanOrEqual", + "threshold": "[parameters('alertCheckoutRefusalThreshold')]" + }, + { + "name": "unhealthy-hosts", + "displayName": "[format('Linux Broker: unhealthy hosts ({0})', parameters('namePrefix'))]", + "description": "For 15 minutes every fleet snapshot showed a Linux host with a stale heartbeat, an unreachable home share or no xrdp service. The workbook lists the hosts' errors.", + "severity": 2, + "query": "[variables('unhealthyHostsAlertQuery')]", + "operator": "GreaterThan", + "threshold": 0 + }, + { + "name": "api-errors", + "displayName": "[format('Linux Broker: API errors ({0})', parameters('namePrefix'))]", + "description": "The broker API answered requests with a 5xx status in the last 15 minutes.", + "severity": 2, + "query": "[variables('apiErrorAlertQuery')]", + "operator": "GreaterThanOrEqual", + "threshold": "[parameters('alertApiErrorThreshold')]" + } + ], + "nfsThrottlingResponseTypes": [ + "SuccessWithThrottling", + "SuccessWithShareIopsThrottling", + "SuccessWithShareEgressThrottling", + "SuccessWithShareIngressThrottling", + "SuccessWithMetadataThrottling", + "ClientThrottlingError", + "ClientShareIopsThrottlingError", + "ClientShareEgressThrottlingError", + "ClientShareIngressThrottlingError" + ], + "workbookBaseQuery": { + "version": "KqlItem/1.0", + "size": 0, + "timeContextFromParameter": "TimeRange", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "crossComponentResources": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]" + ] + }, + "workbookContent": { + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "name": "introduction", + "content": { + "json": "## Linux Broker fleet\nThe scaling run logs a fleet snapshot every five minutes. Host logs arrive once deploy/Enable-HostMonitoring.ps1 has installed the Azure Monitor agent on the hosts." + } + }, + { + "type": 9, + "name": "parameters", + "content": { + "version": "KqlParameterItem/1.0", + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.operationalinsights/workspaces", + "parameters": [ + { + "id": "5d0c8f3e-2a41-4b8e-9f6d-1c7a3b2e4f10", + "version": "KqlParameterItem/1.0", + "name": "TimeRange", + "label": "Time range", + "type": 4, + "isRequired": true, + "value": { + "durationMs": 86400000 + }, + "typeSettings": { + "allowCustom": true, + "selectableValues": [ + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + }, + { + "durationMs": 2592000000 + } + ] + } + } + ] + } + }, + { + "type": 3, + "name": "latest-snapshot", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Latest fleet snapshot', 'query', variables('latestSnapshotQuery'), 'visualization', 'table', 'noDataMessage', 'No fleet snapshot in this time range. The API logs one every five minutes once SQL script 157 is applied.'))]" + }, + { + "type": 3, + "name": "fleet-trend", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Hosts over time (fewest ready, most in use, waiting, booting and powered on)', 'query', variables('fleetTrendQuery'), 'visualization', 'timechart'))]" + }, + { + "type": 3, + "name": "checkout-outcomes", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Checkouts by outcome', 'query', variables('checkoutOutcomeQuery'), 'visualization', 'timechart'))]" + }, + { + "type": 3, + "name": "checkout-latency", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Checkout time in milliseconds (50th and 95th percentile)', 'query', variables('checkoutLatencyQuery'), 'visualization', 'timechart'))]" + }, + { + "type": 3, + "name": "releases", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Releases and returns', 'query', variables('releaseQuery'), 'visualization', 'timechart'))]" + }, + { + "type": 3, + "name": "start-on-demand", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Hosts started on demand', 'query', variables('startOnDemandQuery'), 'visualization', 'timechart'))]" + }, + { + "type": 3, + "name": "settings-failures", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Host settings that could not be applied', 'query', variables('settingsFailureQuery'), 'visualization', 'table', 'noDataMessage', 'No host setting errors or warnings in this time range.'))]" + }, + { + "type": 3, + "name": "host-errors", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Errors and warnings by Linux host', 'query', variables('hostErrorQuery'), 'visualization', 'table', 'noDataMessage', 'No host errors in this time range, or the Azure Monitor agent is not installed yet.'))]" + }, + { + "type": 3, + "name": "nfs-failures", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'NFS home share failures', 'query', variables('nfsFailureQuery'), 'visualization', 'table', 'noDataMessage', 'No NFS mount failures in this time range.'))]" + }, + { + "type": 3, + "name": "avd-events", + "customWidth": "50", + "content": "[union(variables('workbookBaseQuery'), createObject('title', 'Errors and warnings by AVD session host', 'query', variables('avdEventQuery'), 'visualization', 'table', 'noDataMessage', 'No AVD session host errors in this time range, or the Azure Monitor agent is not installed yet.'))]" + } + ], + "fallbackResourceIds": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]" + ], + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" + } + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/tables", + "apiVersion": "2023-09-01", + "name": "[format('{0}/{1}', parameters('logAnalyticsWorkspaceName'), variables('hostLogTableName'))]", + "properties": { + "plan": "Analytics", + "schema": { + "name": "[variables('hostLogTableName')]", + "columns": [ + { + "name": "TimeGenerated", + "type": "dateTime" + }, + { + "name": "RawData", + "type": "string" + }, + { + "name": "FilePath", + "type": "string" + }, + { + "name": "Computer", + "type": "string" + } + ] + } + } + }, + { + "type": "Microsoft.Insights/dataCollectionRules", + "apiVersion": "2023-03-11", + "name": "[take(format('dcr-{0}-linux', parameters('namePrefix')), 64)]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "kind": "Linux", + "properties": { + "description": "Linux Broker logs, xrdp logs and syslog from the Linux hosts.", + "streamDeclarations": { + "[format('{0}', variables('hostLogStreamName'))]": { + "columns": [ + { + "name": "TimeGenerated", + "type": "datetime" + }, + { + "name": "RawData", + "type": "string" + }, + { + "name": "FilePath", + "type": "string" + }, + { + "name": "Computer", + "type": "string" + } + ] + } + }, + "dataSources": { + "logFiles": [ + { + "name": "linuxBrokerLogs", + "streams": [ + "[variables('hostLogStreamName')]" + ], + "filePatterns": "[variables('linuxHostLogFiles')]", + "format": "text", + "settings": { + "text": { + "recordStartTimestampFormat": "YYYY-MM-DD HH:MM:SS" + } + } + } + ], + "syslog": [ + { + "name": "authentication", + "streams": [ + "Microsoft-Syslog" + ], + "facilityNames": [ + "auth", + "authpriv" + ], + "logLevels": [ + "Info", + "Notice", + "Warning", + "Error", + "Critical", + "Alert", + "Emergency" + ] + }, + { + "name": "kernelAndUser", + "streams": [ + "Microsoft-Syslog" + ], + "facilityNames": [ + "kern", + "user" + ], + "logLevels": [ + "Notice", + "Warning", + "Error", + "Critical", + "Alert", + "Emergency" + ] + }, + { + "name": "otherFacilities", + "streams": [ + "Microsoft-Syslog" + ], + "facilityNames": [ + "cron", + "daemon", + "ftp", + "local0", + "local1", + "local2", + "local3", + "local4", + "local5", + "local6", + "local7", + "lpr", + "mail", + "news", + "syslog", + "uucp" + ], + "logLevels": [ + "Warning", + "Error", + "Critical", + "Alert", + "Emergency" + ] + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "name": "workspace", + "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "[variables('hostLogStreamName')]" + ], + "destinations": [ + "workspace" + ], + "transformKql": "source", + "outputStream": "[variables('hostLogStreamName')]" + }, + { + "streams": [ + "Microsoft-Syslog" + ], + "destinations": [ + "workspace" + ] + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.OperationalInsights/workspaces/tables', parameters('logAnalyticsWorkspaceName'), variables('hostLogTableName'))]" + ] + }, + { + "type": "Microsoft.Insights/dataCollectionRules", + "apiVersion": "2023-03-11", + "name": "[take(format('dcr-{0}-avd', parameters('namePrefix')), 64)]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "kind": "Windows", + "properties": { + "description": "Connect-LinuxBroker.ps1 events and Remote Desktop client problems from the AVD session hosts.", + "dataSources": { + "windowsEventLogs": [ + { + "name": "linuxBrokerEvents", + "streams": [ + "Microsoft-Event" + ], + "xPathQueries": [ + "Application!*[System[Provider[@Name='LinuxBrokerScript']]]", + "Microsoft-Windows-TerminalServices-RDPClient/Operational!*[System[(Level=1 or Level=2 or Level=3)]]" + ] + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "name": "workspace", + "workspaceResourceId": "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-Event" + ], + "destinations": [ + "workspace" + ] + } + ] + } + }, + { + "type": "Microsoft.Insights/workbooks", + "apiVersion": "2023-06-01", + "name": "[guid(resourceGroup().id, parameters('namePrefix'), 'linuxbroker-fleet-workbook')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "kind": "shared", + "properties": { + "displayName": "[format('Linux Broker fleet ({0})', parameters('namePrefix'))]", + "category": "workbook", + "sourceId": "[toLower(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName')))]", + "version": "Notebook/1.0", + "serializedData": "[string(variables('workbookContent'))]" + } + }, + { + "condition": "[not(empty(variables('alertEmailList')))]", + "type": "Microsoft.Insights/actionGroups", + "apiVersion": "2023-01-01", + "name": "[format('ag-{0}', parameters('namePrefix'))]", + "location": "Global", + "tags": "[parameters('tags')]", + "properties": { + "copy": [ + { + "name": "emailReceivers", + "count": "[length(variables('alertEmailList'))]", + "input": { + "name": "[format('email-{0}', add(copyIndex('emailReceivers'), 1))]", + "emailAddress": "[variables('alertEmailList')[copyIndex('emailReceivers')]]", + "useCommonAlertSchema": true + } + } + ], + "groupShortName": "LinuxBroker", + "enabled": true + } + }, + { + "copy": { + "name": "logAlertRules", + "count": "[length(variables('logAlerts'))]" + }, + "type": "Microsoft.Insights/scheduledQueryRules", + "apiVersion": "2023-12-01", + "name": "[format('alert-{0}-{1}', parameters('namePrefix'), variables('logAlerts')[copyIndex()].name)]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "kind": "LogAlert", + "properties": { + "displayName": "[variables('logAlerts')[copyIndex()].displayName]", + "description": "[variables('logAlerts')[copyIndex()].description]", + "severity": "[variables('logAlerts')[copyIndex()].severity]", + "enabled": true, + "evaluationFrequency": "PT5M", + "windowSize": "PT15M", + "scopes": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWorkspaceName'))]" + ], + "skipQueryValidation": true, + "autoMitigate": true, + "criteria": { + "allOf": [ + { + "query": "[variables('logAlerts')[copyIndex()].query]", + "timeAggregation": "Count", + "operator": "[variables('logAlerts')[copyIndex()].operator]", + "threshold": "[variables('logAlerts')[copyIndex()].threshold]", + "failingPeriods": { + "numberOfEvaluationPeriods": 1, + "minFailingPeriodsToAlert": 1 + } + } + ] + }, + "actions": { + "actionGroups": "[variables('alertActionGroupIds')]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Insights/actionGroups', format('ag-{0}', parameters('namePrefix')))]" + ] + }, + { + "condition": "[variables('monitorNfsShare')]", + "type": "Microsoft.Insights/metricAlerts", + "apiVersion": "2018-03-01", + "name": "[format('alert-{0}-nfs-throttling', parameters('namePrefix'))]", + "location": "global", + "tags": "[parameters('tags')]", + "properties": { + "copy": [ + { + "name": "actions", + "count": "[length(variables('alertActionGroupIds'))]", + "input": { + "actionGroupId": "[variables('alertActionGroupIds')[copyIndex('actions')]]" + } + } + ], + "description": "Azure Files throttled requests to the NFS home share in the last 15 minutes. Users may see slow sign-ins and applications; see the sizing guide in DEPLOYMENT.md.", + "severity": 2, + "enabled": true, + "scopes": [ + "[variables('nfsFileServiceId')]" + ], + "evaluationFrequency": "PT5M", + "windowSize": "PT15M", + "autoMitigate": true, + "criteria": { + "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria", + "allOf": [ + { + "criterionType": "StaticThresholdCriterion", + "name": "ThrottledTransactions", + "metricNamespace": "Microsoft.Storage/storageAccounts/fileServices", + "metricName": "Transactions", + "dimensions": [ + { + "name": "ResponseType", + "operator": "Include", + "values": "[variables('nfsThrottlingResponseTypes')]" + } + ], + "operator": "GreaterThan", + "threshold": 0, + "timeAggregation": "Total" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Insights/actionGroups', format('ag-{0}', parameters('namePrefix')))]" + ] + }, + { + "condition": "[variables('monitorNfsShare')]", + "type": "Microsoft.Insights/metricAlerts", + "apiVersion": "2018-03-01", + "name": "[format('alert-{0}-nfs-latency', parameters('namePrefix'))]", + "location": "global", + "tags": "[parameters('tags')]", + "properties": { + "copy": [ + { + "name": "actions", + "count": "[length(variables('alertActionGroupIds'))]", + "input": { + "actionGroupId": "[variables('alertActionGroupIds')[copyIndex('actions')]]" + } + } + ], + "description": "The NFS home share answered slowly on average over the last 15 minutes.", + "severity": 3, + "enabled": true, + "scopes": [ + "[variables('nfsFileServiceId')]" + ], + "evaluationFrequency": "PT5M", + "windowSize": "PT15M", + "autoMitigate": true, + "criteria": { + "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria", + "allOf": [ + { + "criterionType": "StaticThresholdCriterion", + "name": "EndToEndLatency", + "metricNamespace": "Microsoft.Storage/storageAccounts/fileServices", + "metricName": "SuccessE2ELatency", + "operator": "GreaterThan", + "threshold": "[parameters('alertNfsLatencyThresholdMs')]", + "timeAggregation": "Average" + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Insights/actionGroups', format('ag-{0}', parameters('namePrefix')))]" + ] + } + ], + "outputs": { + "linuxHostDataCollectionRuleId": { + "type": "string", + "value": "[resourceId('Microsoft.Insights/dataCollectionRules', take(format('dcr-{0}-linux', parameters('namePrefix')), 64))]" + }, + "avdHostDataCollectionRuleId": { + "type": "string", + "value": "[resourceId('Microsoft.Insights/dataCollectionRules', take(format('dcr-{0}-avd', parameters('namePrefix')), 64))]" + } + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'nfsStorage')]", + "[resourceId('Microsoft.Resources/deployments', 'observability')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "containerRegistry", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[parameters('location')]" + }, + "tags": { + "value": "[parameters('tags')]" + }, + "containerRegistryName": { + "value": "[variables('containerRegistryName')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.44.1.10279", + "templateHash": "3588414506722776755" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]" + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "containerRegistryName": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2023-07-01", + "name": "[parameters('containerRegistryName')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "sku": { + "name": "Basic" + }, + "properties": { + "adminUserEnabled": false, + "publicNetworkAccess": "Enabled" + } + } + ], + "outputs": { + "id": { + "type": "string", + "value": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]" + }, + "loginServer": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName')), '2023-07-01').loginServer]" + }, + "name": { + "type": "string", + "value": "[parameters('containerRegistryName')]" + } + } + } + } + }, + { "type": "Microsoft.Resources/deployments", "apiVersion": "2025-04-01", "name": "storageAccount", @@ -2161,6 +3313,9 @@ }, "skuName": { "value": "[parameters('appServicePlanSku')]" + }, + "capacity": { + "value": "[parameters('appServicePlanCapacity')]" } }, "template": { @@ -2170,7 +3325,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "1639916006898550317" + "templateHash": "10348306834059684448" } }, "parameters": { @@ -2188,6 +3343,15 @@ "skuName": { "type": "string", "defaultValue": "P2mv3" + }, + "capacity": { + "type": "int", + "defaultValue": 1, + "minValue": 1, + "maxValue": 30, + "metadata": { + "description": "Instances the plan runs. The portal, the API and the task function all run on every instance." + } } }, "resources": [ @@ -2201,7 +3365,7 @@ "name": "[parameters('skuName')]", "tier": "PremiumV3", "size": "[parameters('skuName')]", - "capacity": 1 + "capacity": "[parameters('capacity')]" }, "kind": "linux", "properties": { @@ -2220,7 +3384,269 @@ } } } - } + } + }, + { + "condition": "[variables('usePortalRedis')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "portalRedis", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "kind": { + "value": "[variables('cloudProfile').portalRedisKind]" + }, + "name": { + "value": "[variables('portalRedisName')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "cacheLocation": "[if(empty(parameters('portalRedisLocation')), createObject('value', parameters('location')), createObject('value', parameters('portalRedisLocation')))]", + "tags": { + "value": "[parameters('tags')]" + }, + "skuName": { + "value": "[parameters('portalRedisSku')]" + }, + "virtualNetworkId": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'networking'), '2025-04-01').outputs.vnetId.value]" + }, + "privateEndpointSubnetId": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'networking'), '2025-04-01').outputs.privateEndpointSubnetId.value]" + }, + "privateDnsZoneName": { + "value": "[variables('cloudProfile').portalRedisPrivateDnsZoneName]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.44.1.10279", + "templateHash": "11134019957083360765" + } + }, + "parameters": { + "kind": { + "type": "string", + "allowedValues": [ + "managed", + "cache" + ], + "metadata": { + "description": "Azure Managed Redis (managed), or Azure Cache for Redis (cache) where Azure Managed Redis is not available, as in Azure Government." + } + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 60, + "metadata": { + "description": "Name of the cache. It becomes part of a public DNS name, so it must be unique." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Region of the private endpoint, which must be the virtual network's region." + } + }, + "cacheLocation": { + "type": "string", + "defaultValue": "[parameters('location')]", + "metadata": { + "description": "Region of the cache. Defaults to the virtual network's region. The portal reaches a cache in another region through the same private endpoint." + } + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "skuName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Leave empty for the default size: Balanced_B0 for Azure Managed Redis, Standard_C1 for Azure Cache for Redis. Azure Cache for Redis sizes are written _, such as Standard_C2 or Premium_P1." + } + }, + "virtualNetworkId": { + "type": "string", + "metadata": { + "description": "Virtual network the portal reaches the cache from." + } + }, + "privateEndpointSubnetId": { + "type": "string", + "metadata": { + "description": "Subnet the cache private endpoint is placed in." + } + }, + "privateDnsZoneName": { + "type": "string", + "metadata": { + "description": "Private DNS zone of the cache private endpoint: privatelink.redis.azure.net for Azure Managed Redis, privatelink.redis.cache. for Azure Cache for Redis." + } + } + }, + "variables": { + "managed": "[equals(parameters('kind'), 'managed')]", + "effectiveSku": "[if(empty(parameters('skuName')), if(variables('managed'), 'Balanced_B0', 'Standard_C1'), parameters('skuName'))]", + "cacheSkuParts": "[split(variables('effectiveSku'), '_')]", + "managedPort": 10000 + }, + "resources": [ + { + "condition": "[variables('managed')]", + "type": "Microsoft.Cache/redisEnterprise", + "apiVersion": "2025-07-01", + "name": "[parameters('name')]", + "location": "[parameters('cacheLocation')]", + "tags": "[parameters('tags')]", + "sku": { + "name": "[variables('effectiveSku')]" + }, + "properties": { + "minimumTlsVersion": "1.2", + "publicNetworkAccess": "Disabled" + } + }, + { + "condition": "[variables('managed')]", + "type": "Microsoft.Cache/redisEnterprise/databases", + "apiVersion": "2025-07-01", + "name": "[format('{0}/{1}', parameters('name'), 'default')]", + "properties": { + "clientProtocol": "Encrypted", + "port": "[variables('managedPort')]", + "clusteringPolicy": "EnterpriseCluster", + "evictionPolicy": "VolatileLRU", + "accessKeysAuthentication": "Disabled" + }, + "dependsOn": [ + "[resourceId('Microsoft.Cache/redisEnterprise', parameters('name'))]" + ] + }, + { + "condition": "[not(variables('managed'))]", + "type": "Microsoft.Cache/redis", + "apiVersion": "2024-11-01", + "name": "[parameters('name')]", + "location": "[parameters('cacheLocation')]", + "tags": "[parameters('tags')]", + "properties": { + "sku": { + "name": "[variables('cacheSkuParts')[0]]", + "family": "[first(variables('cacheSkuParts')[1])]", + "capacity": "[int(skip(variables('cacheSkuParts')[1], 1))]" + }, + "enableNonSslPort": false, + "minimumTlsVersion": "1.2", + "publicNetworkAccess": "Disabled", + "disableAccessKeyAuthentication": true, + "redisConfiguration": { + "aad-enabled": "true", + "maxmemory-policy": "volatile-lru" + } + } + }, + { + "type": "Microsoft.Network/privateEndpoints", + "apiVersion": "2024-05-01", + "name": "[format('pe-{0}', parameters('name'))]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "subnet": { + "id": "[parameters('privateEndpointSubnetId')]" + }, + "privateLinkServiceConnections": [ + { + "name": "redis", + "properties": { + "privateLinkServiceId": "[if(variables('managed'), resourceId('Microsoft.Cache/redisEnterprise', parameters('name')), resourceId('Microsoft.Cache/redis', parameters('name')))]", + "groupIds": [ + "[if(variables('managed'), 'redisEnterprise', 'redisCache')]" + ] + } + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.Cache/redis', parameters('name'))]", + "[resourceId('Microsoft.Cache/redisEnterprise', parameters('name'))]", + "[resourceId('Microsoft.Cache/redisEnterprise/databases', parameters('name'), 'default')]" + ] + }, + { + "type": "Microsoft.Network/privateDnsZones", + "apiVersion": "2024-06-01", + "name": "[parameters('privateDnsZoneName')]", + "location": "global", + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "apiVersion": "2024-06-01", + "name": "[format('{0}/{1}', parameters('privateDnsZoneName'), format('link-{0}', last(split(parameters('virtualNetworkId'), '/'))))]", + "location": "global", + "tags": "[parameters('tags')]", + "properties": { + "registrationEnabled": false, + "virtualNetwork": { + "id": "[parameters('virtualNetworkId')]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/privateDnsZones', parameters('privateDnsZoneName'))]" + ] + }, + { + "type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups", + "apiVersion": "2024-05-01", + "name": "[format('{0}/{1}', format('pe-{0}', parameters('name')), 'default')]", + "properties": { + "privateDnsZoneConfigs": [ + { + "name": "redis", + "properties": { + "privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', parameters('privateDnsZoneName'))]" + } + } + ] + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/privateDnsZones', parameters('privateDnsZoneName'))]", + "[resourceId('Microsoft.Network/privateEndpoints', format('pe-{0}', parameters('name')))]" + ] + } + ], + "outputs": { + "name": { + "type": "string", + "value": "[parameters('name')]" + }, + "hostName": { + "type": "string", + "value": "[if(variables('managed'), reference(resourceId('Microsoft.Cache/redisEnterprise', parameters('name')), '2025-07-01').hostName, reference(resourceId('Microsoft.Cache/redis', parameters('name')), '2024-11-01').hostName)]" + }, + "port": { + "type": "int", + "value": "[if(variables('managed'), variables('managedPort'), reference(resourceId('Microsoft.Cache/redis', parameters('name')), '2024-11-01').sslPort)]" + } + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'networking')]" + ] }, { "type": "Microsoft.Resources/deployments", @@ -2254,7 +3680,7 @@ "value": "[reference(resourceId('Microsoft.Resources/deployments', 'observability'), '2025-04-01').outputs.applicationInsightsConnectionString.value]" }, "appSettings": { - "value": "[variables('frontendSettings')]" + "value": "[union(variables('frontendSettings'), if(variables('usePortalRedis'), union(createObject('SESSION_BACKEND', 'redis', 'REDIS_HOST', reference(resourceId('Microsoft.Resources/deployments', 'portalRedis'), '2025-04-01').outputs.hostName.value, 'REDIS_PORT', string(reference(resourceId('Microsoft.Resources/deployments', 'portalRedis'), '2025-04-01').outputs.port.value)), if(empty(variables('cloudProfile').portalRedisEntraResource), createObject(), createObject('REDIS_ENTRA_RESOURCE', variables('cloudProfile').portalRedisEntraResource))), createObject('SESSION_BACKEND', 'filesystem')))]" }, "authSettings": { "value": "[variables('frontendAuthSettings')]" @@ -2267,6 +3693,10 @@ }, "useManagedIdentityForRegistry": { "value": true + }, + "virtualNetworkSubnetId": "[if(variables('usePortalRedis'), createObject('value', reference(resourceId('Microsoft.Resources/deployments', 'networking'), '2025-04-01').outputs.appSubnetId.value), createObject('value', ''))]", + "clientAffinityEnabled": { + "value": "[not(variables('usePortalRedis'))]" } }, "template": { @@ -2276,7 +3706,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "8972138919102563065" + "templateHash": "1104947881100529762" } }, "parameters": { @@ -2329,6 +3759,13 @@ "metadata": { "description": "Delegated subnet for regional VNet integration. Leave empty to keep the app off the virtual network." } + }, + "clientAffinityEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Send each browser back to the instance that served it first (ARR affinity). Only an app that keeps state on the instance needs it." + } } }, "variables": { @@ -2348,6 +3785,7 @@ "properties": { "serverFarmId": "[parameters('serverFarmId')]", "httpsOnly": true, + "clientAffinityEnabled": "[parameters('clientAffinityEnabled')]", "virtualNetworkSubnetId": "[if(empty(parameters('virtualNetworkSubnetId')), null(), parameters('virtualNetworkSubnetId'))]", "siteConfig": "[variables('webSiteConfig')]" } @@ -2424,7 +3862,93 @@ "dependsOn": [ "[resourceId('Microsoft.Resources/deployments', 'appServicePlan')]", "[resourceId('Microsoft.Resources/deployments', 'containerRegistry')]", - "[resourceId('Microsoft.Resources/deployments', 'observability')]" + "[resourceId('Microsoft.Resources/deployments', 'networking')]", + "[resourceId('Microsoft.Resources/deployments', 'observability')]", + "[resourceId('Microsoft.Resources/deployments', 'portalRedis')]" + ] + }, + { + "condition": "[variables('usePortalRedis')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "portalRedisAccess", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "kind": { + "value": "[variables('cloudProfile').portalRedisKind]" + }, + "redisName": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'portalRedis'), '2025-04-01').outputs.name.value]" + }, + "principalId": { + "value": "[reference(resourceId('Microsoft.Resources/deployments', 'frontendApp'), '2025-04-01').outputs.principalId.value]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.44.1.10279", + "templateHash": "11694917012265494470" + } + }, + "parameters": { + "kind": { + "type": "string", + "allowedValues": [ + "managed", + "cache" + ] + }, + "redisName": { + "type": "string" + }, + "principalId": { + "type": "string", + "metadata": { + "description": "Object ID of the portal's system-assigned managed identity." + } + } + }, + "variables": { + "assignmentName": "[format('portal{0}', uniqueString(parameters('principalId')))]" + }, + "resources": [ + { + "condition": "[equals(parameters('kind'), 'managed')]", + "type": "Microsoft.Cache/redisEnterprise/databases/accessPolicyAssignments", + "apiVersion": "2025-07-01", + "name": "[format('{0}/{1}/{2}', parameters('redisName'), 'default', variables('assignmentName'))]", + "properties": { + "accessPolicyName": "default", + "user": { + "objectId": "[parameters('principalId')]" + } + } + }, + { + "condition": "[equals(parameters('kind'), 'cache')]", + "type": "Microsoft.Cache/redis/accessPolicyAssignments", + "apiVersion": "2024-11-01", + "name": "[format('{0}/{1}', parameters('redisName'), variables('assignmentName'))]", + "properties": { + "accessPolicyName": "Data Contributor", + "objectId": "[parameters('principalId')]", + "objectIdAlias": "[parameters('principalId')]" + } + } + ] + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'frontendApp')]", + "[resourceId('Microsoft.Resources/deployments', 'portalRedis')]" ] }, { @@ -2501,6 +4025,9 @@ }, "virtualNetworkSubnetId": { "value": "[reference(resourceId('Microsoft.Resources/deployments', 'networking'), '2025-04-01').outputs.appSubnetId.value]" + }, + "clientAffinityEnabled": { + "value": false } }, "template": { @@ -2510,7 +4037,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "8972138919102563065" + "templateHash": "1104947881100529762" } }, "parameters": { @@ -2563,6 +4090,13 @@ "metadata": { "description": "Delegated subnet for regional VNet integration. Leave empty to keep the app off the virtual network." } + }, + "clientAffinityEnabled": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Send each browser back to the instance that served it first (ARR affinity). Only an app that keeps state on the instance needs it." + } } }, "variables": { @@ -2582,6 +4116,7 @@ "properties": { "serverFarmId": "[parameters('serverFarmId')]", "httpsOnly": true, + "clientAffinityEnabled": "[parameters('clientAffinityEnabled')]", "virtualNetworkSubnetId": "[if(empty(parameters('virtualNetworkSubnetId')), null(), parameters('virtualNetworkSubnetId'))]", "siteConfig": "[variables('webSiteConfig')]" } @@ -2964,6 +4499,9 @@ }, "desktop": { "value": "[parameters('linuxHostDesktop')]" + }, + "skipExtensionVmNames": { + "value": "[parameters('hostNamesNotRunning')]" } }, "template": { @@ -2973,7 +4511,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "11700213154129506841" + "templateHash": "4186961051720753174" } }, "parameters": { @@ -3063,6 +4601,13 @@ "metadata": { "description": "Desktop the hosts run in xrdp sessions. Changing it changes the extension command, which runs the bootstrap again on existing hosts." } + }, + "skipExtensionVmNames": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Hosts whose bootstrap extension this deployment leaves out, because Azure refuses to change an extension on a VM that is not running. The preprovision hook passes the ones that are deallocated or stopped." + } } }, "variables": { @@ -3071,6 +4616,11 @@ "name": "vmNames", "count": "[length(range(1, parameters('numberOfVMs')))]", "input": "[format('{0}-{1}', parameters('vmNamePrefix'), padLeft(range(1, parameters('numberOfVMs'))[copyIndex('vmNames')], 2, '0'))]" + }, + { + "name": "skipExtensionVmNamesLower", + "count": "[length(parameters('skipExtensionVmNames'))]", + "input": "[toLower(parameters('skipExtensionVmNames')[copyIndex('skipExtensionVmNamesLower')])]" } ], "normalizedScriptSourceRoot": "[if(endsWith(parameters('scriptSourceRoot'), '/'), take(parameters('scriptSourceRoot'), sub(length(parameters('scriptSourceRoot')), 1)), parameters('scriptSourceRoot'))]", @@ -3226,6 +4776,7 @@ "name": "linuxCustomScriptExtension", "count": "[length(variables('vmNames'))]" }, + "condition": "[not(contains(variables('skipExtensionVmNamesLower'), toLower(variables('vmNames')[copyIndex()])))]", "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2022-03-01", "name": "[format('{0}/customScript', variables('vmNames')[copyIndex()])]", @@ -3313,6 +4864,21 @@ }, "avdUsersGroupId": { "value": "[parameters('avdUsersGroupId')]" + }, + "linuxDesktopFullScreen": { + "value": "[parameters('avdLinuxDesktopFullScreen')]" + }, + "linuxDesktopMultiMonitor": { + "value": "[parameters('avdLinuxDesktopMultiMonitor')]" + }, + "startVmOnConnect": { + "value": "[parameters('avdStartVmOnConnect')]" + }, + "skipExtensionVmNames": { + "value": "[parameters('hostNamesNotRunning')]" + }, + "scalingExclusions": { + "value": "[parameters('avdScalingExclusions')]" } }, "template": { @@ -3322,7 +4888,7 @@ "_generator": { "name": "bicep", "version": "0.44.1.10279", - "templateHash": "16448105144039973911" + "templateHash": "945210851556391725" } }, "parameters": { @@ -3465,10 +5031,50 @@ "metadata": { "description": "Display name of the RemoteApp that connects users to a Linux host." } + }, + "linuxDesktopFullScreen": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Opens the Linux desktop full screen. False opens it in a window on one monitor. Turn it off only once every session host runs Connect-LinuxBroker.ps1 2.0.0 or later: older scripts refuse the argument and the Linux Desktop app fails to open." + } + }, + "linuxDesktopMultiMonitor": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Spreads a full-screen Linux desktop across every monitor. False keeps it on one monitor. Turn it off only once every session host runs Connect-LinuxBroker.ps1 2.0.0 or later: older scripts refuse the argument and the Linux Desktop app fails to open." + } + }, + "startVmOnConnect": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Starts a deallocated session host when a user opens the Linux Desktop app and no running session host can take the session. Azure Virtual Desktop needs Desktop Virtualization Power On Contributor, or Power On Off Contributor, to start it." + } + }, + "skipExtensionVmNames": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Session hosts whose VM extensions this deployment leaves out, because Azure refuses to change an extension on a VM that is not running. The preprovision hook passes the ones that are deallocated or stopped." + } + }, + "scalingExclusions": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Lowercase session host names, each mapped to the value of the excludeFromScaling tag it carries. A deployment replaces the tags of every VM it declares, so these are written back." + } } }, "variables": { "copy": [ + { + "name": "skipExtensionVmNamesLower", + "count": "[length(parameters('skipExtensionVmNames'))]", + "input": "[toLower(parameters('skipExtensionVmNames')[copyIndex('skipExtensionVmNamesLower')])]" + }, { "name": "vmNames", "count": "[length(range(1, parameters('sessionHostCount')))]", @@ -3479,6 +5085,7 @@ "linuxBrokerConfigScriptUri": "[format('{0}/custom_script_extensions/Configure-AVD-Host.ps1', variables('normalizedScriptSourceRoot'))]", "desktopVirtualizationUserRoleId": "1d18fff3-a72a-46b5-b4a9-0b38a3cd7e63", "virtualMachineUserLoginRoleId": "fb879df8-f326-4884-b1cf-06f3ad86be52", + "linuxDesktopDisplayArguments": "[format('{0}{1}', if(parameters('linuxDesktopFullScreen'), '', ' -FullScreen Off'), if(parameters('linuxDesktopMultiMonitor'), '', ' -MultiMonitor Off'))]", "osImage": "microsoftwindowsdesktop:Windows-11:win11-24h2-avd:latest", "storageAccountName": "wvdportalstorageblob", "containerName": "galleryartifacts", @@ -3502,7 +5109,7 @@ "loadBalancerType": "[parameters('loadBalancerType')]", "maxSessionLimit": "[parameters('maxSessionLimit')]", "customRdpProperty": "[parameters('customRdpProperty')]", - "startVMOnConnect": false, + "startVMOnConnect": "[parameters('startVmOnConnect')]", "validationEnvironment": false, "agentUpdate": "[parameters('agentUpdate')]", "registrationInfo": { @@ -3606,7 +5213,7 @@ "apiVersion": "2024-11-01", "name": "[variables('vmNames')[copyIndex()]]", "location": "[parameters('location')]", - "tags": "[parameters('tags')]", + "tags": "[if(contains(parameters('scalingExclusions'), toLower(variables('vmNames')[copyIndex()])), union(parameters('tags'), createObject('excludeFromScaling', parameters('scalingExclusions')[toLower(variables('vmNames')[copyIndex()])])), parameters('tags'))]", "identity": { "type": "SystemAssigned" }, @@ -3665,6 +5272,7 @@ "name": "entraloginExtension", "count": "[length(variables('vmNames'))]" }, + "condition": "[not(contains(variables('skipExtensionVmNamesLower'), toLower(variables('vmNames')[copyIndex()])))]", "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2024-11-01", "name": "[format('{0}/AADLoginForWindows', variables('vmNames')[copyIndex()])]", @@ -3686,6 +5294,7 @@ "name": "avdDscExtension", "count": "[length(variables('vmNames'))]" }, + "condition": "[not(contains(variables('skipExtensionVmNamesLower'), toLower(variables('vmNames')[copyIndex()])))]", "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2024-11-01", "name": "[format('{0}/Microsoft.PowerShell.DSC', variables('vmNames')[copyIndex()])]", @@ -3729,6 +5338,7 @@ "name": "linuxBrokerConfig", "count": "[length(variables('vmNames'))]" }, + "condition": "[not(contains(variables('skipExtensionVmNamesLower'), toLower(variables('vmNames')[copyIndex()])))]", "type": "Microsoft.Compute/virtualMachines/extensions", "apiVersion": "2024-11-01", "name": "[format('{0}/CustomScriptExtension', variables('vmNames')[copyIndex()])]", @@ -3782,7 +5392,7 @@ "applicationType": "InBuilt", "filePath": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", "commandLineSetting": "Require", - "commandLineArguments": "-ExecutionPolicy Bypass -NoProfile -WindowStyle Hidden -File C:\\Temp\\Connect-LinuxBroker.ps1", + "commandLineArguments": "[format('-ExecutionPolicy Bypass -NoProfile -WindowStyle Hidden -File C:\\Temp\\Connect-LinuxBroker.ps1{0}', variables('linuxDesktopDisplayArguments'))]", "iconPath": "C:\\Windows\\System32\\mstsc.exe", "iconIndex": 0, "showInPortal": true @@ -3830,7 +5440,7 @@ "value": "[parameters('customRdpProperty')]" }, "startVMOnConnect": { - "value": "[reference(resourceId('Microsoft.DesktopVirtualization/hostPools', parameters('hostPoolName')), '2024-04-03').startVMOnConnect]" + "value": "[parameters('startVmOnConnect')]" }, "validationEnvironment": { "value": "[reference(resourceId('Microsoft.DesktopVirtualization/hostPools', parameters('hostPoolName')), '2024-04-03').validationEnvironment]" @@ -4024,11 +5634,286 @@ "nfsSharePath": { "type": "string", "value": "[variables('effectiveNfsShare')]" + }, + "linuxHostDataCollectionRuleId": { + "type": "string", + "value": "[if(parameters('deployHostMonitoring'), reference(resourceId('Microsoft.Resources/deployments', 'hostMonitoring'), '2025-04-01').outputs.linuxHostDataCollectionRuleId.value, '')]" + }, + "avdHostDataCollectionRuleId": { + "type": "string", + "value": "[if(parameters('deployHostMonitoring'), reference(resourceId('Microsoft.Resources/deployments', 'hostMonitoring'), '2025-04-01').outputs.avdHostDataCollectionRuleId.value, '')]" + }, + "portalSessionStore": { + "type": "string", + "value": "[if(variables('usePortalRedis'), 'redis', 'filesystem')]" + }, + "portalRedisName": { + "type": "string", + "value": "[if(variables('usePortalRedis'), reference(resourceId('Microsoft.Resources/deployments', 'portalRedis'), '2025-04-01').outputs.name.value, '')]" + } + } + } + }, + "dependsOn": [ + "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('effectiveResourceGroupName'))]" + ] + }, + { + "condition": "[variables('deployAvdSessionHosts')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "avdScalingPlan", + "resourceGroup": "[variables('effectiveResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[parameters('location')]" + }, + "tags": { + "value": "[union(parameters('tags'), createObject('broker-role', 'avd-host'))]" + }, + "name": { + "value": "[take(format('{0}-scaling-plan', parameters('avdHostPoolName')), 64)]" + }, + "hostPoolName": { + "value": "[parameters('avdHostPoolName')]" + }, + "enabled": { + "value": "[parameters('avdScalingPlanEnabled')]" + }, + "timeZone": { + "value": "[parameters('avdScalingPlanTimeZone')]" + }, + "rampUpStartTime": { + "value": "[parameters('avdScalingPlanRampUpStart')]" + }, + "peakStartTime": { + "value": "[parameters('avdScalingPlanPeakStart')]" + }, + "rampDownStartTime": { + "value": "[parameters('avdScalingPlanRampDownStart')]" + }, + "offPeakStartTime": { + "value": "[parameters('avdScalingPlanOffPeakStart')]" + }, + "rampUpMinimumHostsPct": { + "value": "[parameters('avdScalingPlanRampUpMinimumHostsPct')]" + }, + "rampUpCapacityThresholdPct": { + "value": "[parameters('avdScalingPlanRampUpCapacityThresholdPct')]" + }, + "rampDownMinimumHostsPct": { + "value": "[parameters('avdScalingPlanRampDownMinimumHostsPct')]" + }, + "rampDownCapacityThresholdPct": { + "value": "[parameters('avdScalingPlanRampDownCapacityThresholdPct')]" + }, + "weekendMinimumHostsPct": { + "value": "[parameters('avdScalingPlanWeekendMinimumHostsPct')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.44.1.10279", + "templateHash": "155344827278101734" + } + }, + "parameters": { + "location": { + "type": "string" + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "name": { + "type": "string", + "minLength": 3, + "maxLength": 64 + }, + "friendlyName": { + "type": "string", + "defaultValue": "[parameters('name')]" + }, + "hostPoolName": { + "type": "string", + "metadata": { + "description": "Name of the pooled host pool the plan scales, in the same resource group." + } + }, + "enabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Scales the host pool. False keeps the plan but assigns it to no host pool, so the session hosts are left as they are and another plan can be assigned instead." + } + }, + "timeZone": { + "type": "string", + "defaultValue": "UTC", + "metadata": { + "description": "Windows time zone ID the schedule times are in, for example UTC or Eastern Standard Time." + } + }, + "exclusionTag": { + "type": "string", + "defaultValue": "excludeFromScaling", + "metadata": { + "description": "Session hosts that carry this tag are never started, stopped or taken out of drain mode by autoscale." + } + }, + "rampUpStartTime": { + "type": "string", + "defaultValue": "07:00", + "metadata": { + "description": "Weekday start of ramp-up, as HH:mm in timeZone. The weekend uses the same times." + } + }, + "peakStartTime": { + "type": "string", + "defaultValue": "09:00", + "metadata": { + "description": "Start of peak hours, as HH:mm. Must come after ramp-up." + } + }, + "rampDownStartTime": { + "type": "string", + "defaultValue": "18:00", + "metadata": { + "description": "Start of ramp-down, as HH:mm. Must come after peak." + } + }, + "offPeakStartTime": { + "type": "string", + "defaultValue": "20:00", + "metadata": { + "description": "Start of off-peak hours, as HH:mm. Must come after ramp-down." + } + }, + "rampUpMinimumHostsPct": { + "type": "int", + "defaultValue": 20, + "minValue": 0, + "maxValue": 100, + "metadata": { + "description": "Share of the session hosts kept on from ramp-up until ramp-down on weekdays, rounded up to whole hosts." + } + }, + "rampUpCapacityThresholdPct": { + "type": "int", + "defaultValue": 60, + "minValue": 1, + "maxValue": 100, + "metadata": { + "description": "Share of the running hosts' sessions that makes autoscale start another host during ramp-up and peak hours." + } + }, + "rampDownMinimumHostsPct": { + "type": "int", + "defaultValue": 10, + "minValue": 0, + "maxValue": 100, + "metadata": { + "description": "Share of the session hosts kept on from ramp-down until the next ramp-up on weekdays, rounded up to whole hosts." + } + }, + "rampDownCapacityThresholdPct": { + "type": "int", + "defaultValue": 90, + "minValue": 1, + "maxValue": 100, + "metadata": { + "description": "Share of the running hosts' sessions that makes autoscale start another host during ramp-down and off-peak hours." + } + }, + "weekendMinimumHostsPct": { + "type": "int", + "defaultValue": 0, + "minValue": 0, + "maxValue": 100, + "metadata": { + "description": "Share of the session hosts kept on through Saturday and Sunday. 0 lets every host turn off, and Start VM on Connect starts one for the first user." + } + } + }, + "variables": { + "rampUpTime": "[split(parameters('rampUpStartTime'), ':')]", + "peakTime": "[split(parameters('peakStartTime'), ':')]", + "rampDownTime": "[split(parameters('rampDownStartTime'), ':')]", + "offPeakTime": "[split(parameters('offPeakStartTime'), ':')]", + "sharedScheduleSettings": { + "rampUpStartTime": { + "hour": "[int(variables('rampUpTime')[0])]", + "minute": "[int(variables('rampUpTime')[1])]" + }, + "rampUpLoadBalancingAlgorithm": "BreadthFirst", + "rampUpCapacityThresholdPct": "[parameters('rampUpCapacityThresholdPct')]", + "peakStartTime": { + "hour": "[int(variables('peakTime')[0])]", + "minute": "[int(variables('peakTime')[1])]" + }, + "peakLoadBalancingAlgorithm": "BreadthFirst", + "rampDownStartTime": { + "hour": "[int(variables('rampDownTime')[0])]", + "minute": "[int(variables('rampDownTime')[1])]" + }, + "rampDownLoadBalancingAlgorithm": "DepthFirst", + "rampDownCapacityThresholdPct": "[parameters('rampDownCapacityThresholdPct')]", + "rampDownForceLogoffUsers": false, + "rampDownStopHostsWhen": "ZeroSessions", + "rampDownWaitTimeMinutes": 30, + "rampDownNotificationMessage": "This session host is about to shut down. Save your work and sign out.", + "offPeakStartTime": { + "hour": "[int(variables('offPeakTime')[0])]", + "minute": "[int(variables('offPeakTime')[1])]" + }, + "offPeakLoadBalancingAlgorithm": "DepthFirst" + } + }, + "resources": [ + { + "type": "Microsoft.DesktopVirtualization/scalingPlans", + "apiVersion": "2024-04-03", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "friendlyName": "[parameters('friendlyName')]", + "description": "Starts and stops the Linux Desktop session hosts on a schedule.", + "hostPoolType": "Pooled", + "timeZone": "[parameters('timeZone')]", + "exclusionTag": "[parameters('exclusionTag')]", + "hostPoolReferences": "[if(parameters('enabled'), createArray(createObject('hostPoolArmPath', resourceId('Microsoft.DesktopVirtualization/hostPools', parameters('hostPoolName')), 'scalingPlanEnabled', true())), createArray())]", + "schedules": [ + "[union(variables('sharedScheduleSettings'), createObject('name', 'weekdays', 'daysOfWeek', createArray('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday'), 'rampUpMinimumHostsPct', parameters('rampUpMinimumHostsPct'), 'rampDownMinimumHostsPct', parameters('rampDownMinimumHostsPct')))]", + "[union(variables('sharedScheduleSettings'), createObject('name', 'weekend', 'daysOfWeek', createArray('Saturday', 'Sunday'), 'rampUpMinimumHostsPct', parameters('weekendMinimumHostsPct'), 'rampDownMinimumHostsPct', parameters('weekendMinimumHostsPct')))]" + ] + } + } + ], + "outputs": { + "name": { + "type": "string", + "value": "[parameters('name')]" + }, + "id": { + "type": "string", + "value": "[resourceId('Microsoft.DesktopVirtualization/scalingPlans', parameters('name'))]" } } } }, "dependsOn": [ + "[subscriptionResourceId('Microsoft.Authorization/roleAssignments', guid(subscription().id, parameters('avdServicePrincipalObjectId'), variables('avdPowerOnOffContributorRoleGuid')))]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('effectiveResourceGroupName')), 'Microsoft.Resources/deployments', 'resources')]", "[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('effectiveResourceGroupName'))]" ] } @@ -4089,6 +5974,22 @@ "nfsSharePath": { "type": "string", "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('effectiveResourceGroupName')), 'Microsoft.Resources/deployments', 'resources'), '2025-04-01').outputs.nfsSharePath.value]" + }, + "linuxHostDataCollectionRuleId": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('effectiveResourceGroupName')), 'Microsoft.Resources/deployments', 'resources'), '2025-04-01').outputs.linuxHostDataCollectionRuleId.value]" + }, + "avdHostDataCollectionRuleId": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('effectiveResourceGroupName')), 'Microsoft.Resources/deployments', 'resources'), '2025-04-01').outputs.avdHostDataCollectionRuleId.value]" + }, + "portalSessionStore": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('effectiveResourceGroupName')), 'Microsoft.Resources/deployments', 'resources'), '2025-04-01').outputs.portalSessionStore.value]" + }, + "portalRedisName": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('effectiveResourceGroupName')), 'Microsoft.Resources/deployments', 'resources'), '2025-04-01').outputs.portalRedisName.value]" } } } \ No newline at end of file diff --git a/deploy/bicep/main.parameters.example.json b/deploy/bicep/main.parameters.example.json index 62d8e53..5363c33 100644 --- a/deploy/bicep/main.parameters.example.json +++ b/deploy/bicep/main.parameters.example.json @@ -11,6 +11,18 @@ "appServicePlanSku": { "value": "P2mv3" }, + "appServicePlanCapacity": { + "value": 1 + }, + "portalSessionStore": { + "value": "redis" + }, + "portalRedisSku": { + "value": "" + }, + "portalRedisLocation": { + "value": "" + }, "appName": { "value": "linuxbroker" }, @@ -140,11 +152,80 @@ "avdMaxSessionLimit": { "value": 5 }, + "avdLinuxDesktopFullScreen": { + "value": true + }, + "avdLinuxDesktopMultiMonitor": { + "value": true + }, + "avdScalingPlanEnabled": { + "value": true + }, + "avdStartVmOnConnect": { + "value": true + }, + "avdScalingPlanTimeZone": { + "value": "UTC" + }, + "avdScalingPlanRampUpStart": { + "value": "07:00" + }, + "avdScalingPlanPeakStart": { + "value": "09:00" + }, + "avdScalingPlanRampDownStart": { + "value": "18:00" + }, + "avdScalingPlanOffPeakStart": { + "value": "20:00" + }, + "avdScalingPlanRampUpMinimumHostsPct": { + "value": 20 + }, + "avdScalingPlanRampUpCapacityThresholdPct": { + "value": 60 + }, + "avdScalingPlanRampDownMinimumHostsPct": { + "value": 10 + }, + "avdScalingPlanRampDownCapacityThresholdPct": { + "value": 90 + }, + "avdScalingPlanWeekendMinimumHostsPct": { + "value": 0 + }, + "avdScalingExclusions": { + "value": {} + }, + "hostNamesNotRunning": { + "value": [] + }, + "avdServicePrincipalObjectId": { + "value": "" + }, + "assignAvdAutoscaleRole": { + "value": true + }, "avdVmNamePrefix": { "value": "avdhost" }, "avdVmSize": { "value": "Standard_D8s_v5" + }, + "deployHostMonitoring": { + "value": true + }, + "alertEmailAddresses": { + "value": "" + }, + "alertCheckoutRefusalThreshold": { + "value": 1 + }, + "alertApiErrorThreshold": { + "value": 5 + }, + "alertNfsLatencyThresholdMs": { + "value": 50 } } } \ No newline at end of file diff --git a/deploy/bicep/main.resources.bicep b/deploy/bicep/main.resources.bicep index 4eef7dc..24188bf 100644 --- a/deploy/bicep/main.resources.bicep +++ b/deploy/bicep/main.resources.bicep @@ -69,6 +69,24 @@ param appServiceDomain string = '' param scriptSourceRoot string = 'https://raw.githubusercontent.com/microsoft/LinuxBrokerForAVDAccess/refs/heads/main' param allowedClientIp string = '' param appServicePlanSku string = 'P2mv3' + +@description('Instances of the App Service plan. The portal, the API and the task function run on every instance.') +@minValue(1) +@maxValue(30) +param appServicePlanCapacity int = 1 + +@description('Where the portal keeps sign-in sessions: redis, a cache every portal instance shares, or filesystem, each instance\'s own disk. AzureCustom always uses filesystem.') +@allowed([ + 'redis' + 'filesystem' +]) +param portalSessionStore string = 'redis' + +@description('Size of the portal session cache. Leave empty for the default.') +param portalRedisSku string = '' + +@description('Region of the portal session cache. Leave empty for the deployment\'s region.') +param portalRedisLocation string = '' param deployLinuxHosts bool = false param deployAvdHosts bool = false param linuxHostVmNamePrefix string = 'lnxhost' @@ -103,6 +121,16 @@ param linuxHostDesktop string = 'gnome' param avdHostPoolName string = '' param avdSessionHostCount int = 0 param avdMaxSessionLimit int = 5 +@description('Opens the Linux desktop full screen.') +param avdLinuxDesktopFullScreen bool = true +@description('Spreads a full-screen Linux desktop across every monitor.') +param avdLinuxDesktopMultiMonitor bool = true +@description('Starts a deallocated AVD session host when a user connects and no running one can take the session.') +param avdStartVmOnConnect bool = true +@description('Lowercase AVD session host names mapped to their excludeFromScaling tag values, which the deployment writes back.') +param avdScalingExclusions object = {} +@description('Linux hosts and AVD session hosts whose VM extensions are left out because the VMs are not running.') +param hostNamesNotRunning array = [] param avdVmNamePrefix string = 'avdhost' @allowed([ 'Standard_DS2_v2' @@ -117,6 +145,17 @@ param avdVmNamePrefix string = 'avdhost' ]) param avdVmSize string = 'Standard_D8s_v5' +@description('Deploy host log collection, the fleet workbook and the alerts.') +param deployHostMonitoring bool = true +@description('Email addresses that receive the alerts, separated by commas or semicolons.') +param alertEmailAddresses string = '' +@minValue(1) +param alertCheckoutRefusalThreshold int = 1 +@minValue(1) +param alertApiErrorThreshold int = 5 +@minValue(1) +param alertNfsLatencyThresholdMs int = 50 + var sanitizedApp = toLower(replace(appName, '-', '')) var sanitizedEnv = toLower(replace(environmentName, '-', '')) var sqlLocation = location == 'eastus2' ? 'eastus' : location @@ -155,6 +194,8 @@ var provisionNfsShare = deployNfsShare && empty(nfsShare) && deployLinuxHosts && var nfsStorageAccountName = take('nfs${sanitizedApp}${suffix}', 24) var nfsShareName = 'home' var effectiveNfsShare = provisionNfsShare ? '${nfsStorageAccountName}.file.${environment().suffixes.storage}:/${nfsStorageAccountName}/${nfsShareName}' : nfsShare +// Part of a public DNS name. The unique suffix is never cut off, and the name never ends in a hyphen. +var portalRedisName = 'redis-${take('${sanitizedApp}${sanitizedEnv}', 40)}-${suffix}' module networking 'modules/core/networking.bicep' = { name: 'networking' @@ -194,6 +235,26 @@ module observability 'modules/core/observability.bicep' = { } } +// deploy/Enable-HostMonitoring.ps1 installs the agent and associates the rules after provisioning. +module hostMonitoring 'modules/core/host-monitoring.bicep' = if (deployHostMonitoring) { + name: 'hostMonitoring' + params: { + location: location + tags: tags + namePrefix: '${appName}-${environmentName}' + logAnalyticsWorkspaceName: observability.outputs.logAnalyticsWorkspaceName + apiAppName: apiAppName + nfsStorageAccountName: provisionNfsShare ? nfsStorageAccountName : '' + alertEmailAddresses: alertEmailAddresses + alertCheckoutRefusalThreshold: alertCheckoutRefusalThreshold + alertApiErrorThreshold: alertApiErrorThreshold + alertNfsLatencyThresholdMs: alertNfsLatencyThresholdMs + } + dependsOn: [ + nfsStorage + ] +} + module containerRegistry 'modules/core/container-registry.bicep' = { name: 'containerRegistry' params: { @@ -255,6 +316,7 @@ module appServicePlan 'modules/core/app-service-plan.bicep' = { tags: tags appServicePlanName: appServicePlanName skuName: appServicePlanSku + capacity: appServicePlanCapacity } } @@ -284,19 +346,33 @@ var cloudProfiles = { graphEndpoint: 'https://graph.microsoft.com' stsIssuerHost: 'https://sts.windows.net' appServiceDomain: 'azurewebsites.net' + // Azure Cache for Redis can no longer be created by new customers in the public cloud. + portalRedisKind: 'managed' + portalRedisPrivateDnsZoneName: 'privatelink.redis.azure.net' + // Empty keeps the portal's default, https://redis.azure.com. + portalRedisEntraResource: '' } AzureUSGovernment: { graphEndpoint: 'https://graph.microsoft.us' stsIssuerHost: 'https://sts.windows.net' appServiceDomain: 'azurewebsites.us' + // Azure Managed Redis is not offered in Azure Government. + portalRedisKind: 'cache' + portalRedisPrivateDnsZoneName: 'privatelink.redis.cache.usgovcloudapi.net' + // The Redis application ID, which unlike its URI does not depend on the cloud. + portalRedisEntraResource: 'acca5fbb-b7e4-4009-81f1-37e38fd66d78' } AzureCustom: { graphEndpoint: '' stsIssuerHost: '' appServiceDomain: '' + portalRedisKind: '' + portalRedisPrivateDnsZoneName: '' + portalRedisEntraResource: '' } } var cloudProfile = cloudProfiles[azureCloudName] +var usePortalRedis = portalSessionStore == 'redis' && !empty(cloudProfile.portalRedisKind) // Carries a trailing slash; both azure-identity and the app config normalize it away. var resolvedAuthorityHost = empty(azureAuthorityHost) ? environment().authentication.loginEndpoint : azureAuthorityHost var resolvedGraphEndpoint = empty(graphEndpoint) ? cloudProfile.graphEndpoint : graphEndpoint @@ -392,6 +468,16 @@ var frontendSettings = { TENANT_ID: tenantId WEBSITE_AUTH_AAD_ALLOWED_TENANTS: tenantId } +// Provisioning replaces every app setting, so these are declared here rather than set by hand. +var portalSessionSettings = usePortalRedis ? union({ + SESSION_BACKEND: 'redis' + REDIS_HOST: portalRedis!.outputs.hostName + REDIS_PORT: string(portalRedis!.outputs.port) +}, empty(cloudProfile.portalRedisEntraResource) ? {} : { + REDIS_ENTRA_RESOURCE: cloudProfile.portalRedisEntraResource +}) : { + SESSION_BACKEND: 'filesystem' +} var apiSettings = { ALLOW_LEGACY_SCOPE_ACCESS: allowLegacyScopeAccess ? 'true' : 'false' AVD_HOST_GROUP_ID: avdHostGroupId @@ -430,6 +516,23 @@ var functionSettings = { WEBSITE_CONTENTSHARE: toLower(take('${taskAppName}content', 63)) } +// Sessions every portal instance shares. The cache takes no public traffic, so the portal +// reaches it through the virtual network. +module portalRedis 'modules/core/redis-session-store.bicep' = if (usePortalRedis) { + name: 'portalRedis' + params: { + kind: cloudProfile.portalRedisKind + name: portalRedisName + location: location + cacheLocation: empty(portalRedisLocation) ? location : portalRedisLocation + tags: tags + skuName: portalRedisSku + virtualNetworkId: networking.outputs.vnetId + privateEndpointSubnetId: networking.outputs.privateEndpointSubnetId + privateDnsZoneName: cloudProfile.portalRedisPrivateDnsZoneName + } +} + module frontendApp 'modules/apps/container-web-app.bicep' = { name: 'frontendApp' params: { @@ -442,11 +545,24 @@ module frontendApp 'modules/apps/container-web-app.bicep' = { containerImageName: frontendImageName containerRegistryLoginServer: containerRegistry.outputs.loginServer applicationInsightsConnectionString: observability.outputs.applicationInsightsConnectionString - appSettings: frontendSettings + appSettings: union(frontendSettings, portalSessionSettings) authSettings: frontendAuthSettings healthCheckPath: '/health' alwaysOn: true useManagedIdentityForRegistry: true + // The plan's apps all share one integration subnet. + virtualNetworkSubnetId: usePortalRedis ? networking.outputs.appSubnetId : '' + // Sessions kept on each instance's disk only survive if the browser returns to that instance. + clientAffinityEnabled: !usePortalRedis + } +} + +module portalRedisAccess 'modules/core/redis-session-access.bicep' = if (usePortalRedis) { + name: 'portalRedisAccess' + params: { + kind: cloudProfile.portalRedisKind + redisName: portalRedis!.outputs.name + principalId: frontendApp.outputs.principalId } } @@ -469,6 +585,8 @@ module apiApp 'modules/apps/container-web-app.bicep' = { useManagedIdentityForRegistry: true // SSH to the Linux hosts goes to private addresses, so the API joins the virtual network. virtualNetworkSubnetId: networking.outputs.appSubnetId + // The API keeps no per-browser state, so any instance can serve any request. + clientAffinityEnabled: false } } @@ -579,6 +697,7 @@ module linuxHosts 'modules/Linux/main.bicep' = if (deployLinuxHosts && linuxHost scriptSourceRoot: scriptSourceRoot disableScreenLock: linuxHostDisableScreenLock desktop: linuxHostDesktop + skipExtensionVmNames: hostNamesNotRunning } } @@ -603,6 +722,11 @@ module avdHosts 'modules/AVD/main.bicep' = if (deployAvdHosts && avdSessionHostC linuxBrokerApiClientId: apiClientId scriptSourceRoot: scriptSourceRoot avdUsersGroupId: avdUsersGroupId + linuxDesktopFullScreen: avdLinuxDesktopFullScreen + linuxDesktopMultiMonitor: avdLinuxDesktopMultiMonitor + startVmOnConnect: avdStartVmOnConnect + skipExtensionVmNames: hostNamesNotRunning + scalingExclusions: avdScalingExclusions } } @@ -619,3 +743,7 @@ output sqlDatabaseName string = sql.outputs.databaseName output virtualNetworkName string = networking.outputs.vnetName output linuxHostDomainName string = effectiveDomainName output nfsSharePath string = effectiveNfsShare +output linuxHostDataCollectionRuleId string = deployHostMonitoring ? hostMonitoring!.outputs.linuxHostDataCollectionRuleId : '' +output avdHostDataCollectionRuleId string = deployHostMonitoring ? hostMonitoring!.outputs.avdHostDataCollectionRuleId : '' +output portalSessionStore string = usePortalRedis ? 'redis' : 'filesystem' +output portalRedisName string = usePortalRedis ? portalRedis!.outputs.name : '' diff --git a/deploy/bicep/modules/AVD/main.bicep b/deploy/bicep/modules/AVD/main.bicep index 301d374..fe002b4 100644 --- a/deploy/bicep/modules/AVD/main.bicep +++ b/deploy/bicep/modules/AVD/main.bicep @@ -61,11 +61,25 @@ param scriptSourceRoot string = 'https://raw.githubusercontent.com/microsoft/Lin param avdUsersGroupId string = '' @description('Display name of the RemoteApp that connects users to a Linux host.') param remoteAppFriendlyName string = 'Linux Desktop' +@description('Opens the Linux desktop full screen. False opens it in a window on one monitor. Turn it off only once every session host runs Connect-LinuxBroker.ps1 2.0.0 or later: older scripts refuse the argument and the Linux Desktop app fails to open.') +param linuxDesktopFullScreen bool = true +@description('Spreads a full-screen Linux desktop across every monitor. False keeps it on one monitor. Turn it off only once every session host runs Connect-LinuxBroker.ps1 2.0.0 or later: older scripts refuse the argument and the Linux Desktop app fails to open.') +param linuxDesktopMultiMonitor bool = true +@description('Starts a deallocated session host when a user opens the Linux Desktop app and no running session host can take the session. Azure Virtual Desktop needs Desktop Virtualization Power On Contributor, or Power On Off Contributor, to start it.') +param startVmOnConnect bool = true +@description('Session hosts whose VM extensions this deployment leaves out, because Azure refuses to change an extension on a VM that is not running. The preprovision hook passes the ones that are deallocated or stopped.') +param skipExtensionVmNames array = [] +@description('Lowercase session host names, each mapped to the value of the excludeFromScaling tag it carries. A deployment replaces the tags of every VM it declares, so these are written back.') +param scalingExclusions object = {} var normalizedScriptSourceRoot = endsWith(scriptSourceRoot, '/') ? take(scriptSourceRoot, length(scriptSourceRoot) - 1) : scriptSourceRoot var linuxBrokerConfigScriptUri = '${normalizedScriptSourceRoot}/custom_script_extensions/Configure-AVD-Host.ps1' var desktopVirtualizationUserRoleId = '1d18fff3-a72a-46b5-b4a9-0b38a3cd7e63' var virtualMachineUserLoginRoleId = 'fb879df8-f326-4884-b1cf-06f3ad86be52' +// The script's defaults are On, so only Off is passed, and a session host that still runs a +// script from before 2.0.0 opens the desktop as it always has. +var linuxDesktopDisplayArguments = '${linuxDesktopFullScreen ? '' : ' -FullScreen Off'}${linuxDesktopMultiMonitor ? '' : ' -MultiMonitor Off'}' +var skipExtensionVmNamesLower = [for vmName in skipExtensionVmNames: toLower(vmName)] var osImage = 'microsoftwindowsdesktop:Windows-11:win11-24h2-avd:latest' var vmNames = [for i in range(1, sessionHostCount): '${vmNamePrefix}-${padLeft(i, 2, '0')}'] @@ -88,7 +102,7 @@ resource hostPool 'Microsoft.DesktopVirtualization/hostPools@2024-04-03' = { loadBalancerType: loadBalancerType maxSessionLimit: maxSessionLimit customRdpProperty: customRdpProperty - startVMOnConnect: false + startVMOnConnect: startVmOnConnect validationEnvironment: false agentUpdate: agentUpdate registrationInfo: { @@ -156,7 +170,9 @@ module hostPoolRegistrationToken 'token.bicep' = { preferredAppGroupType: hostPool.properties.preferredAppGroupType maxSessionLimit: hostPool.properties.maxSessionLimit customRdpProperty: customRdpProperty - startVMOnConnect: hostPool.properties.startVMOnConnect + // Passed as declared rather than read back from the host pool, in case creating the host pool + // ignores it: this second PUT is always an update. + startVMOnConnect: startVmOnConnect validationEnvironment: hostPool.properties.validationEnvironment agentUpdate: hostPool.properties.agentUpdate } @@ -207,7 +223,13 @@ resource vmSessionHost 'Microsoft.Compute/virtualMachines@2024-11-01' = [ name: name location: location // The broker-role tag is how the post-provision hook finds the session hosts to add to the AVD host group. - tags: tags + // The deployment replaces a VM's tags, so an excludeFromScaling tag an administrator added, + // which keeps autoscale away from the host, is written back. + tags: contains(scalingExclusions, toLower(name)) + ? union(tags, { + excludeFromScaling: scalingExclusions[toLower(name)] + }) + : tags identity: { type: 'SystemAssigned' } @@ -262,8 +284,12 @@ resource vmSessionHost 'Microsoft.Compute/virtualMachines@2024-11-01' = [ } ] +// Azure refuses to change an extension on a VM that is not running, and the DSC extension's +// registration token changes on every deployment, so the extensions of session hosts that +// autoscale has deallocated are left out. A session host that is registered keeps working +// without them. resource entraloginExtension 'Microsoft.Compute/virtualMachines/extensions@2024-11-01' = [ - for (name, i) in vmNames: { + for (name, i) in vmNames: if (!contains(skipExtensionVmNamesLower, toLower(name))) { name: '${name}/AADLoginForWindows' location: resourceGroup().location properties: { @@ -285,7 +311,7 @@ resource entraloginExtension 'Microsoft.Compute/virtualMachines/extensions@2024- ] resource avdDscExtension 'Microsoft.Compute/virtualMachines/extensions@2024-11-01' = [ - for (name, i) in vmNames: { + for (name, i) in vmNames: if (!contains(skipExtensionVmNamesLower, toLower(name))) { name: '${name}/Microsoft.PowerShell.DSC' location: resourceGroup().location properties: { @@ -323,7 +349,7 @@ resource avdDscExtension 'Microsoft.Compute/virtualMachines/extensions@2024-11-0 ] resource linuxBrokerConfig 'Microsoft.Compute/virtualMachines/extensions@2024-11-01' = [ - for (name, i) in vmNames: { + for (name, i) in vmNames: if (!contains(skipExtensionVmNamesLower, toLower(name))) { name: '${name}/CustomScriptExtension' location: location properties: { @@ -371,7 +397,7 @@ resource linuxDesktopApp 'Microsoft.DesktopVirtualization/applicationGroups/appl applicationType: 'InBuilt' filePath: 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' commandLineSetting: 'Require' - commandLineArguments: '-ExecutionPolicy Bypass -NoProfile -WindowStyle Hidden -File C:\\Temp\\Connect-LinuxBroker.ps1' + commandLineArguments: '-ExecutionPolicy Bypass -NoProfile -WindowStyle Hidden -File C:\\Temp\\Connect-LinuxBroker.ps1${linuxDesktopDisplayArguments}' iconPath: 'C:\\Windows\\System32\\mstsc.exe' iconIndex: 0 showInPortal: true diff --git a/deploy/bicep/modules/AVD/scaling-plan.bicep b/deploy/bicep/modules/AVD/scaling-plan.bicep new file mode 100644 index 0000000..c7d1e4f --- /dev/null +++ b/deploy/bicep/modules/AVD/scaling-plan.bicep @@ -0,0 +1,151 @@ +// Power-management autoscale for the pooled host pool. Azure Virtual Desktop starts and +// deallocates the session hosts on this schedule, which needs the Azure Virtual Desktop service +// principal to hold Desktop Virtualization Power On Off Contributor on the subscription +// (assigned from main.bicep). + +param location string +param tags object = {} + +@minLength(3) +@maxLength(64) +param name string +param friendlyName string = name + +@description('Name of the pooled host pool the plan scales, in the same resource group.') +param hostPoolName string + +@description('Scales the host pool. False keeps the plan but assigns it to no host pool, so the session hosts are left as they are and another plan can be assigned instead.') +param enabled bool = true + +@description('Windows time zone ID the schedule times are in, for example UTC or Eastern Standard Time.') +param timeZone string = 'UTC' + +@description('Session hosts that carry this tag are never started, stopped or taken out of drain mode by autoscale.') +param exclusionTag string = 'excludeFromScaling' + +@description('Weekday start of ramp-up, as HH:mm in timeZone. The weekend uses the same times.') +param rampUpStartTime string = '07:00' + +@description('Start of peak hours, as HH:mm. Must come after ramp-up.') +param peakStartTime string = '09:00' + +@description('Start of ramp-down, as HH:mm. Must come after peak.') +param rampDownStartTime string = '18:00' + +@description('Start of off-peak hours, as HH:mm. Must come after ramp-down.') +param offPeakStartTime string = '20:00' + +@description('Share of the session hosts kept on from ramp-up until ramp-down on weekdays, rounded up to whole hosts.') +@minValue(0) +@maxValue(100) +param rampUpMinimumHostsPct int = 20 + +@description('Share of the running hosts\' sessions that makes autoscale start another host during ramp-up and peak hours.') +@minValue(1) +@maxValue(100) +param rampUpCapacityThresholdPct int = 60 + +@description('Share of the session hosts kept on from ramp-down until the next ramp-up on weekdays, rounded up to whole hosts.') +@minValue(0) +@maxValue(100) +param rampDownMinimumHostsPct int = 10 + +@description('Share of the running hosts\' sessions that makes autoscale start another host during ramp-down and off-peak hours.') +@minValue(1) +@maxValue(100) +param rampDownCapacityThresholdPct int = 90 + +@description('Share of the session hosts kept on through Saturday and Sunday. 0 lets every host turn off, and Start VM on Connect starts one for the first user.') +@minValue(0) +@maxValue(100) +param weekendMinimumHostsPct int = 0 + +var rampUpTime = split(rampUpStartTime, ':') +var peakTime = split(peakStartTime, ':') +var rampDownTime = split(rampDownStartTime, ':') +var offPeakTime = split(offPeakStartTime, ':') + +// Breadth-first while hosts are coming on spreads users out; depth-first once demand falls packs +// them onto fewer hosts, so the rest can stop. Users are never signed out: a host stops only once +// it has no session at all, disconnected ones included. +var sharedScheduleSettings = { + rampUpStartTime: { + hour: int(rampUpTime[0]) + minute: int(rampUpTime[1]) + } + rampUpLoadBalancingAlgorithm: 'BreadthFirst' + rampUpCapacityThresholdPct: rampUpCapacityThresholdPct + peakStartTime: { + hour: int(peakTime[0]) + minute: int(peakTime[1]) + } + peakLoadBalancingAlgorithm: 'BreadthFirst' + rampDownStartTime: { + hour: int(rampDownTime[0]) + minute: int(rampDownTime[1]) + } + rampDownLoadBalancingAlgorithm: 'DepthFirst' + rampDownCapacityThresholdPct: rampDownCapacityThresholdPct + rampDownForceLogoffUsers: false + rampDownStopHostsWhen: 'ZeroSessions' + rampDownWaitTimeMinutes: 30 + rampDownNotificationMessage: 'This session host is about to shut down. Save your work and sign out.' + offPeakStartTime: { + hour: int(offPeakTime[0]) + minute: int(offPeakTime[1]) + } + offPeakLoadBalancingAlgorithm: 'DepthFirst' +} + +resource hostPool 'Microsoft.DesktopVirtualization/hostPools@2024-04-03' existing = { + name: hostPoolName +} + +// Azure checks that the Azure Virtual Desktop service principal can reach the host pool before it +// assigns the plan to it, and answers 400 BadRequest when the role is missing. +resource scalingPlan 'Microsoft.DesktopVirtualization/scalingPlans@2024-04-03' = { + name: name + location: location + tags: tags + properties: { + friendlyName: friendlyName + description: 'Starts and stops the Linux Desktop session hosts on a schedule.' + hostPoolType: 'Pooled' + timeZone: timeZone + exclusionTag: exclusionTag + hostPoolReferences: enabled + ? [ + { + hostPoolArmPath: hostPool.id + scalingPlanEnabled: true + } + ] + : [] + schedules: [ + union(sharedScheduleSettings, { + name: 'weekdays' + daysOfWeek: [ + 'Monday' + 'Tuesday' + 'Wednesday' + 'Thursday' + 'Friday' + ] + rampUpMinimumHostsPct: rampUpMinimumHostsPct + rampDownMinimumHostsPct: rampDownMinimumHostsPct + }) + union(sharedScheduleSettings, { + name: 'weekend' + daysOfWeek: [ + 'Saturday' + 'Sunday' + ] + rampUpMinimumHostsPct: weekendMinimumHostsPct + rampDownMinimumHostsPct: weekendMinimumHostsPct + }) + ] + } +} + +output name string = scalingPlan.name +output id string = scalingPlan.id diff --git a/deploy/bicep/modules/Linux/main.bicep b/deploy/bicep/modules/Linux/main.bicep index ce54c42..51423f2 100644 --- a/deploy/bicep/modules/Linux/main.bicep +++ b/deploy/bicep/modules/Linux/main.bicep @@ -46,6 +46,9 @@ param disableScreenLock bool = true @description('Desktop the hosts run in xrdp sessions. Changing it changes the extension command, which runs the bootstrap again on existing hosts.') param desktop string = 'gnome' +@description('Hosts whose bootstrap extension this deployment leaves out, because Azure refuses to change an extension on a VM that is not running. The preprovision hook passes the ones that are deallocated or stopped.') +param skipExtensionVmNames array = [] + var normalizedScriptSourceRoot = endsWith(scriptSourceRoot, '/') ? take(scriptSourceRoot, length(scriptSourceRoot) - 1) : scriptSourceRoot var bootstrapArgs = '"${linuxBrokerApiBaseUrl}" "${linuxBrokerApiClientId}"' // GNOME is what the bootstrap installs without LINUXBROKER_DESKTOP, so leaving the variable out @@ -54,6 +57,7 @@ var desktopEnv = desktop == 'gnome' ? '' : ' LINUXBROKER_DESKTOP="${desktop}"' var bootstrapEnv = 'LINUXBROKER_SCRIPT_SOURCE_ROOT="${normalizedScriptSourceRoot}" LINUXBROKER_DISABLE_SCREEN_LOCK="${disableScreenLock ? 'true' : 'false'}"${desktopEnv}' var vmNames = [for i in range(1, numberOfVMs): '${vmNamePrefix}-${padLeft(i, 2, '0')}'] +var skipExtensionVmNamesLower = [for vmName in skipExtensionVmNames: toLower(vmName)] var adminCredentials = authType == 'Password' ? { adminPassword: adminPassword } : {} @@ -236,8 +240,10 @@ resource vmLinuxHost 'Microsoft.Compute/virtualMachines@2022-03-01' = [ } ] +// Scaling and start on demand leave hosts deallocated, and Azure refuses to change an extension on +// a VM that is not running, even to the same settings, so those hosts' extensions are left out. resource linuxCustomScriptExtension 'Microsoft.Compute/virtualMachines/extensions@2022-03-01' = [ - for (name, i) in vmNames: { + for (name, i) in vmNames: if (!contains(skipExtensionVmNamesLower, toLower(name))) { name: '${name}/customScript' location: location properties: { diff --git a/deploy/bicep/modules/apps/container-web-app.bicep b/deploy/bicep/modules/apps/container-web-app.bicep index 403c7a6..577103e 100644 --- a/deploy/bicep/modules/apps/container-web-app.bicep +++ b/deploy/bicep/modules/apps/container-web-app.bicep @@ -12,6 +12,8 @@ param alwaysOn bool = true param useManagedIdentityForRegistry bool = true @description('Delegated subnet for regional VNet integration. Leave empty to keep the app off the virtual network.') param virtualNetworkSubnetId string = '' +@description('Send each browser back to the instance that served it first (ARR affinity). Only an app that keeps state on the instance needs it.') +param clientAffinityEnabled bool = false var webSiteConfig = union({ alwaysOn: alwaysOn @@ -33,6 +35,7 @@ resource webApp 'Microsoft.Web/sites@2023-12-01' = { properties: { serverFarmId: serverFarmId httpsOnly: true + clientAffinityEnabled: clientAffinityEnabled // Only private (RFC 1918) traffic is routed into the VNet; SQL, Key Vault, Graph and ACR stay on the public path. virtualNetworkSubnetId: empty(virtualNetworkSubnetId) ? null : virtualNetworkSubnetId siteConfig: webSiteConfig diff --git a/deploy/bicep/modules/core/app-service-plan.bicep b/deploy/bicep/modules/core/app-service-plan.bicep index bea9775..7e41b2a 100644 --- a/deploy/bicep/modules/core/app-service-plan.bicep +++ b/deploy/bicep/modules/core/app-service-plan.bicep @@ -3,6 +3,11 @@ param tags object = {} param appServicePlanName string param skuName string = 'P2mv3' +@description('Instances the plan runs. The portal, the API and the task function all run on every instance.') +@minValue(1) +@maxValue(30) +param capacity int = 1 + resource appServicePlan 'Microsoft.Web/serverfarms@2023-12-01' = { name: appServicePlanName location: location @@ -11,7 +16,7 @@ resource appServicePlan 'Microsoft.Web/serverfarms@2023-12-01' = { name: skuName tier: 'PremiumV3' size: skuName - capacity: 1 + capacity: capacity } kind: 'linux' properties: { diff --git a/deploy/bicep/modules/core/host-monitoring.bicep b/deploy/bicep/modules/core/host-monitoring.bicep new file mode 100644 index 0000000..fc1c8d5 --- /dev/null +++ b/deploy/bicep/modules/core/host-monitoring.bicep @@ -0,0 +1,815 @@ +// Host log collection, the fleet workbook and the alerts. +// +// deploy/Enable-HostMonitoring.ps1 installs the Azure Monitor agent and associates these data +// collection rules with the hosts. Bicep cannot do it: adding a VM extension fails on a +// deallocated VM, and scaling powers hosts off. + +@description('Location of the data collection rules, the workbook and the log alerts.') +param location string = resourceGroup().location +param tags object = {} + +@description('Prefix of the resource names, -.') +param namePrefix string + +@description('Log Analytics workspace that Application Insights writes to. The host logs go there too.') +param logAnalyticsWorkspaceName string + +@description('Name of the broker API web app, which is the cloud role name of its telemetry.') +param apiAppName string + +@description('Storage account of the NFS share the deployment provisions, or empty when it provisions none.') +param nfsStorageAccountName string = '' + +@description('Email addresses that receive the alerts, separated by commas or semicolons. Leave empty to deploy the alerts without notifications.') +param alertEmailAddresses string = '' + +@description('Refused checkouts (HTTP 409) in 15 minutes that raise an alert.') +@minValue(1) +param alertCheckoutRefusalThreshold int = 1 + +@description('API responses with a 5xx status in 15 minutes that raise an alert.') +@minValue(1) +param alertApiErrorThreshold int = 5 + +@description('Average end-to-end latency of the NFS share over 15 minutes, in milliseconds, that raises an alert.') +@minValue(1) +param alertNfsLatencyThresholdMs int = 50 + +var hostLogTableName = 'LinuxBrokerHost_CL' +var hostLogStreamName = 'Custom-${hostLogTableName}' + +// Each broker log line starts with a YYYY-MM-DD HH:MM:SS timestamp, which starts a record. xrdp +// starts its lines with a timestamp in brackets instead, [YYYYMMDD-HH:MM:SS] in 0.9 and +// [YYYY-MM-DDTHH:MM:SS.mmm+zzzz] in 0.10, and a file in which no line starts with the format is +// split into records at each line end. +var linuxHostLogFiles = [ + '/var/log/release-session.log' + '/var/log/release-session-watcher.log' + '/var/log/linuxbroker-host-settings.log' + '/var/log/createuser.log' + '/var/log/linuxbroker-session-control.log' + '/var/log/linuxbroker-patch.log' + '/var/log/xrdp.log' + '/var/log/xrdp-sesman.log' +] + +var alertEmailList = filter(map(split(replace(alertEmailAddresses, ';', ','), ','), address => trim(address)), address => !empty(address)) +var alertActionGroupIds = empty(alertEmailList) ? [] : [alertActionGroup.id] +var monitorNfsShare = !empty(nfsStorageAccountName) +var nfsFileServiceId = resourceId('Microsoft.Storage/storageAccounts/fileServices', monitorNfsShare ? nfsStorageAccountName : 'none', 'default') + +// The queries are built from single-line strings so that main.json is the same whichever line +// endings the checkout has. Workbook queries bin by the time range's grain, but never below the +// five minutes between fleet snapshots. +var fleetSnapshots = [ + 'AppTraces' + '| where AppRoleName == "${apiAppName}" and Message == "fleet snapshot"' +] +var apiRequests = [ + 'AppRequests' + '| where AppRoleName == "${apiAppName}"' +] +// The request's path, or the route in the span name when the URL is missing. +var requestPath = '| extend Path = coalesce(tostring(parse_url(Url).Path), tostring(split(Name, " ")[-1]))' +var workbookBin = 'bin(TimeGenerated, max_of(5m, {TimeRange:grain}))' +var hostLogLevel = '| extend Level = case(RawData has_cs "ERROR", "Error", RawData has_cs "WARNING" or RawData has_cs "WARN", "Warning", RawData has_any ("failed", "cannot", "unable") or RawData has "could not", "Error", "")' + +var latestSnapshotQuery = join(concat(fleetSnapshots, [ + '| top 1 by TimeGenerated desc' + '| project TimeGenerated, Ready = toint(Properties.ReadyHosts), InUse = toint(Properties.InUse), Waiting = toint(Properties.Waiting), Booting = toint(Properties.Booting), PoweredOn = toint(Properties.PoweredOn), Total = toint(Properties.TotalHosts), MinVMs = toint(Properties.EffectiveMinVMs), MaxVMs = toint(Properties.MaxVMs), StaleHeartbeats = toint(Properties.StaleHeartbeats), NfsUnreachable = toint(Properties.NfsUnreachable), XrdpInactive = toint(Properties.XrdpInactive), Draining = toint(Properties.Draining), Maintenance = toint(Properties.Maintenance), StartOnDemand = iff(toint(Properties.StartOnDemandEnabled) == 1, "On", "Off")' +]), '\n') + +var fleetTrendQuery = join(concat(fleetSnapshots, [ + '| summarize Ready = min(toint(Properties.ReadyHosts)), InUse = max(toint(Properties.InUse)), Waiting = max(toint(Properties.Waiting)), Booting = max(toint(Properties.Booting)), PoweredOn = max(toint(Properties.PoweredOn)) by ${workbookBin}' + '| order by TimeGenerated asc' +]), '\n') + +var checkoutOutcomeQuery = join(concat(apiRequests, [ + requestPath + '| where Path == "/api/vms/checkout"' + '| extend Outcome = case(ResultCode == "200", "Assigned (200)", ResultCode == "202", "Starting a host (202)", ResultCode == "409", "Refused (409)", toint(ResultCode) >= 500, "Failed (5xx)", strcat("Other (", ResultCode, ")"))' + '| summarize Checkouts = count() by Outcome, ${workbookBin}' + '| order by TimeGenerated asc' +]), '\n') + +var checkoutLatencyQuery = join(concat(apiRequests, [ + '| where ResultCode == "200"' + requestPath + '| where Path == "/api/vms/checkout"' + '| summarize P50 = percentile(DurationMs, 50), P95 = percentile(DurationMs, 95) by ${workbookBin}' + '| order by TimeGenerated asc' +]), '\n') + +var releaseQuery = join(concat(apiRequests, [ + requestPath + '| extend Action = case(Path matches regex @"^/api/vms/[^/]+/release$", "Release", Path matches regex @"^/api/vms/[^/]+/return$", "Return", "")' + '| where isnotempty(Action)' + '| extend Series = strcat(Action, iff(toint(ResultCode) < 400, " succeeded", strcat(" failed (", ResultCode, ")")))' + '| summarize Requests = count() by Series, ${workbookBin}' + '| order by TimeGenerated asc' +]), '\n') + +var startOnDemandQuery = join([ + 'union' + ' (AppTraces' + ' | where AppRoleName == "${apiAppName}" and tostring(Properties.audit_action) == "vm.start_on_demand"' + ' | extend Series = iff(tostring(Properties.audit_outcome) == "success", "Hosts started", "Starts that failed")),' + ' (AppRequests' + ' | where AppRoleName == "${apiAppName}" and ResultCode == "202"' + ' ${requestPath}' + ' | where Path == "/api/vms/checkout"' + ' | extend Series = "Checkouts told to wait (202)")' + '| summarize Events = count() by Series, ${workbookBin}' + '| order by TimeGenerated asc' +], '\n') + +var settingsFailureQuery = join([ + 'LinuxBrokerHost_CL' + '| where FilePath endswith "linuxbroker-host-settings.log"' + '| where RawData has_cs "ERROR" or RawData has_cs "WARNING"' + '| project TimeGenerated, Computer, Level = iff(RawData has_cs "ERROR", "Error", "Warning"), Message = RawData' + '| order by TimeGenerated desc' + '| take 200' +], '\n') + +var hostErrorQuery = join([ + 'union isfuzzy=true' + ' (LinuxBrokerHost_CL' + ' ${hostLogLevel}' + ' | where isnotempty(Level)' + ' | extend Source = extract(@"([^/]+)$", 1, FilePath), Message = RawData),' + ' (Syslog' + ' | where SeverityLevel in ("emerg", "alert", "crit", "err")' + ' | extend Level = "Error", Source = iff(isempty(ProcessName), strcat("syslog ", Facility), strcat("syslog ", Facility, " ", ProcessName)), Message = SyslogMessage)' + '| summarize Events = count(), arg_max(TimeGenerated, Message) by Computer, Source, Level' + '| project Computer, Source, Level, Events, LastSeen = TimeGenerated, LatestMessage = Message' + '| order by Events desc' +], '\n') + +var nfsFailureQuery = join([ + 'union isfuzzy=true' + ' (LinuxBrokerHost_CL' + ' | where FilePath endswith "createuser.log" and RawData has "Failed to mount NFS share"' + ' | extend Source = "create-user.sh"),' + ' (LinuxBrokerHost_CL' + ' | where FilePath endswith "linuxbroker-session-control.log" and RawData has "Could not mount the profile share"' + ' | extend Source = "session-control.sh"),' + ' (Syslog' + ' | where Facility == "kern" and SyslogMessage has "nfs" and (SyslogMessage has "not responding" or SyslogMessage has "timed out")' + ' | extend Source = "kernel", RawData = SyslogMessage)' + '| project TimeGenerated, Computer, Source, Message = RawData' + '| order by TimeGenerated desc' + '| take 500' +], '\n') + +var avdEventQuery = join([ + 'Event' + '| where Source == "LinuxBrokerScript" or EventLog == "Microsoft-Windows-TerminalServices-RDPClient/Operational"' + '| where EventLevelName in ("Critical", "Error", "Warning")' + '| summarize Events = count(), arg_max(TimeGenerated, RenderedDescription) by Computer, Source, Level = EventLevelName' + '| project Computer, Source, Level, Events, LastSeen = TimeGenerated, LatestMessage = RenderedDescription' + '| order by Events desc' +], '\n') + +// Each fleet alert needs every snapshot in its 15-minute window to show the problem, so a host +// that is briefly busy or booting does not raise one. +var noReadyHostsAlertQuery = join(concat(fleetSnapshots, [ + '| extend Ready = toint(Properties.ReadyHosts), MinVMs = toint(Properties.EffectiveMinVMs), Waiting = toint(Properties.Waiting)' + '| summarize Snapshots = count(), Short = countif(Ready == 0 and (MinVMs > 0 or Waiting > 0)), MostWaiting = max(Waiting)' + '| where Snapshots > 0 and Short == Snapshots' +]), '\n') + +var unhealthyHostsAlertQuery = join(concat(fleetSnapshots, [ + '| extend Stale = toint(Properties.StaleHeartbeats), Nfs = toint(Properties.NfsUnreachable), Xrdp = toint(Properties.XrdpInactive)' + '| summarize Snapshots = count(), Lowest = min(Stale + Nfs + Xrdp), StaleHeartbeats = max(Stale), NfsUnreachable = max(Nfs), XrdpInactive = max(Xrdp)' + '| where Snapshots > 0 and Lowest > 0' +]), '\n') + +var refusedCheckoutAlertQuery = join(concat(apiRequests, [ + '| where ResultCode == "409"' + requestPath + '| where Path == "/api/vms/checkout"' +]), '\n') + +var apiErrorAlertQuery = join(concat(apiRequests, [ + '| where toint(ResultCode) >= 500' +]), '\n') + +var logAlerts = [ + { + name: 'no-ready-hosts' + displayName: 'Linux Broker: no ready hosts (${namePrefix})' + description: 'For 15 minutes every fleet snapshot showed no ready Linux host while the scaling minimum is above 0 or users are waiting. Check the Linux hosts and the scaling settings.' + severity: 1 + query: noReadyHostsAlertQuery + operator: 'GreaterThan' + threshold: 0 + } + { + name: 'fleet-snapshot-missing' + displayName: 'Linux Broker: no fleet snapshot (${namePrefix})' + description: 'The scaling run has logged no fleet snapshot for 15 minutes. The task function, the API or the database may be down, or SQL script 157 is not applied yet.' + severity: 2 + query: join(fleetSnapshots, '\n') + operator: 'LessThan' + threshold: 1 + } + { + name: 'checkouts-refused' + displayName: 'Linux Broker: checkouts refused (${namePrefix})' + description: 'The broker refused checkouts (HTTP 409) in the last 15 minutes, so users could not get a Linux desktop.' + severity: 2 + query: refusedCheckoutAlertQuery + operator: 'GreaterThanOrEqual' + threshold: alertCheckoutRefusalThreshold + } + { + name: 'unhealthy-hosts' + displayName: 'Linux Broker: unhealthy hosts (${namePrefix})' + description: 'For 15 minutes every fleet snapshot showed a Linux host with a stale heartbeat, an unreachable home share or no xrdp service. The workbook lists the hosts\' errors.' + severity: 2 + query: unhealthyHostsAlertQuery + operator: 'GreaterThan' + threshold: 0 + } + { + name: 'api-errors' + displayName: 'Linux Broker: API errors (${namePrefix})' + description: 'The broker API answered requests with a 5xx status in the last 15 minutes.' + severity: 2 + query: apiErrorAlertQuery + operator: 'GreaterThanOrEqual' + threshold: alertApiErrorThreshold + } +] + +var nfsThrottlingResponseTypes = [ + 'SuccessWithThrottling' + 'SuccessWithShareIopsThrottling' + 'SuccessWithShareEgressThrottling' + 'SuccessWithShareIngressThrottling' + 'SuccessWithMetadataThrottling' + 'ClientThrottlingError' + 'ClientShareIopsThrottlingError' + 'ClientShareEgressThrottlingError' + 'ClientShareIngressThrottlingError' +] + +resource workspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' existing = { + name: logAnalyticsWorkspaceName +} + +resource hostLogTable 'Microsoft.OperationalInsights/workspaces/tables@2023-09-01' = { + parent: workspace + name: hostLogTableName + properties: { + plan: 'Analytics' + schema: { + name: hostLogTableName + columns: [ + { + name: 'TimeGenerated' + type: 'dateTime' + } + { + name: 'RawData' + type: 'string' + } + { + name: 'FilePath' + type: 'string' + } + { + name: 'Computer' + type: 'string' + } + ] + } + } +} + +resource linuxHostRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: take('dcr-${namePrefix}-linux', 64) + location: location + tags: tags + kind: 'Linux' + properties: { + description: 'Linux Broker logs, xrdp logs and syslog from the Linux hosts.' + streamDeclarations: { + '${hostLogStreamName}': { + columns: [ + { + name: 'TimeGenerated' + type: 'datetime' + } + { + name: 'RawData' + type: 'string' + } + { + name: 'FilePath' + type: 'string' + } + { + name: 'Computer' + type: 'string' + } + ] + } + } + dataSources: { + logFiles: [ + { + name: 'linuxBrokerLogs' + streams: [ + hostLogStreamName + ] + filePatterns: linuxHostLogFiles + format: 'text' + settings: { + text: { + recordStartTimestampFormat: 'YYYY-MM-DD HH:MM:SS' + } + } + } + ] + syslog: [ + { + name: 'authentication' + streams: [ + 'Microsoft-Syslog' + ] + facilityNames: [ + 'auth' + 'authpriv' + ] + logLevels: [ + 'Info' + 'Notice' + 'Warning' + 'Error' + 'Critical' + 'Alert' + 'Emergency' + ] + } + { + // The kernel reports an NFS server that stops answering at notice level. + name: 'kernelAndUser' + streams: [ + 'Microsoft-Syslog' + ] + facilityNames: [ + 'kern' + 'user' + ] + logLevels: [ + 'Notice' + 'Warning' + 'Error' + 'Critical' + 'Alert' + 'Emergency' + ] + } + { + name: 'otherFacilities' + streams: [ + 'Microsoft-Syslog' + ] + facilityNames: [ + 'cron' + 'daemon' + 'ftp' + 'local0' + 'local1' + 'local2' + 'local3' + 'local4' + 'local5' + 'local6' + 'local7' + 'lpr' + 'mail' + 'news' + 'syslog' + 'uucp' + ] + logLevels: [ + 'Warning' + 'Error' + 'Critical' + 'Alert' + 'Emergency' + ] + } + ] + } + destinations: { + logAnalytics: [ + { + name: 'workspace' + workspaceResourceId: workspace.id + } + ] + } + dataFlows: [ + { + streams: [ + hostLogStreamName + ] + destinations: [ + 'workspace' + ] + transformKql: 'source' + outputStream: hostLogStreamName + } + { + streams: [ + 'Microsoft-Syslog' + ] + destinations: [ + 'workspace' + ] + } + ] + } + dependsOn: [ + hostLogTable + ] +} + +resource avdHostRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: take('dcr-${namePrefix}-avd', 64) + location: location + tags: tags + kind: 'Windows' + properties: { + description: 'Connect-LinuxBroker.ps1 events and Remote Desktop client problems from the AVD session hosts.' + dataSources: { + windowsEventLogs: [ + { + name: 'linuxBrokerEvents' + streams: [ + 'Microsoft-Event' + ] + xPathQueries: [ + 'Application!*[System[Provider[@Name=\'LinuxBrokerScript\']]]' + 'Microsoft-Windows-TerminalServices-RDPClient/Operational!*[System[(Level=1 or Level=2 or Level=3)]]' + ] + } + ] + } + destinations: { + logAnalytics: [ + { + name: 'workspace' + workspaceResourceId: workspace.id + } + ] + } + dataFlows: [ + { + streams: [ + 'Microsoft-Event' + ] + destinations: [ + 'workspace' + ] + } + ] + } +} + +var workbookBaseQuery = { + version: 'KqlItem/1.0' + size: 0 + timeContextFromParameter: 'TimeRange' + queryType: 0 + resourceType: 'microsoft.operationalinsights/workspaces' + crossComponentResources: [ + workspace.id + ] +} + +var workbookContent = { + version: 'Notebook/1.0' + items: [ + { + type: 1 + name: 'introduction' + content: { + json: '## Linux Broker fleet\nThe scaling run logs a fleet snapshot every five minutes. Host logs arrive once deploy/Enable-HostMonitoring.ps1 has installed the Azure Monitor agent on the hosts.' + } + } + { + type: 9 + name: 'parameters' + content: { + version: 'KqlParameterItem/1.0' + style: 'pills' + queryType: 0 + resourceType: 'microsoft.operationalinsights/workspaces' + parameters: [ + { + id: '5d0c8f3e-2a41-4b8e-9f6d-1c7a3b2e4f10' + version: 'KqlParameterItem/1.0' + name: 'TimeRange' + label: 'Time range' + type: 4 + isRequired: true + value: { + durationMs: 86400000 + } + typeSettings: { + allowCustom: true + selectableValues: [ + { + durationMs: 3600000 + } + { + durationMs: 14400000 + } + { + durationMs: 43200000 + } + { + durationMs: 86400000 + } + { + durationMs: 259200000 + } + { + durationMs: 604800000 + } + { + durationMs: 2592000000 + } + ] + } + } + ] + } + } + { + type: 3 + name: 'latest-snapshot' + content: union(workbookBaseQuery, { + title: 'Latest fleet snapshot' + query: latestSnapshotQuery + visualization: 'table' + noDataMessage: 'No fleet snapshot in this time range. The API logs one every five minutes once SQL script 157 is applied.' + }) + } + { + type: 3 + name: 'fleet-trend' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Hosts over time (fewest ready, most in use, waiting, booting and powered on)' + query: fleetTrendQuery + visualization: 'timechart' + }) + } + { + type: 3 + name: 'checkout-outcomes' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Checkouts by outcome' + query: checkoutOutcomeQuery + visualization: 'timechart' + }) + } + { + type: 3 + name: 'checkout-latency' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Checkout time in milliseconds (50th and 95th percentile)' + query: checkoutLatencyQuery + visualization: 'timechart' + }) + } + { + type: 3 + name: 'releases' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Releases and returns' + query: releaseQuery + visualization: 'timechart' + }) + } + { + type: 3 + name: 'start-on-demand' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Hosts started on demand' + query: startOnDemandQuery + visualization: 'timechart' + }) + } + { + type: 3 + name: 'settings-failures' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Host settings that could not be applied' + query: settingsFailureQuery + visualization: 'table' + noDataMessage: 'No host setting errors or warnings in this time range.' + }) + } + { + type: 3 + name: 'host-errors' + content: union(workbookBaseQuery, { + title: 'Errors and warnings by Linux host' + query: hostErrorQuery + visualization: 'table' + noDataMessage: 'No host errors in this time range, or the Azure Monitor agent is not installed yet.' + }) + } + { + type: 3 + name: 'nfs-failures' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'NFS home share failures' + query: nfsFailureQuery + visualization: 'table' + noDataMessage: 'No NFS mount failures in this time range.' + }) + } + { + type: 3 + name: 'avd-events' + customWidth: '50' + content: union(workbookBaseQuery, { + title: 'Errors and warnings by AVD session host' + query: avdEventQuery + visualization: 'table' + noDataMessage: 'No AVD session host errors in this time range, or the Azure Monitor agent is not installed yet.' + }) + } + ] + fallbackResourceIds: [ + workspace.id + ] + '$schema': 'https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json' +} + +resource fleetWorkbook 'Microsoft.Insights/workbooks@2023-06-01' = { + name: guid(resourceGroup().id, namePrefix, 'linuxbroker-fleet-workbook') + location: location + tags: tags + kind: 'shared' + properties: { + displayName: 'Linux Broker fleet (${namePrefix})' + category: 'workbook' + sourceId: toLower(workspace.id) + version: 'Notebook/1.0' + serializedData: string(workbookContent) + } +} + +resource alertActionGroup 'Microsoft.Insights/actionGroups@2023-01-01' = if (!empty(alertEmailList)) { + name: 'ag-${namePrefix}' + location: 'Global' + tags: tags + properties: { + groupShortName: 'LinuxBroker' + enabled: true + emailReceivers: [ + for (address, i) in alertEmailList: { + name: 'email-${i + 1}' + emailAddress: address + useCommonAlertSchema: true + } + ] + } +} + +resource logAlertRules 'Microsoft.Insights/scheduledQueryRules@2023-12-01' = [ + for alert in logAlerts: { + name: 'alert-${namePrefix}-${alert.name}' + location: location + tags: tags + kind: 'LogAlert' + properties: { + displayName: alert.displayName + description: alert.description + severity: alert.severity + enabled: true + evaluationFrequency: 'PT5M' + windowSize: 'PT15M' + scopes: [ + workspace.id + ] + // Application Insights creates its tables only when its first telemetry arrives. + skipQueryValidation: true + autoMitigate: true + criteria: { + allOf: [ + { + query: alert.query + timeAggregation: 'Count' + operator: alert.operator + threshold: alert.threshold + failingPeriods: { + numberOfEvaluationPeriods: 1 + minFailingPeriodsToAlert: 1 + } + } + ] + } + actions: { + actionGroups: alertActionGroupIds + } + } + } +] + +resource nfsThrottlingAlert 'Microsoft.Insights/metricAlerts@2018-03-01' = if (monitorNfsShare) { + name: 'alert-${namePrefix}-nfs-throttling' + location: 'global' + tags: tags + properties: { + description: 'Azure Files throttled requests to the NFS home share in the last 15 minutes. Users may see slow sign-ins and applications; see the sizing guide in DEPLOYMENT.md.' + severity: 2 + enabled: true + scopes: [ + nfsFileServiceId + ] + evaluationFrequency: 'PT5M' + windowSize: 'PT15M' + autoMitigate: true + criteria: { + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + name: 'ThrottledTransactions' + metricNamespace: 'Microsoft.Storage/storageAccounts/fileServices' + metricName: 'Transactions' + dimensions: [ + { + name: 'ResponseType' + operator: 'Include' + values: nfsThrottlingResponseTypes + } + ] + operator: 'GreaterThan' + threshold: 0 + timeAggregation: 'Total' + } + ] + } + actions: [ + for actionGroupId in alertActionGroupIds: { + actionGroupId: actionGroupId + } + ] + } +} + +resource nfsLatencyAlert 'Microsoft.Insights/metricAlerts@2018-03-01' = if (monitorNfsShare) { + name: 'alert-${namePrefix}-nfs-latency' + location: 'global' + tags: tags + properties: { + description: 'The NFS home share answered slowly on average over the last 15 minutes.' + severity: 3 + enabled: true + scopes: [ + nfsFileServiceId + ] + evaluationFrequency: 'PT5M' + windowSize: 'PT15M' + autoMitigate: true + criteria: { + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + name: 'EndToEndLatency' + metricNamespace: 'Microsoft.Storage/storageAccounts/fileServices' + metricName: 'SuccessE2ELatency' + operator: 'GreaterThan' + threshold: alertNfsLatencyThresholdMs + timeAggregation: 'Average' + } + ] + } + actions: [ + for actionGroupId in alertActionGroupIds: { + actionGroupId: actionGroupId + } + ] + } +} + +output linuxHostDataCollectionRuleId string = linuxHostRule.id +output avdHostDataCollectionRuleId string = avdHostRule.id diff --git a/deploy/bicep/modules/core/observability.bicep b/deploy/bicep/modules/core/observability.bicep index abecf4f..53eb958 100644 --- a/deploy/bicep/modules/core/observability.bicep +++ b/deploy/bicep/modules/core/observability.bicep @@ -33,3 +33,4 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { output applicationInsightsConnectionString string = applicationInsights.properties.ConnectionString output applicationInsightsName string = applicationInsights.name output logAnalyticsWorkspaceId string = logAnalyticsWorkspace.id +output logAnalyticsWorkspaceName string = logAnalyticsWorkspace.name diff --git a/deploy/bicep/modules/core/redis-session-access.bicep b/deploy/bicep/modules/core/redis-session-access.bicep new file mode 100644 index 0000000..7590826 --- /dev/null +++ b/deploy/bicep/modules/core/redis-session-access.bicep @@ -0,0 +1,53 @@ +// Gives the portal's managed identity data access to the session cache. Separate from +// redis-session-store.bicep because the portal's app settings need the cache's host name +// first, and this needs the portal's identity. + +@allowed([ + 'managed' + 'cache' +]) +param kind string +param redisName string + +@description('Object ID of the portal\'s system-assigned managed identity.') +param principalId string + +// Alphanumeric, which both kinds of assignment accept. A new identity gets its own assignment +// rather than changing the old one. +var assignmentName = 'portal${uniqueString(principalId)}' + +resource managedRedis 'Microsoft.Cache/redisEnterprise@2025-07-01' existing = { + name: redisName +} + +resource managedRedisDatabase 'Microsoft.Cache/redisEnterprise/databases@2025-07-01' existing = { + parent: managedRedis + name: 'default' +} + +resource managedRedisAccess 'Microsoft.Cache/redisEnterprise/databases/accessPolicyAssignments@2025-07-01' = if (kind == 'managed') { + parent: managedRedisDatabase + name: assignmentName + properties: { + // The only policy Azure Managed Redis offers: full data access. + accessPolicyName: 'default' + user: { + objectId: principalId + } + } +} + +resource cache 'Microsoft.Cache/redis@2024-11-01' existing = { + name: redisName +} + +resource cacheAccess 'Microsoft.Cache/redis/accessPolicyAssignments@2024-11-01' = if (kind == 'cache') { + parent: cache + name: assignmentName + properties: { + accessPolicyName: 'Data Contributor' + objectId: principalId + // The alias is also the user name the portal signs in with, which is its object ID. + objectIdAlias: principalId + } +} diff --git a/deploy/bicep/modules/core/redis-session-store.bicep b/deploy/bicep/modules/core/redis-session-store.bicep new file mode 100644 index 0000000..198ac6c --- /dev/null +++ b/deploy/bicep/modules/core/redis-session-store.bicep @@ -0,0 +1,151 @@ +@description('Azure Managed Redis (managed), or Azure Cache for Redis (cache) where Azure Managed Redis is not available, as in Azure Government.') +@allowed([ + 'managed' + 'cache' +]) +param kind string + +@description('Name of the cache. It becomes part of a public DNS name, so it must be unique.') +@minLength(1) +@maxLength(60) +param name string + +@description('Region of the private endpoint, which must be the virtual network\'s region.') +param location string = resourceGroup().location + +@description('Region of the cache. Defaults to the virtual network\'s region. The portal reaches a cache in another region through the same private endpoint.') +param cacheLocation string = location +param tags object = {} + +@description('Leave empty for the default size: Balanced_B0 for Azure Managed Redis, Standard_C1 for Azure Cache for Redis. Azure Cache for Redis sizes are written _, such as Standard_C2 or Premium_P1.') +param skuName string = '' + +@description('Virtual network the portal reaches the cache from.') +param virtualNetworkId string + +@description('Subnet the cache private endpoint is placed in.') +param privateEndpointSubnetId string + +@description('Private DNS zone of the cache private endpoint: privatelink.redis.azure.net for Azure Managed Redis, privatelink.redis.cache. for Azure Cache for Redis.') +param privateDnsZoneName string + +var managed = kind == 'managed' +// Microsoft recommends C1 or larger outside dev/test, because C0 shares a CPU core. +var effectiveSku = empty(skuName) ? (managed ? 'Balanced_B0' : 'Standard_C1') : skuName +// Standard_C1 is tier Standard, family C, capacity 1. +var cacheSkuParts = split(effectiveSku, '_') +var managedPort = 10000 + +// Sessions are the only data, and the portal reads and writes them with the managed identity. +// Access keys are off, and so is public network access: the portal reaches the cache through +// its virtual network integration. +resource managedRedis 'Microsoft.Cache/redisEnterprise@2025-07-01' = if (managed) { + name: name + location: cacheLocation + tags: tags + sku: { + name: effectiveSku + } + properties: { + minimumTlsVersion: '1.2' + publicNetworkAccess: 'Disabled' + } +} + +resource managedRedisDatabase 'Microsoft.Cache/redisEnterprise/databases@2025-07-01' = if (managed) { + parent: managedRedis + name: 'default' + properties: { + clientProtocol: 'Encrypted' + port: managedPort + // One endpoint for clients that do not speak the cluster protocol, as the portal's does not. + clusteringPolicy: 'EnterpriseCluster' + // Every session expires, so under memory pressure the least recently used go first. + evictionPolicy: 'VolatileLRU' + accessKeysAuthentication: 'Disabled' + } +} + +resource cache 'Microsoft.Cache/redis@2024-11-01' = if (!managed) { + name: name + location: cacheLocation + tags: tags + properties: { + sku: { + name: cacheSkuParts[0] + family: first(cacheSkuParts[1]) + capacity: int(skip(cacheSkuParts[1], 1)) + } + enableNonSslPort: false + minimumTlsVersion: '1.2' + publicNetworkAccess: 'Disabled' + disableAccessKeyAuthentication: true + redisConfiguration: { + 'aad-enabled': 'true' + 'maxmemory-policy': 'volatile-lru' + } + } +} + +resource privateEndpoint 'Microsoft.Network/privateEndpoints@2024-05-01' = { + name: 'pe-${name}' + location: location + tags: tags + properties: { + subnet: { + id: privateEndpointSubnetId + } + privateLinkServiceConnections: [ + { + name: 'redis' + properties: { + privateLinkServiceId: managed ? managedRedis.id : cache.id + groupIds: [ + managed ? 'redisEnterprise' : 'redisCache' + ] + } + } + ] + } + dependsOn: [ + managedRedisDatabase + ] +} + +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { + name: privateDnsZoneName + location: 'global' + tags: tags +} + +resource privateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = { + parent: privateDnsZone + name: 'link-${last(split(virtualNetworkId, '/'))}' + location: 'global' + tags: tags + properties: { + registrationEnabled: false + virtualNetwork: { + id: virtualNetworkId + } + } +} + +resource privateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2024-05-01' = { + parent: privateEndpoint + name: 'default' + properties: { + privateDnsZoneConfigs: [ + { + name: 'redis' + properties: { + privateDnsZoneId: privateDnsZone.id + } + } + ] + } +} + +output name string = name +output hostName string = managed ? managedRedis!.properties.hostName : cache!.properties.hostName +output port int = managed ? managedPort : cache!.properties.sslPort diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4c035af..838d1fa 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -822,8 +822,8 @@ there before Phase 3. | Item | Why | Direction | | --- | --- | --- | -| Checks with `mstsc` | The test client couldn't present several monitors or play sound, and full screen and multi-monitor depend on the user's `Default.rdp`, because the launcher runs `mstsc /v:` | Through AVD, on an Ubuntu and a RHEL host: resize and maximize the window, use full screen and two monitors, and play sound on Ubuntu GNOME | -| Resizing on Ubuntu | A live resize dropped FreeRDP 3.31 on xrdp 0.9.24 (neutrinolabs/xrdp#3877), and with **Keep sessions alive** off the agent then ended the session. RHEL 9's xrdp 0.10.6 resized fine. | If `mstsc` drops too, have the launcher write an `.rdp` file with `dynamic resolution:i:0` (4.8), or ship a fixed xrdp | +| Checks with `mstsc` | The test client couldn't present several monitors or play sound. The launcher now opens the desktop full screen across every monitor (4.8), which nobody has checked through AVD yet, including whether a RemoteApp session gives the inner `mstsc` more than one monitor | Through AVD, on an Ubuntu and a RHEL host: resize and maximize the window, use full screen and two monitors, and play sound on Ubuntu GNOME | +| Resizing on Ubuntu | A live resize dropped FreeRDP 3.31 on xrdp 0.9.24 (neutrinolabs/xrdp#3877), and with **Keep sessions alive** off the agent then ended the session. RHEL 9's xrdp 0.10.6 resized fine. | If `mstsc` drops too, ship a fixed xrdp, or evaluate turning dynamic resolution off in the user's `Default.rdp`, which `mstsc /v:` reads without the warning that a written `.rdp` file now brings (4.8) | | RDP audio | RHEL, Rocky and AlmaLinux package no xrdp audio module. On Ubuntu, Xfce runs PulseAudio, which has no xrdp module in the archive, and MATE starts no sound server, so only GNOME, with PipeWire and `pipewire-module-xrdp`, can play sound. | Build or ship `pipewire-module-xrdp` for the RHEL family, and run PipeWire in Xfce and MATE sessions | | Idle timeout on the RHEL family | RHEL, Rocky and AlmaLinux don't package `xprintidle`, even in EPEL, so those hosts neither warn nor disconnect idle sessions | Read the X idle time another way, for example with `python3` calling `XScreenSaverQueryInfo` in `libXss` through `ctypes` | | Host reboot during a lease (*pre-existing*) | The lease file survives the reboot, but the home isn't mounted again and the agent has no record of the user, so the host stays checked out until someone returns it, and a reconnect lands in an empty local home | Reconcile leases at boot: mount the home again, or release the lease | @@ -842,9 +842,64 @@ there before Phase 3. ## Phase 4: strategic scale +**Status: In progress.** The foundations, items 4.1 and 4.5–4.9, shipped together, with host +agent 1.3.0, and the home directory share now mounts on `/nfs_profiles`. Each has a **Shipped** +note on where it differs from the design below it, and a **Validated** note on what the Phase 4 +validation confirmed in a public-cloud test environment with two RHEL 9 hosts and a Windows +session host, which had no interactive AVD client. 4.3 and 4.4 come next, together, then 4.2 and +4.10. + ### 4.1 Start a host on demand at checkout -**Status: Planned** · depends on Phase 1 (readiness and scaling fixes) +**Status: Done** · depends on Phase 1 (readiness and scaling fixes) + +**Shipped.** A checkout that finds no ready host starts one and answers `202`, the AVD script +waits for it, and a rule or window may keep a minimum of 0 hosts. Where it differs from the +design below: + +- `ReserveVmForStart` runs under the scaling application lock, so it never races a scaling + run. It starts at most one host per waiting user, and at most `MaxPendingStarts` at once (2 by + default, 1–20), never past the active phase's `MaxVMs`. A user who arrives while enough hosts + are starting waits for one of them. The API commits before it asks Azure, and records the host + as off again if Azure refuses. +- `retryAfterSeconds` isn't fixed at 60: it's the median start-to-reachable time over the last + week, less how long the oldest starting host has taken, kept between 30 and 120 seconds, and + the `Retry-After` header carries it too. While a user waits, each request probes the starting + hosts on port 22, so a host is handed out without waiting for the task's probe, which runs + every two minutes. +- Start on demand is on by default. The **Start on demand** card on the Scaling page turns it + off and sets the pending-start limit. A minimum of 0 can be saved only while it's on, and if + it's turned off later, scaling reads 0 as 1. +- The AVD script (2.0.0) waits in a modeless window with **Cancel**, up to `-MaxWaitSeconds` + (600 by default). It asks again after `Retry-After`, or 30, 30, then 60 seconds when the + broker gives none, gives up after three transient failures in a row, and renews a refused + token once. + It sends `clientVersion` with each checkout, and the Scaling page names the AVD hosts whose + latest checkout reported none, because their script can't wait. +- `Update-AvdHostBrokerScript.ps1` replaces the script on existing AVD hosts through Run + Command, and `Migrate-ExistingEnvironment.ps1` runs it after the Linux host migration. +- Checkout events gain a `Starting` outcome. Scaling counts waiting users as demand and keeps a + host for each of them when scaling down, and the dashboard reports waits, how many were + served, and their median and 95th percentile. + +**Validated** with the minimum at 0 and both hosts stopped: + +- The first checkout from the 2.0.0 script got `202` and started one host. The API probed the + host on the script's next requests, and the script had it 67 seconds after the first request. + The dashboard showed the wait, with a median of 65 seconds, and the audit entry carried the + script's version. +- With `MaxPendingStarts` at 1, a second waiting user got `202` and no second start; at 2, a + second host started. At `MaxVMs`, a checkout got `409`. The `Retry-After` header matched the + body each time. +- A minimum of 0 was refused while start on demand was off. Once the hosts were returned, scaling + stopped both again. The broker listed the two test clients, which sent no version, among the + session hosts whose script can't wait. +- One user on the only running host fills the pool, so the next scaling run started + `ScaleUpIncrement` more hosts, as it does for any full pool. `README.md` and `DEPLOYMENT.md` now + say so. + +**Open.** From a real AVD session: **Cancel** in the waiting window, and what a script older than +2.0.0 shows while a host starts for its user. **Why.** When the pool is exhausted, the user sees "No Linux host is available right now. Try again in a few minutes." (`Connect-LinuxBroker.ps1`). The script retries three times @@ -936,9 +991,50 @@ different image. ### 4.5 AVD scaling plan for the pass-through host pool -**Status: Planned** · no dependencies +**Status: Done** · no dependencies -The Windows AVD hosts only run `mstsc`, but they're deployed without any autoscale +**Shipped.** The deployment creates a scaling plan for the AVD host pool, turns on Start VM on +Connect, and gives the Azure Virtual Desktop service principal the role both need. Where it +differs from the design below: + +- **The role is on the subscription, not the host resource group.** Autoscale works only with + **Desktop Virtualization Power On Off Contributor** on the whole subscription, so the deployment + assigns it there. That needs Owner or User Access Administrator, and lets Azure Virtual Desktop + start and stop any session host in the subscription. `preprovision` finds the service principal + by its app ID only, never by its display name, and checks whether it holds the role already, + directly, through a group or from a management group, and whether the deploying account can + assign it. When the role is there it assigns nothing, and when it can't be assigned the plan is + deployed assigned to no host pool and `preprovision` prints the command for an Owner, instead + of the deployment failing. `assignAvdAutoscaleRole=false` leaves the role to an administrator. +- Microsoft documents one app ID for Azure Virtual Desktop, `9cdead84-a844-4324-93f2-b2e6bb768d07`. + `avdServicePrincipalAppId` still overrides it, and `avdServicePrincipalObjectId` skips the lookup. +- **Start VM on Connect,** so the weekend minimum can be 0. The first user after a quiet spell + waits while it starts a session host, and can then wait again for a Linux host: two starts. +- **Weekday and weekend schedules** with the same times, 07:00, 09:00, 18:00 and 20:00 in UTC by + default. Ramp-up and peak spread new sessions breadth-first, and ramp-down and off-peak pack them + depth-first. Nobody is signed out: a session host stops only once it has no sessions, + disconnected ones included. +- **Stopped hosts no longer block a deployment.** Azure refuses to change a VM extension on a VM + that isn't running, and autoscale, like the broker's own scaling and start on demand, leaves + hosts deallocated. `preprovision` lists the Linux hosts and session hosts that aren't running, + and the deployment leaves out their extensions. A deployment also replaces a VM's tags, so it + writes back the `excludeFromScaling` tag that keeps autoscale away from a session host. + +**Validated.** `azd provision` created the plan in `UTC`, with both schedules and +`excludeFromScaling` as its exclusion tag, assigned it to the host pool and turned on Start VM on +Connect. +`preprovision` found that the Azure Virtual Desktop service principal already held the role on the +subscription, from an earlier deployment, and assigned nothing. The deployment left out the +extensions of the session host that was stopped, and kept the `excludeFromScaling` tag the +validation had given it. + +**Open.** Check that the plan can be assigned right after the role, in a subscription where the +service principal doesn't hold it yet, and that Start VM on Connect starts a session host for a +real client. `Update-AvdHostBrokerScript.ps1` skips session hosts that +autoscale has stopped, so updating one means tagging, starting and updating it by hand; the +script could start them itself, or session hosts could update the script when they boot. + +**Design.** The Windows AVD hosts only run `mstsc`, but they're deployed without any autoscale (`deploy/bicep/modules/AVD/main.bicep` has no `scalingPlans` resource). Add a `Microsoft.DesktopVirtualization/scalingPlans` resource for the pooled host pool, with ramp-up, peak, ramp-down and off-peak parameters and a time zone. Assign **Desktop @@ -947,7 +1043,52 @@ the host resource group; its app ID differs in sovereign clouds, so parameterize ### 4.6 NFS home performance -**Status: Planned** · informed by 3.4 +**Status: Done** · informed by 3.4 + +**Shipped.** A new host script, `install-host-config.sh`, which the bootstraps and the host +migration install and run as root, keeps users' caches off the share, tunes the NFS read-ahead and +rotates the broker's logs, and `deploy/DEPLOYMENT.md` has a sizing guide for the share. Where it +differs from the design below: + +- **The caches are in `/var/cache/linuxbroker/users/`, not `/var/tmp/xdg-cache/$USER`.** Only + root can create entries in that directory, so no user can create or link another user's cache + ahead of them, as they could in world-writable `/var/tmp`. `create-user.sh` creates the cache at + checkout, owned by the user with mode 700, and replaces anything else it finds there without + following links. `manage-lease.sh` deletes it when the broker returns the host, and + `systemd-tmpfiles` empties the directory at every boot. The session launcher sets + `XDG_CACHE_HOME` only when the directory is the user's own, and + `/etc/profile.d/linuxbroker-cache.sh` does the same in login shells. +- **Browsers aren't configured one by one.** Firefox from the distribution's packages and + Chromium-based browsers already follow `XDG_CACHE_HOME`. The Firefox snap on Ubuntu doesn't, + because a snap sets its own, so its cache stays on the share. +- **The read-ahead is Microsoft's 15 MiB,** from Microsoft's udev rule in + `/etc/udev/rules.d/99-nfs.rules`, and the script applies it at once to NFS mounts that already + exist. The rule is Microsoft's but for one character: systemd 255 reports Microsoft's `$4` as an + invalid substitution each time udev loads its rules, while `$$4` passes `udevadm verify` and + still gives awk `$4`. Like Microsoft's, it applies to every NFS mount on the host. +- **The Bicep default stays at 100 GiB.** Its 3,100 baseline and 10,000 burst IOPS carry a small + pool, and the guide says how to see throttling and grow the share. The I/O of a sign-in hasn't + been measured, so the guide gives no figure per user. +- **Log rotation came with it,** though planned for 4.7: `/etc/logrotate.d/linuxbroker` rotates + five broker logs in `/var/log` weekly, or at 50 MB, and keeps four. `linuxbroker-patch.log` + trims itself. +- **The share mounts on `/nfs_profiles`,** renamed from `/awipsprofiles`, which the script removes + when it is empty and nothing is mounted on it. +- The alerts on throttling and latency are part of 4.7. + +**Validated** on both hosts after the host migration. The share's read-ahead was 15,360 KiB, +`/awipsprofiles` was gone, and `/nfs_profiles` was an empty directory with nothing mounted on it +between checkouts. A checkout created the user's cache with mode 700 and the user as its owner, a +login shell pointed `XDG_CACHE_HOME` at it, and the return removed it with the account. The share's +`Transactions` metric splits by `ResponseType`, which the throttling alert filters on, and its +end-to-end latency averaged about 3 ms, and 31 ms at most, over the day of the validation. + +**Open.** Measure a sign-in's I/O on the share and add figures per user to the sizing guide. +Offer the provisioned v2 model, which Microsoft now recommends for new shares and which provisions +IOPS apart from capacity; a v1 storage account can't be converted, so an existing environment +would need a new share and a copy of the profiles. SELinux labels the caches `var_t`, which does no +harm while the broker's users are unconfined, as they are by default on RHEL; a host that confines +them would need a file context for the directory. **Why.** Every user's home, including caches, is on a Premium Azure Files NFS share that defaults to 100 GiB (`deploy/bicep/modules/core/nfs-storage.bicep`). Premium performance @@ -972,7 +1113,78 @@ Tracker that kept failing on an index written by another distribution's Tracker ### 4.7 Host log shipping and observability -**Status: Planned** · pairs with 2.2 +**Status: Done** · pairs with 2.2 + +**Shipped.** A new Bicep module, `modules/core/host-monitoring.bicep`, adds a data collection rule +for the Linux hosts and one for the AVD session hosts, the **Linux Broker fleet** workbook and the +alerts, and `deploy/Enable-HostMonitoring.ps1`, which `postprovision` runs, connects the hosts to +the rules. `deployHostMonitoring` leaves all of it out, and an `AzureCustom` cloud has to opt in. +Where it differs from the design below: + +- **A script installs the agent, not the deployment.** Azure refuses to add an extension to a VM + that isn't running, and scaling, start on demand and AVD autoscale all stop hosts, so an agent + extension in Bicep would fail `azd provision` whenever a host is off. The script associates each + VM tagged `broker-role=linux-host` or `broker-role=avd-host` with its rule, which works on a + stopped VM, and installs the Azure Monitor agent, with automatic upgrades, on the running ones. + It names each host that isn't running, to be connected with `-HostNames` once it is started, and + `postprovision` reports its failure as a warning. The agent authenticates with each VM's + system-assigned identity and needs no data collection endpoint. +- **More logs.** The Linux rule also collects `linuxbroker-session-control.log` and + `linuxbroker-patch.log`, from 2.3 and 2.9. Each line of the broker's logs starts with a + `YYYY-MM-DD HH:MM:SS` timestamp, which starts a record, so a message of several lines stays one + record; `createuser.log` had no timestamps and now has them. xrdp starts its lines in a format the + agent doesn't recognize, so each line of `xrdp.log` and `xrdp-sesman.log` is a record of its own. + Syslog comes from `auth` and `authpriv` at `info` and up, from `kern` and `user` at `notice` and + up, because the kernel reports an NFS server that stops answering at `notice`, and from every + other facility at `warning` and up. The AVD rule collects the events of `Connect-LinuxBroker.ps1` + and the Remote Desktop client's critical events, errors and warnings. +- **The `syslog` group can read the release agent's logs.** They name the users signed in to the + host, so only root could read them. At each run, the release agent and its watcher now give them + mode 640 and the `syslog` group where the host has one, and keep them at 600 where it doesn't. +- **A fleet snapshot from the database.** The heartbeats from 2.2 go to the broker's database, not + to Log Analytics, so SQL script `157` adds `GetFleetSnapshot`, and at the end of each scaling run, + every five minutes, the API logs its figures as a `fleet snapshot` trace: ready, powered-on, + in-use and booting hosts, waiting users, stale heartbeats, unreachable shares, missing xrdp, and + the scaling minimum and maximum in effect, among others. The workbook's fleet charts and three of + the alerts read it. An API ahead of the database logs once per process that the procedure is + missing. +- **The alerts.** "No ready hosts" fires only while the scaling minimum is above 0 or users are + waiting, so a pool that 4.1 scaled to zero raises nothing. "A host with no heartbeat" became + "Unhealthy hosts", which also covers a host that reports the share unreachable or no xrdp. Both + need every snapshot in the 15-minute window to show the problem, so a host that is briefly busy + or starting raises nothing. "No fleet snapshot" is new, and fires when the scaling runs stop or + the API can't read the snapshot. Refused checkouts and API errors count `409` and `5xx` responses + against thresholds that are parameters. The throttling and latency alerts watch only a share the + deployment creates. +- **Email only.** One action group notifies the addresses in `alertEmailAddresses`, and a Teams + channel through its email address. Without addresses, the alerts still appear in Azure Monitor. +- **Log rotation shipped with 4.6.** + +**Validated** on the two RHEL 9 hosts and the Windows session host: + +- `Enable-HostMonitoring.ps1` installed agent 1.45 on the Linux hosts, with automatic upgrades, + and associated all three VMs with their rules, the stopped session host included. Once that host + was started, `-HostNames` installed the Windows agent on it in under a minute. +- The agent reads the text logs as root, so their modes don't matter. Installing it created the + `syslog` user and group on RHEL 9, which the hosts didn't have before. +- Every broker log that had new lines arrived in `LinuxBrokerHost_CL`, `createuser.log` with its + new timestamps, and so did `xrdp.log`. The agent sent only the lines written after it started, + not the files' history. xrdp 0.10 starts its lines with `[2026-09-27T06:17:15.790+0000]` rather + than 0.9's format, and each line arrived as a record of its own. `authpriv` messages arrived in + `Syslog`, and test events under the script's `LinuxBrokerScript` source, an information event and + a warning, in `Event`. +- Every workbook query ran without error over 24 hours. +- "Checkouts refused" fired four and a half minutes after a refused checkout. With both hosts + checked out and a minimum of 2, "no ready hosts" fired 19 minutes later, once its window held + only snapshots with no ready host. With xrdp stopped on one host, "unhealthy hosts" fired + 14 minutes later. The share's `Transactions` metric has the `ResponseType` dimension that the + throttling alert filters on. + +**Open.** `linuxbroker-patch.log` and `xrdp-sesman.log` got no lines during the validation, so +check that the agent picks up the patch log after it trims itself. "No fleet snapshot" and +"API errors" weren't fired. + +**Design.** - Deploy the Azure Monitor Agent to Linux hosts, plus a Data Collection Rule for `/var/log/release-session.log`, `/var/log/release-session-watcher.log`, @@ -985,9 +1197,49 @@ Tracker that kept failing on an index written by another distribution's Tracker ### 4.8 Tuning the double RDP hop -**Status: Planned** - -The user's session is RDP inside RDP: AVD outer, `mstsc` to xrdp inner. +**Status: Done, except the checks through AVD and the CPU measurements** · both need an +interactive AVD client, which the Phase 4 validation didn't have + +**Shipped.** `Connect-LinuxBroker.ps1` opens the Linux desktop full screen across every monitor of +the user's AVD session, with `mstsc /v:
/f /multimon`. Where it differs from the design +below: + +- **Switches, not an `.rdp` file.** Since the April 2026 update (CVE-2026-26151), Remote Desktop + Connection warns about every `.rdp` file that no trusted publisher signed, and turns off every + redirection the file asks for, the clipboard included, until the user turns each one back on. + Connections that don't start from a file are unaffected, so the launcher keeps `mstsc /v:`. The + properties that have no mstsc switch (`session bpp`, `networkautodetect`, `bandwidthautodetect`, + `connection type`, wallpaper and font smoothing) stay as the user's own Remote Desktop + Connection settings have them. Signed connection + files are in the hardening backlog. Until they exist, the session hosts must stay out of + Microsoft's high-security RDP file configuration, which blocks `mstsc /v:` too, and + `DEPLOYMENT.md` says so. +- **Both on by default, and each can be turned off.** `-FullScreen Off` opens the desktop in a + window on one monitor, and `-MultiMonitor Off` keeps it full screen on one. The deployment sets + them from `avdLinuxDesktopFullScreen` and `avdLinuxDesktopMultiMonitor` and passes only `Off`, + so the default RemoteApp command line is unchanged, and scripts older than 2.0.0, which refuse + the new parameters, keep working with it. +- The script refuses an address from the broker that isn't a host name or an IP address, before it + changes anything, because mstsc reads anything else on its command line as another switch or a + connection file. +- **No H.264 on the Linux hosts.** What each distribution's xrdp offers: + + | Distribution | xrdp | Graphics pipeline | + | --- | --- | --- | + | Ubuntu 24.04 | 0.9.24 | None: it arrived in 0.10 | + | RHEL 9, Rocky Linux 9, AlmaLinux 9 | 0.10.6.1 from EPEL | RFX. The package brings the `noopenh264` stub, which xrdp detects and doesn't use | + | RHEL 8 | 0.10.6.1 from EPEL | RFX. The package is built without openh264 | + + Without a GPU, H.264 in xrdp was more sluggish and blurrier than RFX + (neutrinolabs/xrdp#3489), so the hosts keep RFX. xrdp's default `gfx.toml` lists H.264 first, + so a host where Cisco's openh264 replaced the stub would switch to H.264. It belongs with GPU + encoding on GPU hosts. + +**Open.** Through AVD, check full screen and two monitors, including whether a RemoteApp session +gives the inner `mstsc` more than one monitor, and measure CPU on both hops with GNOME and with +Xfce, which feeds 3.2 and 4.3. Both belong with the 3.8 `mstsc` checks. + +**Design.** The user's session is RDP inside RDP: AVD outer, `mstsc` to xrdp inner. - xrdp ≥ 0.10.2 supports **H.264** in the graphics pipeline. Check the version on each distribution (`xrdp --version`) and enable GFX/H.264 in `xrdp.ini` where available. - Have `Connect-LinuxBroker.ps1` write an `.rdp` file instead of `mstsc /v:`, to set @@ -997,7 +1249,76 @@ The user's session is RDP inside RDP: AVD outer, `mstsc` to xrdp inner. ### 4.9 Scaling out the portal and API -**Status: Planned** +**Status: Done, except the checks in Azure Government** · they need a Government environment + +**Shipped.** The portal keeps its sessions in a Redis cache that every instance shares, the API +reuses its database connections, and `appServicePlanCapacity` sets how many instances the App +Service plan runs, 1 by default. Where it differs from the design below: + +- **A shared session store, with affinity only as the fallback.** `portalSessionStore` defaults to + `redis`. The deployment creates a cache with access keys off, reached only through a private + endpoint, gives the portal's managed identity access to it, integrates the portal with the app + subnet, and turns ARR affinity off for the portal. It turns affinity off for the API too. With + `filesystem`, and on an `AzureCustom` cloud, the portal keeps sessions on disk with affinity on. +- **Redis differs by cloud.** The public cloud gets Azure Managed Redis `Balanced_B0`, with its + high availability on, because new customers can't create Azure Cache for Redis there since + 1 April 2026. Azure Government gets Azure Cache for Redis, because Azure Managed Redis isn't + offered there; move it when it is, and before Azure Cache for Redis retires on 30 September 2028. + The Government default is `Standard_C1`, not `Standard_C0`, which shares a CPU core and which + Microsoft recommends only for dev/test. `portalRedisSku` picks another size, and + `portalRedisLocation` another region, for where the deployment's region doesn't offer the cache + or has no room for it. The private endpoint stays in the virtual network, so the portal still + reaches a cache in another region privately. +- **Token resource in Azure Government.** The portal asks for tokens for `https://redis.azure.com` + by default. In Azure Government the deployment asks for Azure Cache for Redis's application ID, + `acca5fbb-b7e4-4009-81f1-37e38fd66d78`, instead, since Microsoft's documentation accepts both in + the public cloud and names no Government URI. +- **The cookie and the lifetime.** The session cookie is now `Secure` and `SameSite=Lax` as well as + `HttpOnly`. A session ends 12 hours after the operator's last request, when the cache deletes it. + `SESSION_LIFETIME_HOURS` changes that, but `azd provision` replaces the portal's app settings, so + it has to be set again after each provision. Only the sign-in routes and `/api/ui` load the + session, so the page shell and `/health` keep working while the cache is down, and the portal + answers `503` with `Retry-After` instead of failing to load. +- **A pool without Entra SQL authentication.** Each API worker process keeps the connections it has + finished with and reuses the newest first. It closes a connection idle for 120 seconds, rather + than 300, because App Service forgets an outbound connection after four minutes without traffic, + or open for 30 minutes. A connection is rolled back when it returns and closed if that fails. Its + next transaction begins + when it is handed out again, not when it returned: the SQL integration tests showed that SQL + Server stamps a change to a system-versioned table with its transaction's start time and refuses + one older than the row's current version (error 13535), which a transaction left open in the + pool would be. `DB_POOL_ENABLED` turns it off. Entra authentication and the mssql-python driver + stay in the hardening backlog. +- **Capacity warnings.** Each instance's API can run 12 database requests at once, so `preprovision` + warns when the Basic database tier, at most 30, is too small for the capacity, and when sign-ins + on several instances would rely on affinity. +- **The task function stays on the plan.** Its timers already run on one instance at a time, + through a lease in its storage account, and each run is a few calls to the API. A plan of its + own, or Flex Consumption, would add cost and a second virtual network integration for no gain. +- The two stray `flask_session/` files are no longer tracked. + +**Validated** in the public cloud: + +- **No room in eastus2.** Azure refused every size of Azure Managed Redis there with + `InsufficientCapacity`, and left a failed cache that failed the deployment and can't be resized. + Nearby regions accepted `Balanced_B0`, so `portalRedisLocation` was added, and the validation's + cache is in eastus, behind a private endpoint in eastus2. +- The cache came up with high availability, TLS 1.2, public access off and access keys off. The + portal's first request, about five minutes after the deployment gave its identity access, signed + in to it as the managed identity. +- With the plan at two instances, 40 requests that carried one session cookie were answered by both + instances, and every one saw the same session. The cookie is `Secure`, `HttpOnly` and + `SameSite=Lax`, and lasts 12 hours. +- With the portal pointed at a port the cache doesn't serve, `/api/ui` answered `503` with + `Retry-After: 30` in about six seconds, while the page shell and `/health` kept answering, and it + answered normally within 36 seconds of the port being put back. + +**Open.** In Azure Government, check the token resource and that the Azure Cache for Redis API +versions are available. Sign in with a browser on one instance and carry on on another, with the +capacity at 2; the validation shared one session between the instances through the portal's API, +without a browser sign-in. + +**Design.** - The BFF uses filesystem sessions (`front_end/app.py`), which rely on ARR affinity when scaled out. Move to a shared session store, or keep affinity and document it. @@ -1007,6 +1328,46 @@ The user's session is RDP inside RDP: AVD outer, `mstsc` to xrdp inner. - Split the task function onto its own plan, or use a Flex Consumption plan, if its timers compete with the API. +### 4.10 GPU-accelerated OpenGL + +**Status: Planned** · depends on 4.4 (host pools); benefits from 4.2 (golden image) and 4.3 + +**Why.** The hosts have no GPU, so OpenGL in an xrdp session renders in software, with Mesa's +llvmpipe, on the same CPU as everything else on the host. That serves office work and browsers, +but 3D visualization, CAD and other OpenGL applications draw slowly, and on a multi-session host +(4.3) one of them slows every session. + +**Design.** + +- **A pool of GPU hosts.** Host pools (4.4) give GPU VMs their own pool, image and RemoteApp, so + only the users who need a GPU get one. The candidates are the sizes Microsoft supports with + NVIDIA's GRID drivers for virtual workstations: NVadsA10 v5, which offers from a sixth of an + NVIDIA A10 up to whole GPUs, and NCasT4_v3, with NVIDIA T4s. NVv3 retires on 30 September 2026. + Check which of them each target region offers, Azure Government's included, and the + subscription's GPU quota, before choosing. +- **Drivers.** Microsoft redistributes the GRID drivers for these sizes with the vGPU license + included, so no NVIDIA license server is needed. They support Ubuntu 20.04 to 24.04, RHEL 8.10 + and RHEL 9.4, 9.6 and 9.7, but not Rocky Linux or AlmaLinux. Microsoft's installation steps need + Secure Boot and vTPM off, while the deployment gives Linux hosts Trusted Launch with both on, so + a GPU pool needs its own security profile. The driver belongs in the pool's image (4.2). It's + built for the running kernel, so rolling maintenance (2.9) must check `nvidia-smi` after a + kernel update. +- **VirtualGL in the session.** xrdp's X server has no GPU, so OpenGL applications run under + VirtualGL's `vglrun`, which renders them on the GPU and copies each finished frame into the + session. Its EGL back end reaches the GPU without a second X server, but it emulates only part of + GLX; an application that needs the rest needs the GLX back end and an X server on the GPU. The + session launcher can set VirtualGL up, and a pool setting can name the applications to wrap, so + users start them from the menu as usual. +- **Encoding stays on the CPU at first.** The frames still reach the user as RFX, encoded on the + host's CPU (4.8). H.264 encoded on the GPU, through xrdp's graphics pipeline, is later work for + once the pool exists. +- **Measure** frame rate, GPU and CPU use for the applications the pool is for, with and without + `vglrun`, on each candidate size, and how many sessions one GPU serves. + +**Open questions.** Which sizes to offer, and in which regions. Whether Secure Boot can stay on. +How many sessions share one GPU partition. How Wayland-only distributions (3.5) change the +approach. + --- ## Security hardening backlog @@ -1015,16 +1376,18 @@ A separate track, prioritized independently of the phases. | Item | Why | Direction | | --- | --- | --- | -| API connects to SQL as the **server admin** | `DB_USERNAME` is the SQL admin login (`deploy/bicep/main.resources.bicep`) | A contained database user with EXECUTE on the broker procedures only, or Entra managed-identity authentication (pyodbc with an access token) | +| API connects to SQL as the **server admin** | `DB_USERNAME` is the SQL admin login (`deploy/bicep/main.resources.bicep`) | A contained database user with EXECUTE on the broker procedures only, or Entra managed-identity authentication, with the mssql-python driver or pyodbc and an access token | | `avdadmin` sudo allowlist is root-equivalent | `usermod`, `userdel`, `groupadd` and `chpasswd` with arbitrary arguments (`Configure-*-Host.sh`) | Once every host runs `create-user.sh --password-stdin` (Phase 1), drop `chpasswd`, `groupadd` and `usermod`. Move `userdel` into `manage-lease.sh`. The allowlist then holds only validated scripts. | | SSH host keys not verified | `StrictHostKeyChecking=no` in `run_remote_command` | Record host keys at provisioning (Key Vault or SQL) and pin them, or use an SSH CA | | RDP server identity not verified | `AuthenticationLevelOverride=0` in `Connect-LinuxBroker.ps1`; xrdp self-signed certificates | Issue xrdp certificates from Key Vault or enterprise PKI, and restore server authentication | +| Launch without a signed connection file | `Connect-LinuxBroker.ps1` starts `mstsc /v:` (4.8), so the AVD session hosts can't take Microsoft's high-security RDP file configuration, which allows only files signed by a trusted publisher, and the connection properties that have no `mstsc` switch stay at their defaults | Have the API sign each checkout's `.rdp` file with a Key Vault certificate that the session hosts trust through the trusted-publisher thumbprint policy. Signing on the session host would expose the key to its users. Pair it with the xrdp certificates above. | +| Every user of a session host can change the launch script | `Configure-AVD-Host.ps1` creates `C:\Temp` with the permissions it inherits from `C:\`, which give Authenticated Users Modify, as the Phase 4 validation found, and `Update-AvdHostBrokerScript.ps1` writes `Connect-LinuxBroker.ps1` there too, so one user of a pooled session host can change the script that the others' **Linux Desktop** runs | Install the script in a folder that only administrators can write, such as `C:\Program Files\LinuxBroker`, from both scripts, and move the RemoteApp's command line there once every session host has it | | Username collisions | `re.sub(r'[^a-zA-Z0-9_]', '', username)` maps `john.smith` and `johnsmith` to the same Linux account and NFS home | Derive usernames from a stable identifier (UPN plus collision check, or object ID) and keep the mapping in `VmUsers` | | Shared SSH private key for the whole fleet | One Key Vault secret (`KEY_NAME`) | Per-host keys or short-lived SSH certificates | | NFS `AUTH_SYS` trust | `sec=sys`, `NoRootSquash` (`nfs-storage.bicep`, `create-user.sh`); root on any host in the subnet can read every home | Restrict private endpoint access to the Linux host subnet with an NSG. Consider Azure NetApp Files with Kerberos (krb5p) for strong isolation. | | Hosts download scripts from `main` at provisioning | `scriptSourceRoot` defaults to the `main` branch | Pin to a release tag or commit and verify checksums, or bake scripts into the image (4.2) | | Checkout abuse | The AvdHost role can check out any username | Rate-limit per AVD host. Optionally verify that the username belongs to the user signed in to that AVD session. | -| Committed test artifacts | `flask_session/` files are tracked, although they're in `.gitignore` | `git rm` them | +| Portal sign-in has no `state`, nonce or PKCE | `/login` builds the sign-in URL with `get_authorization_request_url` alone, and `/getAToken` redeems any `code` it receives (`front_end/route_authentication.py`), so another site can finish a sign-in in an operator's browser with an authorization code of its own (login CSRF) | Start the sign-in with MSAL's `initiate_auth_code_flow` and keep the flow in the session, which every instance now shares (4.9), then redeem it with `acquire_token_by_auth_code_flow`, which checks `state` and the nonce and sends the PKCE verifier | --- @@ -1064,6 +1427,19 @@ A separate track, prioritized independently of the phases. | 2026-09 | Ubuntu desktop hosts keep systemd-networkd as the network renderer, at the cost of GNOME's network indicator, so a package install can't take a host off the network. | | 2026-09 | Idle time never counts from before the user's current connection, so a resumed session isn't disconnected for the time it spent disconnected. | | 2026-09 | Ubuntu 26.04 and RHEL 10 wait for 3.5 or 3.8. Rocky Linux 9 is offered through its Marketplace image, and `alma-9` is recommended where Marketplace purchases are blocked. | +| 2026-09 | Phase 4 ships in stages. The first PR covers the foundations: 4.1 start on demand, 4.5 the AVD scaling plan, 4.6 NFS home performance, 4.7 host log shipping, 4.8 the double RDP hop and 4.9 portal and API scale-out, with one host agent rollout (1.3.0). 4.3 and 4.4 follow in one PR, then 4.2. | +| 2026-09 | Start on demand is **on by default**. It starts at most one host for each waiting user, and at most `MaxPendingStarts` (2 by default) at once, never past `MaxVMs`. `MinVMs` may be 0 only while it is on, and if it is turned off later, scaling reads 0 as 1. This replaces the `MinVMs ≥ 1` rule. | +| 2026-09 | While a user waits, each checkout request probes the starting hosts on port 22 and records those that answer as reachable, rather than waiting for the task's probe. Readiness still comes from a TCP probe, never from the heartbeat. | +| 2026-09 | The AVD script reports its version with each checkout, kept with the checkout event, and the **Start on demand** card names the session hosts whose latest checkout reported none, because their script can't wait. That's the check before setting a minimum of 0. | +| 2026-09 | The AVD autoscale role is assigned on the **subscription**, because autoscale needs it there, not on the host resource group. `preprovision` finds the Azure Virtual Desktop service principal by app ID only, assigns nothing when the role is already held, and prints the command for an Owner instead of failing when it can't assign it. | +| 2026-09 | A script installs the Azure Monitor agent, not the deployment, because Azure refuses extension changes on a stopped VM and scaling, start on demand and autoscale all stop VMs. For the same reason, a deployment leaves out the extensions of the hosts that aren't running. | +| 2026-09 | The home directory share mounts on `/nfs_profiles` while `create-user.sh` creates a home, and users' caches stay on the host's disk in `/var/cache/linuxbroker/users/`, whose parent only root can write, rather than in world-writable `/var/tmp`. | +| 2026-09 | The launcher keeps `mstsc /v:` and adds `/f` and `/multimon`, both on by default, rather than writing an `.rdp` file, because since the April 2026 update every unsigned `.rdp` file opens with its redirections off. Signed connection files are in the hardening backlog. The Linux hosts keep RFX, and H.264 waits for GPU hosts (4.10). | +| 2026-09 | The portal's sessions move to **Redis**, with access keys off and the portal's managed identity: Azure Managed Redis in the public cloud, where new customers can't create Azure Cache for Redis since April 2026, and Azure Cache for Redis `Standard_C1` in Azure Government, which doesn't offer Azure Managed Redis. `portalSessionStore=filesystem` keeps sessions on disk, with ARR affinity. | +| 2026-09 | The API pools its `pymssql` connections in each worker process, and the task function stays on the shared App Service plan. Entra SQL authentication and the mssql-python driver stay in the hardening backlog. | +| 2026-09 | GPU-accelerated OpenGL (4.10) goes to a pool of GPU hosts once host pools (4.4) exist, rather than to every host. | +| 2026-09 | The portal's session cache may be in another region than the rest of the deployment (`portalRedisLocation`), behind a private endpoint in the deployment's virtual network, because Azure had no room for Azure Managed Redis in eastus2 during the Phase 4 validation. The default stays the deployment's region. | +| 2026-09 | A pool that start on demand wakes from zero scales up like any full pool: its first user fills it, so the next scaling run starts `ScaleUpIncrement` more hosts, ready for the users after them. This is documented rather than special-cased. | ## Glossary diff --git a/flask_session/1c6fd4407b2df568cb24e167aa24bda3 b/flask_session/1c6fd4407b2df568cb24e167aa24bda3 deleted file mode 100644 index f0db5fc..0000000 Binary files a/flask_session/1c6fd4407b2df568cb24e167aa24bda3 and /dev/null differ diff --git a/flask_session/2029240f6d1128be89ddc32729463129 b/flask_session/2029240f6d1128be89ddc32729463129 deleted file mode 100644 index ffb2cd9..0000000 Binary files a/flask_session/2029240f6d1128be89ddc32729463129 and /dev/null differ diff --git a/front_end/README.md b/front_end/README.md index 74d3df1..20cbad4 100644 --- a/front_end/README.md +++ b/front_end/README.md @@ -30,12 +30,39 @@ Why the BFF stays: - Flask serves the SPA shell for **every** non-API path, so a bookmarked deep link or a hard refresh still resolves and React Router renders the right page. +### Sessions + +The session holds the operator's access token, so it stays on the server, and the browser carries +only an opaque id in the session cookie, which is `Secure`, `HttpOnly` and `SameSite=Lax`. +`session_store.py` picks where it lives: + +- **Local runs and tests** keep sessions on disk, under `flask_session/` in the working directory, + which Git ignores. +- **Deployments** keep them in Redis, so that every instance of the portal finds the session a + sign-in created on another: Azure Managed Redis, or Azure Cache for Redis in Azure Government, + reached through a private endpoint. The portal signs in to Redis as the web app's system-assigned + managed identity, through `redis-entraid`, so the Redis store works only where the portal has + one. See [Portal And API Scale-Out](../deploy/DEPLOYMENT.md#portal-and-api-scale-out). + +Only `/login`, `/getAToken`, `/logout` and `/api/ui/*` load the session; `SESSION_ROUTES` in +`session_store.py` lists the routes outside `/api/ui`. The SPA shell, the static assets and +`/health` never touch the store, so they keep working while it is down. Writing to the session on +any other route raises an error naming the route, so a new route that needs a session fails in the +tests rather than losing its data. + +When the store can't be reached, a request that needs the session gets `503` with +`Retry-After: 30`: JSON with an `error` on `/api/ui/*`, plain text on the sign-in routes. The +portal logs the cause through `linuxbroker.frontend`, which Application Insights collects. When a +request only loses the refresh of the session's expiry, its response stands, and the portal logs a +warning. + ## Directory Layout | Path | Purpose | | --- | --- | | `app.py` | Creates the Flask app, serves the SPA shell, exposes `/api/ui/session` and `/api/ui/dashboard`, and defines the JSON and SPA error handlers. | | `config.py` | Reads cloud, Entra ID, and Broker API settings from environment variables. | +| `session_store.py` | Server-side sessions: the disk or Redis store, the cookie policy, which routes load the session, and the `503` when the store can't be reached. | | `function_authentication.py` | `@login_required`. Returns `401` JSON for `/api/ui/*` and redirects page requests to `/login`. | | `function_api.py` | Authenticated Broker API helpers, request timeouts, JSON decoding, dashboard VM summary retrieval, history filter parsing, and paged history calls. | | `function_bff.py` | Shared JSON plumbing: the `@broker_endpoint` error decorator, request-body helpers, and the paged history envelope. | @@ -99,8 +126,8 @@ a path that serves the SPA shell. | POST | `/api/ui/scaling/rules//delete` | | | GET | `/api/ui/scaling/log` | Paged. | | GET | `/api/ui/scaling/rules/history` | Paged. | -| GET | `/api/ui/scaling/policy` | The time zone, the default rule, the windows, and what applies now and next. `{Available: false}` when the broker predates it, so the Scaling section offers the scaling rules instead. | -| POST | `/api/ui/scaling/policy` | Body `{timezone}`. | +| GET | `/api/ui/scaling/policy` | The time zone, the default rule, the windows, what applies now and next, the start on demand settings, and the broker script versions the AVD hosts run. `{Available: false}` when the broker predates it, so the Scaling section offers the scaling rules instead. | +| POST | `/api/ui/scaling/policy` | Body with any of `timezone`, `startondemandenabled` and `maxpendingstarts`, applied together. | | GET | `/api/ui/scaling/timezones` | | | POST | `/api/ui/scaling/schedules` | Adds a window. | | POST | `/api/ui/scaling/schedules//update` | | @@ -335,6 +362,19 @@ Then open . `APPLY_TIMEOUT_SECONDS` controls the BFF timeout for `/api/ui/hosts/settings/apply` and defaults to `110` seconds. +These settings control the [sessions](#sessions). A local run needs none of them: + +| Variable | Default | Purpose | +| --- | --- | --- | +| `SESSION_BACKEND` | `filesystem` | `filesystem` keeps sessions under `flask_session/`. `redis` keeps them in Redis and needs `REDIS_HOST`. | +| `SESSION_LIFETIME_HOURS` | `12` | How long an idle session lasts, from 1 to 720. Each request that loads the session starts the period again. | +| `SESSION_COOKIE_SECURE` | `true` | Marks the session cookie `Secure`. Edge, Chrome and Firefox accept it from `http://localhost`, so set `false` only to reach the portal over plain HTTP by another name. | +| `REDIS_HOST` | | The Redis host name. | +| `REDIS_PORT` | `10000` | 10000 for Azure Managed Redis, 6380 for Azure Cache for Redis. | +| `REDIS_ENTRA_RESOURCE` | `https://redis.azure.com` | The Entra resource the managed identity gets tokens for. The deployment sets `acca5fbb-b7e4-4009-81f1-37e38fd66d78`, the application ID of Azure Cache for Redis, in Azure Government. | + +The portal checks them at startup and stops with a message naming the setting that is wrong. + The app reads environment variables directly; it does not load `.env` files by itself. For Azure US Government set `AZURE_CLOUD_NAME=AzureUSGovernment`. For custom or sovereign clouds without a built-in profile set `AZURE_CLOUD_NAME=AzureCustom` and provide `AZURE_AUTHORITY_HOST`. @@ -362,6 +402,9 @@ gunicorn --bind 0.0.0.0:8000 app:app the `web/` sources are removed. Nothing generated is committed; `package-lock.json` is, so `npm ci` is reproducible. +`requirements.txt` pins every Python package. `redis-entraid` 1.2.1 requires PyJWT 2.13, so `pyjwt` +can move past 2.13 only with a `redis-entraid` release that allows it. + ## Testing Two suites, both run in `.github/workflows/front-end-tests.yml`. @@ -384,6 +427,11 @@ sends a `POST` with that header attached, and the `spa_bundle` fixture supplies the SPA-serving tests do not depend on whether anyone has run `npm run build`. The Python suite therefore needs no Node toolchain. +`tests/test_session_store.py` covers the session settings, the cookie policy, which routes load the +session, the Redis client, a sign-in on one instance honoured by another, and the `503` when the +store can't be reached. It fakes the managed identity and the Redis server, so nothing leaves the +machine. + Behaviour is tested on whichever side now owns it. Broadly: date conversion, ignore-filter semantics, pagination parameters, the legacy bare-list fallback, dashboard summary preference and degradation, CSRF, and the error-status mapping are **pytest**; VM lifecycle rules, pagination diff --git a/front_end/app.py b/front_end/app.py index e88ae41..e11fe05 100644 --- a/front_end/app.py +++ b/front_end/app.py @@ -9,12 +9,12 @@ import requests from flask import Flask, jsonify, request, send_from_directory, session -from flask_session import Session from flask_wtf.csrf import CSRFProtect, CSRFError, generate_csrf from function_api import NotAuthenticated, api_get, api_post, fetch_vm_summary from function_authentication import login_required from function_bff import API_PREFIX, json_error +from session_store import SessionStoreUnavailable, configure_sessions from route_authentication import register_route_authentication from route_vm_management import register_route_vm_management from route_scaling_management import register_route_scaling_management @@ -29,12 +29,13 @@ app = Flask(__name__) app.config['SECRET_KEY'] = os.environ.get('FLASK_KEY') -app.config['SESSION_TYPE'] = 'filesystem' -app.config['VERSION'] = '0.121' +app.config['VERSION'] = '0.122' # Tokens stay valid for the life of the session rather than expiring after an # hour, so a long-lived management page does not start rejecting submissions. app.config['WTF_CSRF_TIME_LIMIT'] = None -Session(app) +# Local disk unless SESSION_BACKEND=redis, which deployments set so every +# instance shares the same sessions. +configure_sessions(app) # Protects the state-changing endpoints. The React client reads the token from # /api/ui/session and sends it back as the X-CSRFToken header, which CSRFProtect @@ -273,6 +274,20 @@ def handle_csrf_error(e): @app.errorhandler(500) def handle_server_error(e): + # A session store outage is raised while the session opens or saves, outside + # any view, so it arrives here wrapped in the 500. Flask has already logged the + # traceback, but through its own logger, which Application Insights doesn't collect. + cause = getattr(e, 'original_exception', None) + if isinstance(cause, SessionStoreUnavailable): + logger.error("%s", cause) + message = ("The portal cannot reach its session store right now, so sign-in is " + "unavailable. Try again in a few minutes.") + headers = {"Retry-After": "30"} + if _wants_json(): + body, status = json_error(message, 503) + return body, status, headers + return message, 503, dict(headers, **{"Content-Type": "text/plain; charset=utf-8"}) + logger.error("Unhandled server error: %s", e) if _wants_json(): return json_error("An unexpected error occurred. The issue has been logged.", 500) diff --git a/front_end/requirements.txt b/front_end/requirements.txt index d38f71a..266c3fe 100644 --- a/front_end/requirements.txt +++ b/front_end/requirements.txt @@ -5,6 +5,8 @@ azure-monitor-opentelemetry==1.8.9 requests==2.34.2 msal==1.37.0 Flask-Session==0.8.0 +redis==8.1.0 +redis-entraid==1.2.1 Flask-WTF==1.3.0 pyjwt==2.13.0 cryptography==50.0.0 diff --git a/front_end/route_scaling_management.py b/front_end/route_scaling_management.py index 4bd213b..bd985c9 100644 --- a/front_end/route_scaling_management.py +++ b/front_end/route_scaling_management.py @@ -92,10 +92,28 @@ def ui_scaling_policy(): @login_required @broker_endpoint("Unable to update the scaling policy. Please try again later.") def ui_update_scaling_policy(): - zone = json_body().get('timezone') - if not isinstance(zone, str) or not zone.strip(): - raise BadRequest("Choose a time zone.") - return jsonify(api_post('/scaling/policy/update', {'timezone': zone.strip()[:64]})) + # Any of the time zone and the start on demand settings; the broker validates them and + # applies them together. + body = json_body() + payload = {} + if 'timezone' in body: + zone = body.get('timezone') + if not isinstance(zone, str) or not zone.strip(): + raise BadRequest("Choose a time zone.") + payload['timezone'] = zone.strip()[:64] + if 'startondemandenabled' in body: + enabled = body.get('startondemandenabled') + if not isinstance(enabled, bool): + raise BadRequest("Choose whether start on demand is on.") + payload['startondemandenabled'] = enabled + if 'maxpendingstarts' in body: + pending = body.get('maxpendingstarts') + if isinstance(pending, bool) or not isinstance(pending, (int, str)) or (isinstance(pending, str) and len(pending) > 8): + raise BadRequest("Enter how many hosts may start at once, from 1 to 20.") + payload['maxpendingstarts'] = pending + if not payload: + raise BadRequest("Choose a time zone or the start on demand settings.") + return jsonify(api_post('/scaling/policy/update', payload)) @app.route(f'{API_PREFIX}/scaling/timezones') @login_required diff --git a/front_end/session_store.py b/front_end/session_store.py new file mode 100644 index 0000000..54bc728 --- /dev/null +++ b/front_end/session_store.py @@ -0,0 +1,244 @@ +"""Server-side sessions for the portal. + +A session holds the operator's access token, so it stays on the server and the +browser only carries an opaque id. One instance can keep sessions on local disk, +but two or more cannot: a sign-in completed on one instance would be unknown to +the next. Deployments therefore keep them in Redis, reached over a private +endpoint and authenticated with the web app's managed identity. Local runs and +tests keep the disk store. + +Only the routes that use the session load it. The page shell, static assets and +the health probe never touch the store, so they keep working while it is down and +the portal can explain the outage instead of failing to load. +""" + +import logging +import os +from dataclasses import dataclass +from datetime import timedelta +from typing import Optional + +from flask import g, has_request_context, request +from flask.sessions import NullSession, SessionInterface +from flask_session import Session + +from function_bff import API_PREFIX + +# A child of the logger app.py sends to Application Insights. +logger = logging.getLogger('linuxbroker.frontend.sessions') + +BACKEND_FILESYSTEM = 'filesystem' +BACKEND_REDIS = 'redis' +BACKENDS = (BACKEND_FILESYSTEM, BACKEND_REDIS) + +DEFAULT_LIFETIME_HOURS = 12 +MAX_LIFETIME_HOURS = 720 +# Azure Managed Redis listens on 10000. Azure Cache for Redis uses 6380. +DEFAULT_REDIS_PORT = 10000 +# The Entra resource Redis tokens are issued for. If a cloud rejects it, set +# REDIS_ENTRA_RESOURCE to the application ID acca5fbb-b7e4-4009-81f1-37e38fd66d78. +DEFAULT_REDIS_ENTRA_RESOURCE = 'https://redis.azure.com' + +# The routes outside the JSON API that read or write the session. +SESSION_ROUTES = frozenset({'/login', '/getAToken', '/logout'}) + +_STORE_FAILED = '_session_store_failed' + + +def uses_session(path): + return path in SESSION_ROUTES or path == API_PREFIX or path.startswith(API_PREFIX + '/') + + +class SessionStoreUnavailable(Exception): + """The session store could not be reached. The portal answers 503.""" + + +class SessionNotUsedHere(NullSession): + """Stands in for the session on routes that never use it. + + Reads find nothing. Writes raise and name the route, rather than being + silently dropped, so a new route that needs a session fails in tests. + """ + + def _refuse(self, *args, **kwargs): + path = request.path if has_request_context() else 'This route' + raise RuntimeError( + f"{path} does not load the session. Add it to session_store.SESSION_ROUTES " + "if it needs one." + ) + + __setitem__ = __delitem__ = clear = pop = popitem = update = setdefault = _refuse + + +class PortalSessionInterface(SessionInterface): + """Wraps the storage backend with the route rule and outage handling.""" + + null_session_class = SessionNotUsedHere + + def __init__(self, backend, store_errors=()): + self.backend = backend + self.store_errors = tuple(store_errors) + + def open_session(self, app, request): + if not uses_session(request.path): + return self.make_null_session(app) + try: + return self.backend.open_session(app, request) + except self.store_errors as error: + raise _unavailable(error) from error + + def save_session(self, app, session, response): + # There is no session when opening it failed. After a failed save, Flask + # saves again while finalizing the error response; that attempt is skipped + # rather than waiting on the store a second time. + if session is None or g.get(_STORE_FAILED): + return + try: + self.backend.save_session(app, session, response) + except self.store_errors as error: + if not session.modified: + # Only the expiry refresh was lost. The view has already acted, so + # its response stands rather than reporting a failure. + logger.warning("Could not refresh the session's expiry in the session store: %s", error) + return + raise _unavailable(error) from error + + +def _unavailable(error): + setattr(g, _STORE_FAILED, True) + return SessionStoreUnavailable( + f"The session store could not be reached ({type(error).__name__}: {error})." + ) + + +@dataclass(frozen=True) +class SessionSettings: + backend: str + lifetime_hours: int + cookie_secure: bool + redis_host: Optional[str] + redis_port: int + redis_entra_resource: str + + +def _text(environ, name): + return (environ.get(name) or '').strip() + + +def _whole_number(environ, name, default, minimum, maximum): + raw = _text(environ, name) + if not raw: + return default + try: + value = int(raw) + except ValueError: + value = None + if value is None or not minimum <= value <= maximum: + raise RuntimeError(f"{name} must be a whole number from {minimum} to {maximum}, not '{raw}'.") + return value + + +def _flag(environ, name, default): + raw = _text(environ, name).lower() + if not raw: + return default + if raw in ('1', 'true', 'yes', 'on'): + return True + if raw in ('0', 'false', 'no', 'off'): + return False + raise RuntimeError(f"{name} must be true or false, not '{raw}'.") + + +def read_settings(environ=None): + environ = os.environ if environ is None else environ + + backend = _text(environ, 'SESSION_BACKEND').lower() or BACKEND_FILESYSTEM + if backend not in BACKENDS: + raise RuntimeError(f"SESSION_BACKEND must be 'filesystem' or 'redis', not '{backend}'.") + + settings = SessionSettings( + backend=backend, + lifetime_hours=_whole_number(environ, 'SESSION_LIFETIME_HOURS', DEFAULT_LIFETIME_HOURS, + 1, MAX_LIFETIME_HOURS), + # Browsers accept Secure cookies from http://localhost, so this only needs + # turning off when the portal is reached over plain HTTP by another name. + cookie_secure=_flag(environ, 'SESSION_COOKIE_SECURE', True), + redis_host=_text(environ, 'REDIS_HOST') or None, + redis_port=_whole_number(environ, 'REDIS_PORT', DEFAULT_REDIS_PORT, 1, 65535), + redis_entra_resource=_text(environ, 'REDIS_ENTRA_RESOURCE') or DEFAULT_REDIS_ENTRA_RESOURCE, + ) + + if settings.backend == BACKEND_REDIS and not settings.redis_host: + raise RuntimeError("SESSION_BACKEND is 'redis' but REDIS_HOST is not set.") + + return settings + + +def build_redis_client(settings): + """A Redis client that signs in as the web app's system-assigned managed identity.""" + # Imported here so the disk store, used by local runs and tests, does not load them. + import redis + from redis.backoff import ExponentialWithJitterBackoff + from redis.retry import Retry + from redis_entraid.cred_provider import create_from_managed_identity + from redis_entraid.identity_provider import ManagedIdentityType + + credentials = create_from_managed_identity( + identity_type=ManagedIdentityType.SYSTEM_ASSIGNED, + resource=settings.redis_entra_resource, + ) + + return redis.Redis( + host=settings.redis_host, + port=settings.redis_port, + ssl=True, + credential_provider=credentials, + # RESP2 signs in with a plain AUTH, which Azure Managed Redis and Azure + # Cache for Redis both accept. It also keeps redis-py's maintenance + # notifications off; they can move a client to an address the private + # endpoint does not serve. + protocol=2, + # Fail within seconds when the store is down, so the operator gets a 503 + # rather than a page that hangs. + socket_connect_timeout=2, + socket_timeout=2, + socket_keepalive=True, + health_check_interval=60, + retry=Retry(ExponentialWithJitterBackoff(base=0.05, cap=0.5), 2), + ) + + +def redis_store_errors(): + from redis.auth.err import InvalidTokenSchemaErr, RequestTokenErr, TokenRenewalErr + from redis.exceptions import RedisError + + # Token failures are not RedisErrors, but they leave the store just as unreachable. + return (RedisError, RequestTokenErr, TokenRenewalErr, InvalidTokenSchemaErr) + + +def configure_sessions(app, environ=None): + """Apply the cookie policy and attach the session store the environment names.""" + settings = read_settings(environ) + + app.config.update( + SESSION_TYPE=settings.backend, + SESSION_PERMANENT=True, + # Each request that loads the session pushes its expiry out again, so this + # is how long the portal remembers an idle operator. Redis drops the entry + # at the same moment. + PERMANENT_SESSION_LIFETIME=timedelta(hours=settings.lifetime_hours), + SESSION_REFRESH_EACH_REQUEST=True, + SESSION_COOKIE_SECURE=settings.cookie_secure, + SESSION_COOKIE_HTTPONLY=True, + # Lax still sends the cookie on the top-level redirect back from Entra ID. + SESSION_COOKIE_SAMESITE='Lax', + ) + + store_errors = () + if settings.backend == BACKEND_REDIS: + app.config['SESSION_REDIS'] = build_redis_client(settings) + store_errors = redis_store_errors() + + Session(app) + app.session_interface = PortalSessionInterface(app.session_interface, store_errors) + return settings diff --git a/front_end/tests/conftest.py b/front_end/tests/conftest.py index ec331c0..dd5d99f 100644 --- a/front_end/tests/conftest.py +++ b/front_end/tests/conftest.py @@ -61,6 +61,10 @@ # Every JSON endpoint the React portal calls. API = "/api/ui" +# Only the API and the sign-in routes load the session, so a test that reads or +# writes it directly must open it on one of them. +UI_SESSION_PATH = f"{API}/session" + HOST_HEALTH = { "ExpectedAgentVersion": "1.0.0", "CurrentSettingsVersion": 3, "StaleAfterSeconds": 180, "Summary": {"Total": 2, "PoweredOn": 2, "Reporting": 1, "Healthy": 1, "Attention": 1, "Off": 0, @@ -386,7 +390,7 @@ def signed_in_client(client): def sign_in(client): # Match the app's naive datetime.utcnow().timestamp() convention exactly. expiry = (datetime.utcnow() + timedelta(hours=1)).timestamp() - with client.session_transaction() as sess: + with client.session_transaction(UI_SESSION_PATH) as sess: sess["user"] = {"name": "Test Operator", "preferred_username": "op@contoso.com", "oid": "0000-1111", "tid": "2222-3333"} sess["access_token"] = "fake-token" diff --git a/front_end/tests/test_bff_api.py b/front_end/tests/test_bff_api.py index 7a3668c..8497d7a 100644 --- a/front_end/tests/test_bff_api.py +++ b/front_end/tests/test_bff_api.py @@ -7,7 +7,7 @@ import pytest -from conftest import API, HISTORY_PATHS, VMS, csrf_token, post +from conftest import API, HISTORY_PATHS, UI_SESSION_PATH, VMS, csrf_token, post # ============================================================== SPA shell @@ -104,7 +104,7 @@ def test_session_endpoint_reports_the_signed_in_user(signed_in_client): def test_session_endpoint_survives_a_malformed_session(signed_in_client): """base.html used to guard against this; a non-mapping user must not turn a handled state into a 500.""" - with signed_in_client.session_transaction() as session: + with signed_in_client.session_transaction(UI_SESSION_PATH) as session: session["user"] = "not-a-mapping" response = signed_in_client.get(f"{API}/session") @@ -284,7 +284,7 @@ def test_filters_are_not_stored_in_the_session(signed_in_client, path): session cannot grow without bound.""" signed_in_client.get(f"{path}?startdate=2026-01-15&limit=37") - with signed_in_client.session_transaction() as session: + with signed_in_client.session_transaction(UI_SESSION_PATH) as session: for key in session.keys(): assert "history" not in key assert "filters" not in key @@ -576,7 +576,7 @@ def server_error(url, **kwargs): def test_an_expired_token_returns_401_json(signed_in_client): from datetime import datetime, timedelta - with signed_in_client.session_transaction() as session: + with signed_in_client.session_transaction(UI_SESSION_PATH) as session: # Matches the app's naive datetime.utcnow().timestamp() convention. session["token_expiry"] = (datetime.utcnow() - timedelta(minutes=5)).timestamp() diff --git a/front_end/tests/test_scaling_policy_bff.py b/front_end/tests/test_scaling_policy_bff.py index 80d8410..eb19413 100644 --- a/front_end/tests/test_scaling_policy_bff.py +++ b/front_end/tests/test_scaling_policy_bff.py @@ -68,6 +68,33 @@ def test_the_time_zone_is_set(signed_in_client, broker_api): assert broker_api.posts[-1]["json"] == {"timezone": "UTC"} +def test_start_on_demand_is_forwarded_with_or_without_the_time_zone(signed_in_client, broker_api): + broker_api.post_replies["/scaling/policy/update"] = (200, {"Result": "Updated", "StartOnDemandEnabled": False}) + + response = post(signed_in_client, f"{API}/scaling/policy", + {"startondemandenabled": False, "maxpendingstarts": 4, "junk": True}) + + assert response.status_code == 200 and response.get_json()["StartOnDemandEnabled"] is False + assert broker_api.posts[-1]["json"] == {"startondemandenabled": False, "maxpendingstarts": 4} + post(signed_in_client, f"{API}/scaling/policy", {"timezone": " UTC ", "maxpendingstarts": "3"}) + assert broker_api.posts[-1]["json"] == {"timezone": "UTC", "maxpendingstarts": "3"} + + +@pytest.mark.parametrize("payload", [ + {}, {"junk": True}, {"startondemandenabled": "yes"}, {"maxpendingstarts": True}, + {"maxpendingstarts": 2.5}, {"maxpendingstarts": "1" * 9}, {"timezone": "UTC", "startondemandenabled": 1}, +]) +def test_an_unusable_policy_update_never_reaches_the_broker(signed_in_client, broker_api, payload): + assert post(signed_in_client, f"{API}/scaling/policy", payload).status_code == 400 + assert broker_api.posts == [] + + +def test_the_brokers_refusal_of_the_pending_limit_is_kept(signed_in_client, broker_api): + broker_api.post_replies["/scaling/policy/update"] = (400, {"error": "maxpendingstarts must be a whole number from 1 to 20."}) + response = post(signed_in_client, f"{API}/scaling/policy", {"maxpendingstarts": 40}) + assert response.status_code == 400 and "1 to 20" in response.get_json()["error"] + + def test_the_preview_forwards_a_time_and_proposed_values(signed_in_client, broker_api): broker_api.get_replies["/scaling/preview"] = (200, {"Action": "None", "Summary": "No change."}) signed_in_client.get(f"{API}/scaling/preview?at=2026-09-21T09:00:00Z") diff --git a/front_end/tests/test_session_store.py b/front_end/tests/test_session_store.py new file mode 100644 index 0000000..36c15a6 --- /dev/null +++ b/front_end/tests/test_session_store.py @@ -0,0 +1,385 @@ +"""Portal session storage: the disk store for local runs, Redis for deployments. + +Covers the settings, the Redis client with the managed identity stubbed out, the +cookie policy, one instance honouring another's sign-in, the routes that never +touch the store, and what the operator sees when the store is down. +""" + +from collections import Counter +from datetime import timedelta + +import pytest +import redis +from flask import Flask, jsonify, session +from flask.sessions import SessionInterface +from flask_session.redis import RedisSessionInterface + +import session_store +from conftest import API, UI_SESSION_PATH, sign_in + +REDIS_ENV = { + "SESSION_BACKEND": "redis", + "REDIS_HOST": "portal.eastus.redis.azure.net", + "REDIS_PORT": "10000", +} + +OUTAGE = "Error 111 connecting to portal.eastus.redis.azure.net:10000. Connection refused." + + +class FakeRedisServer: + """Answers for every redis.Redis client in a test, so nothing leaves the machine.""" + + def __init__(self): + self.data = {} + self.sets = [] + self.failing = set() + self.attempts = Counter() + + def _call(self, operation): + self.attempts[operation] += 1 + if operation in self.failing: + raise redis.exceptions.ConnectionError(OUTAGE) + + def install(self, monkeypatch): + server = self + + def get(client, name): + server._call("get") + return server.data.get(name) + + def set_(client, name, value, ex=None, **kwargs): + server._call("set") + server.sets.append((name, ex)) + server.data[name] = value + return True + + def delete(client, *names): + server._call("delete") + return sum(server.data.pop(name, None) is not None for name in names) + + monkeypatch.setattr(redis.Redis, "get", get) + monkeypatch.setattr(redis.Redis, "set", set_) + monkeypatch.setattr(redis.Redis, "delete", delete) + + +class UnreachableStore(SessionInterface): + """A backend that fails the way Redis does when it cannot be reached.""" + + def __init__(self): + self.calls = 0 + + def open_session(self, app, request): + self.calls += 1 + raise redis.exceptions.ConnectionError(OUTAGE) + + def save_session(self, app, session, response): + self.calls += 1 + raise redis.exceptions.ConnectionError(OUTAGE) + + +@pytest.fixture +def managed_identity(monkeypatch): + """Stand in for the managed identity; nothing may ask IMDS for a token in tests.""" + import redis_entraid.cred_provider as cred_provider + + calls = [] + + def create_from_managed_identity(**kwargs): + calls.append(kwargs) + return redis.credentials.UsernamePasswordCredentialProvider("portal-object-id", "entra-token") + + monkeypatch.setattr(cred_provider, "create_from_managed_identity", create_from_managed_identity) + return calls + + +@pytest.fixture +def fake_redis(monkeypatch): + server = FakeRedisServer() + server.install(monkeypatch) + return server + + +def _answer_like_production(app, monkeypatch): + # TESTING propagates exceptions into the test; production hands them to the + # error handlers, which is what these tests are about. + monkeypatch.setitem(app.config, "PROPAGATE_EXCEPTIONS", False) + + +@pytest.fixture +def store_down(app, monkeypatch): + backend = UnreachableStore() + monkeypatch.setattr(app, "session_interface", + session_store.PortalSessionInterface(backend, (redis.exceptions.RedisError,))) + _answer_like_production(app, monkeypatch) + return backend + + +@pytest.fixture +def redis_backed(app, monkeypatch, managed_identity, fake_redis): + """Run the portal on Flask-Session's real Redis store, talking to the fake server.""" + client = session_store.build_redis_client(session_store.read_settings(REDIS_ENV)) + interface = session_store.PortalSessionInterface(RedisSessionInterface(app, client=client), + session_store.redis_store_errors()) + monkeypatch.setattr(app, "session_interface", interface) + _answer_like_production(app, monkeypatch) + return fake_redis + + +def make_instance(environ): + """A bare app with the portal's session setup, standing in for one web app instance.""" + instance = Flask(__name__) + instance.config.update(SECRET_KEY="test-secret-key", TESTING=True) + session_store.configure_sessions(instance, environ) + + @instance.route(f"{API}/remember", methods=["POST"]) + def remember(): + session["user"] = {"name": "Test Operator"} + return jsonify(ok=True) + + @instance.route(f"{API}/recall") + def recall(): + return jsonify(user=session.get("user")) + + return instance + + +def session_cookie(response): + return next(header for header in response.headers.getlist("Set-Cookie") + if header.startswith("session=")) + + +# ================================================================ settings + + +def test_local_runs_default_to_the_disk_store(): + settings = session_store.read_settings({}) + + assert settings.backend == "filesystem" + assert settings.lifetime_hours == 12 + assert settings.cookie_secure is True + assert settings.redis_host is None + assert settings.redis_port == 10000 + assert settings.redis_entra_resource == "https://redis.azure.com" + + +def test_redis_settings_come_from_the_environment(): + settings = session_store.read_settings({ + "SESSION_BACKEND": " Redis ", + "REDIS_HOST": "portal.redis.cache.usgovcloudapi.net", + "REDIS_PORT": "6380", + "REDIS_ENTRA_RESOURCE": "acca5fbb-b7e4-4009-81f1-37e38fd66d78", + "SESSION_LIFETIME_HOURS": "8", + "SESSION_COOKIE_SECURE": "false", + }) + + assert settings.backend == "redis" + assert settings.redis_host == "portal.redis.cache.usgovcloudapi.net" + assert settings.redis_port == 6380 + assert settings.redis_entra_resource == "acca5fbb-b7e4-4009-81f1-37e38fd66d78" + assert settings.lifetime_hours == 8 + assert settings.cookie_secure is False + + +@pytest.mark.parametrize("environ, named", [ + ({"SESSION_BACKEND": "memcached"}, "SESSION_BACKEND"), + ({"SESSION_BACKEND": "redis"}, "REDIS_HOST"), + ({"SESSION_LIFETIME_HOURS": "0"}, "SESSION_LIFETIME_HOURS"), + ({"SESSION_LIFETIME_HOURS": "721"}, "SESSION_LIFETIME_HOURS"), + ({"SESSION_LIFETIME_HOURS": "twelve"}, "SESSION_LIFETIME_HOURS"), + ({"SESSION_COOKIE_SECURE": "sometimes"}, "SESSION_COOKIE_SECURE"), + ({**REDIS_ENV, "REDIS_PORT": "70000"}, "REDIS_PORT"), +]) +def test_a_bad_setting_stops_the_portal_with_a_message_naming_it(environ, named): + with pytest.raises(RuntimeError, match=named): + session_store.read_settings(environ) + + +@pytest.mark.parametrize("path, expected", [ + ("/api/ui/session", True), + ("/api/ui/vms/1/start", True), + ("/api/ui", True), + ("/login", True), + ("/getAToken", True), + ("/logout", True), + ("/", False), + ("/vms", False), + ("/health", False), + ("/favicon.ico", False), + ("/static/dist/assets/index.js", False), + ("/api/uiother", False), + ("/login/again", False), +]) +def test_only_the_api_and_sign_in_routes_use_the_session(path, expected): + assert session_store.uses_session(path) is expected + + +# ============================================================ Redis client + + +def test_the_redis_client_signs_in_as_the_managed_identity_over_tls(managed_identity): + from redis_entraid.identity_provider import ManagedIdentityType + + settings = session_store.read_settings( + {**REDIS_ENV, "REDIS_ENTRA_RESOURCE": "acca5fbb-b7e4-4009-81f1-37e38fd66d78"}) + client = session_store.build_redis_client(settings) + + assert managed_identity == [{"identity_type": ManagedIdentityType.SYSTEM_ASSIGNED, + "resource": "acca5fbb-b7e4-4009-81f1-37e38fd66d78"}] + + pool = client.connection_pool + connection = pool.connection_kwargs + assert issubclass(pool.connection_class, redis.SSLConnection) + assert connection["host"] == "portal.eastus.redis.azure.net" + assert connection["port"] == 10000 + assert connection["protocol"] == 2 + assert connection["credential_provider"].get_credentials() == ("portal-object-id", "entra-token") + assert connection["socket_connect_timeout"] == 2 + assert connection["socket_timeout"] == 2 + assert client.get_retry().get_retries() == 2 + + +def test_configuring_redis_wraps_flask_sessions_store(managed_identity): + instance = Flask(__name__) + instance.config["SECRET_KEY"] = "test-secret-key" + + settings = session_store.configure_sessions(instance, REDIS_ENV) + + interface = instance.session_interface + assert settings.backend == "redis" + assert isinstance(interface, session_store.PortalSessionInterface) + assert isinstance(interface.backend, RedisSessionInterface) + assert interface.backend.client.connection_pool.connection_kwargs["host"] == REDIS_ENV["REDIS_HOST"] + assert redis.exceptions.RedisError in interface.store_errors + assert instance.permanent_session_lifetime == timedelta(hours=12) + + +def test_the_portal_runs_locally_on_the_wrapped_disk_store(app): + from flask_session.filesystem import FileSystemSessionInterface + + assert isinstance(app.session_interface, session_store.PortalSessionInterface) + assert isinstance(app.session_interface.backend, FileSystemSessionInterface) + assert app.session_interface.store_errors == () + + +def test_a_sign_in_on_one_instance_is_honoured_by_another(managed_identity, fake_redis): + first, second = make_instance(REDIS_ENV).test_client(), make_instance(REDIS_ENV).test_client() + + assert first.post(f"{API}/remember").status_code == 200 + session_id = first.get_cookie("session").value + + second.set_cookie("session", session_id) + assert second.get(f"{API}/recall").get_json() == {"user": {"name": "Test Operator"}} + # Redis forgets the session when its cookie would expire. + assert fake_redis.sets[0] == (f"session:{session_id}", 12 * 3600) + + +# =========================================================== cookie policy + + +def test_the_session_cookie_is_secure_http_only_and_same_site_lax(client): + attributes = [part.strip().lower() for part in session_cookie(client.get(UI_SESSION_PATH)).split(";")] + + assert "secure" in attributes + assert "httponly" in attributes + assert "samesite=lax" in attributes + assert any(attribute.startswith("expires=") for attribute in attributes) + + +def test_an_idle_session_lasts_twelve_hours_by_default(app): + assert app.permanent_session_lifetime == timedelta(hours=12) + + +# ============================================== routes without a session + + +@pytest.mark.parametrize("path", ["/health", "/", "/vms/1/update", "/favicon.ico", + "/static/dist/index.html"]) +def test_routes_that_do_not_use_the_session_work_while_the_store_is_down(client, store_down, + spa_bundle, path): + client.set_cookie("session", "an-existing-session-id") + + response = client.get(path) + + assert response.status_code == 200 + assert store_down.calls == 0 + assert "Set-Cookie" not in response.headers + + +def test_writing_the_session_on_a_route_that_does_not_load_it_fails_loudly(app): + with app.test_request_context("/vms"): + with pytest.raises(RuntimeError, match="/vms does not load the session"): + session["user"] = {"name": "Test Operator"} + + +# ========================================================== store outage + + +def test_an_unreachable_store_answers_503_json_on_the_api(client, store_down): + client.set_cookie("session", "an-existing-session-id") + + response = client.get(UI_SESSION_PATH) + + assert response.status_code == 503 + assert response.headers["Retry-After"] == "30" + assert "session store" in response.get_json()["error"] + + +def test_an_unreachable_store_answers_503_on_the_sign_in_routes(client, store_down): + response = client.get("/logout") + + assert response.status_code == 503 + assert response.mimetype == "text/plain" + assert "session store" in response.get_data(as_text=True) + + +def test_a_signed_in_session_is_read_back_from_redis(client, redis_backed): + sign_in(client) + + assert client.get(f"{API}/vms").status_code == 200 + assert [key for key in redis_backed.data if key.startswith("session:")] + + +def test_a_failed_read_from_redis_answers_503(client, redis_backed): + sign_in(client) + redis_backed.failing = {"get", "set"} + + response = client.get(f"{API}/vms") + + assert response.status_code == 503 + assert "session store" in response.get_json()["error"] + + +def test_an_outage_is_logged_with_its_cause_where_application_insights_collects_it( + client, redis_backed, caplog): + sign_in(client) + redis_backed.failing = {"get", "set"} + + with caplog.at_level("ERROR", logger="linuxbroker.frontend"): + client.get(f"{API}/vms") + + # Flask logs the traceback too, but under its own logger, which is not exported. + exported = [record.getMessage() for record in caplog.records + if record.name.startswith("linuxbroker.frontend")] + assert exported == [f"The session store could not be reached (ConnectionError: {OUTAGE})."] + + +def test_a_failed_save_of_a_changed_session_answers_503_once(client, redis_backed): + # With no cookie yet, the first store call is saving the new CSRF token. + redis_backed.failing = {"set"} + + response = client.get(UI_SESSION_PATH) + + assert response.status_code == 503 + # Finalizing the error response must not wait on the store a second time. + assert redis_backed.attempts["set"] == 1 + + +def test_losing_only_the_expiry_refresh_keeps_the_views_answer(client, redis_backed, caplog): + sign_in(client) + redis_backed.failing = {"set"} + + with caplog.at_level("WARNING", logger="linuxbroker.frontend.sessions"): + response = client.get(f"{API}/vms") + + assert response.status_code == 200 + assert "expiry" in caplog.text diff --git a/front_end/web/package-lock.json b/front_end/web/package-lock.json index 5676b44..cb27265 100644 --- a/front_end/web/package-lock.json +++ b/front_end/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "linux-broker-portal", - "version": "0.121.0", + "version": "0.122.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "linux-broker-portal", - "version": "0.121.0", + "version": "0.122.0", "dependencies": { "@tanstack/react-query": "^5.62.0", "react": "^18.3.1", diff --git a/front_end/web/package.json b/front_end/web/package.json index 134f111..7d22d54 100644 --- a/front_end/web/package.json +++ b/front_end/web/package.json @@ -1,7 +1,7 @@ { "name": "linux-broker-portal", "private": true, - "version": "0.121.0", + "version": "0.122.0", "type": "module", "description": "Service Management Portal for the Linux Broker for AVD Access solution.", "scripts": { diff --git a/front_end/web/src/App.test.tsx b/front_end/web/src/App.test.tsx index 9d56f61..64bd4a2 100644 --- a/front_end/web/src/App.test.tsx +++ b/front_end/web/src/App.test.tsx @@ -232,6 +232,13 @@ const SCALING_POLICY = { Schedules: [{ ...SCHEDULE_BASE, ScheduleID: 1, Name: 'Business hours', StartTime: '08:00', EndTime: '18:00' }], NextChange: { InMinutes: 465, AtLocal: 'Thursday 18:00', PhaseName: 'Default rule', ScheduleID: null }, LastRun: null, + StartOnDemandEnabled: true, + MaxPendingStarts: 2, + ZeroMinimumCount: 0, + AvdHostScripts: { + Seen: 3, Outdated: 1, OutdatedHostnames: ['avd-03'], CurrentVersion: '2.0.0', + Versions: [{ ClientVersion: '2.0.0', AvdHosts: 2, Current: true }], + }, }; let scalingPolicy: typeof SCALING_POLICY | { Available: false } = SCALING_POLICY; @@ -411,9 +418,14 @@ function stubFetch() { if (url.startsWith('/api/ui/scaling/log')) return jsonResponse(EMPTY_PAGE); if (url.startsWith('/api/ui/scaling/rules')) return jsonResponse(RULES); if (url.startsWith('/api/ui/scaling/policy')) { - return method === 'POST' - ? jsonResponse({ TimeZone: 'Eastern Standard Time', message: 'Schedules are now read in Eastern Standard Time.' }) - : jsonResponse(scalingPolicy); + if (method !== 'POST') return jsonResponse(scalingPolicy); + const body = JSON.parse(String(init?.body)) as Record; + return 'startondemandenabled' in body + ? jsonResponse({ + Result: 'Updated', StartOnDemandEnabled: body.startondemandenabled, MaxPendingStarts: body.maxpendingstarts, + ZeroMinimumCount: 0, message: `Start on demand is now ${body.startondemandenabled ? 'on' : 'off'}.`, + }) + : jsonResponse({ Result: 'Updated', TimeZone: 'Eastern Standard Time', message: 'Schedules are now read in Eastern Standard Time.' }); } if (url.startsWith('/api/ui/scaling/preview')) return jsonResponse(method === 'POST' ? PROPOSED_PREVIEW : PREVIEW); if (url.startsWith('/api/ui/scaling/timezones')) return jsonResponse(TIME_ZONES); @@ -1390,6 +1402,94 @@ describe('App', () => { expect(screen.queryByRole('link', { name: 'Add a window' })).not.toBeInTheDocument(); expect(screen.queryByRole('button', { name: 'Edit Business hours' })).not.toBeInTheDocument(); expect(screen.queryByLabelText('Time zone')).not.toBeInTheDocument(); + expect(screen.getByText('On. Up to 2 hosts start at once for waiting users.')).toBeInTheDocument(); + expect(screen.queryByRole('switch', { name: 'Start a host when a user finds none free' })).not.toBeInTheDocument(); + }); + + it('shows start on demand and which AVD hosts cannot wait for a host', async () => { + renderApp('/scaling'); + + expect(await screen.findByRole('heading', { name: 'Start on demand' })).toBeInTheDocument(); + expect(screen.getByRole('switch', { name: 'Start a host when a user finds none free' })).toBeChecked(); + expect(screen.getByText(/1 AVD host runs a script that cannot wait for a host to start: avd-03\./)).toBeInTheDocument(); + expect(screen.getByText('deploy/Update-AvdHostBrokerScript.ps1')).toBeInTheDocument(); + const scripts = screen.getByRole('region', { name: 'AVD host scripts' }); + expect(within(scripts).getByText('Current')).toBeInTheDocument(); + expect(within(scripts).getByText('Cannot wait')).toBeInTheDocument(); + }); + + it('turns start on demand off only after warning about minimums of 0', async () => { + scalingPolicy = { ...SCALING_POLICY, ZeroMinimumCount: 2 }; + renderApp('/scaling'); + + await userEvent.click(await screen.findByRole('switch', { name: 'Start a host when a user finds none free' })); + await userEvent.click(screen.getByRole('button', { name: 'Save start on demand' })); + + const dialog = await screen.findByRole('dialog'); + expect(within(dialog).getByText(/2 rules and windows keep a minimum of 0 hosts/)).toBeInTheDocument(); + const posted = () => + (globalThis.fetch as ReturnType).mock.calls.find( + ([url, init]) => String(url) === '/api/ui/scaling/policy' && init?.method === 'POST', + ); + expect(posted()).toBeUndefined(); + await userEvent.click(within(dialog).getByRole('button', { name: 'Turn it off' })); + + await waitFor(() => expect(JSON.parse(String(posted()?.[1]?.body))).toEqual({ startondemandenabled: false, maxpendingstarts: 2 })); + expect(await screen.findByText('Start on demand is now off.')).toBeInTheDocument(); + }); + + it('saves how many hosts may start at once and refuses a limit outside 1 to 20', async () => { + renderApp('/scaling'); + const limit = await screen.findByLabelText('Hosts that may start at once'); + const save = screen.getByRole('button', { name: 'Save start on demand' }); + expect(save).toBeDisabled(); + + await userEvent.clear(limit); + await userEvent.type(limit, '25'); + expect(screen.getByText('Enter a whole number from 1 to 20.')).toBeInTheDocument(); + expect(save).toBeDisabled(); + + await userEvent.clear(limit); + await userEvent.type(limit, '4'); + await userEvent.click(save); + await waitFor(() => { + const call = (globalThis.fetch as ReturnType).mock.calls.find( + ([url, init]) => String(url) === '/api/ui/scaling/policy' && init?.method === 'POST', + ); + expect(JSON.parse(String(call?.[1]?.body))).toEqual({ startondemandenabled: true, maxpendingstarts: 4 }); + }); + // Leaving it on needs no confirmation. + expect(screen.queryByRole('dialog')).not.toBeInTheDocument(); + }); + + it('says when a broker cannot start hosts on demand yet', async () => { + const older: Record = { ...SCALING_POLICY }; + for (const key of ['StartOnDemandEnabled', 'MaxPendingStarts', 'ZeroMinimumCount', 'AvdHostScripts']) { + delete older[key]; + } + scalingPolicy = older as typeof SCALING_POLICY; + renderApp('/scaling'); + expect(await screen.findByText(/This broker does not support start on demand yet/)).toBeInTheDocument(); + expect(screen.queryByRole('switch', { name: 'Start a host when a user finds none free' })).not.toBeInTheDocument(); + }); + + it('lets a rule keep a minimum of 0 only while start on demand is on', async () => { + renderApp('/scaling/rules/create'); + const minimum = await screen.findByLabelText('Minimum VMs'); + await waitFor(() => expect(minimum).toHaveAttribute('min', '0')); + await userEvent.type(minimum, '0'); + expect(screen.getByText(/With a minimum of 0, no host runs while nobody needs one/)).toBeInTheDocument(); + expect(minimum).not.toHaveAttribute('aria-invalid'); + }); + + it('flags a minimum of 0 while start on demand is off', async () => { + scalingPolicy = { ...SCALING_POLICY, StartOnDemandEnabled: false }; + renderApp('/scaling/rules/create'); + const minimum = await screen.findByLabelText('Minimum VMs'); + await userEvent.type(minimum, '0'); + expect(await screen.findByText(/A minimum of 0 needs start on demand, which is off/)).toBeInTheDocument(); + expect(minimum).toHaveAttribute('aria-invalid', 'true'); + expect(minimum).toHaveAttribute('min', '1'); }); it('changes the policy time zone', async () => { diff --git a/front_end/web/src/components/dashboard/AttentionPanel.tsx b/front_end/web/src/components/dashboard/AttentionPanel.tsx index 22a4094..e966bc4 100644 --- a/front_end/web/src/components/dashboard/AttentionPanel.tsx +++ b/front_end/web/src/components/dashboard/AttentionPanel.tsx @@ -38,7 +38,12 @@ export function describeAttention(item: AttentionItem): Described { switch (item.Kind) { case 'no-ready-hosts': return { - text: <>No host can take a new user: anyone who needs one now will be turned away., + text: ( + <> + No host can take a new user and none is starting: anyone who needs one now must wait for one to start, or + is turned away. + + ), to: '/vms', action: 'View hosts', }; diff --git a/front_end/web/src/components/dashboard/CapacityCard.tsx b/front_end/web/src/components/dashboard/CapacityCard.tsx index 9821161..cc9cda7 100644 --- a/front_end/web/src/components/dashboard/CapacityCard.tsx +++ b/front_end/web/src/components/dashboard/CapacityCard.tsx @@ -63,6 +63,7 @@ export function describeCapacity(series: UtilizationPoint[], hours: UtilizationH const serviceable = range(series.map((point) => point.Serviceable)); const maximum = range(series.map((point) => point.MaxVMs)); const denied = series.reduce((total, point) => total + point.Denied, 0); + const waited = series.reduce((most, point) => Math.max(most, point.Waited ?? 0), 0); const parts = [`Capacity over ${windowPhrase(hours)}.`]; if (peak) parts.push(`At most ${peak.max} hosts were in use at once.`); @@ -74,10 +75,17 @@ export function describeCapacity(series: UtilizationPoint[], hours: UtilizationH ); } if (maximum) parts.push(`The scaling maximum was ${maximum.max === maximum.min ? maximum.max : `${maximum.min} to ${maximum.max}`}.`); + if (waited) parts.push(`Users waited for a host to start, at most ${waited} in one interval.`); parts.push(denied ? `${denied} checkout${denied === 1 ? '' : 's'} found no host.` : 'Every checkout found a host.'); return parts.join(' '); } +/** The latest scaling run left no host powered on, as a minimum of 0 allows. */ +export function scaledToZero(series: UtilizationPoint[]) { + const latest = [...series].reverse().find((point) => point.Runs > 0); + return latest !== undefined && latest.PoweredOn === 0 && latest.MinVMs === 0; +} + export interface CapacityCardProps { hours: UtilizationHours; onHoursChange: (hours: UtilizationHours) => void; @@ -93,7 +101,9 @@ export function CapacityCard({ hours, onHoursChange, data, busy, error, classNam // While another window loads, the previous one stays on screen and is described as such. const shownHours = data?.Hours ?? hours; const hasRuns = series.some((point) => point.Runs > 0); - const hasCheckouts = series.some((point) => point.Checkouts > 0); + const hasCheckouts = series.some((point) => point.Checkouts > 0 || (point.Waited ?? 0) > 0); + const hasWaits = series.some((point) => (point.Waited ?? 0) > 0); + const atZero = scaledToZero(series); const lines = useMemo( () => [ @@ -128,8 +138,21 @@ export function CapacityCard({ hours, onHoursChange, data, busy, error, classNam step: true, width: 1.5, }, + ...(hasWaits + ? [ + { + key: 'waited', + label: 'Users who waited for a host', + values: series.map((point) => point.Waited ?? 0), + colour: 'var(--lb-info-fg)', + dash: '6 2 1 2', + step: true, + width: 1.5, + }, + ] + : []), ], - [series], + [series, hasWaits], ); const markers = useMemo( () => series.flatMap((point, index) => (point.Denied > 0 ? [{ index, count: point.Denied }] : [])), @@ -159,15 +182,26 @@ export function CapacityCard({ hours, onHoursChange, data, busy, error, classNam icon="activity" /> ) : ( - point.BucketStartUtc)} - bucketMinutes={data.BucketMinutes ?? 60} - series={lines} - markers={markers} - markerLabel="checkouts found no host" - summary={describeCapacity(series, shownHours)} - caption={`Capacity over ${windowPhrase(shownHours)}, by ${data.BucketMinutes ?? 60}-minute interval`} - /> + <> + point.BucketStartUtc)} + bucketMinutes={data.BucketMinutes ?? 60} + series={lines} + markers={markers} + markerLabel="checkouts found no host" + summary={describeCapacity(series, shownHours)} + caption={`Capacity over ${windowPhrase(shownHours)}, by ${data.BucketMinutes ?? 60}-minute interval`} + /> + {atZero ? ( +

+ + + Scaled to zero: no host was powered on after the latest scaling run. Start on demand starts one for + the next user, who waits a minute or two. + +

+ ) : null} + )} diff --git a/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx b/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx index 13a786d..f6a04b8 100644 --- a/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx +++ b/front_end/web/src/components/dashboard/CheckoutHealthCard.tsx @@ -44,6 +44,27 @@ function Figure({ ); } +/** + * Users who waited while start on demand started a host: how long those who got one waited, + * and how many are waiting now. + */ +export function describeWaits(stats: CheckoutStats): string { + const waits = stats.Waits ?? 0; + const served = stats.WaitsServed ?? 0; + const waitingNow = stats.WaitingNow ?? 0; + const parts: string[] = []; + if (served) { + parts.push(`Median ${formatDuration(stats.WaitP50Seconds)} · 95% within ${formatDuration(stats.WaitP95Seconds)}`); + } + if (waits > served) { + parts.push(`${formatNumber(waits - served)} without a host yet`); + } + if (waitingNow) { + parts.push(`${formatNumber(waitingNow)} waiting now`); + } + return parts.length ? parts.join(' · ') : 'Nobody waited for a host to start'; +} + /** How checkouts went: unmet demand, how long users waited, and how long hosts take to start. */ export function CheckoutHealthCard({ stats, @@ -98,6 +119,13 @@ export function CheckoutHealthCard({ : 'No host was started' } /> + {stats.Waits !== undefined ? ( +
+ ) : null} {failed ? (
{ it('says when every checkout found a host', () => { expect(describeCapacity([point({})], 24)).toContain('Every checkout found a host.'); }); + + it('says when users waited for a host to start', () => { + const series = [point({ Waited: 1 }), point({ Waited: 3 }), point({})]; + expect(describeCapacity(series, 24)).toContain('Users waited for a host to start, at most 3 in one interval.'); + expect(describeCapacity([point({ Waited: 0 })], 24)).not.toContain('waited'); + }); +}); + +describe('scaledToZero', () => { + it('is true when the latest run left no host on under a minimum of 0', () => { + expect(scaledToZero([point({}), point({ PoweredOn: 0, MinVMs: 0 }), point({ Runs: 0, PoweredOn: null, MinVMs: null })])).toBe(true); + }); + + it('is false while a host is on, the minimum is above 0, or nothing ran', () => { + expect(scaledToZero([point({ PoweredOn: 0, MinVMs: 0 }), point({ PoweredOn: 1, MinVMs: 0 })])).toBe(false); + expect(scaledToZero([point({ PoweredOn: 0, MinVMs: 1 })])).toBe(false); + expect(scaledToZero([point({ Runs: 0, PoweredOn: 0, MinVMs: 0 })])).toBe(false); + expect(scaledToZero([])).toBe(false); + }); +}); + +describe('CapacityCard', () => { + const metrics = (series: UtilizationPoint[]) => ({ Available: true as const, Hours: 24 as const, BucketMinutes: 60, Series: series }); + + it('charts the users who waited and says when the pool scaled to zero', () => { + render( + undefined} + data={metrics([point({ Waited: 2 }), point({ PoweredOn: 0, InUse: 0, Serviceable: 0, MinVMs: 0 })])} + busy={false} + error={null} + />, + ); + expect(screen.getByText('Users who waited for a host')).toBeInTheDocument(); + expect(screen.getByText(/Scaled to zero: no host was powered on after the latest scaling run/)).toBeInTheDocument(); + }); + + it('leaves the waits out when nobody waited', () => { + render( undefined} data={metrics([point({})])} busy={false} error={null} />); + expect(screen.queryByText('Users who waited for a host')).not.toBeInTheDocument(); + expect(screen.queryByText(/Scaled to zero/)).not.toBeInTheDocument(); + }); }); describe('CheckoutHealthCard', () => { @@ -53,6 +96,29 @@ describe('CheckoutHealthCard', () => { expect(screen.getByText('No checkouts yet')).toBeInTheDocument(); expect(screen.getByText('No host was started')).toBeInTheDocument(); expect(screen.queryByText('Failed checkouts')).not.toBeInTheDocument(); + // A broker that predates start on demand reports no waits, so the figure is left out. + expect(screen.queryByText('Waited for a host')).not.toBeInTheDocument(); + }); + + it('shows how long users waited while a host started', () => { + render( + , + ); + expect(screen.getByText('Waited for a host')).toBeInTheDocument(); + expect(screen.getByText('4')).toBeInTheDocument(); + expect(screen.getByText('Median 1 min · 95% within 3 min · 1 without a host yet · 1 waiting now')).toBeInTheDocument(); + }); + + it('describes waits', () => { + const none = { ...STATS, Starting: 0, Waits: 0, WaitsServed: 0, WaitingNow: 0, WaitP50Seconds: null, WaitP95Seconds: null }; + expect(describeWaits(none)).toBe('Nobody waited for a host to start'); + expect(describeWaits({ ...none, WaitingNow: 2 })).toBe('2 waiting now'); + expect(describeWaits({ ...none, Waits: 2, WaitsServed: 2, WaitP50Seconds: 45, WaitP95Seconds: 90 })).toBe( + 'Median 45 s · 95% within 2 min', + ); }); it('formats times', () => { diff --git a/front_end/web/src/components/scaling/ScalingPreviewCard.tsx b/front_end/web/src/components/scaling/ScalingPreviewCard.tsx index 3680f77..942d1f4 100644 --- a/front_end/web/src/components/scaling/ScalingPreviewCard.tsx +++ b/front_end/web/src/components/scaling/ScalingPreviewCard.tsx @@ -63,11 +63,24 @@ export function ScalingPreviewCard({ preview, title, busy = false, error = null, + {preview.Counts.Waiting ? ( +

+ + + {preview.Counts.Waiting.toLocaleString()} {preview.Counts.Waiting === 1 ? 'user is' : 'users are'} waiting + for a host to start, counted as in use. + +

+ ) : null} + {preview.Phase.MinVMs !== null ? (

Using {preview.Phase.Name ?? 'the default rule'}: keep{' '} {preview.Phase.MinVMs}–{preview.Phase.MaxVMs} hosts, scale up at {preview.Phase.ScaleUpRatio}% and down at{' '} {preview.Phase.ScaleDownRatio}%. + {preview.Phase.MinVMs === 0 && preview.StartOnDemandEnabled ? ( + <> With a minimum of 0, the last idle host can stop; start on demand starts one for the next user. + ) : null} {preview.Phase.MaintenanceSurge ? ( <> {' '} diff --git a/front_end/web/src/components/scaling/StartOnDemandCard.tsx b/front_end/web/src/components/scaling/StartOnDemandCard.tsx new file mode 100644 index 0000000..6d57cd5 --- /dev/null +++ b/front_end/web/src/components/scaling/StartOnDemandCard.tsx @@ -0,0 +1,229 @@ +import { useEffect, useState } from 'react'; + +import { Badge } from '../ui/Badge'; +import { Button } from '../ui/Button'; +import { Notice } from '../ui/Feedback'; +import { Switch, TextField } from '../ui/Field'; +import { GlassCard } from '../ui/GlassCard'; +import { useToast } from '../ui/Toast'; +import { useSetStartOnDemand } from '../../hooks/useBroker'; +import { useConfirm } from '../../hooks/useConfirm'; +import { errorMessage } from '../../lib/api'; +import type { AvdHostScripts, ScalingPolicy } from '../../types/broker'; + +const DEFAULT_MAX_PENDING = 2; +const MAX_PENDING_LIMIT = 20; +const HOSTNAMES_SHOWN = 5; + +function plural(count: number, one: string, many: string) { + return `${count.toLocaleString()} ${count === 1 ? one : many}`; +} + +function validMaxPending(text: string) { + return /^\d{1,2}$/.test(text.trim()) && Number(text) >= 1 && Number(text) <= MAX_PENDING_LIMIT; +} + +function listHostnames(hostnames: string[], total: number) { + const shown = hostnames.slice(0, HOSTNAMES_SHOWN).join(', '); + const more = total - Math.min(total, HOSTNAMES_SHOWN); + return more > 0 ? `${shown} and ${more.toLocaleString()} more` : shown; +} + +/** Which broker script the AVD session hosts run, since only a current one waits for a host. */ +function AvdHostScriptSummary({ scripts }: { scripts: AvdHostScripts }) { + return ( +

+

+ AVD host scripts +

+

+ The broker script on each AVD session host that asked for a Linux host in the last seven days. Version{' '} + {scripts.CurrentVersion} or later waits while a host starts; an older script turns the user away. +

+ + {scripts.Seen === 0 ? ( +

No AVD session host has asked for a Linux host in the last seven days.

+ ) : ( + <> + {scripts.Outdated > 0 ? ( + + {plural(scripts.Outdated, 'AVD host runs', 'AVD hosts run')} a script that cannot wait for a host to start + {scripts.OutdatedHostnames.length ? `: ${listHostnames(scripts.OutdatedHostnames, scripts.Outdated)}` : ''}. + Their users are turned away when no host is free. Update them with{' '} + deploy/Update-AvdHostBrokerScript.ps1. + + ) : null} +
    + {scripts.Versions.map((version) => ( +
  • + + Version {version.ClientVersion} + + + {plural(version.AvdHosts, 'host', 'hosts')} + {version.Current ? ( + Current + ) : ( + Older + )} + +
    • + ))} + {scripts.Outdated > 0 ? ( +
  • + No version reported + + {plural(scripts.Outdated, 'host', 'hosts')} + Cannot wait + +
    • + ) : null} +
+ + )} +
+ ); +} + +function StartOnDemandEditor({ policy }: { policy: ScalingPolicy }) { + const setStartOnDemand = useSetStartOnDemand(); + const { showToast } = useToast(); + const { confirm, dialog } = useConfirm(); + const enabled = policy.StartOnDemandEnabled === true; + const savedMaxPending = policy.MaxPendingStarts ?? DEFAULT_MAX_PENDING; + const zeroMinimums = policy.ZeroMinimumCount ?? 0; + const [on, setOn] = useState(enabled); + const [maxPending, setMaxPending] = useState(String(savedMaxPending)); + + useEffect(() => setOn(enabled), [enabled]); + useEffect(() => setMaxPending(String(savedMaxPending)), [savedMaxPending]); + + const valid = validMaxPending(maxPending); + const changed = on !== enabled || (valid && Number(maxPending) !== savedMaxPending); + + async function save() { + try { + const result = await setStartOnDemand.mutateAsync({ + startondemandenabled: on, + maxpendingstarts: Number(maxPending), + }); + showToast(result.message, 'success'); + } catch (cause) { + showToast(errorMessage(cause, 'Unable to change start on demand.'), 'danger'); + } + } + + function submit() { + if (!valid || !changed) { + return; + } + if (enabled && !on) { + confirm({ + title: 'Turn off start on demand?', + body: + zeroMinimums > 0 + ? `${plural(zeroMinimums, 'rule or window keeps', 'rules and windows keep')} a minimum of 0 hosts. While start on demand is off the scaler keeps at least 1 host on for them, and a user who finds no free host is turned away instead of waiting.` + : 'A user who finds no free host is turned away instead of waiting while a host starts.', + confirmLabel: 'Turn it off', + variant: 'warning', + onConfirm: save, + }); + return; + } + void save(); + } + + return ( + <> +
{ + event.preventDefault(); + submit(); + }} + > + + setMaxPending(event.target.value)} + /> + + + {dialog} + + ); +} + +/** Whether a user who finds no free host waits while one starts, and what the AVD hosts run. */ +export function StartOnDemandCard({ policy, admin }: { policy: ScalingPolicy; admin: boolean }) { + const supported = policy.StartOnDemandEnabled !== null && policy.StartOnDemandEnabled !== undefined; + const enabled = policy.StartOnDemandEnabled === true; + const zeroMinimums = policy.ZeroMinimumCount ?? 0; + const scripts = policy.AvdHostScripts; + + return ( + +
+
+

Start on demand

+

+ When no host is free, the broker starts a stopped one and the user waits for it instead of being turned + away. It also lets a rule or window keep a minimum of 0 hosts, so idle hosts can stop. +

+
+ {supported ? ( + enabled ? ( + On + ) : ( + Off + ) + ) : null} +
+ + {!supported ? ( +

+ This broker does not support start on demand yet. Upgrade the broker API and run the database migration to + use it. +

+ ) : ( + <> + {!enabled && zeroMinimums > 0 ? ( + + {plural(zeroMinimums, 'rule or window keeps', 'rules and windows keep')} a minimum of 0 hosts, which needs + start on demand. While it is off the scaler keeps at least 1 host on for them. + + ) : null} +
+ {admin ? ( + + ) : ( +

+ {enabled + ? `On. Up to ${plural(policy.MaxPendingStarts ?? DEFAULT_MAX_PENDING, 'host starts', 'hosts start')} at once for waiting users.` + : 'Off. A user who finds no free host is turned away.'} +

+ )} + {scripts ? : null} +
+ + )} + + ); +} diff --git a/front_end/web/src/components/ui/Badge.test.tsx b/front_end/web/src/components/ui/Badge.test.tsx index ccca4a8..e0f9565 100644 --- a/front_end/web/src/components/ui/Badge.test.tsx +++ b/front_end/web/src/components/ui/Badge.test.tsx @@ -49,6 +49,7 @@ describe('status badges', () => { ['Scale Up', 'Scale up'], ['Scale Down', 'Scale down'], ['No Action', 'No action'], + ['Start On Demand', 'Start on demand'], ])('renders the scaling action %s as %s', (value, label) => { render(); expect(screen.getByText(label)).toBeInTheDocument(); diff --git a/front_end/web/src/components/ui/Badge.tsx b/front_end/web/src/components/ui/Badge.tsx index 4427a1d..1cdd94e 100644 --- a/front_end/web/src/components/ui/Badge.tsx +++ b/front_end/web/src/components/ui/Badge.tsx @@ -76,6 +76,7 @@ const SCALING_ACTION: Record = { 'Scale Up': { tone: 'ok', icon: 'arrow-up', label: 'Scale up' }, 'Scale Down': { tone: 'warn', icon: 'arrow-down', label: 'Scale down' }, 'No Action': { tone: 'neutral', icon: 'dash-circle', label: 'No action' }, + 'Start On Demand': { tone: 'info', icon: 'power', label: 'Start on demand' }, }; export const VmStatusBadge = ({ value }: { value: string | null | undefined }) => diff --git a/front_end/web/src/hooks/useBroker.ts b/front_end/web/src/hooks/useBroker.ts index c54c67f..847ae77 100644 --- a/front_end/web/src/hooks/useBroker.ts +++ b/front_end/web/src/hooks/useBroker.ts @@ -23,6 +23,7 @@ import type { MaintenanceRunsPage, MessageResult, Paged, + PolicyUpdateResult, PowerActionResult, PowerSyncResult, ProfileResetResult, @@ -34,6 +35,7 @@ import type { ScheduleInput, SessionsPage, SignOutResult, + StartOnDemandInput, TimeZoneOption, UserSearchResult, UtilizationHours, @@ -490,6 +492,17 @@ export function useScalingPolicy() { }); } +/** + * Whether a rule may keep a minimum of 0 VMs, which needs start on demand. Undefined until the + * policy has loaded, so a form does not flag a 0 it cannot yet judge. + */ +export function useZeroMinimumAllowed(): boolean | undefined { + const { data, isError } = useScalingPolicy(); + if (isError) return false; + if (!data) return undefined; + return data.Available !== false && data.StartOnDemandEnabled === true; +} + /** What the next scaling run would do now. Refreshed every minute while shown. */ export function useScalingPreview() { return useQuery({ @@ -518,7 +531,16 @@ export function useTimeZones() { export function useSetPolicyTimeZone() { const invalidate = useRuleInvalidation(); return useMutation({ - mutationFn: (timezone: string) => apiPost<{ message: string; TimeZone: string }>('/scaling/policy', { timezone }), + mutationFn: (timezone: string) => apiPost('/scaling/policy', { timezone }), + onSuccess: invalidate, + }); +} + +/** Turn start on demand on or off, and set how many hosts may start at once for waiting users. */ +export function useSetStartOnDemand() { + const invalidate = useRuleInvalidation(); + return useMutation({ + mutationFn: (input: StartOnDemandInput) => apiPost('/scaling/policy', input), onSuccess: invalidate, }); } diff --git a/front_end/web/src/pages/scaling/CreateRule.tsx b/front_end/web/src/pages/scaling/CreateRule.tsx index 9eac21e..d61b32d 100644 --- a/front_end/web/src/pages/scaling/CreateRule.tsx +++ b/front_end/web/src/pages/scaling/CreateRule.tsx @@ -5,7 +5,7 @@ import { ButtonLink } from '../../components/ui/Button'; import { PageHeader } from '../../components/ui/Feedback'; import { useToast } from '../../components/ui/Toast'; import { useConfirm } from '../../hooks/useConfirm'; -import { useCreateScalingRule } from '../../hooks/useBroker'; +import { useCreateScalingRule, useZeroMinimumAllowed } from '../../hooks/useBroker'; import { errorMessage } from '../../lib/api'; import { EMPTY_RULE, RuleForm } from './RuleForm'; @@ -13,6 +13,7 @@ export function CreateRule() { const navigate = useNavigate(); const { showToast } = useToast(); const createRule = useCreateScalingRule(); + const zeroMinimumAllowed = useZeroMinimumAllowed(); const { confirm, dialog } = useConfirm(); const [form, setForm] = useState(EMPTY_RULE); @@ -60,6 +61,7 @@ export function CreateRule() { submitLabel="Create rule" busy={createRule.isPending} cancelTo="/scaling" + zeroMinimumAllowed={zeroMinimumAllowed} /> {dialog} diff --git a/front_end/web/src/pages/scaling/RuleForm.tsx b/front_end/web/src/pages/scaling/RuleForm.tsx index 62fb3a3..b0e63b9 100644 --- a/front_end/web/src/pages/scaling/RuleForm.tsx +++ b/front_end/web/src/pages/scaling/RuleForm.tsx @@ -26,7 +26,7 @@ const FIELDS: FieldSpec[] = [ { key: 'minvms', label: 'Minimum VMs', - help: 'Keep at least this many VMs powered on for baseline capacity. Must be at least 1.', + help: 'Keep at least this many VMs powered on for baseline capacity. Must be at least 1, or 0 while start on demand is on.', }, { key: 'maxvms', @@ -66,6 +66,8 @@ export interface RuleFormProps { submitLabel: string; busy?: boolean; cancelTo: string; + /** Start on demand is on, so the minimum may be 0. Undefined while the policy loads. */ + zeroMinimumAllowed?: boolean; } type RuleValues = Omit & { stopmode: ScalingRuleInput['stopmode'] | '' }; @@ -75,10 +77,21 @@ export interface RuleFieldsProps { onChange: (value: T) => void; /** Offer "the default rule's" as the stop mode, for a schedule window. */ inheritStopMode?: boolean; + /** Start on demand is on, so the minimum may be 0. Undefined while the policy loads. */ + zeroMinimumAllowed?: boolean; } +const ZERO_MINIMUM_HELP = + 'Keep at least this many VMs powered on. 0 lets idle hosts stop until someone needs one: start on demand then starts a host for the first user, who waits a minute or two.'; + /** The scaling values and stop mode, shared by the default rule and schedule windows. */ -export function RuleFields({ value, onChange, inheritStopMode = false }: RuleFieldsProps) { +export function RuleFields({ + value, + onChange, + inheritStopMode = false, + zeroMinimumAllowed, +}: RuleFieldsProps) { + const zeroMinimum = String(value.minvms).trim() !== '' && Number(value.minvms) === 0; return ( <>
@@ -86,9 +99,14 @@ export function RuleFields({ value, onChange, inheritStopM ({ value, onChange, inheritStopM />
+ {zeroMinimum && zeroMinimumAllowed ? ( + + With a minimum of 0, no host runs while nobody needs one. The first user to connect waits while a host + starts, typically one to three minutes, and their AVD session connects when it is ready. + + ) : null} + {value.stopmode === 'Deallocate' ? ( Deallocate stops compute billing, but disks and IPs can still bill. Starts take longer, @@ -134,6 +159,7 @@ export function RuleForm({ submitLabel, busy = false, cancelTo, + zeroMinimumAllowed, }: RuleFormProps) { return ( @@ -144,7 +170,7 @@ export function RuleForm({ onSubmit(); }} > - +