+ >
+ );
+}
+```
+
+- [ ] **Step 2: Add index messages**
+
+Replace `Settings.pages.index.emptyState` with:
+
+```json
+"states": {
+ "available": "Available",
+ "notConfigured": "Not configured"
+},
+"sections": {
+ "profile": {
+ "title": "Profile",
+ "description": "Current CloudStack user, account, domain, and console preferences."
+ },
+ "security": {
+ "title": "Security",
+ "description": "Authentication source, two-factor status, API key access, and session posture."
+ },
+ "apiTokens": {
+ "title": "API tokens",
+ "description": "CloudStack API key visibility and scoped key-pair generation."
+ },
+ "integrations": {
+ "title": "Integrations",
+ "description": "Identity, monitoring, and automation integrations for this scope."
+ },
+ "billing": {
+ "title": "Billing",
+ "description": "Invoices, usage exports, and payment configuration."
+ },
+ "notifications": {
+ "title": "Notifications",
+ "description": "Email, event, and operational alert preferences."
+ },
+ "advanced": {
+ "title": "Advanced",
+ "description": "Low-level console and experimental operator controls."
+ }
+}
+```
+
+- [ ] **Step 3: Verify and commit**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack/.worktrees/phase5j-settings-index/web
+npm run test:unit
+npm run typecheck
+npm run lint
+cd ..
+git diff --check
+git add 'web/app/(app)/settings/page.tsx' web/messages/en.json web/lib/settings-messages.test.ts
+git commit -m "Add settings landing surface"
+```
+
+## Task 5: Settings Browser Coverage
+
+**Files:**
+- Modify: `web/tests/e2e/settings-pages.spec.ts`
+- Optional modify: `web/tests/e2e/README.md`
+
+- [ ] **Step 1: Update e2e test framing**
+
+Replace the current `test.describe("settings placeholder pages", ...)` with:
+
+```ts
+import { expect, test } from "./fixtures/cloudstack-bff";
+
+test.describe("settings surfaces", () => {
+ test("settings index links to active and not-configured sections", async ({ page }) => {
+ await page.goto("/settings");
+
+ await expect(page.getByRole("heading", { name: "Settings" })).toBeVisible();
+ await expect(page.getByRole("link", { name: /Profile/ })).toBeVisible();
+ await expect(page.getByRole("link", { name: /Security/ })).toBeVisible();
+ await expect(page.getByRole("link", { name: /API tokens/ })).toBeVisible();
+ await expect(page.getByText("Available")).toHaveCount(3);
+ });
+
+ test("profile page renders current user identity from listUsers", async ({ page, mockCloudStackBff }) => {
+ mockCloudStackBff.use("listUsers", {
+ listusersresponse: {
+ count: 1,
+ user: [{
+ id: "mock-uuid-alex",
+ username: "alex",
+ firstname: "Alex",
+ lastname: "Kim",
+ email: "alex@example.test",
+ account: "admin",
+ domain: "ROOT",
+ timezone: "Australia/Perth",
+ usersource: "native",
+ state: "enabled",
+ apikeyaccess: true,
+ is2faenabled: true,
+ }],
+ },
+ });
+
+ await page.goto("/settings/profile");
+
+ await expect(page.getByRole("heading", { name: "Profile" })).toBeVisible();
+ await expect(page.getByText("Alex Kim")).toBeVisible();
+ await expect(page.getByText("alex@example.test")).toBeVisible();
+ expect(mockCloudStackBff.calls("listUsers").at(-1)?.params.get("id")).toBe("mock-uuid-alex");
+ });
+
+ test("security page renders current user security flags from listUsers", async ({ page, mockCloudStackBff }) => {
+ mockCloudStackBff.use("listUsers", {
+ listusersresponse: {
+ count: 1,
+ user: [{
+ id: "mock-uuid-alex",
+ username: "alex",
+ usersource: "native",
+ state: "enabled",
+ apikeyaccess: true,
+ is2faenabled: true,
+ is2famandated: false,
+ }],
+ },
+ });
+
+ await page.goto("/settings/security");
+
+ await expect(page.getByRole("heading", { name: "Security" })).toBeVisible();
+ await expect(page.getByText("Authentication source")).toBeVisible();
+ await expect(page.getByText("native")).toBeVisible();
+ });
+
+ test("API token page renders getUserKeys status and can request key generation", async ({ page, mockCloudStackBff }) => {
+ mockCloudStackBff.use("getUserKeys", {
+ getuserkeysresponse: {
+ userkeys: {
+ apikeyaccess: true,
+ apikey: "api-key-12345678",
+ secretkey: "secret-key-abcdefgh",
+ },
+ },
+ });
+ mockCloudStackBff.use("registerUserKeys", {
+ registeruserkeysresponse: {
+ userkeys: {
+ id: "keypair-generated",
+ apikey: "generated-api-key",
+ secretkey: "generated-secret-key",
+ },
+ },
+ });
+
+ await page.goto("/settings/api-tokens");
+
+ await expect(page.getByRole("heading", { name: "API tokens" })).toBeVisible();
+ await expect(page.getByText("api-...5678")).toBeVisible();
+ await page.getByRole("button", { name: "Generate key pair" }).click();
+ await expect(page.getByRole("status")).toContainText("Generated key pair");
+ expect(mockCloudStackBff.calls("registerUserKeys").at(-1)?.json).toMatchObject({
+ id: "mock-uuid-alex",
+ });
+ });
+});
+
+const placeholderPages = [
+ ["/settings/integrations", "Integrations", "No integrations configured"],
+ ["/settings/billing", "Billing", "No billing settings available"],
+ ["/settings/notifications", "Notifications", "No notification preferences configured"],
+ ["/settings/advanced", "Advanced", "No advanced controls available"],
+] as const;
+
+test.describe("settings not-configured pages", () => {
+ for (const [path, heading, emptyTitle] of placeholderPages) {
+ test(`${path} keeps stable not-configured state`, async ({ page }) => {
+ await page.goto(path);
+ await expect(page.getByRole("heading", { name: heading })).toBeVisible();
+ await expect(page.getByText(emptyTitle)).toBeVisible();
+ });
+ }
+});
+```
+
+If the e2e branch runs before page branches merge, keep assertions compatible by allowing either the new operational text or the old empty-state text, then tighten assertions after coordinator merge.
+
+- [ ] **Step 2: Verify and commit**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack/.worktrees/phase5j-settings-e2e/web
+PLAYWRIGHT_PORT=3156 npm run test:e2e -- tests/e2e/settings-pages.spec.ts
+npm run typecheck
+cd ..
+git diff --check
+git add web/tests/e2e/settings-pages.spec.ts web/tests/e2e/README.md
+git commit -m "Add settings surface browser coverage"
+```
+
+If Playwright fails with `EPERM` binding `127.0.0.1`, rerun the same command with sandbox escalation.
+
+## Task 6: Optional Package Metadata Warning Cleanup
+
+**Files:**
+- Modify only if safe: `web/package.json`
+
+- [ ] **Step 1: Reproduce the warning**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack/.worktrees/phase5j-package-metadata/web
+npm run test:unit
+```
+
+Expected current warning:
+
+```text
+MODULE_TYPELESS_PACKAGE_JSON
+```
+
+- [ ] **Step 2: Add ESM package metadata**
+
+Modify `web/package.json`:
+
+```json
+{
+ "name": "cloudstack-web",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "description": "Modern role-based Apache CloudStack UI (Phase 5 rebuild)"
+}
+```
+
+Keep all existing scripts and dependencies unchanged.
+
+- [ ] **Step 3: Run full web verification in the branch**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack/.worktrees/phase5j-package-metadata/web
+npm run test:unit
+npm run typecheck
+npm run lint
+npm run build
+PLAYWRIGHT_PORT=3157 npm run test:e2e -- tests/e2e/app-shell.spec.ts
+```
+
+Expected:
+
+```text
+# unit tests pass without MODULE_TYPELESS_PACKAGE_JSON warning
+# typecheck/lint/build pass
+# focused e2e passes
+```
+
+- [ ] **Step 4: Abort if config churn starts**
+
+If adding `"type": "module"` requires changing Next config, Playwright config, TS config, imports, or more than `web/package.json`, revert this branch:
+
+```bash
+cd /Users/damian/Claude/cloudstack/.worktrees/phase5j-package-metadata
+git diff
+git restore web/package.json
+git status -sb
+```
+
+Expected:
+
+```text
+## phase5j-package-metadata
+```
+
+- [ ] **Step 5: Commit only if verification is green**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack/.worktrees/phase5j-package-metadata
+git diff --check
+git add web/package.json
+git commit -m "Mark web package as ESM"
+```
+
+If no commit is made, report `no commit; cleanup unsafe or unnecessary`.
+
+## Task 7: Coordinator Merge And Verification
+
+**Files:**
+- Modify during merge: `web/messages/en.json`
+- Modify during merge: `web/lib/settings-messages.test.ts`
+- Modify after green batch: `/Users/damian/Claude/HANDOVER.md`
+
+- [ ] **Step 1: Merge profile first**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack
+git checkout modernize-2026
+git merge --no-ff phase5j-profile-settings -m "Merge Phase 5j profile settings"
+```
+
+Expected:
+
+```text
+Merge made by the 'ort' strategy.
+```
+
+- [ ] **Step 2: Merge API token slice**
+
+Run:
+
+```bash
+git merge --no-ff phase5j-api-token-settings -m "Merge Phase 5j API token settings"
+```
+
+If `web/messages/en.json` conflicts, resolve by preserving both `Settings.pages.profile` and `Settings.pages.apiTokens`.
+
+- [ ] **Step 3: Merge security slice**
+
+Run:
+
+```bash
+git merge --no-ff phase5j-security-settings -m "Merge Phase 5j security settings"
+```
+
+If `security-settings.ts` duplicated types from `users.ts`, refactor to import from `./users.ts` before committing the merge resolution.
+
+- [ ] **Step 4: Merge settings index**
+
+Run:
+
+```bash
+git merge --no-ff phase5j-settings-index -m "Merge Phase 5j settings index"
+```
+
+Preserve all `Settings.pages.index`, `profile`, `security`, and `apiTokens` keys.
+
+- [ ] **Step 5: Merge settings e2e**
+
+Run:
+
+```bash
+git merge --no-ff phase5j-settings-e2e -m "Merge Phase 5j settings browser coverage"
+```
+
+Tighten e2e assertions if the branch used compatibility assertions before the page slices landed.
+
+- [ ] **Step 6: Merge or discard package metadata**
+
+If the package metadata branch has a green commit:
+
+```bash
+git merge --no-ff phase5j-package-metadata -m "Merge Phase 5j package metadata cleanup"
+```
+
+If it has no commit:
+
+```bash
+git branch -D phase5j-package-metadata
+```
+
+Only delete the branch if its worktree has already been removed or if Git allows it.
+
+- [ ] **Step 7: Run full web verification**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack/web
+npm run test:unit
+npm run typecheck
+npm run lint
+npm run build
+PLAYWRIGHT_PORT=3158 npm run test:e2e
+```
+
+Expected:
+
+```text
+# all unit tests pass
+# typecheck passes
+# lint passes
+# Next build passes
+# Playwright Chromium tests pass
+```
+
+- [ ] **Step 8: Run root verification**
+
+Run:
+
+```bash
+cd /Users/damian/Claude/cloudstack
+git diff --check
+git status -sb
+git log --oneline -12
+```
+
+Expected:
+
+```text
+## modernize-2026
+```
+
+- [ ] **Step 9: Push**
+
+Run:
+
+```bash
+git push origin modernize-2026
+```
+
+Expected:
+
+```text
+modernize-2026 -> modernize-2026
+```
+
+- [ ] **Step 10: Update handover**
+
+Add a new top section to `/Users/damian/Claude/HANDOVER.md`:
+
+```markdown
+=== Latest Codex handback (Phase 5j settings surfaces landed) ===
+
+HEAD at handback checkpoint: (`origin/modernize-2026` pushed after verification)
+
+What landed:
+- CloudStack-backed Profile settings page via `listUsers`.
+- CloudStack-backed Security settings status via `listUsers`.
+- CloudStack API-token status and key generation via `getUserKeys`/`registerUserKeys`.
+- Practical Settings landing page.
+- Settings browser coverage for active and not-configured settings routes.
+- Optional package metadata cleanup: .
+
+Verification:
+- `cd web && npm run test:unit`
+- `cd web && npm run typecheck`
+- `cd web && npm run lint`
+- `cd web && npm run build`
+- `cd web && PLAYWRIGHT_PORT=3158 npm run test:e2e`
+- `git diff --check`
+
+Next useful work:
+1. Implement integrations, notifications, billing settings only once real CloudStack endpoints are selected.
+2. Continue converting mock fallback surfaces to explicit unavailable/error UI where operator confidence matters.
+3. Clean old Phase 5d/e temporary worktrees in a separate explicit cleanup pass.
+```
+
+## Cleanup After Merge
+
+Only after all useful Phase 5j work is merged and pushed:
+
+```bash
+git worktree remove .worktrees/phase5j-profile-settings
+git worktree remove .worktrees/phase5j-api-token-settings
+git worktree remove .worktrees/phase5j-security-settings
+git worktree remove .worktrees/phase5j-settings-index
+git worktree remove .worktrees/phase5j-settings-e2e
+git worktree remove .worktrees/phase5j-package-metadata
+git branch -d phase5j-profile-settings phase5j-api-token-settings phase5j-security-settings phase5j-settings-index phase5j-settings-e2e phase5j-package-metadata
+```
+
+Do not remove old `.worktrees/phase5d-*` or `.worktrees/phase5e-*` as part of Phase 5j unless the user explicitly asks for that cleanup.
+
+## Acceptance Criteria
+
+Phase 5j is complete when:
+
+- `/settings` is no longer an empty placeholder and links to all settings sections.
+- `/settings/profile` renders current user identity/scope from `listUsers` with mock fallback.
+- `/settings/security` renders current user security status from `listUsers` with mock fallback.
+- `/settings/api-tokens` renders key status from `getUserKeys` and can generate a key pair through `registerUserKeys`, or generation is explicitly deferred with read-only status landed.
+- Settings e2e tests cover active settings pages and the still-not-configured pages.
+- Full web verification passes.
+- `modernize-2026` is pushed.
+- `/Users/damian/Claude/HANDOVER.md` points to the new checkpoint and this plan file.
+
+## Self-Review
+
+- Spec coverage: The plan covers the user-requested Phase 5j hold point, the queued settings slices, the optional package warning cleanup, merge/push, handover, and cleanup.
+- Placeholder scan: The plan avoids TBD/TODO language. Optional items have explicit abort criteria and do not block completion.
+- Type consistency: `users.ts` owns reusable user/listUsers types; `security-settings.ts` imports them during final merge. API-token names match CloudStack commands and response envelope casing already present in the repo.
diff --git a/engine/api/pom.xml b/engine/api/pom.xml
index cb50ef0cd46b..a84f7580006b 100644
--- a/engine/api/pom.xml
+++ b/engine/api/pom.xml
@@ -59,18 +59,18 @@
${project.version}
- javax.xml.bind
- jaxb-api
+ jakarta.xml.bind
+ jakarta.xml.bind-api${cs.jaxb.version}
- com.sun.xml.bind
+ org.glassfish.jaxbjaxb-core${cs.jaxb.version}
- com.sun.xml.bind
- jaxb-impl
+ org.glassfish.jaxb
+ jaxb-runtime${cs.jaxb.impl.version}
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/cloud/entity/api/VirtualMachineEntity.java b/engine/api/src/main/java/org/apache/cloudstack/engine/cloud/entity/api/VirtualMachineEntity.java
index 55fb5d8d5b78..5415f0ef22f8 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/cloud/entity/api/VirtualMachineEntity.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/cloud/entity/api/VirtualMachineEntity.java
@@ -21,11 +21,11 @@
import java.util.List;
import java.util.Map;
-import javax.ws.rs.BeanParam;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.xml.bind.annotation.XmlRootElement;
+import jakarta.ws.rs.BeanParam;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.xml.bind.annotation.XmlRootElement;
import org.apache.cloudstack.engine.entity.api.CloudStackEntity;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/DataCenterResourceEntity.java b/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/DataCenterResourceEntity.java
index bc532c30f294..c2458d9fdd2f 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/DataCenterResourceEntity.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/DataCenterResourceEntity.java
@@ -18,9 +18,9 @@
*/
package org.apache.cloudstack.engine.datacenter.entity.api;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Produces;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Produces;
import org.apache.cloudstack.engine.entity.api.CloudStackEntity;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/ZoneEntity.java b/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/ZoneEntity.java
index 6750bf11508f..9a329226aa13 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/ZoneEntity.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/ZoneEntity.java
@@ -20,10 +20,10 @@
import java.util.List;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.xml.bind.annotation.XmlRootElement;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.xml.bind.annotation.XmlRootElement;
import org.apache.cloudstack.engine.service.api.ProvisioningService;
import org.apache.cloudstack.framework.ws.jackson.Url;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/entity/api/CloudStackEntity.java b/engine/api/src/main/java/org/apache/cloudstack/engine/entity/api/CloudStackEntity.java
index 7891b90e2aa2..4e3557576268 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/entity/api/CloudStackEntity.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/entity/api/CloudStackEntity.java
@@ -23,7 +23,7 @@
import java.util.List;
import java.util.Map;
-import javax.ws.rs.GET;
+import jakarta.ws.rs.GET;
/**
* All entities returned by the Cloud Orchestration Platform must implement
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ClusterRestService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ClusterRestService.java
index a38b623e0cb4..bd58dadaf46c 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ClusterRestService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ClusterRestService.java
@@ -20,13 +20,13 @@
import java.util.List;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.datacenter.entity.api.ClusterEntity;
import org.apache.cloudstack.engine.service.api.ProvisioningService;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/NetworkRestService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/NetworkRestService.java
index 8e470854456d..b9a53c16a614 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/NetworkRestService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/NetworkRestService.java
@@ -20,13 +20,13 @@
import java.util.List;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.cloud.entity.api.NetworkEntity;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/PodRestService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/PodRestService.java
index 9bad3c75cf33..8ea779308220 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/PodRestService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/PodRestService.java
@@ -18,13 +18,13 @@
*/
package org.apache.cloudstack.engine.rest.service.api;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.datacenter.entity.api.PodEntity;
import org.apache.cloudstack.engine.service.api.ProvisioningService;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VirtualMachineRestService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VirtualMachineRestService.java
index b731d36abcf1..24bc8824cb78 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VirtualMachineRestService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VirtualMachineRestService.java
@@ -20,12 +20,12 @@
import java.util.List;
-import javax.ws.rs.GET;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VolumeRestService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VolumeRestService.java
index 1de87aecae7f..63c746c774fd 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VolumeRestService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/VolumeRestService.java
@@ -20,14 +20,14 @@
import java.util.List;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ZoneRestService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ZoneRestService.java
index 17f59afd4bc5..2cf7ddce38a2 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ZoneRestService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/rest/service/api/ZoneRestService.java
@@ -20,14 +20,14 @@
import java.util.List;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.datacenter.entity.api.ZoneEntity;
import org.apache.cloudstack.engine.service.api.ProvisioningService;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java
index 6be71b3cb250..9deeb4894d39 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java
@@ -24,11 +24,11 @@
import java.util.List;
import java.util.Map;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
import org.apache.cloudstack.engine.cloud.entity.api.NetworkEntity;
import org.apache.cloudstack.engine.cloud.entity.api.TemplateEntity;
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/type/VolumeTypeHelper.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/type/VolumeTypeHelper.java
index 9fcc17cfb522..779f8dd054a7 100644
--- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/type/VolumeTypeHelper.java
+++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/type/VolumeTypeHelper.java
@@ -20,7 +20,7 @@
import java.util.List;
import java.util.Map;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
public class VolumeTypeHelper {
diff --git a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java
index 1c88c7df124c..166900f3bece 100644
--- a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java
+++ b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java
@@ -22,8 +22,8 @@
import java.util.HashMap;
import java.util.Map;
-import javax.annotation.PostConstruct;
-import javax.inject.Inject;
+import jakarta.annotation.PostConstruct;
+import jakarta.inject.Inject;
import com.cloud.network.Network;
import org.apache.commons.collections.MapUtils;
diff --git a/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java b/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java
index 107e177ef579..be70ed77692f 100644
--- a/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java
+++ b/engine/components-api/src/main/java/com/cloud/network/NetworkStateListener.java
@@ -22,7 +22,7 @@
import java.util.HashMap;
import java.util.Map;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.events.EventDistributor;
diff --git a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java
index 792a3a6b397f..0bfe28ecd7ec 100644
--- a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java
+++ b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java
@@ -31,6 +31,7 @@
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.network.IpAddress;
+import com.cloud.network.element.VpcProvider;
import com.cloud.network.Network;
import com.cloud.network.Network.Provider;
import com.cloud.network.Network.Service;
@@ -216,4 +217,11 @@ public interface VpcManager {
* Returns true if the network is part of a VPC, and the VPC is created from conserve mode enabled VPC offering
*/
boolean isNetworkOnVpcEnabledConserveMode(Network network);
+
+ /**
+ * Returns the cached list of {@link VpcProvider} network elements (lazily initialized from
+ * the VPC virtual-router provider). Used by static-route, ACL and other VPC sub-services
+ * that need to apply state through the configured provider.
+ */
+ List getVpcElements();
}
diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageUtil.java b/engine/components-api/src/main/java/com/cloud/storage/StorageUtil.java
index 40e4a0f3dfc4..444a2a34abc7 100644
--- a/engine/components-api/src/main/java/com/cloud/storage/StorageUtil.java
+++ b/engine/components-api/src/main/java/com/cloud/storage/StorageUtil.java
@@ -18,7 +18,7 @@
import java.util.List;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
index 1215829d92f8..5c147610c927 100644
--- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java
@@ -39,7 +39,7 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.utils.StringUtils;
diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
index 38a198b73040..9049c58a59e7 100644
--- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java
@@ -37,7 +37,7 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import javax.naming.ConfigurationException;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLEngine;
diff --git a/engine/orchestration/src/main/java/com/cloud/cluster/agentlb/ClusterBasedAgentLoadBalancerPlanner.java b/engine/orchestration/src/main/java/com/cloud/cluster/agentlb/ClusterBasedAgentLoadBalancerPlanner.java
index 5b05b4df0423..0da0ef665444 100644
--- a/engine/orchestration/src/main/java/com/cloud/cluster/agentlb/ClusterBasedAgentLoadBalancerPlanner.java
+++ b/engine/orchestration/src/main/java/com/cloud/cluster/agentlb/ClusterBasedAgentLoadBalancerPlanner.java
@@ -24,7 +24,7 @@
import java.util.List;
import java.util.Map;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import com.cloud.cluster.ManagementServerHostVO;
import org.springframework.stereotype.Component;
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java
index b8995f6aa4a5..d9deaba32308 100644
--- a/engine/orchestration/src/main/java/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java
@@ -19,7 +19,7 @@
import java.util.List;
import java.util.Map;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.cluster.ClusterManager;
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
index 17ddf8706702..6cda36fa3eed 100755
--- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
@@ -18,47 +18,29 @@
package com.cloud.vm;
import static com.cloud.configuration.ConfigurationManagerImpl.EXPOSE_ERRORS_TO_USER;
-import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS;
-import java.lang.reflect.Field;
import java.net.URI;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
import java.util.Set;
-import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import javax.naming.ConfigurationException;
-import javax.persistence.EntityExistsException;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
-import org.apache.cloudstack.annotation.AnnotationService;
-import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
-import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin;
-import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd;
-import org.apache.cloudstack.backup.BackupManager;
import org.apache.cloudstack.backup.dao.BackupDao;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.context.CallContext;
@@ -78,14 +60,11 @@
import org.apache.cloudstack.framework.extensions.dao.ExtensionDetailsDao;
import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager;
import org.apache.cloudstack.framework.extensions.vo.ExtensionDetailsVO;
-import org.apache.cloudstack.framework.jobs.AsyncJob;
import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
import org.apache.cloudstack.framework.jobs.AsyncJobManager;
import org.apache.cloudstack.framework.jobs.Outcome;
import org.apache.cloudstack.framework.jobs.dao.VmWorkJobDao;
-import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
import org.apache.cloudstack.framework.jobs.impl.JobSerializerHelper;
-import org.apache.cloudstack.framework.jobs.impl.OutcomeImpl;
import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.MessageDispatcher;
@@ -94,12 +73,10 @@
import org.apache.cloudstack.jobs.JobInfo;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.reservation.dao.ReservationDao;
-import org.apache.cloudstack.resource.ResourceCleanupService;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.cloudstack.utils.cache.SingleCache;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
import org.apache.cloudstack.vm.UnmanagedVMsManager;
@@ -119,45 +96,18 @@
import com.cloud.agent.api.ClusterVMMetaDataSyncAnswer;
import com.cloud.agent.api.ClusterVMMetaDataSyncCommand;
import com.cloud.agent.api.Command;
-import com.cloud.agent.api.GetVmDiskStatsAnswer;
-import com.cloud.agent.api.GetVmDiskStatsCommand;
-import com.cloud.agent.api.GetVmNetworkStatsAnswer;
-import com.cloud.agent.api.GetVmNetworkStatsCommand;
-import com.cloud.agent.api.GetVmStatsAnswer;
-import com.cloud.agent.api.GetVmStatsCommand;
import com.cloud.agent.api.MigrateCommand;
-import com.cloud.agent.api.MigrateVmToPoolAnswer;
-import com.cloud.agent.api.ModifyTargetsCommand;
import com.cloud.agent.api.PingRoutingCommand;
-import com.cloud.agent.api.PlugNicAnswer;
-import com.cloud.agent.api.PlugNicCommand;
-import com.cloud.agent.api.PrepareExternalProvisioningAnswer;
-import com.cloud.agent.api.PrepareExternalProvisioningCommand;
import com.cloud.agent.api.PrepareForMigrationAnswer;
import com.cloud.agent.api.PrepareForMigrationCommand;
-import com.cloud.agent.api.RebootAnswer;
import com.cloud.agent.api.RebootCommand;
-import com.cloud.agent.api.RecreateCheckpointsCommand;
-import com.cloud.agent.api.ReplugNicAnswer;
-import com.cloud.agent.api.ReplugNicCommand;
-import com.cloud.agent.api.RestoreVMSnapshotAnswer;
-import com.cloud.agent.api.RestoreVMSnapshotCommand;
-import com.cloud.agent.api.ScaleVmCommand;
import com.cloud.agent.api.StartAnswer;
import com.cloud.agent.api.StartCommand;
import com.cloud.agent.api.StartupCommand;
import com.cloud.agent.api.StartupRoutingCommand;
import com.cloud.agent.api.StopAnswer;
import com.cloud.agent.api.StopCommand;
-import com.cloud.agent.api.UnPlugNicAnswer;
-import com.cloud.agent.api.UnPlugNicCommand;
import com.cloud.agent.api.UnmanageInstanceCommand;
-import com.cloud.agent.api.UnregisterVMCommand;
-import com.cloud.agent.api.UpdateVmNicAnswer;
-import com.cloud.agent.api.UpdateVmNicCommand;
-import com.cloud.agent.api.VmDiskStatsEntry;
-import com.cloud.agent.api.VmNetworkStatsEntry;
-import com.cloud.agent.api.VmStatsEntry;
import com.cloud.agent.api.routing.NetworkElementCommand;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
@@ -169,19 +119,11 @@
import com.cloud.agent.manager.allocator.HostAllocator;
import com.cloud.alert.AlertManager;
import com.cloud.api.ApiDBUtils;
-import com.cloud.api.query.dao.DomainRouterJoinDao;
-import com.cloud.api.query.dao.UserVmJoinDao;
-import com.cloud.api.query.vo.DomainRouterJoinVO;
-import com.cloud.api.query.vo.UserVmJoinVO;
import com.cloud.capacity.CapacityManager;
import com.cloud.configuration.Resource;
-import com.cloud.dc.ClusterDetailsDao;
-import com.cloud.dc.ClusterDetailsVO;
-import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
-import com.cloud.dc.Pod;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.dc.dao.HostPodDao;
@@ -192,13 +134,9 @@
import com.cloud.deploy.DeploymentPlanner.ExcludeList;
import com.cloud.deploy.DeploymentPlanningManager;
import com.cloud.deploy.DeploymentPlanningManagerImpl;
-import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
import com.cloud.domain.Domain;
-import com.cloud.domain.dao.DomainDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
-import com.cloud.event.UsageEventUtils;
-import com.cloud.event.UsageEventVO;
import com.cloud.exception.AffinityConflictException;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.ConcurrentOperationException;
@@ -210,7 +148,6 @@
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.StorageAccessException;
-import com.cloud.exception.StorageUnavailableException;
import com.cloud.ha.HighAvailabilityManager;
import com.cloud.ha.HighAvailabilityManager.WorkType;
import com.cloud.host.Host;
@@ -220,44 +157,26 @@
import com.cloud.host.dao.HostDetailsDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.HypervisorGuru;
-import com.cloud.hypervisor.HypervisorGuruBase;
import com.cloud.hypervisor.HypervisorGuruManager;
import com.cloud.network.Network;
import com.cloud.network.NetworkModel;
import com.cloud.network.NetworkService;
-import com.cloud.network.Networks;
import com.cloud.network.dao.NetworkDao;
-import com.cloud.network.dao.NetworkDetailVO;
import com.cloud.network.dao.NetworkDetailsDao;
-import com.cloud.network.dao.NetworkVO;
import com.cloud.network.router.VirtualRouter;
import com.cloud.network.security.SecurityGroupManager;
-import com.cloud.network.vpc.VpcVO;
-import com.cloud.network.vpc.dao.VpcDao;
import com.cloud.offering.DiskOffering;
import com.cloud.offering.DiskOfferingInfo;
-import com.cloud.offering.NetworkOffering;
import com.cloud.offering.ServiceOffering;
-import com.cloud.offerings.NetworkOfferingVO;
-import com.cloud.offerings.dao.NetworkOfferingDao;
import com.cloud.org.Cluster;
import com.cloud.resource.ResourceManager;
-import com.cloud.resource.ResourceState;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
-import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Snapshot;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
-import com.cloud.storage.VMTemplateVO;
-import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.storage.Volume;
-import com.cloud.storage.Volume.Type;
-import com.cloud.storage.VolumeApiService;
-import com.cloud.storage.VolumeApiServiceImpl;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.GuestOSCategoryDao;
@@ -271,15 +190,12 @@
import com.cloud.user.Account;
import com.cloud.user.ResourceLimitService;
import com.cloud.user.User;
-import com.cloud.user.dao.AccountDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.DateUtil;
import com.cloud.utils.Journal;
import com.cloud.utils.LogUtils;
import com.cloud.utils.Pair;
-import com.cloud.utils.Predicate;
import com.cloud.utils.ReflectionUse;
-import com.cloud.utils.StringUtils;
import com.cloud.utils.Ternary;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -289,8 +205,6 @@
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionCallbackWithException;
-import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
-import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.exception.ExecutionException;
@@ -298,18 +212,16 @@
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.vm.ItWorkVO.Step;
import com.cloud.vm.VirtualMachine.Event;
-import com.cloud.vm.VirtualMachine.PowerState;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.vm.snapshot.VMSnapshotManager;
-import com.cloud.vm.snapshot.VMSnapshotVO;
-import com.cloud.vm.snapshot.dao.VMSnapshotDao;
import com.google.gson.Gson;
-public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable {
+public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable, VmStateMachineActions,
+ VmNetworkAttachmentOrchestrationService.BackendNicOperations {
public static final String VM_WORK_JOB_HANDLER = VirtualMachineManagerImpl.class.getSimpleName();
@@ -376,8 +288,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Inject
private StoragePoolHostDao _poolHostDao;
@Inject
- private VMSnapshotDao _vmSnapshotDao;
- @Inject
private AffinityGroupVMMapDao _affinityGroupVMMapDao;
@Inject
private EntityManager _entityMgr;
@@ -396,8 +306,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Inject
private VMSnapshotManager _vmSnapshotMgr;
@Inject
- private ClusterDetailsDao _clusterDetailsDao;
- @Inject
private VMInstanceDetailsDao vmInstanceDetailsDao;
@Inject
private VolumeOrchestrationService volumeMgr;
@@ -420,36 +328,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
@Inject
private SecurityGroupManager _securityGroupManager;
@Inject
- private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao;
- @Inject
- private UserVmJoinDao userVmJoinDao;
- @Inject
- private NetworkOfferingDao networkOfferingDao;
- @Inject
- private DomainRouterJoinDao domainRouterJoinDao;
- @Inject
- private AnnotationDao annotationDao;
- @Inject
- private AccountDao accountDao;
- @Inject
- private VpcDao vpcDao;
- @Inject
- private DomainDao domainDao;
- @Inject
public NetworkService networkService;
@Inject
- ResourceCleanupService resourceCleanupService;
- @Inject
VmWorkJobDao vmWorkJobDao;
@Inject
DataStoreProviderManager dataStoreProviderManager;
@Inject
- BackupManager backupManager;
- @Inject
BackupDao backupDao;
- private SingleCache> vmIdsInProgressCache;
-
@Inject
private SnapshotDataStoreDao snapshotDataStoreDao;
@@ -462,6 +348,62 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
ExtensionsManager extensionsManager;
@Inject
ExtensionDetailsDao extensionDetailsDao;
+ @Inject
+ private VmServiceOfferingUpgradeManager vmServiceOfferingUpgradeManager;
+ @Inject
+ private VmIscsiTargetManager vmIscsiTargetManager;
+ @Inject
+ private VmStatsCollector vmStatsCollector;
+ @Inject
+ protected VmExternalProvisioningManager vmExternalProvisioningManager;
+ @Inject
+ protected VmVolumeMigrationPlanningService vmVolumeMigrationPlanningService;
+ @Inject
+ protected VmVolumeMigrationPlanningServiceImpl vmVolumeMigrationPlanningServiceImpl;
+ @Inject
+ protected VmOfflineStorageMigrationService vmOfflineStorageMigrationService;
+ @Inject
+ protected VmOfflineStorageMigrationServiceImpl vmOfflineStorageMigrationServiceImpl;
+ @Inject
+ protected VmDiskOfferingSuitabilityService vmDiskOfferingSuitabilityService;
+ @Inject
+ protected VmCommandSpecPostProcessingService vmCommandSpecPostProcessingService;
+ @Inject
+ protected VmWorkJobQueueService vmWorkJobQueueService;
+ @Inject
+ protected VmExpungeOrchestrationService vmExpungeOrchestrationService;
+ @Inject
+ protected VmDestroyOrchestrationService vmDestroyOrchestrationService;
+ @Inject
+ protected VmMetadataSyncService vmMetadataSyncService;
+ @Inject
+ protected VmNetworkNameMappingService vmNetworkNameMappingService;
+ @Inject
+ protected VmStartProfilePreparationService vmStartProfilePreparationService;
+ @Inject
+ protected VmVlanPersistenceMappingService vmVlanPersistenceMappingService;
+ @Inject
+ protected VmStopCommandService vmStopCommandService;
+ @Inject
+ protected VmStopOrchestrationService vmStopOrchestrationService;
+ @Inject
+ protected VmMigrationCheckpointService vmMigrationCheckpointService;
+ @Inject
+ protected VmRebootOrchestrationService vmRebootOrchestrationService;
+ @Inject
+ protected VmPowerStateSyncManager vmPowerStateSyncManager;
+ @Inject
+ protected VmNicUpdateService vmNicUpdateService;
+ @Inject
+ protected VmAllocationOrchestrationService vmAllocationOrchestrationService;
+ @Inject
+ protected VmNicBackendCommandService vmNicBackendCommandService;
+ @Inject
+ protected VmMigrateAwayPlanningService vmMigrateAwayPlanningService;
+ @Inject
+ protected VmScaleReconfigurationService vmScaleReconfigurationService;
+ @Inject
+ protected VmNetworkAttachmentOrchestrationService vmNetworkAttachmentOrchestrationService;
VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
@@ -507,8 +449,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
Long.class, "systemvm.root.disk.size", "-1",
"Size of root volume (in GB) of system VMs and virtual routers", true);
- private boolean syncTransitioningVmPowerState;
-
ScheduledExecutorService _executor = null;
private long _nodeId;
@@ -539,114 +479,22 @@ public void allocate(final String vmInstanceName, final VirtualMachineTemplate t
final LinkedHashMap extends Network, List extends NicProfile>> auxiliaryNetworks,final DeploymentPlan plan, final HypervisorType hyperType,
final Map> extraDhcpOptions, final Map datadiskTemplateToDiskOfferingMap, Volume volume, Snapshot snapshot)
throws InsufficientCapacityException {
-
- logger.info("Allocating Instance from Template: {} with hostname: {} and {} networks", template, vmInstanceName, auxiliaryNetworks.size());
- VMInstanceVO persistedVm = null;
- try {
- final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName);
- final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
-
- logger.debug("Allocating entries for VM: " + vm);
-
- vm.setDataCenterId(plan.getDataCenterId());
- if (plan.getPodId() != null) {
- vm.setPodIdToDeployIn(plan.getPodId());
- }
- assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet";
- persistedVm = _vmDao.persist(vm);
-
- final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(persistedVm, template, serviceOffering, null, null);
-
- Long rootDiskSize = rootDiskOfferingInfo.getSize();
- if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) {
- rootDiskSize = SystemVmRootDiskSize.value();
- }
- final Long rootDiskSizeFinal = rootDiskSize;
-
- logger.debug("Allocating NICs for {}", persistedVm);
-
- try {
- if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) {
- _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions);
- }
- } catch (final ConcurrentOperationException e) {
- throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e);
- }
-
- logger.debug("Allocating disks for {}", persistedVm);
-
- allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot);
-
- // Create new Volume context and inject event resource type, id and details to generate VOLUME.CREATE event for the ROOT disk.
- CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume);
- try {
- if (dataDiskOfferings != null) {
- int index = 0;
- for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) {
- Long deviceId = dataDiskDeviceIds.get(index++);
- String volumeName = deviceId == null ? "DATA-" + persistedVm.getId() : "DATA-" + persistedVm.getId() + "-" + String.valueOf(deviceId);
- volumeMgr.allocateRawVolume(Type.DATADISK, volumeName, dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(),
- dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId, true);
- }
- }
- if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) {
- Long diskNumber = 1L;
- for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) {
- DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue();
- long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024);
- VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey());
- volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf( diskNumber), diskOffering, diskOfferingSize, null, null,
- persistedVm, dataDiskTemplate, owner, diskNumber, true);
- diskNumber++;
- }
- }
- } finally {
- // Remove volumeContext and pop vmContext back
- CallContext.unregister();
- }
-
- logger.debug("Allocation completed for VM: " + persistedVm);
- } catch (InsufficientCapacityException | CloudRuntimeException e) {
- // Failed VM will be in Stopped. Transition it to Error, so it can be expunged by ExpungeTask or similar
- try {
- if (persistedVm != null) {
- stateTransitTo(persistedVm, VirtualMachine.Event.OperationFailedToError, null);
- }
- } catch (NoTransitionException nte) {
- logger.error("Failed to transition {} in {} state to Error state", persistedVm, persistedVm.getState().toString());
- }
- throw e;
- }
+ vmAllocationOrchestrationService.allocate(vmInstanceName, template, serviceOffering, rootDiskOfferingInfo, dataDiskOfferings,
+ dataDiskDeviceIds, auxiliaryNetworks, plan, hyperType, extraDhcpOptions, datadiskTemplateToDiskOfferingMap, volume, snapshot);
}
- private void allocateRootVolume(VMInstanceVO vm, VirtualMachineTemplate template, DiskOfferingInfo rootDiskOfferingInfo, Account owner, Long rootDiskSizeFinal, Volume volume, Snapshot snapshot) {
- // Create new Volume context and inject event resource type, id and details to generate VOLUME.CREATE event for the ROOT disk.
- CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume);
- try {
- String rootVolumeName = String.format("ROOT-%s", vm.getId());
- if (template.getFormat() == ImageFormat.ISO) {
- volumeMgr.allocateRawVolume(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(),
- rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null, true);
- } else if (Arrays.asList(ImageFormat.BAREMETAL, ImageFormat.EXTERNAL).contains(template.getFormat())) {
- logger.debug("{} has format [{}]. Skipping ROOT volume [{}] allocation.", template, template.getFormat(), rootVolumeName);
- } else {
- volumeMgr.allocateTemplatedVolumes(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskSizeFinal,
- rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), template, vm, owner, volume, snapshot);
- }
- } finally {
- // Remove volumeContext and pop vmContext back
- CallContext.unregister();
- }
+ protected void allocateRootVolume(VMInstanceVO vm, VirtualMachineTemplate template, DiskOfferingInfo rootDiskOfferingInfo, Account owner, Long rootDiskSizeFinal, Volume volume, Snapshot snapshot) {
+ vmAllocationOrchestrationService.allocateRootVolume(vm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot);
}
@Override
public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering,
final LinkedHashMap extends Network, List extends NicProfile>> networks, final DeploymentPlan plan, final HypervisorType hyperType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException {
- DiskOffering diskOffering = _diskOfferingDao.findById(serviceOffering.getDiskOfferingId());
- allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(diskOffering), new ArrayList<>(), new ArrayList<>(), networks, plan, hyperType, null, null, volume, snapshot);
+ vmAllocationOrchestrationService.allocate(vmInstanceName, template, serviceOffering, networks, plan, hyperType, volume, snapshot);
}
- VirtualMachineGuru getVmGuru(final VirtualMachine vm) {
+ @Override
+ public VirtualMachineGuru getVmGuru(final VirtualMachine vm) {
if(vm != null) {
return _vmGurus.get(vm.getType());
}
@@ -655,238 +503,20 @@ VirtualMachineGuru getVmGuru(final VirtualMachine vm) {
@Override
public void expunge(final String vmUuid) throws ResourceUnavailableException {
- try {
- advanceExpunge(vmUuid);
- } catch (final OperationTimedoutException e) {
- throw new CloudRuntimeException("Operation timed out", e);
- } catch (final ConcurrentOperationException e) {
- throw new CloudRuntimeException("Concurrent operation ", e);
- }
+ vmExpungeOrchestrationService.expunge(vmUuid);
}
@Override
public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
- final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
- advanceExpunge(vm);
- }
-
- private boolean isValidSystemVMType(VirtualMachine vm) {
- return VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) ||
- VirtualMachine.Type.ConsoleProxy.equals(vm.getType());
- }
-
- private boolean isVmDestroyed(VMInstanceVO vm) {
- if (vm == null || vm.getRemoved() != null) {
- logger.debug("Unable to find vm or vm is expunged: " + vm);
- return true;
- }
- return false;
+ vmExpungeOrchestrationService.advanceExpunge(vmUuid);
}
protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
- if (isVmDestroyed(vm)) {
- return;
- }
-
- if (HypervisorType.External.equals(vm.getHypervisorType())) {
- UserVmVO userVM = _userVmDao.findById(vm.getId());
- _userVmDao.loadDetails(userVM);
- userVM.setDetail(VmDetailConstants.EXPUNGE_EXTERNAL_VM, Boolean.TRUE.toString());
- _userVmDao.saveDetails(userVM);
- }
-
- advanceStop(vm.getUuid(), VmDestroyForcestop.value());
- vm = _vmDao.findByUuid(vm.getUuid());
-
- try {
- if (!stateTransitTo(vm, VirtualMachine.Event.ExpungeOperation, vm.getHostId())) {
- logger.debug("Unable to expunge the vm because it is not in the correct state: " + vm);
- throw new CloudRuntimeException("Unable to expunge " + vm);
-
- }
- } catch (final NoTransitionException e) {
- logger.debug("Unable to expunge the vm because it is not in the correct state: " + vm);
- throw new CloudRuntimeException("Unable to expunge " + vm, e);
- }
-
- logger.debug("Expunging vm " + vm);
-
- final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
-
- final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType());
-
- List vmNics = profile.getNics();
- logger.debug("Cleaning up NICS [{}] of {}.", vmNics.stream().map(nic -> nic.toString()).collect(Collectors.joining(", ")),vm.toString());
- final List nicExpungeCommands = hvGuru.finalizeExpungeNics(vm, profile.getNics());
- _networkMgr.cleanupNics(profile);
-
- logger.debug("Cleaning up hypervisor data structures (ex. SRs in XenServer) for managed storage. Data from {}.", vm.toString());
-
- final List volumeExpungeCommands = hvGuru.finalizeExpungeVolumes(vm);
-
- final Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId();
-
- List
- *
- *
When a host is UP, a state report for the VMs will typically be received. However, certain scenarios
- * (e.g., out-of-band changes or behavior specific to hypervisors like XenServer or KVM) might result in
- * missing reports, preventing the state-sync logic from running. To address this, the method scans VMs
- * based on their last update timestamp. If a VM remains stalled without a status update while its host is UP,
- * it is assumed to be powered off, which is generally a safe assumption.
- *
- * @param hostId the ID of the host to scan for stalled VMs in transition states.
+ * This method helps constructing vmSpec for Unmanage operation for Stopped Instance
+ * @param vmId
+ * @param hostId
+ * @return VirtualMachineTO
*/
- private void scanStalledVMInTransitionStateOnUpHost(final long hostId) {
- if (!syncTransitioningVmPowerState) {
- return;
- }
- if (!_hostDao.isHostUp(hostId)) {
- return;
- }
- final long stallThresholdInMs = VmJobStateReportInterval.value() * 2;
- final long cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs).getTime();
- final List hostTransitionVms = _vmDao.listByHostAndState(hostId, State.Starting, State.Stopping, State.Migrating);
+ protected VirtualMachineTO prepVmSpecForUnmanageCmd(Long vmId, Long hostId) {
+ final VMInstanceVO vm = _vmDao.findById(vmId);
+ final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
+ final ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
+ final VirtualMachineTemplate template = _entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vm.getTemplateId());
+ Host host = _hostDao.findById(hostId);
+ VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, template, offering, owner, null);
+ updateOverCommitRatioForVmProfile(vmProfile, host.getClusterId());
+ final List nics = _nicsDao.listByVmId(vmProfile.getId());
+ Collections.sort(nics, (nic1, nic2) -> {
+ Long nicId1 = Long.valueOf(nic1.getDeviceId());
+ Long nicId2 = Long.valueOf(nic2.getDeviceId());
+ return nicId1.compareTo(nicId2);
+ });
- final List mostLikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostTransitionVms, cutTime);
- for (final VMInstanceVO vm : mostLikelyStoppedVMs) {
- handlePowerOffReportWithNoPendingJobsOnVM(vm);
+ for (final NicVO nic : nics) {
+ final Network network = _networkModel.getNetwork(nic.getNetworkId());
+ final NicProfile nicProfile =
+ new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), null, _networkModel.isSecurityGroupSupportedInNetwork(network),
+ _networkModel.getNetworkTag(vmProfile.getHypervisorType(), network));
+ vmProfile.addNic(nicProfile);
}
- final List vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostTransitionVms, cutTime);
- for (final VMInstanceVO vm : vmsWithRecentReport) {
- if (vm.getPowerState() == PowerState.PowerOn) {
- handlePowerOnReportWithNoPendingJobsOnVM(vm);
- } else {
- handlePowerOffReportWithNoPendingJobsOnVM(vm);
- }
+ List volumes = _volsDao.findUsableVolumesForInstance(vmId);
+ for (VolumeVO vol: volumes) {
+ VolumeInfo volumeInfo = volumeDataFactory.getVolume(vol.getId());
+ DataTO dataTO = volumeInfo.getTO();
+ DiskTO disk = storageMgr.getDiskWithThrottling(dataTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId());
+ vmProfile.addDisk(disk);
}
- }
+ Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vmId,
+ List.of(VirtualMachineProfile.Param.BootType.getName(), VirtualMachineProfile.Param.BootMode.getName(),
+ VirtualMachineProfile.Param.UefiFlag.getName()));
- private void scanStalledVMInTransitionStateOnDisconnectedHosts() {
- final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - VmOpWaitInterval.value() * 1000);
- final List stuckAndUncontrollableVMs = listStalledVMInTransitionStateOnDisconnectedHosts(cutTime);
- for (final Long vmId : stuckAndUncontrollableVMs) {
- final VMInstanceVO vm = _vmDao.findById(vmId);
-
- _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(),
- VM_SYNC_ALERT_SUBJECT, String.format("VM %s(%s) is stuck in %s state and its host is unreachable for too long",
- vm.getHostName(), vm, vm.getState()));
+ if (details.containsKey(VirtualMachineProfile.Param.BootType.getName())) {
+ vmProfile.getParameters().put(VirtualMachineProfile.Param.BootType, details.get(VirtualMachineProfile.Param.BootType.getName()));
}
- }
- private List listStalledVMInTransitionStateOnUpHost(
- final List transitioningVms, final long cutTime) {
- if (CollectionUtils.isEmpty(transitioningVms)) {
- return transitioningVms;
+ if (details.containsKey(VirtualMachineProfile.Param.BootMode.getName())) {
+ vmProfile.getParameters().put(VirtualMachineProfile.Param.BootMode, details.get(VirtualMachineProfile.Param.BootMode.getName()));
}
- List vmIdsInProgress = vmIdsInProgressCache.get();
- return transitioningVms.stream()
- .filter(v -> v.getPowerStateUpdateTime().getTime() < cutTime && !vmIdsInProgress.contains(v.getId()))
- .collect(Collectors.toList());
- }
- private List listVMInTransitionStateWithRecentReportOnUpHost(
- final List transitioningVms, final long cutTime) {
- if (CollectionUtils.isEmpty(transitioningVms)) {
- return transitioningVms;
+ if (details.containsKey(VirtualMachineProfile.Param.UefiFlag.getName())) {
+ vmProfile.getParameters().put(VirtualMachineProfile.Param.UefiFlag, details.get(VirtualMachineProfile.Param.UefiFlag.getName()));
}
- List vmIdsInProgress = vmIdsInProgressCache.get();
- return transitioningVms.stream()
- .filter(v -> v.getPowerStateUpdateTime().getTime() > cutTime && !vmIdsInProgress.contains(v.getId()))
- .collect(Collectors.toList());
- }
-
- private List listStalledVMInTransitionStateOnDisconnectedHosts(final Date cutTime) {
- final String sql = "SELECT i.* " +
- "FROM vm_instance AS i " +
- "INNER JOIN host AS h ON i.host_id = h.id " +
- "WHERE h.status != 'UP' " +
- " AND i.power_state_update_time < ? " +
- " AND i.state IN ('Starting', 'Stopping', 'Migrating') " +
- " AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job AS w " +
- " INNER JOIN async_job AS j ON w.id = j.id " +
- " WHERE j.job_status = ?) " +
- " AND i.removed IS NULL";
-
- final List l = new ArrayList<>();
- TransactionLegacy txn = TransactionLegacy.currentTxn();
- String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
- int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
- try {
- PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
+ return toVmTO(vmProfile);
+ }
- pstmt.setString(1, cutTimeStr);
- pstmt.setInt(2, jobStatusInProgress);
- final ResultSet rs = pstmt.executeQuery();
- while (rs.next()) {
- l.add(rs.getLong(1));
+ protected VirtualMachineTO getVmTO(Long vmId) {
+ final VMInstanceVO vm = _vmDao.findById(vmId);
+ final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+ final List nics = _nicsDao.listByVmId(profile.getId());
+ Collections.sort(nics, new Comparator() {
+ @Override
+ public int compare(NicVO nic1, NicVO nic2) {
+ Long nicId1 = Long.valueOf(nic1.getDeviceId());
+ Long nicId2 = Long.valueOf(nic2.getDeviceId());
+ return nicId1.compareTo(nicId2);
}
- } catch (final SQLException e) {
- logger.error("Unable to execute SQL [{}] with params {\"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
+ });
+
+ for (final NicVO nic : nics) {
+ final Network network = _networkModel.getNetwork(nic.getNetworkId());
+ final NicProfile nicProfile =
+ new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), null, _networkModel.isSecurityGroupSupportedInNetwork(network),
+ _networkModel.getNetworkTag(profile.getHypervisorType(), network));
+ profile.addNic(nicProfile);
}
- return l;
+ final VirtualMachineTO to = toVmTO(profile);
+ return to;
}
- public class VmStateSyncOutcome extends OutcomeImpl {
- private long _vmId;
+ public Command cleanup(final VirtualMachine vm, Map dpdkInterfaceMapping) {
+ return vmStopCommandService.buildCleanupCommand(vm, getExecuteInSequence(vm.getHypervisorType()), dpdkInterfaceMapping);
+ }
- public VmStateSyncOutcome(final AsyncJob job, final PowerState desiredPowerState, final long vmId, final Long srcHostIdForMigration) {
- super(VirtualMachine.class, job, VmJobCheckInterval.value(), new Predicate() {
- @Override
- public boolean checkCondition() {
- final AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId());
- return jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS;
- }
- }, Topics.VM_POWER_STATE, AsyncJob.Topics.JOB_STATE);
- _vmId = vmId;
- }
+ public Command cleanup(final String vmName) {
+ return vmStopCommandService.buildCleanupCommand(vmName, getExecuteInSequence(null));
+ }
- @Override
- protected VirtualMachine retrieve() {
- return _vmDao.findById(_vmId);
- }
+ public void syncVMMetaData(final Map vmMetadatum) {
+ vmMetadataSyncService.syncVMMetaData(vmMetadatum);
}
- public class VmJobVirtualMachineOutcome extends OutcomeImpl {
- private long _vmId;
+ @Override
+ public boolean isRecurring() {
+ return true;
+ }
- public VmJobVirtualMachineOutcome(final AsyncJob job, final long vmId) {
- super(VirtualMachine.class, job, VmJobCheckInterval.value(), new Predicate() {
- @Override
- public boolean checkCondition() {
- final AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, job.getId());
- return jobVo == null || jobVo.getStatus() != JobInfo.Status.IN_PROGRESS;
+ @Override
+ public boolean processAnswers(final long agentId, final long seq, final Answer[] answers) {
+ for (final Answer answer : answers) {
+ if ( answer instanceof ClusterVMMetaDataSyncAnswer) {
+ final ClusterVMMetaDataSyncAnswer cvms = (ClusterVMMetaDataSyncAnswer)answer;
+ if (!cvms.isExecuted()) {
+ syncVMMetaData(cvms.getVMMetaDatum());
+ cvms.setExecuted();
}
- }, AsyncJob.Topics.JOB_STATE);
- _vmId = vmId;
- }
-
- @Override
- protected VirtualMachine retrieve() {
- return _vmDao.findById(_vmId);
+ }
}
+ return true;
}
- public Outcome startVmThroughJobQueue(final String vmUuid,
- final Map params,
- final DeploymentPlan planToDeploy, final DeploymentPlanner planner) {
- String commandName = VmWorkStart.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, VmWorkJobVO.Step.Starting, vmId);
+ @Override
+ public boolean processTimeout(final long agentId, final long seq) {
+ return true;
+ }
- workJob = newVmWorkJobAndInfo.first();
- VmWorkStart workInfo = new VmWorkStart(newVmWorkJobAndInfo.second());
+ @Override
+ public int getTimeout() {
+ return -1;
+ }
- workInfo.setPlan(planToDeploy);
- workInfo.setParams(params);
- if (planner != null) {
- workInfo.setDeploymentPlanner(planner.getName());
+ @Override
+ public boolean processCommands(final long agentId, final long seq, final Command[] cmds) {
+ boolean processed = false;
+ for (final Command cmd : cmds) {
+ if (cmd instanceof PingRoutingCommand) {
+ final PingRoutingCommand ping = (PingRoutingCommand)cmd;
+ if (ping.getHostVmStateReport() != null) {
+ _syncMgr.processHostVmStatePingReport(agentId, ping.getHostVmStateReport(), ping.getOutOfBand());
+ }
+ vmPowerStateSyncManager.scanStalledVMInTransitionStateOnUpHost(agentId);
+ processed = true;
}
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
}
-
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmStateSyncOutcome(workJob,
- VirtualMachine.PowerState.PowerOn, vmId, null);
+ return processed;
}
- public Outcome stopVmThroughJobQueue(final String vmUuid, final boolean cleanup) {
- String commandName = VmWorkStop.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(null, vmUuid, null, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, VmWorkJobVO.Step.Prepare, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkStop workInfo = new VmWorkStop(newVmWorkJobAndInfo.second(), cleanup);
-
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
-
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmStateSyncOutcome(workJob,
- VirtualMachine.PowerState.PowerOff, vmId, null);
+ @Override
+ public AgentControlAnswer processControlCommand(final long agentId, final AgentControlCommand cmd) {
+ return null;
}
- public Outcome rebootVmThroughJobQueue(final String vmUuid,
- final Map params) {
- String commandName = VmWorkReboot.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, VmWorkJobVO.Step.Prepare, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkReboot workInfo = new VmWorkReboot(newVmWorkJobAndInfo.second(), params);
+ @Override
+ public boolean processDisconnect(final long agentId, final Status state) {
+ return true;
+ }
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
+ @Override
+ public void processHostAboutToBeRemoved(long hostId) {
+ }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
+ @Override
+ public void processHostRemoved(long hostId, long clusterId) {
+ }
- return new VmJobVirtualMachineOutcome(workJob,
- vmId);
+ @Override
+ public void processHostAdded(long hostId) {
}
- public Outcome migrateVmThroughJobQueue(final String vmUuid, final long srcHostId, final DeployDestination dest) {
- Map volumeStorageMap = dest.getStorageForDisks();
- if (volumeStorageMap != null) {
- for (Volume vol : volumeStorageMap.keySet()) {
- checkConcurrentJobsPerDatastoreThreshhold(volumeStorageMap.get(vol));
- }
+ @Override
+ public void processConnect(final Host agent, final StartupCommand cmd, final boolean forRebalance) throws ConnectionException {
+ if (!(cmd instanceof StartupRoutingCommand)) {
+ return;
}
- VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
- Long vmId = vm.getId();
-
- String commandName = VmWorkMigrate.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, vmUuid, VirtualMachine.Type.Instance, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
+ logger.debug("Received startup command from hypervisor host. host: {}", agent);
- workJob = newVmWorkJobAndInfo.first();
- VmWorkMigrate workInfo = new VmWorkMigrate(newVmWorkJobAndInfo.second(), srcHostId, dest);
+ _syncMgr.resetHostSyncState(agent);
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
+ if (forRebalance) {
+ logger.debug("Not processing listener {} as connect happens on rebalance process", this);
+ return;
}
+ final Long clusterId = agent.getClusterId();
+ final long agentId = agent.getId();
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmStateSyncOutcome(workJob,
- VirtualMachine.PowerState.PowerOn, vmId, vm.getPowerHostId());
- }
-
- public Outcome migrateVmAwayThroughJobQueue(final String vmUuid, final long srcHostId) {
- VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
- Long vmId = vm.getId();
-
- String commandName = VmWorkMigrateAway.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, vmUuid, VirtualMachine.Type.Instance, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkMigrateAway workInfo = new VmWorkMigrateAway(newVmWorkJobAndInfo.second(), srcHostId);
-
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
+ if (agent.getHypervisorType() == HypervisorType.XenServer) {
+ final ClusterVMMetaDataSyncCommand syncVMMetaDataCmd = new ClusterVMMetaDataSyncCommand(ClusterVMMetaDataSyncInterval.value(), clusterId);
+ try {
+ final long seq_no = _agentMgr.send(agentId, new Commands(syncVMMetaDataCmd), this);
+ logger.debug("Cluster VM metadata sync started with jobid {}", seq_no);
+ } catch (final AgentUnavailableException e) {
+ logger.fatal("The Cluster VM metadata sync process failed for cluster {} with {}", _clusterDao.findById(clusterId), e);
+ }
}
-
-
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmStateSyncOutcome(workJob, VirtualMachine.PowerState.PowerOn, vmId, vm.getPowerHostId());
}
- public Outcome migrateVmWithStorageThroughJobQueue(
- final String vmUuid, final long srcHostId, final long destHostId,
- final Map volumeToPool) {
- String commandName = VmWorkMigrateWithStorage.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
+ protected class TransitionTask extends ManagedContextRunnable {
+ @Override
+ protected void runInContext() {
+ final GlobalLock lock = GlobalLock.getInternLock("TransitionChecking");
+ if (lock == null) {
+ logger.debug("Couldn't get the global lock");
+ return;
+ }
- workJob = newVmWorkJobAndInfo.first();
- VmWorkMigrateWithStorage workInfo = new VmWorkMigrateWithStorage(newVmWorkJobAndInfo.second(), srcHostId, destHostId, volumeToPool);
+ if (!lock.lock(30)) {
+ logger.debug("Couldn't lock the db");
+ return;
+ }
+ try {
+ vmPowerStateSyncManager.scanStalledVMInTransitionStateOnDisconnectedHosts();
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
+ final List instances = _vmDao.findVMInTransition(new Date(DateUtil.currentGMTTime().getTime() - AgentManager.Wait.value() * 1000), State.Starting, State.Stopping);
+ for (final VMInstanceVO instance : instances) {
+ final State state = instance.getState();
+ if (state == State.Stopping) {
+ _haMgr.scheduleStop(instance, instance.getHostId(), WorkType.CheckStop);
+ } else if (state == State.Starting) {
+ _haMgr.scheduleRestart(instance, true);
+ }
+ }
+ } catch (final Exception e) {
+ logger.warn("Caught the following exception on transition checking", e);
+ } finally {
+ lock.unlock();
+ }
}
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmStateSyncOutcome(workJob,
- VirtualMachine.PowerState.PowerOn, vmId, destHostId);
}
- public Outcome migrateVmForScaleThroughJobQueue(
- final String vmUuid, final long srcHostId, final DeployDestination dest, final Long newSvcOfferingId) {
- String commandName = VmWorkMigrateForScale.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkMigrateForScale workInfo = new VmWorkMigrateForScale(newVmWorkJobAndInfo.second(), srcHostId, dest, newSvcOfferingId);
-
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmJobVirtualMachineOutcome(workJob, vmId);
+ @Override
+ public VMInstanceVO findById(final long vmId) {
+ return _vmDao.findById(vmId);
}
- private void checkConcurrentJobsPerDatastoreThreshhold(final StoragePool destPool) {
- final Long threshold = VolumeApiService.ConcurrentMigrationsThresholdPerDatastore.value();
- if (threshold != null && threshold > 0) {
- long count = _jobMgr.countPendingJobs("\"storageid\":\"" + destPool.getUuid() + "\"", MigrateVMCmd.class.getName(), MigrateVolumeCmd.class.getName(), MigrateVolumeCmdByAdmin.class.getName());
- if (count > threshold) {
- throw new CloudRuntimeException("Number of concurrent migration jobs per datastore exceeded the threshold: " + threshold.toString() + ". Please try again after some time.");
- }
- }
+ @Override
+ public void checkIfCanUpgrade(final VirtualMachine vmInstance, final ServiceOffering newServiceOffering) {
+ vmServiceOfferingUpgradeManager.checkIfCanUpgrade(vmInstance, newServiceOffering);
}
- public Outcome migrateVmStorageThroughJobQueue(final String vmUuid, final Map volumeToPool) {
- Collection poolIds = volumeToPool.values();
- Set uniquePoolIds = new HashSet<>(poolIds);
- for (Long poolId : uniquePoolIds) {
- StoragePoolVO pool = _storagePoolDao.findById(poolId);
- checkConcurrentJobsPerDatastoreThreshhold(pool);
- }
-
- String commandName = VmWorkStorageMigration.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkStorageMigration workInfo = new VmWorkStorageMigration(newVmWorkJobAndInfo.second(), volumeToPool);
+ /**
+ * Throws an InvalidParameterValueException in case the new service offerings does not match the storage scope (e.g. local or shared).
+ */
+ protected void checkIfNewOfferingStorageScopeMatchesStoragePool(VirtualMachine vmInstance, DiskOffering newDiskOffering) {
+ vmServiceOfferingUpgradeManager.checkIfNewOfferingStorageScopeMatchesStoragePool(vmInstance, newDiskOffering);
+ }
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
+ public boolean isRootVolumeOnLocalStorage(long vmId) {
+ return vmServiceOfferingUpgradeManager.isRootVolumeOnLocalStorage(vmId);
+ }
- return new VmJobVirtualMachineOutcome(workJob, vmId);
+ @Override
+ public boolean upgradeVmDb(final long vmId, final ServiceOffering newServiceOffering, ServiceOffering currentServiceOffering) {
+ return vmServiceOfferingUpgradeManager.upgradeVmDb(vmId, newServiceOffering, currentServiceOffering);
}
- public Outcome addVmToNetworkThroughJobQueue(
- final VirtualMachine vm, final Network network, final NicProfile requested) {
- Long vmId = vm.getId();
- String commandName = VmWorkAddVmToNetwork.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName);
+ @Override
+ public NicProfile addVmToNetwork(final VirtualMachine vm, final Network network, final NicProfile requested)
+ throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException {
- final CallContext context = CallContext.current();
- final User user = context.getCallingUser();
- final Account account = context.getCallingAccount();
+ final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
+ if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
+ VmWorkJobVO placeHolder = vmWorkJobQueueService.createPlaceHolderWork(vm.getId(), network.getUuid());
+ try {
+ return orchestrateAddVmToNetwork(vm, network, requested);
+ } finally {
+ vmWorkJobQueueService.expungePlaceHolderWork(placeHolder);
+ }
+ } else {
+ final Outcome outcome = vmWorkJobQueueService.addVmToNetworkThroughJobQueue(vm, network, requested);
- final List pendingWorkJobs = _workJobDao.listPendingWorkJobs(
- VirtualMachine.Type.Instance, vm.getId(),
- VmWorkAddVmToNetwork.class.getName(), network.getUuid());
+ vmWorkJobQueueService.retrieveVmFromJobOutcome(outcome, vm.getUuid(), "addVmToNetwork");
- VmWorkJobVO workJob = null;
- if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) {
- if (pendingWorkJobs.size() > 1) {
- throw new CloudRuntimeException(String.format("The number of jobs to add network %s to vm %s are %d", network.getUuid(), vm.getInstanceName(), pendingWorkJobs.size()));
+ Object jobResult = vmWorkJobQueueService.retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
+
+ if (jobResult != null && jobResult instanceof NicProfile) {
+ return (NicProfile) jobResult;
}
- workJob = pendingWorkJobs.get(0);
- } else {
- logger.trace("no jobs to add network {} for vm {} yet", network, vm);
- workJob = createVmWorkJobToAddNetwork(vm, network, requested, context, user, account);
+ throw new RuntimeException("null job execution result");
}
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmJobVirtualMachineOutcome(workJob, vm.getId());
}
- private VmWorkJobVO createVmWorkJobToAddNetwork(
- VirtualMachine vm,
- Network network,
- NicProfile requested,
- CallContext context,
- User user,
- Account account) {
- VmWorkJobVO workJob;
- workJob = new VmWorkJobVO(context.getContextId());
-
- workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER);
- workJob.setCmd(VmWorkAddVmToNetwork.class.getName());
+ private NicProfile orchestrateAddVmToNetwork(final VirtualMachine vm, final Network network, final NicProfile requested) throws ConcurrentOperationException, ResourceUnavailableException,
+ InsufficientCapacityException {
+ return vmNetworkAttachmentOrchestrationService.addVmToNetwork(vm, network, requested, this);
+ }
- workJob.setAccountId(account.getId());
- workJob.setUserId(user.getId());
- workJob.setVmType(VirtualMachine.Type.Instance);
- workJob.setVmInstanceId(vm.getId());
- workJob.setRelated(AsyncJobExecutionContext.getOriginJobId());
- workJob.setSecondaryObjectIdentifier(network.getUuid());
+ @Override
+ public NicTO toNicTO(final NicProfile nic, final HypervisorType hypervisorType) {
+ return vmNetworkAttachmentOrchestrationService.toNicTO(nic, hypervisorType);
+ }
- // save work context info as there might be some duplicates
- final VmWorkAddVmToNetwork workInfo = new VmWorkAddVmToNetwork(user.getId(), account.getId(), vm.getId(),
- VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, network.getId(), requested);
- workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo));
+ @Override
+ public boolean removeNicFromVm(final VirtualMachine vm, final Nic nic)
+ throws ConcurrentOperationException, ResourceUnavailableException {
- try {
- _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId());
- } catch (CloudRuntimeException e) {
- if (e.getCause() instanceof EntityExistsException) {
- String msg = String.format("A job to add a nic for network %s to vm %s already exists", network.getUuid(), vm.getUuid());
- logger.warn(msg, e);
+ final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
+ if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
+ VmWorkJobVO placeHolder = vmWorkJobQueueService.createPlaceHolderWork(vm.getId());
+ try {
+ return orchestrateRemoveNicFromVm(vm, nic);
+ } finally {
+ vmWorkJobQueueService.expungePlaceHolderWork(placeHolder);
}
- throw e;
- }
-
- return workJob;
- }
-
- public Outcome removeNicFromVmThroughJobQueue(
- final VirtualMachine vm, final Nic nic) {
- Long vmId = vm.getId();
- String commandName = VmWorkRemoveNicFromVm.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName);
- VmWorkJobVO workJob = pendingWorkJob.first();
+ } else {
+ final Outcome outcome = vmWorkJobQueueService.removeNicFromVmThroughJobQueue(vm, nic);
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
+ vmWorkJobQueueService.retrieveVmFromJobOutcome(outcome, vm.getUuid(), "removeNicFromVm");
- workJob = newVmWorkJobAndInfo.first();
- VmWorkRemoveNicFromVm workInfo = new VmWorkRemoveNicFromVm(newVmWorkJobAndInfo.second(), nic.getId());
+ try {
+ Object jobResult = vmWorkJobQueueService.retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
+ if (jobResult != null && jobResult instanceof Boolean) {
+ return (Boolean) jobResult;
+ }
+ } catch (InsufficientCapacityException ex) {
+ throw new RuntimeException("Unexpected exception", ex);
+ }
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
+ throw new RuntimeException("Job failed with un-handled exception");
}
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
+ }
+
+ private boolean orchestrateRemoveNicFromVm(final VirtualMachine vm, final Nic nic) throws ConcurrentOperationException, ResourceUnavailableException {
+ return vmNetworkAttachmentOrchestrationService.removeNicFromVm(vm, nic, this);
+ }
- return new VmJobVirtualMachineOutcome(workJob, vmId);
+ @Override
+ @DB
+ public boolean removeVmFromNetwork(final VirtualMachine vm, final Network network, final URI broadcastUri) throws ConcurrentOperationException, ResourceUnavailableException {
+ return orchestrateRemoveVmFromNetwork(vm, network, broadcastUri);
}
- public Outcome removeVmFromNetworkThroughJobQueue(
- final VirtualMachine vm, final Network network, final URI broadcastUri) {
- Long vmId = vm.getId();
- String commandName = VmWorkRemoveVmFromNetwork.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName);
+ @DB
+ private boolean orchestrateRemoveVmFromNetwork(final VirtualMachine vm, final Network network, final URI broadcastUri) throws ConcurrentOperationException, ResourceUnavailableException {
+ return vmNetworkAttachmentOrchestrationService.removeVmFromNetwork(vm, network, broadcastUri, this);
+ }
- VmWorkJobVO workJob = pendingWorkJob.first();
+ @Override
+ public void findHostAndMigrate(final String vmUuid, final Long newSvcOfferingId, final Map customParameters, final ExcludeList excludes) throws InsufficientCapacityException, ConcurrentOperationException,
+ ResourceUnavailableException {
+ vmScaleReconfigurationService.findHostAndMigrate(vmUuid, newSvcOfferingId, customParameters, excludes);
+ }
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
+ @Override
+ public void migrateForScale(final String vmUuid, final long srcHostId, final DeployDestination dest, final Long oldSvcOfferingId)
+ throws ResourceUnavailableException, ConcurrentOperationException {
+ vmScaleReconfigurationService.migrateForScale(vmUuid, srcHostId, dest, oldSvcOfferingId);
+ }
- workJob = newVmWorkJobAndInfo.first();
- VmWorkRemoveVmFromNetwork workInfo = new VmWorkRemoveVmFromNetwork(newVmWorkJobAndInfo.second(), network, broadcastUri);
+ @Override
+ public boolean replugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final Host host) throws ConcurrentOperationException,
+ ResourceUnavailableException, InsufficientCapacityException {
+ return vmNicBackendCommandService.replugNic(network, nic, vm, host);
+ }
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
+ @Override
+ public boolean plugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException,
+ ResourceUnavailableException, InsufficientCapacityException {
+ return vmNicBackendCommandService.plugNic(network, nic, vm, context, dest);
+ }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
+ @Override
+ public boolean unplugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest) throws ConcurrentOperationException,
+ ResourceUnavailableException {
+ return vmNicBackendCommandService.unplugNic(network, nic, vm, context, dest);
+ }
- return new VmJobVirtualMachineOutcome(workJob, vmId);
+ @Override
+ public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final ServiceOffering newServiceOffering,
+ Map customParameters, final boolean reconfiguringOnExistingHost)
+ throws ResourceUnavailableException, InsufficientServerCapacityException, ConcurrentOperationException {
+ return vmScaleReconfigurationService.reConfigureVm(vmUuid, oldServiceOffering, newServiceOffering, customParameters, reconfiguringOnExistingHost);
}
- public Outcome reconfigureVmThroughJobQueue(
- final String vmUuid, final ServiceOffering oldServiceOffering, final ServiceOffering newServiceOffering, Map customParameters, final boolean reconfiguringOnExistingHost) {
- String commandName = VmWorkReconfigure.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmUuid, commandName);
+ @Override
+ public String getConfigComponentName() {
+ return VirtualMachineManager.class.getSimpleName();
+ }
- VmWorkJobVO workJob = pendingWorkJob.first();
- Long vmId = pendingWorkJob.second();
+ @Override
+ public ConfigKey>[] getConfigKeys() {
+ return new ConfigKey>[] { ClusterDeltaSyncInterval, StartRetry, VmDestroyForcestop, VmOpCancelInterval, VmOpCleanupInterval, VmOpCleanupWait,
+ VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval,
+ VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
+ HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
+ AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName,
+ VmSyncPowerStateTransitioning, SystemVmEnableUserData
+ };
+ }
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
+ public List getStoragePoolAllocators() {
+ return _storagePoolAllocators;
+ }
- workJob = newVmWorkJobAndInfo.first();
- VmWorkReconfigure workInfo = new VmWorkReconfigure(newVmWorkJobAndInfo.second(), oldServiceOffering.getId(), newServiceOffering.getId(), customParameters, reconfiguringOnExistingHost);
+ @Inject
+ public void setStoragePoolAllocators(final List storagePoolAllocators) {
+ _storagePoolAllocators = storagePoolAllocators;
+ }
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
+ /**
+ * PowerState report handling for out-of-band changes and handling of left-over transitional VM states
+ */
- return new VmJobVirtualMachineOutcome(workJob, vmId);
+ @MessageHandler(topic = Topics.VM_POWER_STATE)
+ protected void HandlePowerStateReport(final String subject, final String senderAddress, final Object args) {
+ assert args != null;
+ vmPowerStateSyncManager.handlePowerStateReport((Long) args);
}
@ReflectionUse
@@ -6052,7 +2656,7 @@ private Pair orchestrateMigrateWithStorage(final VmWorkM
@ReflectionUse
private Pair orchestrateMigrateForScale(final VmWorkMigrateForScale work) throws Exception {
VMInstanceVO vm = findVmById(work.getVmId());
- orchestrateMigrateForScale(vm.getUuid(),
+ vmScaleReconfigurationService.orchestrateMigrateForScale(vm.getUuid(),
work.getSrcHostId(),
work.getDeployDestination(),
work.getNewServiceOfferringId());
@@ -6062,7 +2666,7 @@ private Pair orchestrateMigrateForScale(final VmWorkMigr
@ReflectionUse
private Pair orchestrateReboot(final VmWorkReboot work) throws Exception {
VMInstanceVO vm = findVmById(work.getVmId());
- orchestrateReboot(vm.getUuid(), work.getParams());
+ vmRebootOrchestrationService.orchestrateReboot(vm.getUuid(), work.getParams());
return new Pair<>(JobInfo.Status.SUCCEEDED, null);
}
@@ -6121,51 +2725,24 @@ public Pair handleVmWorkJob(final VmWork work) throws Ex
return _jobHandlerProxy.handleVmWorkJob(work);
}
- private VmWorkJobVO createPlaceHolderWork(final long instanceId) {
- return createPlaceHolderWork(instanceId, null);
- }
-
- private VmWorkJobVO createPlaceHolderWork(final long instanceId, String secondaryObjectIdentifier) {
- final VmWorkJobVO workJob = new VmWorkJobVO("");
-
- workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_PLACEHOLDER);
- workJob.setCmd("");
- workJob.setCmdInfo("");
-
- workJob.setAccountId(0);
- workJob.setUserId(0);
- workJob.setStep(VmWorkJobVO.Step.Starting);
- workJob.setVmType(VirtualMachine.Type.Instance);
- workJob.setVmInstanceId(instanceId);
- if(org.apache.commons.lang3.StringUtils.isNotBlank(secondaryObjectIdentifier)) {
- workJob.setSecondaryObjectIdentifier(secondaryObjectIdentifier);
- }
- workJob.setInitMsid(ManagementServerNode.getManagementServerId());
-
- _workJobDao.persist(workJob);
-
- return workJob;
- }
@Override
public UserVm restoreVirtualMachine(final long vmId, final Long newTemplateId, final Long rootDiskOfferingId, final boolean expunge, final Map details) throws ResourceUnavailableException, InsufficientCapacityException {
final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
VmWorkJobVO placeHolder = null;
- placeHolder = createPlaceHolderWork(vmId);
+ placeHolder = vmWorkJobQueueService.createPlaceHolderWork(vmId);
try {
return orchestrateRestoreVirtualMachine(vmId, newTemplateId, rootDiskOfferingId, expunge, details);
} finally {
- if (placeHolder != null) {
- _workJobDao.expunge(placeHolder.getId());
- }
+ vmWorkJobQueueService.expungePlaceHolderWork(placeHolder);
}
} else {
- final Outcome outcome = restoreVirtualMachineThroughJobQueue(vmId, newTemplateId, rootDiskOfferingId, expunge, details);
+ final Outcome outcome = vmWorkJobQueueService.restoreVirtualMachineThroughJobQueue(vmId, newTemplateId, rootDiskOfferingId, expunge, details);
- retrieveVmFromJobOutcome(outcome, String.valueOf(vmId), "restoreVirtualMachine");
+ vmWorkJobQueueService.retrieveVmFromJobOutcome(outcome, String.valueOf(vmId), "restoreVirtualMachine");
- Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
+ Object jobResult = vmWorkJobQueueService.retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
if (jobResult != null && jobResult instanceof HashMap) {
HashMap passwordMap = (HashMap)jobResult;
@@ -6185,24 +2762,6 @@ private UserVm orchestrateRestoreVirtualMachine(final long vmId, final Long newT
return _userVmService.restoreVirtualMachine(account, vmId, newTemplateId, rootDiskOfferingId, expunge, details);
}
- public Outcome restoreVirtualMachineThroughJobQueue(final long vmId, final Long newTemplateId, final Long rootDiskOfferingId, final boolean expunge, Map details) {
- String commandName = VmWorkRestore.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkRestore workInfo = new VmWorkRestore(newVmWorkJobAndInfo.second(), newTemplateId, rootDiskOfferingId, expunge, details);
-
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmJobVirtualMachineOutcome(workJob, vmId);
- }
@ReflectionUse
private Pair orchestrateRestoreVirtualMachine(final VmWorkRestore work) throws Exception {
@@ -6218,21 +2777,19 @@ public Boolean updateDefaultNicForVM(final VirtualMachine vm, final Nic nic, fin
final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
- VmWorkJobVO placeHolder = createPlaceHolderWork(vm.getId());
+ VmWorkJobVO placeHolder = vmWorkJobQueueService.createPlaceHolderWork(vm.getId());
try {
- return orchestrateUpdateDefaultNicForVM(vm, nic, defaultNic);
+ return vmNicUpdateService.updateDefaultNicForVM(vm, nic, defaultNic);
} finally {
- if (placeHolder != null) {
- _workJobDao.expunge(placeHolder.getId());
- }
+ vmWorkJobQueueService.expungePlaceHolderWork(placeHolder);
}
} else {
- final Outcome outcome = updateDefaultNicForVMThroughJobQueue(vm, nic, defaultNic);
+ final Outcome outcome = vmWorkJobQueueService.updateDefaultNicForVMThroughJobQueue(vm, nic, defaultNic);
- retrieveVmFromJobOutcome(outcome, vm.getUuid(), "updateDefaultNicForVM");
+ vmWorkJobQueueService.retrieveVmFromJobOutcome(outcome, vm.getUuid(), "updateDefaultNicForVM");
try {
- Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
+ Object jobResult = vmWorkJobQueueService.retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
if (jobResult != null && jobResult instanceof Boolean) {
return (Boolean)jobResult;
@@ -6245,44 +2802,6 @@ public Boolean updateDefaultNicForVM(final VirtualMachine vm, final Nic nic, fin
}
}
- private Boolean orchestrateUpdateDefaultNicForVM(final VirtualMachine vm, final Nic nic, final Nic defaultNic) {
-
- logger.debug("Updating default nic of vm {} from nic {} to nic {}", vm, defaultNic.getUuid(), nic.getUuid());
- Integer chosenID = nic.getDeviceId();
- Integer existingID = defaultNic.getDeviceId();
- NicVO nicVO = _nicsDao.findById(nic.getId());
- NicVO defaultNicVO = _nicsDao.findById(defaultNic.getId());
-
- nicVO.setDefaultNic(true);
- nicVO.setDeviceId(existingID);
- defaultNicVO.setDefaultNic(false);
- defaultNicVO.setDeviceId(chosenID);
-
- _nicsDao.persist(nicVO);
- _nicsDao.persist(defaultNicVO);
- return true;
- }
-
- public Outcome updateDefaultNicForVMThroughJobQueue(final VirtualMachine vm, final Nic nic, final Nic defaultNic) {
- Long vmId = vm.getId();
- String commandName = VmWorkUpdateDefaultNic.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkUpdateDefaultNic workInfo = new VmWorkUpdateDefaultNic(newVmWorkJobAndInfo.second(), nic.getId(), defaultNic.getId());
-
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmJobVirtualMachineOutcome(workJob, vmId);
- }
-
@ReflectionUse
private Pair orchestrateUpdateDefaultNic(final VmWorkUpdateDefaultNic work) throws Exception {
VMInstanceVO vm = findVmById(work.getVmId());
@@ -6294,19 +2813,19 @@ private Pair orchestrateUpdateDefaultNic(final VmWorkUpd
if (defaultNic == null) {
throw new CloudRuntimeException("Unable to find default nic " + work.getDefaultNicId());
}
- final boolean result = orchestrateUpdateDefaultNicForVM(vm, nic, defaultNic);
+ final boolean result = vmNicUpdateService.updateDefaultNicForVM(vm, nic, defaultNic);
return new Pair<>(JobInfo.Status.SUCCEEDED,
_jobMgr.marshallResultObject(result));
}
@Override
public boolean updateVmNic(VirtualMachine vm, Nic nic, Boolean enabled) {
- Outcome outcome = updateVmNicThroughJobQueue(vm, nic, enabled);
+ Outcome outcome = vmWorkJobQueueService.updateVmNicThroughJobQueue(vm, nic, enabled);
- retrieveVmFromJobOutcome(outcome, vm.getUuid(), "updateVmNic");
+ vmWorkJobQueueService.retrieveVmFromJobOutcome(outcome, vm.getUuid(), "updateVmNic");
try {
- Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
+ Object jobResult = vmWorkJobQueueService.retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
if (jobResult instanceof Boolean) {
return BooleanUtils.isTrue((Boolean) jobResult);
}
@@ -6316,52 +2835,6 @@ public boolean updateVmNic(VirtualMachine vm, Nic nic, Boolean enabled) {
throw new CloudRuntimeException("Unexpected job execution result.");
}
- private boolean orchestrateUpdateVmNic(final VirtualMachine vm, final Nic nic, final Boolean enabled) throws ResourceUnavailableException {
- if (vm.getState() == State.Running) {
- try {
- UpdateVmNicCommand updateVmNicCmd = new UpdateVmNicCommand(nic.getMacAddress(), vm.getName(), enabled);
- Commands cmds = new Commands(Command.OnError.Stop);
- cmds.addCommand("updatevmnic", updateVmNicCmd);
-
- _agentMgr.send(vm.getHostId(), cmds);
-
- UpdateVmNicAnswer updateVmNicAnswer = cmds.getAnswer(UpdateVmNicAnswer.class);
- if (updateVmNicAnswer == null || !updateVmNicAnswer.getResult()) {
- logger.warn("Unable to update VM %s NIC [{}].", vm.getName(), nic.getUuid());
- return false;
- }
- } catch (final OperationTimedoutException e) {
- throw new AgentUnavailableException(String.format("Unable to update NIC %s for VM %s.", nic.getUuid(), vm.getUuid()), vm.getHostId(), e);
- }
- }
-
- NicVO nicVo = _nicsDao.findById(nic.getId());
- nicVo.setEnabled(enabled);
- _nicsDao.persist(nicVo);
-
- return true;
- }
-
- public Outcome updateVmNicThroughJobQueue(final VirtualMachine vm, final Nic nic, final Boolean isNicEnabled) {
- Long vmId = vm.getId();
- String commandName = VmWorkUpdateNic.class.getName();
- Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName);
-
- VmWorkJobVO workJob = pendingWorkJob.first();
-
- if (workJob == null) {
- Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId);
-
- workJob = newVmWorkJobAndInfo.first();
- VmWorkUpdateNic workInfo = new VmWorkUpdateNic(newVmWorkJobAndInfo.second(), nic.getId(), isNicEnabled);
-
- setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId);
- }
- AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId());
-
- return new VmJobVirtualMachineOutcome(workJob, vmId);
- }
-
@ReflectionUse
private Pair orchestrateUpdateVmNic(final VmWorkUpdateNic work) throws Exception {
VMInstanceVO vm = findVmById(work.getVmId());
@@ -6369,180 +2842,18 @@ private Pair orchestrateUpdateVmNic(final VmWorkUpdateNi
if (nic == null) {
throw new CloudRuntimeException(String.format("Unable to find NIC with ID %s.", work.getNicId()));
}
- final boolean result = orchestrateUpdateVmNic(vm, nic, work.isEnabled());
+ final boolean result = vmNicUpdateService.updateVmNic(vm, nic, work.isEnabled());
return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result));
}
- private Pair findClusterAndHostIdForVmFromVolumes(long vmId) {
- Long clusterId = null;
- Long hostId = null;
- List volumes = _volsDao.findByInstance(vmId);
- for (VolumeVO volume : volumes) {
- if (Volume.State.Ready.equals(volume.getState()) &&
- volume.getPoolId() != null) {
- StoragePoolVO pool = _storagePoolDao.findById(volume.getPoolId());
- if (pool != null && pool.getClusterId() != null) {
- clusterId = pool.getClusterId();
- // hostId to be used only for sending commands, capacity check skipped
- List hosts = _hostDao.findHypervisorHostInCluster(pool.getClusterId());
- if (CollectionUtils.isNotEmpty(hosts)) {
- hostId = hosts.get(0).getId();
- break;
- }
- }
- }
- }
- return new Pair<>(clusterId, hostId);
- }
-
@Override
public Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm) {
- Long hostId = null;
- Host host = null;
- if (!skipCurrentHostForStartingVm || !State.Starting.equals(vm.getState())) {
- hostId = vm.getHostId();
- }
- Long clusterId = null;
- if (hostId == null) {
- if (vm.getLastHostId() == null) {
- return findClusterAndHostIdForVmFromVolumes(vm.getId());
- }
- hostId = vm.getLastHostId();
- host = _hostDao.findById(hostId);
- logger.debug("host id is null, using last host {} with id {}", host, hostId);
- }
- host = host == null ? _hostDao.findById(hostId) : host;
- if (host != null) {
- clusterId = host.getClusterId();
- return new Pair<>(clusterId, hostId);
- }
- return findClusterAndHostIdForVmFromVolumes(vm.getId());
- }
-
- private Pair findClusterAndHostIdForVm(VirtualMachine vm) {
- return findClusterAndHostIdForVm(vm, false);
+ return vmDiskOfferingSuitabilityService.findClusterAndHostIdForVm(vm, skipCurrentHostForStartingVm);
}
@Override
public Pair findClusterAndHostIdForVm(long vmId) {
- VMInstanceVO vm = _vmDao.findById(vmId);
- if (vm == null) {
- return new Pair<>(null, null);
- }
- return findClusterAndHostIdForVm(vm);
- }
-
- protected VirtualMachine retrieveVmFromJobOutcome(Outcome jobOutcome, String vmUuid, String jobName) {
- try {
- return jobOutcome.get();
- } catch (InterruptedException | java.util.concurrent.ExecutionException e) {
- throw new RuntimeException(String.format("Unable to retrieve result from job \"%s\" due to [%s]. VM {\"uuid\": \"%s\"}.", jobName, e.getMessage(), vmUuid), e);
- }
- }
-
- protected Object retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(Outcome outcome) throws ResourceUnavailableException, InsufficientCapacityException{
- Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob());
-
- if (jobResult == null) {
- return null;
- }
-
- if (jobResult instanceof AgentUnavailableException) {
- throw (AgentUnavailableException) jobResult;
- }
-
- if (jobResult instanceof InsufficientServerCapacityException) {
- throw (InsufficientServerCapacityException) jobResult;
- }
-
- if (jobResult instanceof ResourceUnavailableException) {
- throw (ResourceUnavailableException) jobResult;
- }
-
- if (jobResult instanceof InsufficientCapacityException) {
- throw (InsufficientCapacityException) jobResult;
- }
-
- if (jobResult instanceof ConcurrentOperationException) {
- throw (ConcurrentOperationException) jobResult;
- }
-
- if (jobResult instanceof RuntimeException) {
- throw (RuntimeException) jobResult;
- }
-
- if (jobResult instanceof Throwable) {
- throw new RuntimeException("Unexpected exception", (Throwable)jobResult);
- }
-
- return jobResult;
- }
-
- protected Pair retrievePendingWorkJob(String vmUuid, String commandName) {
- return retrievePendingWorkJob(null, vmUuid, VirtualMachine.Type.Instance, commandName);
- }
-
- protected Pair retrievePendingWorkJob(Long id, String commandName) {
- return retrievePendingWorkJob(id, null, VirtualMachine.Type.Instance, commandName);
- }
-
- protected Pair retrievePendingWorkJob(Long vmId, String vmUuid, VirtualMachine.Type vmType, String commandName) {
- if (vmId == null) {
- VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
-
- if (vm == null) {
- String message = String.format("Could not find a VM with the uuid [%s]. Unable to continue validations with command [%s] through job queue.", vmUuid, commandName);
- logger.error(message);
- throw new RuntimeException(message);
- }
-
- vmId = vm.getId();
-
- if (vmType == null) {
- vmType = vm.getType();
- }
- }
-
- List pendingWorkJobs = _workJobDao.listPendingWorkJobs(vmType, vmId, commandName);
-
- if (CollectionUtils.isNotEmpty(pendingWorkJobs)) {
- return new Pair<>(pendingWorkJobs.get(0), vmId);
- }
-
- return new Pair<>(null, vmId);
- }
-
- protected Pair createWorkJobAndWorkInfo(String commandName, Long vmId) {
- return createWorkJobAndWorkInfo(commandName, null, vmId);
- }
-
- protected Pair createWorkJobAndWorkInfo(String commandName, VmWorkJobVO.Step step, Long vmId) {
- CallContext context = CallContext.current();
- long userId = context.getCallingUser().getId();
- long accountId = context.getCallingAccount().getId();
-
- VmWorkJobVO workJob = new VmWorkJobVO(context.getContextId());
- workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER);
- workJob.setCmd(commandName);
- workJob.setAccountId(accountId);
- workJob.setUserId(userId);
-
- if (step != null) {
- workJob.setStep(step);
- }
-
- workJob.setVmType(VirtualMachine.Type.Instance);
- workJob.setVmInstanceId(vmId);
- workJob.setRelated(AsyncJobExecutionContext.getOriginJobId());
-
- VmWork workInfo = new VmWork(userId, accountId, vmId, VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER);
-
- return new Pair<>(workJob, workInfo);
- }
-
- protected void setCmdInfoAndSubmitAsyncJob(VmWorkJobVO workJob, VmWork workInfo, Long vmId) {
- workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo));
- _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vmId);
+ return vmDiskOfferingSuitabilityService.findClusterAndHostIdForVm(vmId);
}
protected VMInstanceVO findVmById(Long vmId) {
@@ -6558,129 +2869,31 @@ protected VMInstanceVO findVmById(Long vmId) {
@Override
public HashMap getVirtualMachineStatistics(Host host, List vmIds) {
- HashMap vmStatsById = new HashMap<>();
- if (CollectionUtils.isEmpty(vmIds)) {
- return vmStatsById;
- }
- Map vmMap = _vmDao.getNameIdMapForVmIds(vmIds);
- return getVirtualMachineStatistics(host, vmMap);
+ return vmStatsCollector.getVirtualMachineStatistics(host, vmIds);
}
@Override
public HashMap getVirtualMachineStatistics(Host host, Map vmInstanceNameIdMap) {
- HashMap vmStatsById = new HashMap<>();
- if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
- return vmStatsById;
- }
- Answer answer = _agentMgr.easySend(host.getId(), new GetVmStatsCommand(
- new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
- if (answer == null || !answer.getResult()) {
- logger.warn("Unable to obtain VM statistics.");
- return vmStatsById;
- } else {
- HashMap vmStatsByName = ((GetVmStatsAnswer)answer).getVmStatsMap();
- if (vmStatsByName == null) {
- logger.warn("Unable to obtain VM statistics.");
- return vmStatsById;
- }
- for (Map.Entry entry : vmStatsByName.entrySet()) {
- vmStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
- }
- }
- return vmStatsById;
+ return vmStatsCollector.getVirtualMachineStatistics(host, vmInstanceNameIdMap);
}
@Override
public HashMap> getVmDiskStatistics(Host host, Map vmInstanceNameIdMap) {
- HashMap> vmDiskStatsById = new HashMap<>();
- if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
- return vmDiskStatsById;
- }
- Answer answer = _agentMgr.easySend(host.getId(), new GetVmDiskStatsCommand(
- new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
- if (answer == null || !answer.getResult()) {
- logger.warn("Unable to obtain VM disk statistics.");
- return vmDiskStatsById;
- } else {
- HashMap> vmDiskStatsByName = ((GetVmDiskStatsAnswer)answer).getVmDiskStatsMap();
- if (vmDiskStatsByName == null) {
- logger.warn("Unable to obtain VM disk statistics.");
- return vmDiskStatsById;
- }
- for (Map.Entry> entry: vmDiskStatsByName.entrySet()) {
- vmDiskStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
- }
- }
- return vmDiskStatsById;
+ return vmStatsCollector.getVmDiskStatistics(host, vmInstanceNameIdMap);
}
@Override
public HashMap> getVmNetworkStatistics(Host host, Map vmInstanceNameIdMap) {
- HashMap> vmNetworkStatsById = new HashMap<>();
- if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
- return vmNetworkStatsById;
- }
- Answer answer = _agentMgr.easySend(host.getId(), new GetVmNetworkStatsCommand(
- new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
- if (answer == null || !answer.getResult()) {
- logger.warn("Unable to obtain VM network statistics.");
- return vmNetworkStatsById;
- } else {
- HashMap> vmNetworkStatsByName = ((GetVmNetworkStatsAnswer)answer).getVmNetworkStatsMap();
- if (vmNetworkStatsByName == null) {
- logger.warn("Unable to obtain VM network statistics.");
- return vmNetworkStatsById;
- }
- for (Map.Entry> entry: vmNetworkStatsByName.entrySet()) {
- vmNetworkStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
- }
- }
- return vmNetworkStatsById;
+ return vmStatsCollector.getVmNetworkStatistics(host, vmInstanceNameIdMap);
}
protected boolean isDiskOfferingSuitableForVm(VMInstanceVO vm, VirtualMachineProfile profile, long podId, long clusterId, long hostId, long diskOfferingId) {
-
- DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId);
- VolumeVO dummyVolume = new VolumeVO("Data", vm.getDataCenterId(), podId, vm.getAccountId(),
- vm.getDomainId(), vm.getId(), null, null, diskOffering.getProvisioningType(), diskOffering.getDiskSize(), Type.DATADISK);
- try {
- Field idField = dummyVolume.getClass().getDeclaredField("id");
- idField.setAccessible(true);
- idField.set(dummyVolume, Volume.DISK_OFFERING_SUITABILITY_CHECK_VOLUME_ID);
- } catch (NoSuchFieldException | IllegalAccessException ignored) {
- return false;
- }
- dummyVolume.setDiskOfferingId(diskOfferingId);
- DiskProfile diskProfile = new DiskProfile(dummyVolume, diskOffering, profile.getHypervisorType());
- diskProfile.setMinIops(diskOffering.getMinIops());
- diskProfile.setMaxIops(diskOffering.getMaxIops());
- ExcludeList avoid = new ExcludeList();
- DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), podId, clusterId, hostId, null, null);
- for (StoragePoolAllocator allocator : _storagePoolAllocators) {
- List poolListFromAllocator = allocator.allocateToPool(diskProfile, profile, plan, avoid, 1);
- if (CollectionUtils.isNotEmpty(poolListFromAllocator)) {
- logger.debug("Found a suitable pool: {} for disk offering: {}", poolListFromAllocator.get(0).getName(), diskOffering.getName());
- return true;
- }
- }
- return false;
+ return vmDiskOfferingSuitabilityService.isDiskOfferingSuitableForVm(vm, profile, podId, clusterId, hostId, diskOfferingId);
}
@Override
public Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds) {
- VMInstanceVO vm = _vmDao.findById(vmId);
- if (vmInstanceDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) {
- return new HashMap<>();
- }
- VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
- Pair clusterAndHost = findClusterAndHostIdForVm(vm, false);
- Long clusterId = clusterAndHost.first();
- Cluster cluster = _clusterDao.findById(clusterId);
- Map result = new HashMap<>();
- for (Long diskOfferingId : diskOfferingIds) {
- result.put(diskOfferingId, isDiskOfferingSuitableForVm(vm, profile, cluster.getPodId(), clusterId, clusterAndHost.second(), diskOfferingId));
- }
- return result;
+ return vmDiskOfferingSuitabilityService.getDiskOfferingSuitabilityForVm(vmId, diskOfferingIds);
}
@Override
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
index 475ed0f37bd2..49b500884443 100644
--- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
@@ -23,7 +23,7 @@
import java.util.Set;
import java.util.stream.Collectors;
-import javax.inject.Inject;
+import jakarta.inject.Inject;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmAllocationOrchestrationService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmAllocationOrchestrationService.java
new file mode 100644
index 000000000000..70faba178f85
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmAllocationOrchestrationService.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.cloud.deploy.DeploymentPlan;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.network.Network;
+import com.cloud.offering.DiskOffering;
+import com.cloud.offering.DiskOfferingInfo;
+import com.cloud.offering.ServiceOffering;
+import com.cloud.storage.Snapshot;
+import com.cloud.storage.Volume;
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.user.Account;
+
+public interface VmAllocationOrchestrationService {
+
+ void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering,
+ DiskOfferingInfo rootDiskOfferingInfo, List dataDiskOfferings, List dataDiskDeviceIds,
+ LinkedHashMap extends Network, List extends NicProfile>> auxiliaryNetworks, DeploymentPlan plan,
+ HypervisorType hyperType, Map> extraDhcpOptions,
+ Map datadiskTemplateToDiskOfferingMap, Volume volume, Snapshot snapshot)
+ throws InsufficientCapacityException;
+
+ void allocate(String vmInstanceName, VirtualMachineTemplate template, ServiceOffering serviceOffering,
+ LinkedHashMap extends Network, List extends NicProfile>> networks, DeploymentPlan plan,
+ HypervisorType hyperType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException;
+
+ void allocateRootVolume(VMInstanceVO vm, VirtualMachineTemplate template, DiskOfferingInfo rootDiskOfferingInfo,
+ Account owner, Long rootDiskSizeFinal, Volume volume, Snapshot snapshot);
+
+ void checkIfTemplateNeededForCreatingVmVolumes(VMInstanceVO vm);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmAllocationOrchestrationServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmAllocationOrchestrationServiceImpl.java
new file mode 100644
index 000000000000..2a1cc24879c0
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmAllocationOrchestrationServiceImpl.java
@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.deploy.DeploymentPlan;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.network.Network;
+import com.cloud.offering.DiskOffering;
+import com.cloud.offering.DiskOfferingInfo;
+import com.cloud.offering.ServiceOffering;
+import com.cloud.storage.Snapshot;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.VMTemplateZoneVO;
+import com.cloud.storage.Volume;
+import com.cloud.storage.Volume.Type;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.DiskOfferingDao;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VMTemplateZoneDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.user.Account;
+import com.cloud.utils.Pair;
+import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.utils.fsm.StateMachine2;
+import com.cloud.vm.VirtualMachine.Event;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmAllocationOrchestrationServiceImpl implements VmAllocationOrchestrationService {
+
+ private static final Logger logger = LogManager.getLogger(VmAllocationOrchestrationServiceImpl.class);
+
+ @Inject
+ protected VMInstanceDao vmDao;
+ @Inject
+ protected EntityManager entityMgr;
+ @Inject
+ protected NetworkOrchestrationService networkMgr;
+ @Inject
+ protected VolumeOrchestrationService volumeMgr;
+ @Inject
+ protected DiskOfferingDao diskOfferingDao;
+ @Inject
+ protected VMTemplateDao templateDao;
+ @Inject
+ protected VMTemplateZoneDao templateZoneDao;
+ @Inject
+ protected VolumeDao volsDao;
+ protected StateMachine2 stateMachine = State.getStateMachine();
+
+ @Override
+ public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering,
+ final DiskOfferingInfo rootDiskOfferingInfo, final List dataDiskOfferings, List dataDiskDeviceIds,
+ final LinkedHashMap extends Network, List extends NicProfile>> auxiliaryNetworks, final DeploymentPlan plan,
+ final HypervisorType hyperType, final Map> extraDhcpOptions,
+ final Map datadiskTemplateToDiskOfferingMap, Volume volume, Snapshot snapshot)
+ throws InsufficientCapacityException {
+
+ logger.info("Allocating Instance from Template: {} with hostname: {} and {} networks", template, vmInstanceName, auxiliaryNetworks.size());
+ VMInstanceVO persistedVm = null;
+ try {
+ final VMInstanceVO vm = vmDao.findVMByInstanceName(vmInstanceName);
+ final Account owner = entityMgr.findById(Account.class, vm.getAccountId());
+
+ logger.debug("Allocating entries for VM: " + vm);
+
+ vm.setDataCenterId(plan.getDataCenterId());
+ if (plan.getPodId() != null) {
+ vm.setPodIdToDeployIn(plan.getPodId());
+ }
+ assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet";
+ persistedVm = vmDao.persist(vm);
+
+ final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(persistedVm, template, serviceOffering, null, null);
+
+ Long rootDiskSize = rootDiskOfferingInfo.getSize();
+ if (vm.getType().isUsedBySystem() && VirtualMachineManagerImpl.SystemVmRootDiskSize.value() != null
+ && VirtualMachineManagerImpl.SystemVmRootDiskSize.value() > 0L) {
+ rootDiskSize = VirtualMachineManagerImpl.SystemVmRootDiskSize.value();
+ }
+ final Long rootDiskSizeFinal = rootDiskSize;
+
+ logger.debug("Allocating NICs for {}", persistedVm);
+
+ try {
+ if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) {
+ networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions);
+ }
+ } catch (final ConcurrentOperationException e) {
+ throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e);
+ }
+
+ logger.debug("Allocating disks for {}", persistedVm);
+
+ allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal, volume, snapshot);
+
+ CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume);
+ try {
+ if (dataDiskOfferings != null) {
+ int index = 0;
+ for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) {
+ Long deviceId = dataDiskDeviceIds.get(index++);
+ String volumeName = deviceId == null ? "DATA-" + persistedVm.getId() : "DATA-" + persistedVm.getId() + "-" + String.valueOf(deviceId);
+ volumeMgr.allocateRawVolume(Type.DATADISK, volumeName, dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(),
+ dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId, true);
+ }
+ }
+ if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) {
+ Long diskNumber = 1L;
+ for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) {
+ DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue();
+ long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024);
+ VMTemplateVO dataDiskTemplate = templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey());
+ volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null,
+ persistedVm, dataDiskTemplate, owner, diskNumber, true);
+ diskNumber++;
+ }
+ }
+ } finally {
+ CallContext.unregister();
+ }
+
+ logger.debug("Allocation completed for VM: " + persistedVm);
+ } catch (InsufficientCapacityException | CloudRuntimeException e) {
+ try {
+ if (persistedVm != null) {
+ stateTransitTo(persistedVm, Event.OperationFailedToError, null);
+ }
+ } catch (NoTransitionException nte) {
+ logger.error("Failed to transition {} in {} state to Error state", persistedVm, persistedVm.getState().toString());
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public void allocate(final String vmInstanceName, final VirtualMachineTemplate template, final ServiceOffering serviceOffering,
+ final LinkedHashMap extends Network, List extends NicProfile>> networks, final DeploymentPlan plan,
+ final HypervisorType hyperType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException {
+ DiskOffering diskOffering = diskOfferingDao.findById(serviceOffering.getDiskOfferingId());
+ allocate(vmInstanceName, template, serviceOffering, new DiskOfferingInfo(diskOffering), new ArrayList<>(), new ArrayList<>(), networks, plan, hyperType, null, null, volume, snapshot);
+ }
+
+ @Override
+ public void allocateRootVolume(VMInstanceVO vm, VirtualMachineTemplate template, DiskOfferingInfo rootDiskOfferingInfo,
+ Account owner, Long rootDiskSizeFinal, Volume volume, Snapshot snapshot) {
+ CallContext volumeContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Volume);
+ try {
+ String rootVolumeName = String.format("ROOT-%s", vm.getId());
+ if (template.getFormat() == ImageFormat.ISO) {
+ volumeMgr.allocateRawVolume(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(),
+ rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null, true);
+ } else if (Arrays.asList(ImageFormat.BAREMETAL, ImageFormat.EXTERNAL).contains(template.getFormat())) {
+ logger.debug("{} has format [{}]. Skipping ROOT volume [{}] allocation.", template, template.getFormat(), rootVolumeName);
+ } else {
+ volumeMgr.allocateTemplatedVolumes(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskSizeFinal,
+ rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), template, vm, owner, volume, snapshot);
+ }
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ @Override
+ public void checkIfTemplateNeededForCreatingVmVolumes(VMInstanceVO vm) {
+ final List existingRootVolumes = volsDao.findReadyRootVolumesByInstance(vm.getId());
+ if (CollectionUtils.isNotEmpty(existingRootVolumes)) {
+ return;
+ }
+ final VMTemplateVO template = templateDao.findById(vm.getTemplateId());
+ if (template == null) {
+ String msg = "Template for the VM instance can not be found, VM instance configuration needs to be updated";
+ logger.error("{}. Template ID: {} seems to be removed", msg, vm.getTemplateId());
+ throw new CloudRuntimeException(msg);
+ }
+ final VMTemplateZoneVO templateZoneVO = templateZoneDao.findByZoneTemplate(vm.getDataCenterId(), template.getId());
+ if (templateZoneVO == null) {
+ String msg = "Template for the VM instance can not be found in the zone ID: %s, VM instance configuration needs to be updated";
+ logger.error("{}. {}", msg, template);
+ throw new CloudRuntimeException(msg);
+ }
+ }
+
+ protected boolean stateTransitTo(final VMInstanceVO vm, final Event event, final Long hostId) throws NoTransitionException {
+ return stateMachine.transitTo(vm, event, new Pair<>(vm.getHostId(), hostId), vmDao);
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmCommandSpecPostProcessingService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmCommandSpecPostProcessingService.java
new file mode 100644
index 000000000000..f2272ef388d2
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmCommandSpecPostProcessingService.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.Map;
+
+import com.cloud.agent.api.StartAnswer;
+import com.cloud.agent.api.to.DiskTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+
+public interface VmCommandSpecPostProcessingService {
+
+ void setEnterSetupMode(VirtualMachineTO vmTo, Map params);
+
+ void addExtraConfig(VirtualMachineTO vmTO);
+
+ void prepareManagedDiskPaths(DiskTO[] disks, HypervisorType hypervisorType);
+
+ void applyStartAnswerDiskMetadata(DiskTO[] disks, Map> iqnToData);
+
+ void syncDiskChainChange(StartAnswer answer);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmCommandSpecPostProcessingServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmCommandSpecPostProcessingServiceImpl.java
new file mode 100644
index 000000000000..1b6d7ebdb95c
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmCommandSpecPostProcessingServiceImpl.java
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.Map;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.api.StartAnswer;
+import com.cloud.agent.api.to.DiskTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VolumeDao;
+
+@Component
+public class VmCommandSpecPostProcessingServiceImpl implements VmCommandSpecPostProcessingService {
+
+ private static final Logger logger = LogManager.getLogger(VmCommandSpecPostProcessingServiceImpl.class);
+
+ @Inject
+ protected VolumeDao volumeDao;
+ @Inject
+ protected VolumeOrchestrationService volumeMgr;
+
+ @Override
+ public void setEnterSetupMode(VirtualMachineTO vmTo, Map params) {
+ Boolean enterSetup = null;
+ if (params != null) {
+ enterSetup = (Boolean) params.get(VirtualMachineProfile.Param.BootIntoSetup);
+ }
+ logger.debug("Orchestrating VM reboot for '{}' {} set to {}", vmTo.getName(), VirtualMachineProfile.Param.BootIntoSetup, enterSetup);
+ vmTo.setEnterHardwareSetup(enterSetup == null ? false : enterSetup);
+ }
+
+ @Override
+ public void addExtraConfig(VirtualMachineTO vmTO) {
+ Map details = vmTO.getDetails();
+ for (String key : details.keySet()) {
+ if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
+ vmTO.addExtraConfig(key, details.get(key));
+ }
+ }
+ }
+
+ @Override
+ public void prepareManagedDiskPaths(final DiskTO[] disks, final HypervisorType hypervisorType) {
+ if (hypervisorType != HypervisorType.KVM) {
+ return;
+ }
+
+ if (disks != null) {
+ for (final DiskTO disk : disks) {
+ final Map details = disk.getDetails();
+ final boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED));
+
+ if (isManaged && disk.getPath() == null) {
+ final Long volumeId = disk.getData().getId();
+ final VolumeVO volume = volumeDao.findById(volumeId);
+
+ disk.setPath(volume.get_iScsiName());
+
+ if (disk.getData() instanceof VolumeObjectTO) {
+ final VolumeObjectTO volTo = (VolumeObjectTO)disk.getData();
+
+ volTo.setPath(volume.get_iScsiName());
+ }
+
+ volume.setPath(volume.get_iScsiName());
+
+ volumeDao.update(volumeId, volume);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void applyStartAnswerDiskMetadata(final DiskTO[] disks, final Map> iqnToData) {
+ if (disks != null && iqnToData != null) {
+ for (final DiskTO disk : disks) {
+ final Map details = disk.getDetails();
+ final boolean isManaged = details != null && Boolean.parseBoolean(details.get(DiskTO.MANAGED));
+
+ if (isManaged) {
+ final Long volumeId = disk.getData().getId();
+ final VolumeVO volume = volumeDao.findById(volumeId);
+ final String iScsiName = volume.get_iScsiName();
+
+ boolean update = false;
+
+ final Map data = iqnToData.get(iScsiName);
+
+ if (data != null) {
+ final String path = data.get(StartAnswer.PATH);
+
+ if (path != null) {
+ volume.setPath(path);
+
+ update = true;
+ }
+
+ final String imageFormat = data.get(StartAnswer.IMAGE_FORMAT);
+
+ if (imageFormat != null) {
+ volume.setFormat(ImageFormat.valueOf(imageFormat));
+
+ update = true;
+ }
+
+ if (update) {
+ volumeDao.update(volumeId, volume);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void syncDiskChainChange(final StartAnswer answer) {
+ final VirtualMachineTO vmSpec = answer.getVirtualMachine();
+
+ for (final DiskTO disk : vmSpec.getDisks()) {
+ if (disk.getType() != Volume.Type.ISO) {
+ final VolumeObjectTO vol = (VolumeObjectTO)disk.getData();
+ final VolumeVO volume = volumeDao.findById(vol.getId());
+ if (vmSpec.getDeployAsIsInfo() != null && org.apache.commons.lang3.StringUtils.isNotBlank(vol.getPath())) {
+ volume.setPath(vol.getPath());
+ volumeDao.update(volume.getId(), volume);
+ }
+
+ if(vol.getPath() != null) {
+ volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID());
+ } else {
+ volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID());
+ }
+ }
+ }
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmDestroyOrchestrationService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmDestroyOrchestrationService.java
new file mode 100644
index 000000000000..9b57d4b58ee0
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmDestroyOrchestrationService.java
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.OperationTimedoutException;
+
+public interface VmDestroyOrchestrationService {
+
+ void destroy(String vmUuid, boolean expunge) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException;
+
+ void deleteVMSnapshots(VMInstanceVO vm, boolean expunge);
+
+ boolean checkVmOnHost(VirtualMachine vm, long hostId) throws AgentUnavailableException, OperationTimedoutException;
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmDestroyOrchestrationServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmDestroyOrchestrationServiceImpl.java
new file mode 100644
index 000000000000..3de983e21770
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmDestroyOrchestrationServiceImpl.java
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.backup.BackupManager;
+import org.apache.cloudstack.gpu.GpuService;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CheckVirtualMachineAnswer;
+import com.cloud.agent.api.CheckVirtualMachineCommand;
+import com.cloud.agent.api.RestoreVMSnapshotAnswer;
+import com.cloud.agent.api.RestoreVMSnapshotCommand;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
+import com.cloud.utils.db.TransactionStatus;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.vm.VirtualMachine.PowerState;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.UserVmDao;
+import com.cloud.vm.dao.VMInstanceDao;
+import com.cloud.vm.snapshot.VMSnapshotManager;
+import com.cloud.vm.snapshot.VMSnapshotVO;
+import com.cloud.vm.snapshot.dao.VMSnapshotDao;
+
+@Component
+public class VmDestroyOrchestrationServiceImpl implements VmDestroyOrchestrationService {
+
+ private static final Logger logger = LogManager.getLogger(VmDestroyOrchestrationServiceImpl.class);
+
+ @Inject
+ protected VMInstanceDao vmDao;
+ @Inject
+ protected UserVmDao userVmDao;
+ @Inject
+ protected VMSnapshotDao vmSnapshotDao;
+ @Inject
+ protected VMSnapshotManager vmSnapshotMgr;
+ @Inject
+ protected AgentManager agentMgr;
+ @Inject
+ protected GpuService gpuService;
+ @Inject
+ protected BackupManager backupManager;
+ @Inject
+ @Lazy
+ protected VirtualMachineManager virtualMachineManager;
+
+ @Override
+ public void destroy(final String vmUuid, final boolean expunge) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException {
+ VMInstanceVO vm = vmDao.findByUuid(vmUuid);
+ if (vm == null || vm.getState() == State.Destroyed || vm.getState() == State.Expunging || vm.getRemoved() != null) {
+ logger.debug("Unable to find vm or vm is destroyed: {}", vm);
+ return;
+ }
+
+ logger.debug("Destroying vm {}, expunge flag {}", vm, (expunge ? "on" : "off"));
+
+ advanceStop(vmUuid);
+
+ deleteVMSnapshots(vm, expunge);
+
+ gpuService.deallocateAllGpuDevicesForVm(vm.getId());
+
+ Transaction.execute(new TransactionCallbackWithExceptionNoReturn() {
+ @Override
+ public void doInTransactionWithoutResult(final TransactionStatus status) throws CloudRuntimeException {
+ VMInstanceVO vm = vmDao.findByUuid(vmUuid);
+ try {
+ if (!virtualMachineManager.stateTransitTo(vm, VirtualMachine.Event.DestroyRequested, vm.getHostId())) {
+ logger.debug("Unable to destroy the vm because it is not in the correct state: {}", vm);
+ throw new CloudRuntimeException("Unable to destroy " + vm);
+ } else {
+ if (expunge) {
+ backupManager.checkAndRemoveBackupOfferingBeforeExpunge(vm);
+ if (!virtualMachineManager.stateTransitTo(vm, VirtualMachine.Event.ExpungeOperation, vm.getHostId())) {
+ logger.debug("Unable to expunge the vm because it is not in the correct state: {}", vm);
+ throw new CloudRuntimeException("Unable to expunge " + vm);
+ }
+ }
+ }
+ } catch (final NoTransitionException e) {
+ String message = String.format("Unable to destroy %s due to [%s].", vm.toString(), e.getMessage());
+ logger.debug(message, e);
+ throw new CloudRuntimeException(message, e);
+ }
+ }
+ });
+ }
+
+ private void advanceStop(final String vmUuid) throws AgentUnavailableException, OperationTimedoutException, ConcurrentOperationException {
+ try {
+ virtualMachineManager.advanceStop(vmUuid, VirtualMachineManagerImpl.VmDestroyForcestop.value());
+ } catch (ResourceUnavailableException e) {
+ if (e instanceof AgentUnavailableException) {
+ throw (AgentUnavailableException)e;
+ }
+ throw new CloudRuntimeException("Unable to stop vm " + vmUuid, e);
+ }
+ }
+
+ /**
+ * Delete vm snapshots depending on vm's hypervisor type. For Vmware, vm snapshots removal is delegated to vm cleanup thread
+ * to reduce tasks sent to hypervisor (one tasks to delete vm snapshots and vm itself
+ * instead of one task for each vm snapshot plus another for the vm)
+ * @param vm vm
+ * @param expunge indicates if vm should be expunged
+ */
+ @Override
+ public void deleteVMSnapshots(VMInstanceVO vm, boolean expunge) {
+ if (!vm.getHypervisorType().equals(HypervisorType.VMware)) {
+ if (!vmSnapshotMgr.deleteAllVMSnapshots(vm.getId(), null)) {
+ logger.debug("Unable to delete all Snapshots for {}", vm);
+ throw new CloudRuntimeException("Unable to delete Instance Snapshots for " + vm);
+ }
+ } else {
+ if (expunge) {
+ vmSnapshotMgr.deleteVMSnapshotsFromDB(vm.getId(), false);
+ }
+ }
+ }
+
+ @Override
+ public boolean checkVmOnHost(final VirtualMachine vm, final long hostId) throws AgentUnavailableException, OperationTimedoutException {
+ final Answer answer = agentMgr.send(hostId, new CheckVirtualMachineCommand(vm.getInstanceName()));
+ if (answer == null || !answer.getResult()) {
+ return false;
+ }
+ if (answer instanceof CheckVirtualMachineAnswer) {
+ final CheckVirtualMachineAnswer vmAnswer = (CheckVirtualMachineAnswer)answer;
+ if (vmAnswer.getState() == PowerState.PowerOff) {
+ return false;
+ }
+ }
+
+ UserVmVO userVm = userVmDao.findById(vm.getId());
+ if (userVm != null) {
+ List vmSnapshots = vmSnapshotDao.findByVm(vm.getId());
+ RestoreVMSnapshotCommand command = vmSnapshotMgr.createRestoreCommand(userVm, vmSnapshots);
+ if (command != null) {
+ RestoreVMSnapshotAnswer restoreVMSnapshotAnswer = (RestoreVMSnapshotAnswer) agentMgr.send(hostId, command);
+ if (restoreVMSnapshotAnswer == null || !restoreVMSnapshotAnswer.getResult()) {
+ logger.warn("Unable to restore the Instance Snapshot from image file after live migration of Instance with vmsnapshots: {}", restoreVMSnapshotAnswer == null ? "null answer" : restoreVMSnapshotAnswer.getDetails());
+ }
+ }
+ }
+
+ return true;
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmDiskOfferingSuitabilityService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmDiskOfferingSuitabilityService.java
new file mode 100644
index 000000000000..fea519bf376f
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmDiskOfferingSuitabilityService.java
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+import java.util.Map;
+
+import com.cloud.utils.Pair;
+
+public interface VmDiskOfferingSuitabilityService {
+
+ Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm);
+
+ Pair findClusterAndHostIdForVm(long vmId);
+
+ boolean isDiskOfferingSuitableForVm(VMInstanceVO vm, VirtualMachineProfile profile, long podId, long clusterId, long hostId, long diskOfferingId);
+
+ Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmDiskOfferingSuitabilityServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmDiskOfferingSuitabilityServiceImpl.java
new file mode 100644
index 000000000000..52fd04ece949
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmDiskOfferingSuitabilityServiceImpl.java
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.deploy.DataCenterDeployment;
+import com.cloud.deploy.DeploymentPlanner.ExcludeList;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.org.Cluster;
+import com.cloud.storage.DiskOfferingVO;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.DiskOfferingDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.utils.Pair;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.VMInstanceDao;
+import com.cloud.vm.dao.VMInstanceDetailsDao;
+
+@Component
+public class VmDiskOfferingSuitabilityServiceImpl implements VmDiskOfferingSuitabilityService {
+
+ private static final Logger logger = LogManager.getLogger(VmDiskOfferingSuitabilityServiceImpl.class);
+
+ @Inject
+ protected VMInstanceDao vmInstanceDao;
+ @Inject
+ protected VMInstanceDetailsDao vmInstanceDetailsDao;
+ @Inject
+ protected VolumeDao volumeDao;
+ @Inject
+ protected PrimaryDataStoreDao storagePoolDao;
+ @Inject
+ protected HostDao hostDao;
+ @Inject
+ protected ClusterDao clusterDao;
+ @Inject
+ protected DiskOfferingDao diskOfferingDao;
+
+ private List storagePoolAllocators;
+
+ @Inject
+ public void setStoragePoolAllocators(final List storagePoolAllocators) {
+ this.storagePoolAllocators = storagePoolAllocators;
+ }
+
+ public List getStoragePoolAllocators() {
+ return storagePoolAllocators;
+ }
+
+ Pair findClusterAndHostIdForVmFromVolumes(long vmId) {
+ Long clusterId = null;
+ Long hostId = null;
+ List volumes = volumeDao.findByInstance(vmId);
+ for (VolumeVO volume : volumes) {
+ if (Volume.State.Ready.equals(volume.getState()) &&
+ volume.getPoolId() != null) {
+ StoragePoolVO pool = storagePoolDao.findById(volume.getPoolId());
+ if (pool != null && pool.getClusterId() != null) {
+ clusterId = pool.getClusterId();
+ // hostId to be used only for sending commands, capacity check skipped
+ List hosts = hostDao.findHypervisorHostInCluster(pool.getClusterId());
+ if (CollectionUtils.isNotEmpty(hosts)) {
+ hostId = hosts.get(0).getId();
+ break;
+ }
+ }
+ }
+ }
+ return new Pair<>(clusterId, hostId);
+ }
+
+ @Override
+ public Pair findClusterAndHostIdForVm(VirtualMachine vm, boolean skipCurrentHostForStartingVm) {
+ Long hostId = null;
+ Host host = null;
+ if (!skipCurrentHostForStartingVm || !State.Starting.equals(vm.getState())) {
+ hostId = vm.getHostId();
+ }
+ Long clusterId = null;
+ if (hostId == null) {
+ if (vm.getLastHostId() == null) {
+ return findClusterAndHostIdForVmFromVolumes(vm.getId());
+ }
+ hostId = vm.getLastHostId();
+ host = hostDao.findById(hostId);
+ logger.debug("host id is null, using last host {} with id {}", host, hostId);
+ }
+ host = host == null ? hostDao.findById(hostId) : host;
+ if (host != null) {
+ clusterId = host.getClusterId();
+ return new Pair<>(clusterId, hostId);
+ }
+ return findClusterAndHostIdForVmFromVolumes(vm.getId());
+ }
+
+ Pair findClusterAndHostIdForVm(VirtualMachine vm) {
+ return findClusterAndHostIdForVm(vm, false);
+ }
+
+ @Override
+ public Pair findClusterAndHostIdForVm(long vmId) {
+ VMInstanceVO vm = vmInstanceDao.findById(vmId);
+ if (vm == null) {
+ return new Pair<>(null, null);
+ }
+ return findClusterAndHostIdForVm(vm);
+ }
+
+ @Override
+ public boolean isDiskOfferingSuitableForVm(VMInstanceVO vm, VirtualMachineProfile profile, long podId, long clusterId, long hostId, long diskOfferingId) {
+ DiskOfferingVO diskOffering = diskOfferingDao.findById(diskOfferingId);
+ VolumeVO dummyVolume = new VolumeVO("Data", vm.getDataCenterId(), podId, vm.getAccountId(),
+ vm.getDomainId(), vm.getId(), null, null, diskOffering.getProvisioningType(), diskOffering.getDiskSize(), Volume.Type.DATADISK);
+ try {
+ Field idField = dummyVolume.getClass().getDeclaredField("id");
+ idField.setAccessible(true);
+ idField.set(dummyVolume, Volume.DISK_OFFERING_SUITABILITY_CHECK_VOLUME_ID);
+ } catch (NoSuchFieldException | IllegalAccessException ignored) {
+ return false;
+ }
+ dummyVolume.setDiskOfferingId(diskOfferingId);
+ DiskProfile diskProfile = new DiskProfile(dummyVolume, diskOffering, profile.getHypervisorType());
+ diskProfile.setMinIops(diskOffering.getMinIops());
+ diskProfile.setMaxIops(diskOffering.getMaxIops());
+ ExcludeList avoid = new ExcludeList();
+ DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), podId, clusterId, hostId, null, null);
+ for (StoragePoolAllocator allocator : storagePoolAllocators) {
+ List poolListFromAllocator = allocator.allocateToPool(diskProfile, profile, plan, avoid, 1);
+ if (CollectionUtils.isNotEmpty(poolListFromAllocator)) {
+ logger.debug("Found a suitable pool: {} for disk offering: {}", poolListFromAllocator.get(0).getName(), diskOffering.getName());
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds) {
+ VMInstanceVO vm = vmInstanceDao.findById(vmId);
+ if (vmInstanceDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) {
+ return new HashMap<>();
+ }
+ VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+ Pair clusterAndHost = findClusterAndHostIdForVm(vm, false);
+ Long clusterId = clusterAndHost.first();
+ Cluster cluster = clusterDao.findById(clusterId);
+ Map result = new HashMap<>();
+ for (Long diskOfferingId : diskOfferingIds) {
+ result.put(diskOfferingId, isDiskOfferingSuitableForVm(vm, profile, cluster.getPodId(), clusterId, clusterAndHost.second(), diskOfferingId));
+ }
+ return result;
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeCommandService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeCommandService.java
new file mode 100644
index 000000000000..aad3ae3a6724
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeCommandService.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+
+import com.cloud.agent.api.Command;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+
+public interface VmExpungeCommandService {
+
+ void sendVolumeExpungeCommands(List volumeExpungeCommands, Long hostId, VMInstanceVO vm)
+ throws OperationTimedoutException, AgentUnavailableException;
+
+ void sendFinalizeExpungeCommands(List finalizeExpungeCommands, List nicExpungeCommands,
+ VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException;
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeCommandServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeCommandServiceImpl.java
new file mode 100644
index 000000000000..f55d4cdc7e55
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeCommandServiceImpl.java
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+
+import jakarta.inject.Inject;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.manager.Commands;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+@Component
+public class VmExpungeCommandServiceImpl implements VmExpungeCommandService {
+
+ private static final Logger logger = LogManager.getLogger(VmExpungeCommandServiceImpl.class);
+
+ @Inject
+ protected AgentManager agentMgr;
+
+ @Override
+ public void sendVolumeExpungeCommands(List volumeExpungeCommands, Long hostId, VMInstanceVO vm)
+ throws OperationTimedoutException, AgentUnavailableException {
+ if (CollectionUtils.isEmpty(volumeExpungeCommands) || hostId == null) {
+ return;
+ }
+
+ final Commands cmds = new Commands(Command.OnError.Stop);
+ for (final Command volumeExpungeCommand : volumeExpungeCommands) {
+ volumeExpungeCommand.setBypassHostMaintenance(isValidSystemVMType(vm));
+ cmds.addCommand(volumeExpungeCommand);
+ }
+
+ agentMgr.send(hostId, cmds);
+ handleUnsuccessfulCommands(cmds, vm);
+ }
+
+ @Override
+ public void sendFinalizeExpungeCommands(List finalizeExpungeCommands, List nicExpungeCommands,
+ VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException {
+ if ((CollectionUtils.isEmpty(finalizeExpungeCommands) && CollectionUtils.isEmpty(nicExpungeCommands)) || hostId == null) {
+ return;
+ }
+
+ final Commands cmds = new Commands(Command.OnError.Stop);
+ addAllExpungeCommandsFromList(finalizeExpungeCommands, cmds, vm);
+ addAllExpungeCommandsFromList(nicExpungeCommands, cmds, vm);
+ agentMgr.send(hostId, cmds);
+ if (!cmds.isSuccessful()) {
+ for (final Answer answer : cmds.getAnswers()) {
+ if (!answer.getResult()) {
+ logger.warn("Failed to expunge vm due to: {}", answer.getDetails());
+ throw new CloudRuntimeException(String.format("Unable to expunge %s due to %s", vm, answer.getDetails()));
+ }
+ }
+ }
+ }
+
+ protected void handleUnsuccessfulCommands(Commands cmds, VMInstanceVO vm) throws CloudRuntimeException {
+ String cmdsStr = cmds.toString();
+ String vmToString = vm.toString();
+
+ if (cmds.isSuccessful()) {
+ logger.debug("The commands [{}] to {} were successful.", cmdsStr, vmToString);
+ return;
+ }
+
+ logger.info("The commands [{}] to {} were unsuccessful. Handling answers.", cmdsStr, vmToString);
+
+ Answer[] answers = cmds.getAnswers();
+ if (answers == null) {
+ logger.debug("There are no answers to commands [{}] to {}.", cmdsStr, vmToString);
+ return;
+ }
+
+ for (Answer answer : answers) {
+ String details = answer.getDetails();
+ if (!answer.getResult()) {
+ String message = String.format("Unable to expunge %s due to [%s].", vmToString, details);
+ logger.error(message);
+ throw new CloudRuntimeException(message);
+ }
+
+ logger.debug("Commands [{}] to {} got answer [{}].", cmdsStr, vmToString, details);
+ }
+ }
+
+ private void addAllExpungeCommandsFromList(List cmdList, Commands cmds, VMInstanceVO vm) {
+ if (CollectionUtils.isEmpty(cmdList)) {
+ return;
+ }
+ for (final Command command : cmdList) {
+ command.setBypassHostMaintenance(isValidSystemVMType(vm));
+ logger.trace("Adding expunge command [{}] for VM [{}]", command.toString(), vm.toString());
+ cmds.addCommand(command);
+ }
+ }
+
+ private boolean isValidSystemVMType(VirtualMachine vm) {
+ return VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType()) ||
+ VirtualMachine.Type.ConsoleProxy.equals(vm.getType());
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeOrchestrationService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeOrchestrationService.java
new file mode 100644
index 000000000000..1e85a35af6bd
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeOrchestrationService.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+
+public interface VmExpungeOrchestrationService {
+
+ void expunge(String vmUuid) throws ResourceUnavailableException;
+
+ void advanceExpunge(String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException;
+
+ void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException;
+
+ boolean isVmDestroyed(VMInstanceVO vm);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeOrchestrationServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeOrchestrationServiceImpl.java
new file mode 100644
index 000000000000..5799f24cb018
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmExpungeOrchestrationServiceImpl.java
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.annotation.AnnotationService;
+import org.apache.cloudstack.annotation.dao.AnnotationDao;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.resource.ResourceCleanupService;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.api.Command;
+import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.HypervisorGuru;
+import com.cloud.hypervisor.HypervisorGuruManager;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.vm.dao.UserVmDao;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmExpungeOrchestrationServiceImpl implements VmExpungeOrchestrationService {
+
+ private static final Logger logger = LogManager.getLogger(VmExpungeOrchestrationServiceImpl.class);
+
+ @Inject
+ protected VMInstanceDao vmDao;
+ @Inject
+ protected UserVmDao userVmDao;
+ @Inject
+ protected HypervisorGuruManager hvGuruMgr;
+ @Inject
+ protected NetworkOrchestrationService networkMgr;
+ @Inject
+ protected VolumeOrchestrationService volumeMgr;
+ @Inject
+ protected VmExpungeCommandService vmExpungeCommandService;
+ @Inject
+ protected UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao;
+ @Inject
+ protected AnnotationDao annotationDao;
+ @Inject
+ protected ResourceCleanupService resourceCleanupService;
+ @Inject
+ protected VmIscsiTargetManager vmIscsiTargetManager;
+ @Inject
+ @Lazy
+ protected VirtualMachineManager virtualMachineManager;
+ @Inject
+ @Lazy
+ protected VmStateMachineActions vmStateMachineActions;
+
+ @Override
+ public void expunge(final String vmUuid) throws ResourceUnavailableException {
+ try {
+ advanceExpunge(vmUuid);
+ } catch (final OperationTimedoutException e) {
+ throw new CloudRuntimeException("Operation timed out", e);
+ } catch (final ConcurrentOperationException e) {
+ throw new CloudRuntimeException("Concurrent operation ", e);
+ }
+ }
+
+ @Override
+ public void advanceExpunge(final String vmUuid) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
+ final VMInstanceVO vm = vmDao.findByUuid(vmUuid);
+ advanceExpunge(vm);
+ }
+
+ @Override
+ public boolean isVmDestroyed(VMInstanceVO vm) {
+ if (vm == null || vm.getRemoved() != null) {
+ logger.debug("Unable to find vm or vm is expunged: {}", vm);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableException, OperationTimedoutException, ConcurrentOperationException {
+ if (isVmDestroyed(vm)) {
+ return;
+ }
+
+ if (HypervisorType.External.equals(vm.getHypervisorType())) {
+ UserVmVO userVM = userVmDao.findById(vm.getId());
+ userVmDao.loadDetails(userVM);
+ userVM.setDetail(VmDetailConstants.EXPUNGE_EXTERNAL_VM, Boolean.TRUE.toString());
+ userVmDao.saveDetails(userVM);
+ }
+
+ virtualMachineManager.advanceStop(vm.getUuid(), VirtualMachineManagerImpl.VmDestroyForcestop.value());
+ vm = vmDao.findByUuid(vm.getUuid());
+
+ try {
+ if (!vmStateMachineActions.stateTransitTo(vm, VirtualMachine.Event.ExpungeOperation, vm.getHostId())) {
+ logger.debug("Unable to expunge the vm because it is not in the correct state: {}", vm);
+ throw new CloudRuntimeException("Unable to expunge " + vm);
+ }
+ } catch (final NoTransitionException e) {
+ logger.debug("Unable to expunge the vm because it is not in the correct state: {}", vm);
+ throw new CloudRuntimeException("Unable to expunge " + vm, e);
+ }
+
+ logger.debug("Expunging vm {}", vm);
+
+ final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+ final HypervisorGuru hvGuru = hvGuruMgr.getGuru(vm.getHypervisorType());
+
+ List vmNics = profile.getNics();
+ logger.debug("Cleaning up NICS [{}] of {}.", vmNics.stream().map(nic -> nic.toString()).collect(Collectors.joining(", ")), vm.toString());
+ final List nicExpungeCommands = hvGuru.finalizeExpungeNics(vm, profile.getNics());
+ networkMgr.cleanupNics(profile);
+
+ logger.debug("Cleaning up hypervisor data structures (ex. SRs in XenServer) for managed storage. Data from {}.", vm.toString());
+
+ final List volumeExpungeCommands = hvGuru.finalizeExpungeVolumes(vm);
+ final Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId();
+ List> targets = getTargets(hostId, vm.getId());
+
+ vmExpungeCommandService.sendVolumeExpungeCommands(volumeExpungeCommands, hostId, vm);
+
+ if (hostId != null) {
+ volumeMgr.revokeAccess(vm.getId(), hostId);
+ }
+
+ volumeMgr.cleanupVolumes(vm.getId());
+
+ if (hostId != null && CollectionUtils.isNotEmpty(targets)) {
+ removeDynamicTargets(hostId, targets);
+ }
+
+ final VirtualMachineGuru guru = vmStateMachineActions.getVmGuru(vm);
+ guru.finalizeExpunge(vm);
+
+ userVmDeployAsIsDetailsDao.removeDetails(vm.getId());
+ annotationDao.removeByEntityType(AnnotationService.EntityType.VM.name(), vm.getUuid());
+
+ final List finalizeExpungeCommands = hvGuru.finalizeExpunge(vm);
+ vmExpungeCommandService.sendFinalizeExpungeCommands(finalizeExpungeCommands, nicExpungeCommands, vm, hostId);
+
+ logger.debug("Expunged {}", vm);
+ resourceCleanupService.purgeExpungedVmResourcesLaterIfNeeded(vm);
+ }
+
+ private List> getTargets(Long hostId, long vmId) {
+ return vmIscsiTargetManager.getTargets(hostId, vmId);
+ }
+
+ private void removeDynamicTargets(long hostId, List> targets) {
+ vmIscsiTargetManager.removeDynamicTargets(hostId, targets);
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmExternalProvisioningManager.java b/engine/orchestration/src/main/java/com/cloud/vm/VmExternalProvisioningManager.java
new file mode 100644
index 000000000000..6ef41422f789
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmExternalProvisioningManager.java
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.vm;
+
+import java.util.Map;
+
+import com.cloud.agent.api.RebootCommand;
+import com.cloud.agent.api.StartCommand;
+import com.cloud.agent.api.StopCommand;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.dc.DataCenter;
+import com.cloud.host.Host;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+/**
+ * External-hypervisor provisioning handshake and command-decoration helpers.
+ *
+ *
Handles the {@code PrepareExternalProvisioning} command/answer round-trip
+ * for External hypervisor VMs (pre-start metadata exchange, NIC/detail updates),
+ * and decorates {@link StartCommand}, {@link StopCommand}, and
+ * {@link RebootCommand} with the per-host external-access details required by
+ * the External hypervisor driver.
+ *
+ *
Extracted from {@link VirtualMachineManagerImpl} as part of the
+ * Phase 4 Spring-component decomposition.
+ */
+public interface VmExternalProvisioningManager {
+
+ /**
+ * Set the {@code metadataManufacturer} and {@code metadataProductName}
+ * fields on {@code vmTO} from the zone-scoped config keys, falling back
+ * to global defaults when the zone-level value is blank.
+ */
+ void updateVmMetadataManufacturerAndProduct(VirtualMachineTO vmTO, VMInstanceVO vm);
+
+ /**
+ * Persist new VM details returned by a PrepareExternalProvisioning answer
+ * back to {@code vmTO} and the {@link UserVmVO} detail table. No-ops when
+ * {@code newDetails} is {@code null} or equal to the details already on
+ * {@code vmTO}.
+ */
+ void updateExternalVmDetailsFromPrepareAnswer(VirtualMachineTO vmTO, UserVmVO userVmVO,
+ Map newDetails);
+
+ /**
+ * Apply VNC-password and detail updates from {@code updatedTO} back into
+ * {@code vmTO} and the underlying {@link UserVmVO} row. No-ops when
+ * neither field changed.
+ */
+ void updateExternalVmDataFromPrepareAnswer(VirtualMachineTO vmTO, VirtualMachineTO updatedTO);
+
+ /**
+ * Reconcile NIC MAC/IP addresses returned by a PrepareExternalProvisioning
+ * answer against the live {@link com.cloud.vm.dao.NicDao} rows. No-ops
+ * when either NIC array is {@code null}.
+ */
+ void updateExternalVmNicsFromPrepareAnswer(VirtualMachineTO vmTO, VirtualMachineTO updatedTO);
+
+ /**
+ * Convenience wrapper: apply both data and NIC updates from a
+ * PrepareExternalProvisioning answer. No-ops when {@code updatedTO}
+ * is {@code null}.
+ */
+ void updateExternalVmFromPrepareAnswer(VirtualMachineTO vmTO, VirtualMachineTO updatedTO);
+
+ /**
+ * Send a {@code PrepareExternalProvisioningCommand} to the host for the
+ * first start of an External VM whose template's extension requires it,
+ * then reconcile any updates returned in the answer.
+ *
+ *
The caller is responsible for pre-computing {@code vmTO} (via
+ * {@code toVmTO(vmProfile)}) so that the {@code @Spy} stub in unit tests
+ * for {@link VirtualMachineManagerImpl} is honoured correctly. The NIC
+ * array on {@code vmTO} will be populated from the database when it is
+ * empty.
+ *
+ * @param firstStart {@code true} only on the very first start of the VM
+ * @param host the destination host; may be {@code null}
+ * @param vmProfile VM profile including NICs
+ * @param dataCenter target zone (used to resolve NIC profiles)
+ * @param vmTO pre-built transfer object for the VM
+ * @throws CloudRuntimeException when the agent call fails or returns an
+ * unexpected/negative answer
+ */
+ void processPrepareExternalProvisioning(boolean firstStart, Host host,
+ VirtualMachineProfile vmProfile, DataCenter dataCenter, VirtualMachineTO vmTO)
+ throws CloudRuntimeException;
+
+ /**
+ * Populate a {@link StartCommand} with the per-host external-access
+ * details and the default-NIC VLAN segment name. No-ops for non-External
+ * hypervisors.
+ */
+ void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command);
+
+ /**
+ * Populate a {@link StopCommand} with the per-host external-access
+ * details and a cleaned-up {@link VirtualMachineTO}. No-ops for
+ * non-External hypervisors or when the VM profile has no host id.
+ *
+ *
The caller is responsible for pre-computing {@code vmTO} (via
+ * {@code ObjectUtils.defaultIfNull(stopCommand.getVirtualMachine(), toVmTO(vmProfile))})
+ * so that the {@code @Spy} stub in unit tests is honoured correctly.
+ *
+ * @param hypervisorType the VM's hypervisor type
+ * @param vmProfile VM profile including the host id
+ * @param stopCommand command to decorate
+ * @param vmTO pre-built (or pre-resolved from command) transfer object
+ */
+ void updateStopCommandForExternalHypervisorType(HypervisorType hypervisorType,
+ VirtualMachineProfile vmProfile, StopCommand stopCommand, VirtualMachineTO vmTO);
+
+ /**
+ * Populate a {@link RebootCommand} with the per-host external-access
+ * details. No-ops for non-External hypervisors.
+ */
+ void updateRebootCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, RebootCommand rebootCmd);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmExternalProvisioningManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmExternalProvisioningManagerImpl.java
new file mode 100644
index 000000000000..8696cad2ce5b
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmExternalProvisioningManagerImpl.java
@@ -0,0 +1,289 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.vm;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.framework.extensions.dao.ExtensionDetailsDao;
+import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.PrepareExternalProvisioningAnswer;
+import com.cloud.agent.api.PrepareExternalProvisioningCommand;
+import com.cloud.agent.api.RebootCommand;
+import com.cloud.agent.api.StartCommand;
+import com.cloud.agent.api.StopCommand;
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.dc.DataCenter;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.HypervisorGuru;
+import com.cloud.hypervisor.HypervisorGuruManager;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.NetworkService;
+import com.cloud.utils.StringUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.dao.NicDao;
+import com.cloud.vm.dao.UserVmDao;
+
+/**
+ * External-hypervisor provisioning handshake and command-decoration — extracted
+ * from {@link VirtualMachineManagerImpl}.
+ *
+ * @see VmExternalProvisioningManager
+ */
+@Component
+public class VmExternalProvisioningManagerImpl implements VmExternalProvisioningManager {
+
+ private static final Logger logger = LogManager.getLogger(VmExternalProvisioningManagerImpl.class);
+
+ @Inject
+ private AgentManager agentMgr;
+ @Inject
+ private NicDao nicsDao;
+ @Inject
+ private UserVmDao userVmDao;
+ @Inject
+ private ExtensionsManager extensionsManager;
+ @Inject
+ private ExtensionDetailsDao extensionDetailsDao;
+ @Inject
+ private NetworkService networkService;
+ @Inject
+ private HostDao hostDao;
+ @Inject
+ private NetworkModel networkModel;
+ @Inject
+ private HypervisorGuruManager hvGuruMgr;
+
+ @Override
+ public void updateVmMetadataManufacturerAndProduct(VirtualMachineTO vmTO, VMInstanceVO vm) {
+ String metadataManufacturer = VirtualMachineManager.VmMetadataManufacturer.valueIn(vm.getDataCenterId());
+ if (StringUtils.isBlank(metadataManufacturer)) {
+ metadataManufacturer = VirtualMachineManager.VmMetadataManufacturer.defaultValue();
+ }
+ vmTO.setMetadataManufacturer(metadataManufacturer);
+ String metadataProduct = VirtualMachineManager.VmMetadataProductName.valueIn(vm.getDataCenterId());
+ if (StringUtils.isBlank(metadataProduct)) {
+ metadataProduct = String.format("CloudStack %s Hypervisor", vm.getHypervisorType().toString());
+ }
+ vmTO.setMetadataProductName(metadataProduct);
+ }
+
+ @Override
+ public void updateExternalVmDetailsFromPrepareAnswer(VirtualMachineTO vmTO, UserVmVO userVmVO,
+ Map newDetails) {
+ if (newDetails == null || newDetails.equals(vmTO.getDetails())) {
+ return;
+ }
+ vmTO.setDetails(newDetails);
+ userVmVO.setDetails(newDetails);
+ userVmDao.saveDetails(userVmVO);
+ }
+
+ @Override
+ public void updateExternalVmDataFromPrepareAnswer(VirtualMachineTO vmTO, VirtualMachineTO updatedTO) {
+ final String vncPassword = updatedTO.getVncPassword();
+ final Map details = updatedTO.getDetails();
+ if ((vncPassword == null || vncPassword.equals(vmTO.getVncPassword())) &&
+ (details == null || details.equals(vmTO.getDetails()))) {
+ return;
+ }
+ UserVmVO userVmVO = userVmDao.findById(vmTO.getId());
+ if (userVmVO == null) {
+ return;
+ }
+ if (vncPassword != null && !vncPassword.equals(userVmVO.getPassword())) {
+ userVmVO.setVncPassword(vncPassword);
+ vmTO.setVncPassword(vncPassword);
+ }
+ updateExternalVmDetailsFromPrepareAnswer(vmTO, userVmVO, updatedTO.getDetails());
+ }
+
+ @Override
+ public void updateExternalVmNicsFromPrepareAnswer(VirtualMachineTO vmTO, VirtualMachineTO updatedTO) {
+ if (ObjectUtils.anyNull(vmTO.getNics(), updatedTO.getNics())) {
+ return;
+ }
+ Map originalNicsByUuid = new HashMap<>();
+ for (NicTO nic : vmTO.getNics()) {
+ originalNicsByUuid.put(nic.getNicUuid(), nic);
+ }
+ for (NicTO updatedNicTO : updatedTO.getNics()) {
+ final String nicUuid = updatedNicTO.getNicUuid();
+ NicTO originalNicTO = originalNicsByUuid.get(nicUuid);
+ if (originalNicTO == null) {
+ continue;
+ }
+ final String mac = updatedNicTO.getMac();
+ final String ip4 = updatedNicTO.getIp();
+ final String ip6 = updatedNicTO.getIp6Address();
+ if (Objects.equals(mac, originalNicTO.getMac()) &&
+ Objects.equals(ip4, originalNicTO.getIp()) &&
+ Objects.equals(ip6, originalNicTO.getIp6Address())) {
+ continue;
+ }
+ NicVO nicVO = nicsDao.findByUuid(nicUuid);
+ if (nicVO == null) {
+ continue;
+ }
+ logger.debug("Updating {} during External VM preparation", nicVO);
+ if (ip4 != null && !ip4.equals(nicVO.getIPv4Address())) {
+ nicVO.setIPv4Address(ip4);
+ originalNicTO.setIp(ip4);
+ }
+ if (ip6 != null && !ip6.equals(nicVO.getIPv6Address())) {
+ nicVO.setIPv6Address(ip6);
+ originalNicTO.setIp6Address(ip6);
+ }
+ if (mac != null && !mac.equals(nicVO.getMacAddress())) {
+ nicVO.setMacAddress(mac);
+ originalNicTO.setMac(mac);
+ }
+ nicsDao.update(nicVO.getId(), nicVO);
+ }
+ }
+
+ @Override
+ public void updateExternalVmFromPrepareAnswer(VirtualMachineTO vmTO, VirtualMachineTO updatedTO) {
+ if (updatedTO == null) {
+ return;
+ }
+ updateExternalVmDataFromPrepareAnswer(vmTO, updatedTO);
+ updateExternalVmNicsFromPrepareAnswer(vmTO, updatedTO);
+ }
+
+ @Override
+ public void processPrepareExternalProvisioning(boolean firstStart, Host host,
+ VirtualMachineProfile vmProfile, DataCenter dataCenter, VirtualMachineTO virtualMachineTO)
+ throws CloudRuntimeException {
+ if (virtualMachineTO.getNics() == null || virtualMachineTO.getNics().length == 0) {
+ List nics = nicsDao.listByVmId(vmProfile.getId());
+ NicTO[] nicTOs = new NicTO[nics.size()];
+ nics.forEach(nicVO -> {
+ NicTO nicTO = toNicTO(networkModel.getNicProfile(vmProfile.getVirtualMachine(), nicVO, dataCenter),
+ host.getHypervisorType());
+ nicTOs[nicTO.getDeviceId()] = nicTO;
+ });
+ virtualMachineTO.setNics(nicTOs);
+ }
+ Map vmDetails = virtualMachineTO.getExternalDetails();
+ Map> externalDetails = extensionsManager.getExternalAccessDetails(host,
+ vmDetails);
+ PrepareExternalProvisioningCommand cmd = new PrepareExternalProvisioningCommand(virtualMachineTO);
+ cmd.setExternalDetails(externalDetails);
+ Answer answer = null;
+ CloudRuntimeException cre = new CloudRuntimeException("Failed to prepare VM");
+ try {
+ answer = agentMgr.send(host.getId(), cmd);
+ } catch (AgentUnavailableException | OperationTimedoutException e) {
+ logger.error("Failed PrepareExternalProvisioningCommand due to : {}", e.getMessage(), e);
+ throw cre;
+ }
+ if (answer == null) {
+ logger.error("Invalid answer received for PrepareExternalProvisioningCommand");
+ throw cre;
+ }
+ if (!(answer instanceof PrepareExternalProvisioningAnswer)) {
+ logger.error("Unexpected answer received for PrepareExternalProvisioningCommand: [result: {}, details: {}]",
+ answer.getResult(), answer.getDetails());
+ throw cre;
+ }
+ PrepareExternalProvisioningAnswer prepareAnswer = (PrepareExternalProvisioningAnswer) answer;
+ if (!prepareAnswer.getResult()) {
+ logger.error("Unexpected answer received for PrepareExternalProvisioningCommand: [result: {}, details: {}]",
+ answer.getResult(), answer.getDetails());
+ throw cre;
+ }
+ updateExternalVmFromPrepareAnswer(virtualMachineTO, prepareAnswer.getVirtualMachineTO());
+ }
+
+ @Override
+ public void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command) {
+ if (!HypervisorType.External.equals(host.getHypervisorType())) {
+ return;
+ }
+ Map vmExternalDetails = vmTO.getExternalDetails();
+ for (NicTO nic : vmTO.getNics()) {
+ if (!nic.isDefaultNic()) {
+ continue;
+ }
+ vmExternalDetails.put(VmDetailConstants.CLOUDSTACK_VLAN, networkService.getNicVlanValueForExternalVm(nic));
+ }
+ Map> externalDetails = extensionsManager.getExternalAccessDetails(host, vmExternalDetails);
+ command.setExternalDetails(externalDetails);
+ }
+
+ @Override
+ public void updateStopCommandForExternalHypervisorType(HypervisorType hypervisorType,
+ VirtualMachineProfile vmProfile, StopCommand stopCommand, VirtualMachineTO vmTO) {
+ if (!HypervisorType.External.equals(hypervisorType) || vmProfile.getHostId() == null) {
+ return;
+ }
+ HostVO host = hostDao.findById(vmProfile.getHostId());
+ if (host == null) {
+ return;
+ }
+ if (MapUtils.isEmpty(vmTO.getGuestOsDetails())) {
+ vmTO.setGuestOsDetails(null);
+ }
+ if (MapUtils.isEmpty(vmTO.getExtraConfig())) {
+ vmTO.setExtraConfig(null);
+ }
+ if (MapUtils.isEmpty(vmTO.getNetworkIdToNetworkNameMap())) {
+ vmTO.setNetworkIdToNetworkNameMap(null);
+ }
+ Map> externalDetails = extensionsManager.getExternalAccessDetails(host, vmTO.getExternalDetails());
+ stopCommand.setVirtualMachine(vmTO);
+ stopCommand.setExternalDetails(externalDetails);
+ }
+
+ @Override
+ public void updateRebootCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, RebootCommand rebootCmd) {
+ if (!HypervisorType.External.equals(host.getHypervisorType())) {
+ return;
+ }
+ Map> externalDetails = extensionsManager.getExternalAccessDetails(host, vmTO.getExternalDetails());
+ rebootCmd.setExternalDetails(externalDetails);
+ }
+
+ /**
+ * Build a {@link NicTO} from the given profile for the specified hypervisor,
+ * delegating to the hypervisor guru — mirrors
+ * {@link VirtualMachineManagerImpl#toNicTO(NicProfile, HypervisorType)}.
+ */
+ protected NicTO toNicTO(NicProfile nicProfile, HypervisorType hypervisorType) {
+ HypervisorGuru hvGuru = hvGuruMgr.getGuru(hypervisorType);
+ return hvGuru.toNicTO(nicProfile);
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmIscsiTargetManager.java b/engine/orchestration/src/main/java/com/cloud/vm/VmIscsiTargetManager.java
new file mode 100644
index 000000000000..f139b49f2387
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmIscsiTargetManager.java
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.vm;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * VMware managed-iSCSI dynamic-target cleanup helpers — gather the
+ * dynamic iSCSI targets a VM's managed volumes contribute to a VMware
+ * host, then ask that host (and its cluster siblings) to drop them from
+ * the iSCSI HBA's dynamic-target list.
+ *
+ *
Only VMware hosts produce non-empty target lists; every other
+ * hypervisor short-circuits to an empty result, leaving the
+ * {@link com.cloud.agent.api.ModifyTargetsCommand} unsent.
+ *
+ *
Extracted from {@link VirtualMachineManagerImpl} as part of the
+ * Phase 4 Spring-component decomposition.
+ */
+public interface VmIscsiTargetManager {
+
+ /**
+ * Collect the host/port/IQN tuples for every managed primary storage
+ * pool backing the VM's volumes. Returns an empty list when the
+ * host is not VMware, the host record is missing, the VM has no
+ * volumes, or none of its pools are flagged as managed.
+ */
+ List> getTargets(Long hostId, long vmId);
+
+ /**
+ * Send a {@link com.cloud.agent.api.ModifyTargetsCommand} that
+ * removes the supplied {@code targets} from the dynamic-target list
+ * on {@code hostId} and every other host in its cluster.
+ */
+ void removeDynamicTargets(long hostId, List> targets);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmIscsiTargetManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmIscsiTargetManagerImpl.java
new file mode 100644
index 000000000000..e7d3b72446c6
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmIscsiTargetManagerImpl.java
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.vm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.ModifyTargetsCommand;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VolumeDao;
+
+/**
+ * VMware managed-iSCSI dynamic-target cleanup — extracted from
+ * {@link VirtualMachineManagerImpl}.
+ *
+ * @see VmIscsiTargetManager
+ */
+@Component
+public class VmIscsiTargetManagerImpl implements VmIscsiTargetManager {
+
+ private static final Logger logger = LogManager.getLogger(VmIscsiTargetManagerImpl.class);
+
+ @Inject
+ private HostDao hostDao;
+ @Inject
+ private VolumeDao volumeDao;
+ @Inject
+ private PrimaryDataStoreDao storagePoolDao;
+ @Inject
+ private AgentManager agentMgr;
+
+ @Override
+ public List> getTargets(Long hostId, long vmId) {
+ List> targets = new ArrayList<>();
+
+ HostVO hostVO = hostDao.findById(hostId);
+
+ if (hostVO == null || hostVO.getHypervisorType() != HypervisorType.VMware) {
+ return targets;
+ }
+
+ List volumes = volumeDao.findByInstance(vmId);
+
+ if (CollectionUtils.isEmpty(volumes)) {
+ return targets;
+ }
+
+ for (VolumeVO volume : volumes) {
+ StoragePoolVO storagePoolVO = storagePoolDao.findById(volume.getPoolId());
+
+ if (storagePoolVO != null && storagePoolVO.isManaged()) {
+ Map target = new HashMap<>();
+
+ target.put(ModifyTargetsCommand.STORAGE_HOST, storagePoolVO.getHostAddress());
+ target.put(ModifyTargetsCommand.STORAGE_PORT, String.valueOf(storagePoolVO.getPort()));
+ target.put(ModifyTargetsCommand.IQN, volume.get_iScsiName());
+
+ targets.add(target);
+ }
+ }
+
+ return targets;
+ }
+
+ @Override
+ public void removeDynamicTargets(long hostId, List> targets) {
+ ModifyTargetsCommand cmd = new ModifyTargetsCommand();
+
+ cmd.setTargets(targets);
+ cmd.setApplyToAllHostsInCluster(true);
+ cmd.setAdd(false);
+ cmd.setTargetTypeToRemove(ModifyTargetsCommand.TargetTypeToRemove.DYNAMIC);
+
+ sendModifyTargetsCommand(cmd, hostId);
+ }
+
+ protected void sendModifyTargetsCommand(ModifyTargetsCommand cmd, long hostId) {
+ Answer answer = agentMgr.easySend(hostId, cmd);
+
+ if (answer == null) {
+ logger.warn("Unable to get an answer to the modify targets command. Targets [{}].",
+ () -> cmd.getTargets().stream().map(target -> target.toString()).collect(Collectors.joining(", ")));
+ return;
+ }
+
+ if (!answer.getResult()) {
+ logger.warn("Unable to modify targets [{}] on the host [{}].",
+ () -> cmd.getTargets().stream().map(target -> target.toString()).collect(Collectors.joining(", ")), () -> hostId);
+ }
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmMetadataSyncService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmMetadataSyncService.java
new file mode 100644
index 000000000000..9f95bb03443b
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmMetadataSyncService.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.Map;
+
+public interface VmMetadataSyncService {
+
+ void syncVMMetaData(Map vmMetadatum);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmMetadataSyncServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmMetadataSyncServiceImpl.java
new file mode 100644
index 000000000000..78c596728f56
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmMetadataSyncServiceImpl.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+import java.util.Map;
+
+import jakarta.inject.Inject;
+
+import org.springframework.stereotype.Component;
+
+import com.cloud.utils.Pair;
+import com.cloud.vm.dao.UserVmDao;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmMetadataSyncServiceImpl implements VmMetadataSyncService {
+
+ @Inject
+ protected UserVmDao userVmDao;
+ @Inject
+ protected VMInstanceDao vmDao;
+
+ @Override
+ public void syncVMMetaData(final Map vmMetadatum) {
+ if (vmMetadatum == null || vmMetadatum.isEmpty()) {
+ return;
+ }
+ List, Pair>> vmDetails = userVmDao.getVmsDetailByNames(vmMetadatum.keySet(), "platform");
+ for (final Map.Entry entry : vmMetadatum.entrySet()) {
+ final String name = entry.getKey();
+ final String platform = entry.getValue();
+ if (platform == null || platform.isEmpty()) {
+ continue;
+ }
+
+ boolean found = false;
+ for(Pair, Pair> vmDetail : vmDetails ) {
+ Pair vmNameTypePair = vmDetail.first();
+ if(vmNameTypePair.first().equals(name)) {
+ found = true;
+ if(vmNameTypePair.second() == VirtualMachine.Type.User) {
+ Pair detailPair = vmDetail.second();
+ String platformDetail = detailPair.second();
+
+ if (platformDetail != null && platformDetail.equals(platform)) {
+ break;
+ }
+ updateVmMetaData(detailPair.first(), platform);
+ }
+ break;
+ }
+ }
+
+ if(!found) {
+ VMInstanceVO vm = vmDao.findVMByInstanceName(name);
+ if(vm != null && vm.getType() == VirtualMachine.Type.User) {
+ updateVmMetaData(vm.getId(), platform);
+ }
+ }
+ }
+ }
+
+ private void updateVmMetaData(Long vmId, String platform) {
+ UserVmVO userVm = userVmDao.findById(vmId);
+ userVmDao.loadDetails(userVm);
+ if ( userVm.details.containsKey(VmDetailConstants.TIME_OFFSET)) {
+ userVm.details.remove(VmDetailConstants.TIME_OFFSET);
+ }
+ userVm.setDetail(VmDetailConstants.PLATFORM, platform);
+ String pvdriver = "xenserver56";
+ if ( platform.contains("device_id")) {
+ pvdriver = "xenserver61";
+ }
+ if (!userVm.details.containsKey(VmDetailConstants.HYPERVISOR_TOOLS_VERSION) || !userVm.details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION).equals(pvdriver)) {
+ userVm.setDetail(VmDetailConstants.HYPERVISOR_TOOLS_VERSION, pvdriver);
+ }
+ userVmDao.saveDetails(userVm);
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmMigrateAwayPlanningService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrateAwayPlanningService.java
new file mode 100644
index 000000000000..b1b0ee257969
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrateAwayPlanningService.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import com.cloud.deploy.DataCenterDeployment;
+import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.deploy.DeploymentPlanner.ExcludeList;
+import com.cloud.exception.InsufficientServerCapacityException;
+import com.cloud.host.Host;
+
+public interface VmMigrateAwayPlanningService {
+
+ void migrateAway(String vmUuid, long srcHostId) throws InsufficientServerCapacityException;
+
+ void orchestrateMigrateAway(String vmUuid, long srcHostId, DeploymentPlanner planner) throws InsufficientServerCapacityException;
+
+ boolean checkIfVmHasClusterWideVolumes(Long vmId);
+
+ DataCenterDeployment getMigrationDeployment(VirtualMachine vm, Host host, Long poolId, ExcludeList excludes);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmMigrateAwayPlanningServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrateAwayPlanningServiceImpl.java
new file mode 100644
index 000000000000..be6b4c37ee46
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrateAwayPlanningServiceImpl.java
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS;
+
+import java.util.List;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
+import org.apache.cloudstack.framework.jobs.Outcome;
+import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import com.cloud.dc.ClusterVO;
+import com.cloud.dc.DataCenter;
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.deploy.DataCenterDeployment;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.deploy.DeploymentPlanner;
+import com.cloud.deploy.DeploymentPlanner.ExcludeList;
+import com.cloud.deploy.DeploymentPlanningManager;
+import com.cloud.exception.AffinityConflictException;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InsufficientServerCapacityException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.ha.HighAvailabilityManager;
+import com.cloud.host.Host;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.service.dao.ServiceOfferingDao;
+import com.cloud.storage.ScopeType;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmMigrateAwayPlanningServiceImpl implements VmMigrateAwayPlanningService {
+
+ private static final Logger logger = LogManager.getLogger(VmMigrateAwayPlanningServiceImpl.class);
+
+ @Inject
+ protected VMInstanceDao vmDao;
+ @Inject
+ protected ServiceOfferingDao offeringDao;
+ @Inject
+ protected HostDao hostDao;
+ @Inject
+ protected VolumeDao volsDao;
+ @Inject
+ protected PrimaryDataStoreDao storagePoolDao;
+ @Inject
+ protected ClusterDao clusterDao;
+ @Inject
+ protected DeploymentPlanningManager dpMgr;
+ @Inject
+ protected HighAvailabilityManager haMgr;
+ @Inject
+ protected VmWorkJobQueueService vmWorkJobQueueService;
+ @Inject
+ @Lazy
+ protected VirtualMachineManagerImpl virtualMachineManager;
+
+ @Override
+ public void migrateAway(final String vmUuid, final long srcHostId) throws InsufficientServerCapacityException {
+ final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
+ if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
+ final VirtualMachine vm = vmDao.findByUuid(vmUuid);
+ VmWorkJobVO placeHolder = vmWorkJobQueueService.createPlaceHolderWork(vm.getId());
+ try {
+ try {
+ orchestrateMigrateAway(vmUuid, srcHostId, null);
+ } catch (final InsufficientServerCapacityException e) {
+ logger.warn("Failed to deploy vm {} with original planner, sending HAPlanner", vmUuid);
+ orchestrateMigrateAway(vmUuid, srcHostId, haMgr.getHAPlanner());
+ }
+ } finally {
+ vmWorkJobQueueService.expungePlaceHolderWork(placeHolder);
+ }
+ } else {
+ final Outcome outcome = vmWorkJobQueueService.migrateVmAwayThroughJobQueue(vmUuid, srcHostId);
+
+ vmWorkJobQueueService.retrieveVmFromJobOutcome(outcome, vmUuid, "migrateVmAway");
+
+ try {
+ vmWorkJobQueueService.retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome);
+ } catch (ResourceUnavailableException | InsufficientCapacityException ex) {
+ throw new RuntimeException("Unexpected exception", ex);
+ }
+ }
+ }
+
+ @Override
+ public void orchestrateMigrateAway(final String vmUuid, final long srcHostId, final DeploymentPlanner planner) throws InsufficientServerCapacityException {
+ final VMInstanceVO vm = vmDao.findByUuid(vmUuid);
+ if (vm == null) {
+ String message = String.format("Unable to find VM with uuid [%s].", vmUuid);
+ logger.warn(message);
+ throw new CloudRuntimeException(message);
+ }
+
+ ServiceOfferingVO offeringVO = offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
+ final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null);
+
+ final Long hostId = vm.getHostId();
+ if (hostId == null) {
+ String message = String.format("Unable to migrate %s due to it does not have a host id.", vm.toString());
+ logger.warn(message);
+ throw new CloudRuntimeException(message);
+ }
+
+ final Host host = hostDao.findById(hostId);
+ Long poolId = null;
+ final List vols = volsDao.findReadyRootVolumesByInstance(vm.getId());
+ for (final VolumeVO rootVolumeOfVm : vols) {
+ final StoragePoolVO rootDiskPool = storagePoolDao.findById(rootVolumeOfVm.getPoolId());
+ if (rootDiskPool != null) {
+ poolId = rootDiskPool.getId();
+ }
+ }
+
+ final ExcludeList excludes = new ExcludeList();
+ excludes.addHost(hostId);
+ DataCenterDeployment plan = getMigrationDeployment(vm, host, poolId, excludes);
+
+ DeployDestination dest = null;
+ while (true) {
+
+ try {
+ plan.setMigrationPlan(true);
+ dest = dpMgr.planDeployment(profile, plan, excludes, planner);
+ } catch (final AffinityConflictException e2) {
+ String message = String.format("Unable to create deployment, affinity rules associated to the %s conflict.", vm.toString());
+ logger.warn(message, e2);
+ throw new CloudRuntimeException(message, e2);
+ }
+ if (dest == null) {
+ logger.warn("Unable to find destination for migrating the vm {}", profile);
+ throw new InsufficientServerCapacityException("Unable to find a server to migrate to.", DataCenter.class, host.getDataCenterId());
+ }
+ logger.debug("Found destination {} for migrating to.", dest);
+
+ excludes.addHost(dest.getHost().getId());
+ try {
+ virtualMachineManager.migrate(vm, srcHostId, dest);
+ return;
+ } catch (ResourceUnavailableException | ConcurrentOperationException e) {
+ logger.warn("Unable to migrate {} to {} due to [{}]", vm.toString(), dest.getHost().toString(), e.getMessage(), e);
+ }
+
+ try {
+ virtualMachineManager.advanceStop(vmUuid, true);
+ throw new CloudRuntimeException("Unable to migrate " + vm);
+ } catch (final ResourceUnavailableException | ConcurrentOperationException | OperationTimedoutException e) {
+ logger.error("Unable to stop {} due to [{}].", vm.toString(), e.getMessage(), e);
+ throw new CloudRuntimeException("Unable to migrate " + vm);
+ }
+ }
+ }
+
+ /**
+ * Check if the virtual machine has any volume in cluster-wide pool
+ * @param vmId id of the virtual machine
+ * @return true if volume exists on cluster-wide pool else false
+ */
+ @Override
+ public boolean checkIfVmHasClusterWideVolumes(Long vmId) {
+ final List volumesList = volsDao.findCreatedByInstance(vmId);
+
+ return volumesList.parallelStream()
+ .anyMatch(vol -> storagePoolDao.findById(vol.getPoolId()).getScope().equals(ScopeType.CLUSTER));
+
+ }
+
+ @Override
+ public DataCenterDeployment getMigrationDeployment(final VirtualMachine vm, final Host host, final Long poolId, final ExcludeList excludes) {
+ if (MIGRATE_VM_ACROSS_CLUSTERS.valueIn(host.getDataCenterId()) &&
+ (HypervisorType.VMware.equals(host.getHypervisorType()) || !checkIfVmHasClusterWideVolumes(vm.getId()))) {
+ logger.info("Searching for hosts in the zone for vm migration");
+ List clustersToExclude = clusterDao.listAllClusterIds(host.getDataCenterId());
+ List clusterList = clusterDao.listByDcHyType(host.getDataCenterId(), host.getHypervisorType().toString());
+ for (ClusterVO cluster : clusterList) {
+ clustersToExclude.remove(cluster.getId());
+ }
+ for (Long clusterId : clustersToExclude) {
+ excludes.addCluster(clusterId);
+ }
+ if (VirtualMachine.systemVMs.contains(vm.getType())) {
+ return new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), null, null, poolId, null);
+ }
+ return new DataCenterDeployment(host.getDataCenterId(), null, null, null, poolId, null);
+ }
+ return new DataCenterDeployment(host.getDataCenterId(), host.getPodId(), host.getClusterId(), null, poolId, null);
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmMigrationCheckpointService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrationCheckpointService.java
new file mode 100644
index 000000000000..31fed485999b
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrationCheckpointService.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.Volume;
+
+public interface VmMigrationCheckpointService {
+
+ void endSnapshotChainForVolumes(Map volumeToPoolMap, HypervisorType hypervisorType);
+
+ void recreateCheckpointsKvmOnVmAfterMigration(VMInstanceVO vm, long hostId);
+
+ List getVmVolumesWithCheckpointsToRecreate(VMInstanceVO vm);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmMigrationCheckpointServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrationCheckpointServiceImpl.java
new file mode 100644
index 000000000000..8a8f80e33d44
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmMigrationCheckpointServiceImpl.java
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.RecreateCheckpointsCommand;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.Volume;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.snapshot.SnapshotManager;
+import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+@Component
+public class VmMigrationCheckpointServiceImpl implements VmMigrationCheckpointService {
+
+ private static final Logger logger = LogManager.getLogger(VmMigrationCheckpointServiceImpl.class);
+
+ @Inject
+ private AgentManager agentManager;
+ @Inject
+ private VolumeOrchestrationService volumeOrchestrationService;
+ @Inject
+ private SnapshotManager snapshotManager;
+ @Inject
+ private VolumeDao volumeDao;
+
+ @Override
+ public void endSnapshotChainForVolumes(Map volumeToPoolMap, HypervisorType hypervisorType) {
+ Set volumes = volumeToPoolMap.keySet();
+ volumes.forEach(volume -> {
+ Volume volumeOnDestination = volumeDao.findByPoolIdName(volumeToPoolMap.get(volume).getId(), volume.getName());
+ snapshotManager.endSnapshotChainForVolume(volumeOnDestination.getId(), hypervisorType);
+ });
+ }
+
+ @Override
+ public void recreateCheckpointsKvmOnVmAfterMigration(VMInstanceVO vm, long hostId) {
+ if (!HypervisorType.KVM.equals(vm.getHypervisorType())) {
+ logger.debug("Will not recreate checkpoint on VM as it is not running on KVM, thus it is not needed.");
+ return;
+ }
+
+ List volumes = getVmVolumesWithCheckpointsToRecreate(vm);
+
+ if (volumes.isEmpty()) {
+ logger.debug("Will not recreate checkpoints on VM as its volumes do not have any checkpoints associated with them.");
+ return;
+ }
+
+ RecreateCheckpointsCommand recreateCheckpointsCommand = new RecreateCheckpointsCommand(volumes, vm.getInstanceName());
+ Answer answer = null;
+ try {
+ logger.debug(String.format("Recreating the volume checkpoints with URLs [%s] of volumes [%s] on %s as part of the migration process.",
+ volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm));
+ answer = agentManager.send(hostId, recreateCheckpointsCommand);
+ } catch (AgentUnavailableException | OperationTimedoutException e) {
+ logger.error(String.format("Exception while sending command to host [%s] to recreate checkpoints with URLs [%s] of volumes [%s] on %s due to: [%s].",
+ hostId, volumes.stream().map(VolumeObjectTO::getCheckpointPaths).collect(Collectors.toList()), volumes, vm, e.getMessage()), e);
+ throw new CloudRuntimeException(e);
+ } finally {
+ if (answer != null && answer.getResult()) {
+ logger.debug(String.format("Successfully recreated checkpoints on VM [%s].", vm));
+ return;
+ }
+
+ logger.debug(String.format("Migration on VM [%s] was successful; however, we weren't able to recreate the checkpoints on it. Marking the snapshot chain as ended." +
+ " Next snapshot will create a new snapshot chain.", vm));
+
+ volumes.forEach(volumeObjectTO -> snapshotManager.endSnapshotChainForVolume(volumeObjectTO.getId(), HypervisorType.KVM));
+ }
+ }
+
+ @Override
+ public List getVmVolumesWithCheckpointsToRecreate(VMInstanceVO vm) {
+ List vmVolumes = volumeDao.findByInstance(vm.getId());
+ List volumes = new ArrayList<>();
+
+ for (VolumeVO volume : vmVolumes) {
+ Pair, Set> volumeCheckpointPathsAndImageStoreUrls =
+ volumeOrchestrationService.getVolumeCheckpointPathsAndImageStoreUrls(volume.getId(), HypervisorType.KVM);
+ if (volumeCheckpointPathsAndImageStoreUrls.first().isEmpty()) {
+ continue;
+ }
+ VolumeObjectTO volumeTo = new VolumeObjectTO();
+ volumeTo.setCheckpointPaths(volumeCheckpointPathsAndImageStoreUrls.first());
+ volumeTo.setCheckpointImageStoreUrls(volumeCheckpointPathsAndImageStoreUrls.second());
+ volumeTo.setPath(volume.getPath());
+ volumes.add(volumeTo);
+ }
+ return volumes;
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkAttachmentOrchestrationService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkAttachmentOrchestrationService.java
new file mode 100644
index 000000000000..55d010d5c8b2
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkAttachmentOrchestrationService.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.net.URI;
+
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.network.Network;
+
+public interface VmNetworkAttachmentOrchestrationService {
+
+ NicProfile addVmToNetwork(VirtualMachine vm, Network network, NicProfile requested, BackendNicOperations backendNicOperations)
+ throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
+
+ void checkIfNetworkExistsForUserVM(VirtualMachine virtualMachine, Network network);
+
+ NicTO toNicTO(NicProfile nic, HypervisorType hypervisorType);
+
+ boolean removeNicFromVm(VirtualMachine vm, Nic nic, BackendNicOperations backendNicOperations)
+ throws ConcurrentOperationException, ResourceUnavailableException;
+
+ boolean removeVmFromNetwork(VirtualMachine vm, Network network, URI broadcastUri, BackendNicOperations backendNicOperations)
+ throws ConcurrentOperationException, ResourceUnavailableException;
+
+ interface BackendNicOperations {
+ boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest)
+ throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
+
+ boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest)
+ throws ConcurrentOperationException, ResourceUnavailableException;
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkAttachmentOrchestrationServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkAttachmentOrchestrationServiceImpl.java
new file mode 100644
index 000000000000..5dc4267a48d0
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkAttachmentOrchestrationServiceImpl.java
@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.net.URI;
+import java.util.List;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.dc.DataCenter;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.event.EventTypes;
+import com.cloud.event.UsageEventUtils;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.HypervisorGuru;
+import com.cloud.hypervisor.HypervisorGuruManager;
+import com.cloud.network.Network;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.NicDao;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmNetworkAttachmentOrchestrationServiceImpl implements VmNetworkAttachmentOrchestrationService {
+
+ private static final Logger logger = LogManager.getLogger(VmNetworkAttachmentOrchestrationServiceImpl.class);
+
+ @Inject
+ protected UserVmManager userVmMgr;
+ @Inject
+ protected NetworkOrchestrationService networkMgr;
+ @Inject
+ protected NetworkModel networkModel;
+ @Inject
+ protected VMInstanceDao vmDao;
+ @Inject
+ protected NicDao nicsDao;
+ @Inject
+ protected HostDao hostDao;
+ @Inject
+ protected NetworkDao networkDao;
+ @Inject
+ protected HypervisorGuruManager hvGuruMgr;
+ @Inject
+ protected EntityManager entityMgr;
+
+ @Override
+ public NicProfile addVmToNetwork(final VirtualMachine vm, final Network network, final NicProfile requested,
+ BackendNicOperations backendNicOperations) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException {
+ final CallContext cctx = CallContext.current();
+
+ checkIfNetworkExistsForUserVM(vm, network);
+ logger.debug("Adding Instance {} to Network {}; requested NIC profile {}", vm, network, requested);
+ final VMInstanceVO vmVO = vmDao.findById(vm.getId());
+ final ReservationContext context = new ReservationContextImpl(null, null, cctx.getCallingUser(), cctx.getCallingAccount());
+
+ final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmVO, null, null, null, null);
+
+ final DataCenter dc = entityMgr.findById(DataCenter.class, network.getDataCenterId());
+ final Host host = hostDao.findById(vm.getHostId());
+ final DeployDestination dest = new DeployDestination(dc, null, null, host);
+
+ if (vm.getState() == State.Running) {
+ final NicProfile nic = networkMgr.createNicForVm(network, requested, context, vmProfile, true);
+
+ final HypervisorGuru hvGuru = hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType());
+ final VirtualMachineTO vmTO = hvGuru.implement(vmProfile);
+
+ final NicTO nicTO = toNicTO(nic, vmProfile.getVirtualMachine().getHypervisorType());
+
+ logger.debug("Plugging NIC for Instance {} in Network {}", vm, network);
+
+ boolean result = false;
+ try {
+ result = backendNicOperations.plugNic(network, nicTO, vmTO, context, dest);
+ if (result) {
+ userVmMgr.setupVmForPvlan(true, vm.getHostId(), nic);
+ logger.debug("Nic is plugged successfully for vm {} in network {}. VM is a part of network now.", vm, network);
+ final long isDefault = nic.isDefaultNic() ? 1 : 0;
+
+ if (VirtualMachine.Type.User.equals(vmVO.getType())) {
+ UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(),
+ Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay());
+ }
+ return nic;
+ } else {
+ logger.warn("Failed to plug NIC to the Instance {} in Network {}", vm, network);
+ return null;
+ }
+ } finally {
+ if (!result) {
+ logger.debug("Removing NIC {} from Instance {} as NIC plug failed on the backend.", nic, vmProfile.getVirtualMachine());
+ networkMgr.removeNic(vmProfile, nicsDao.findById(nic.getId()));
+ }
+ }
+ } else if (vm.getState() == State.Stopped) {
+ return networkMgr.createNicForVm(network, requested, context, vmProfile, false);
+ } else {
+ logger.warn("Unable to add vm {} to network {}", vm, network);
+ throw new ResourceUnavailableException("Unable to add vm " + vm + " to network, is not in the right state", DataCenter.class, vm.getDataCenterId());
+ }
+ }
+
+ /**
+ * duplicated in {@see UserVmManagerImpl} for a {@see UserVmVO}
+ */
+ @Override
+ public void checkIfNetworkExistsForUserVM(VirtualMachine virtualMachine, Network network) {
+ if (virtualMachine.getType() != VirtualMachine.Type.User) {
+ return; // others may have multiple nics in the same network
+ }
+ List allNics = nicsDao.listByVmId(virtualMachine.getId());
+ for (NicVO nic : allNics) {
+ if (nic.getNetworkId() == network.getId()) {
+ throw new CloudRuntimeException("A NIC already exists for VM:" + virtualMachine.getInstanceName() + " in network: " + network.getUuid());
+ }
+ }
+ }
+
+ @Override
+ public NicTO toNicTO(final NicProfile nic, final HypervisorType hypervisorType) {
+ final HypervisorGuru hvGuru = hvGuruMgr.getGuru(hypervisorType);
+ return hvGuru.toNicTO(nic);
+ }
+
+ @Override
+ public boolean removeNicFromVm(final VirtualMachine vm, final Nic nic, BackendNicOperations backendNicOperations)
+ throws ConcurrentOperationException, ResourceUnavailableException {
+ final CallContext cctx = CallContext.current();
+ final VMInstanceVO vmVO = vmDao.findById(vm.getId());
+ final NetworkVO network = networkDao.findById(nic.getNetworkId());
+ final ReservationContext context = new ReservationContextImpl(null, null, cctx.getCallingUser(), cctx.getCallingAccount());
+
+ final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmVO, null, null, null, null);
+
+ final DataCenter dc = entityMgr.findById(DataCenter.class, network.getDataCenterId());
+ final Host host = hostDao.findById(vm.getHostId());
+ final DeployDestination dest = new DeployDestination(dc, null, null, host);
+ final HypervisorGuru hvGuru = hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType());
+ final VirtualMachineTO vmTO = hvGuru.implement(vmProfile);
+
+ final NicProfile nicProfile =
+ new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), networkModel.getNetworkRate(network.getId(), vm.getId()),
+ networkModel.isSecurityGroupSupportedInNetwork(network), networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network));
+
+ if (vm.getState() == State.Running) {
+ final NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType());
+ logger.debug("Un-plugging NIC {} for Instance {} from Network {}.", nic, vm, network);
+ final boolean result = backendNicOperations.unplugNic(network, nicTO, vmTO, context, dest);
+ if (result) {
+ userVmMgr.setupVmForPvlan(false, vm.getHostId(), nicProfile);
+ logger.debug("NIC is unplugged successfully for Instance {} in Network {}.", vm, network);
+ final long isDefault = nic.isDefaultNic() ? 1 : 0;
+ UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(),
+ Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplay());
+ } else {
+ logger.warn("Failed to unplug NIC for the Instance {} from Network {}.", vm, network);
+ return false;
+ }
+ } else if (vm.getState() != State.Stopped) {
+ logger.warn("Unable to remove Instance {} from Network {}", vm, network);
+ throw new ResourceUnavailableException("Unable to remove Instance " + vm + " from Network, is not in the right state", DataCenter.class, vm.getDataCenterId());
+ }
+
+ networkMgr.releaseNic(vmProfile, nic);
+ logger.debug("Successfully released NIC {} for Instance {}", nic, vm);
+
+ networkMgr.removeNic(vmProfile, nic);
+ nicsDao.remove(nic.getId());
+ return true;
+ }
+
+ @Override
+ @DB
+ public boolean removeVmFromNetwork(final VirtualMachine vm, final Network network, final URI broadcastUri, BackendNicOperations backendNicOperations)
+ throws ConcurrentOperationException, ResourceUnavailableException {
+ final CallContext cctx = CallContext.current();
+ final VMInstanceVO vmVO = vmDao.findById(vm.getId());
+ final ReservationContext context = new ReservationContextImpl(null, null, cctx.getCallingUser(), cctx.getCallingAccount());
+
+ final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmVO, null, null, null, null);
+
+ final DataCenter dc = entityMgr.findById(DataCenter.class, network.getDataCenterId());
+ final Host host = hostDao.findById(vm.getHostId());
+ final DeployDestination dest = new DeployDestination(dc, null, null, host);
+ final HypervisorGuru hvGuru = hvGuruMgr.getGuru(vmProfile.getVirtualMachine().getHypervisorType());
+ final VirtualMachineTO vmTO = hvGuru.implement(vmProfile);
+
+ Nic nic = null;
+ if (broadcastUri != null) {
+ nic = nicsDao.findByNetworkIdInstanceIdAndBroadcastUri(network.getId(), vm.getId(), broadcastUri.toString());
+ } else {
+ nic = networkModel.getNicInNetwork(vm.getId(), network.getId());
+ }
+
+ if (nic == null) {
+ logger.warn("Could not get a NIC with {}", network);
+ return false;
+ }
+
+ if (nic.isDefaultNic() && vm.getType() == VirtualMachine.Type.User) {
+ logger.warn("Failed to remove NIC from {} in {}, NIC is default.", vm, network);
+ throw new CloudRuntimeException("Failed to remove NIC from " + vm + " in " + network + ", NIC is default.");
+ }
+
+ final Nic lock = nicsDao.acquireInLockTable(nic.getId());
+ if (lock == null) {
+ if (nicsDao.findById(nic.getId()) == null) {
+ logger.debug("Not need to remove the vm {} from network {} as the vm doesn't have nic in this network.", vm, network);
+ return true;
+ }
+ throw new ConcurrentOperationException(String.format("Unable to lock nic %s", nic));
+ }
+
+ logger.debug("Lock is acquired for nic {} as a part of remove vm {} from network {}", lock, vm, network);
+
+ try {
+ final NicProfile nicProfile =
+ new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), networkModel.getNetworkRate(network.getId(), vm.getId()),
+ networkModel.isSecurityGroupSupportedInNetwork(network), networkModel.getNetworkTag(vmProfile.getVirtualMachine().getHypervisorType(), network));
+
+ if (vm.getState() == State.Running) {
+ final NicTO nicTO = toNicTO(nicProfile, vmProfile.getVirtualMachine().getHypervisorType());
+ logger.debug("Un-plugging nic for vm {} from network {}", vm, network);
+ final boolean result = backendNicOperations.unplugNic(network, nicTO, vmTO, context, dest);
+ if (result) {
+ logger.debug("Nic is unplugged successfully for vm {} in network {}", vm, network);
+ } else {
+ logger.warn("Failed to unplug nic for the vm {} from network {}", vm, network);
+ return false;
+ }
+ } else if (vm.getState() != State.Stopped) {
+ logger.warn("Unable to remove vm {} from network {}", vm, network);
+ throw new ResourceUnavailableException("Unable to remove vm " + vm + " from network, is not in the right state", DataCenter.class, vm.getDataCenterId());
+ }
+
+ networkMgr.releaseNic(vmProfile, nic);
+ logger.debug("Successfully released nic {} for vm {}", nic, vm);
+
+ networkMgr.removeNic(vmProfile, nic);
+ return true;
+ } finally {
+ nicsDao.releaseFromLockTable(lock.getId());
+ logger.debug("Lock is released for nic {} as a part of remove vm {} from network {}", lock, vm, network);
+ }
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkNameMappingService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkNameMappingService.java
new file mode 100644
index 000000000000..fd8ddb0e3201
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkNameMappingService.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import com.cloud.agent.api.to.VirtualMachineTO;
+
+public interface VmNetworkNameMappingService {
+
+ void setVmNetworkDetails(VMInstanceVO vm, VirtualMachineTO vmTO);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkNameMappingServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkNameMappingServiceImpl.java
new file mode 100644
index 000000000000..4a38f1cee033
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNetworkNameMappingServiceImpl.java
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import jakarta.inject.Inject;
+
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.api.query.dao.DomainRouterJoinDao;
+import com.cloud.api.query.dao.UserVmJoinDao;
+import com.cloud.api.query.vo.DomainRouterJoinVO;
+import com.cloud.api.query.vo.UserVmJoinVO;
+import com.cloud.dc.DataCenter;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.domain.Domain;
+import com.cloud.domain.dao.DomainDao;
+import com.cloud.network.Networks;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
+import com.cloud.network.vpc.VpcVO;
+import com.cloud.network.vpc.dao.VpcDao;
+import com.cloud.user.Account;
+import com.cloud.user.dao.AccountDao;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+@Component
+public class VmNetworkNameMappingServiceImpl implements VmNetworkNameMappingService {
+
+ @Inject
+ protected UserVmJoinDao userVmJoinDao;
+ @Inject
+ protected DomainRouterJoinDao domainRouterJoinDao;
+ @Inject
+ protected NetworkDao networkDao;
+ @Inject
+ protected AccountDao accountDao;
+ @Inject
+ protected DomainDao domainDao;
+ @Inject
+ protected DataCenterDao dataCenterDao;
+ @Inject
+ protected VpcDao vpcDao;
+
+ @Override
+ public void setVmNetworkDetails(VMInstanceVO vm, VirtualMachineTO vmTO) {
+ Map networkToNetworkNameMap = new HashMap<>();
+ if (VirtualMachine.Type.User.equals(vm.getType())) {
+ List userVmJoinVOs = userVmJoinDao.searchByIds(vm.getId());
+ if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) {
+ for (UserVmJoinVO userVmJoinVO : userVmJoinVOs) {
+ addToNetworkNameMap(userVmJoinVO.getNetworkId(), vm.getDataCenterId(), networkToNetworkNameMap);
+ }
+ vmTO.setNetworkIdToNetworkNameMap(networkToNetworkNameMap);
+ }
+ } else if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) {
+ List routerJoinVO = domainRouterJoinDao.getRouterByIdAndTrafficType(vm.getId(), Networks.TrafficType.Guest);
+ for (DomainRouterJoinVO router : routerJoinVO) {
+ NetworkVO guestNetwork = networkDao.findById(router.getNetworkId());
+ if (guestNetwork.getVpcId() == null && guestNetwork.getBroadcastDomainType() == Networks.BroadcastDomainType.NSX) {
+ addToNetworkNameMap(router.getNetworkId(), vm.getDataCenterId(), networkToNetworkNameMap);
+ }
+ }
+ vmTO.setNetworkIdToNetworkNameMap(networkToNetworkNameMap);
+ }
+ }
+
+ private void addToNetworkNameMap(long networkId, long dataCenterId, Map networkToNetworkNameMap) {
+ NetworkVO networkVO = networkDao.findById(networkId);
+ Account acc = accountDao.findById(networkVO.getAccountId());
+ Domain domain = domainDao.findById(networkVO.getDomainId());
+ DataCenter zone = dataCenterDao.findById(dataCenterId);
+ if (Objects.isNull(zone)) {
+ throw new CloudRuntimeException(String.format("Failed to find zone with ID: %s", dataCenterId));
+ }
+ if (Objects.isNull(acc)) {
+ throw new CloudRuntimeException(String.format("Failed to find account with ID: %s", networkVO.getAccountId()));
+ }
+ if (Objects.isNull(domain)) {
+ throw new CloudRuntimeException(String.format("Failed to find domain with ID: %s", networkVO.getDomainId()));
+ }
+ String networkName = String.format("D%s-A%s-Z%s", domain.getId(), acc.getId(), zone.getId());
+ if (Objects.isNull(networkVO.getVpcId())) {
+ networkName += "-S" + networkVO.getId();
+ } else {
+ VpcVO vpc = vpcDao.findById(networkVO.getVpcId());
+ if (Objects.isNull(vpc)) {
+ throw new CloudRuntimeException(String.format("Failed to find VPC with ID: %s", networkVO.getVpcId()));
+ }
+ networkName = String.format("%s-V%s-S%s", networkName, vpc.getId(), networkVO.getId());
+ }
+ networkToNetworkNameMap.put(networkVO.getId(), networkName);
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNicBackendCommandService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNicBackendCommandService.java
new file mode 100644
index 000000000000..8aee2345ecd3
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNicBackendCommandService.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import com.cloud.network.Network;
+
+public interface VmNicBackendCommandService {
+
+ boolean replugNic(Network network, NicTO nic, VirtualMachineTO vm, Host host) throws ConcurrentOperationException,
+ ResourceUnavailableException, InsufficientCapacityException;
+
+ boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException,
+ ResourceUnavailableException, InsufficientCapacityException;
+
+ boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException,
+ ResourceUnavailableException;
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNicBackendCommandServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNicBackendCommandServiceImpl.java
new file mode 100644
index 000000000000..0a95491e580c
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNicBackendCommandServiceImpl.java
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.commons.collections.MapUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.PlugNicAnswer;
+import com.cloud.agent.api.PlugNicCommand;
+import com.cloud.agent.api.ReplugNicAnswer;
+import com.cloud.agent.api.ReplugNicCommand;
+import com.cloud.agent.api.UnPlugNicAnswer;
+import com.cloud.agent.api.UnPlugNicCommand;
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.agent.manager.Commands;
+import com.cloud.dc.DataCenter;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.host.Host;
+import com.cloud.network.Network;
+import com.cloud.network.dao.NetworkDetailVO;
+import com.cloud.network.dao.NetworkDetailsDao;
+import com.cloud.offering.NetworkOffering;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.UserVmDao;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmNicBackendCommandServiceImpl implements VmNicBackendCommandService {
+
+ private static final Logger logger = LogManager.getLogger(VmNicBackendCommandServiceImpl.class);
+
+ @Inject
+ protected AgentManager agentMgr;
+ @Inject
+ protected VMInstanceDao vmDao;
+ @Inject
+ protected UserVmDao userVmDao;
+ @Inject
+ protected UserVmService userVmService;
+ @Inject
+ protected NetworkDetailsDao networkDetailsDao;
+ @Inject
+ protected VmVlanPersistenceMappingService vmVlanPersistenceMappingService;
+
+ @Override
+ public boolean replugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final Host host) throws ConcurrentOperationException,
+ ResourceUnavailableException, InsufficientCapacityException {
+ boolean result = true;
+
+ final VMInstanceVO router = vmDao.findById(vm.getId());
+ if (router.getState() == State.Running) {
+ try {
+ final ReplugNicCommand replugNicCmd = new ReplugNicCommand(nic, vm.getName(), vm.getType(), vm.getDetails());
+ final Commands cmds = new Commands(Command.OnError.Stop);
+ cmds.addCommand("replugnic", replugNicCmd);
+ agentMgr.send(host.getId(), cmds);
+ final ReplugNicAnswer replugNicAnswer = cmds.getAnswer(ReplugNicAnswer.class);
+ if (replugNicAnswer == null || !replugNicAnswer.getResult()) {
+ logger.warn("Unable to replug nic for vm {}", vm.getName());
+ result = false;
+ }
+ } catch (final OperationTimedoutException e) {
+ throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, host.getId(), e);
+ }
+ } else {
+ String message = String.format("Unable to apply ReplugNic, VM [%s] is not in the right state (\"Running\"). VM state [%s].", router.toString(), router.getState());
+ logger.warn(message);
+
+ throw new ResourceUnavailableException(message, DataCenter.class, router.getDataCenterId());
+ }
+
+ return result;
+ }
+
+ @Override
+ public boolean plugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest)
+ throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException {
+ boolean result = true;
+
+ final VMInstanceVO router = vmDao.findById(vm.getId());
+ if (router.getState() == State.Running) {
+ try {
+ NetworkDetailVO pvlanTypeDetail = networkDetailsDao.findDetail(network.getId(), ApiConstants.ISOLATED_PVLAN_TYPE);
+ if (pvlanTypeDetail != null) {
+ Map nicDetails = nic.getDetails() == null ? new HashMap<>() : nic.getDetails();
+ logger.debug("Found PVLAN type: {} on network details, adding it as part of the PlugNicCommand", pvlanTypeDetail.getValue());
+ nicDetails.putIfAbsent(NetworkOffering.Detail.pvlanType, pvlanTypeDetail.getValue());
+ nic.setDetails(nicDetails);
+ }
+ final PlugNicCommand plugNicCmd = new PlugNicCommand(nic, vm.getName(), vm.getType(), vm.getDetails());
+ final Commands cmds = new Commands(Command.OnError.Stop);
+ cmds.addCommand("plugnic", plugNicCmd);
+ agentMgr.send(dest.getHost().getId(), cmds);
+ final PlugNicAnswer plugNicAnswer = cmds.getAnswer(PlugNicAnswer.class);
+ if (plugNicAnswer == null || !plugNicAnswer.getResult()) {
+ logger.warn("Unable to plug nic for vm {}", vm.getName());
+ result = false;
+ }
+ } catch (final OperationTimedoutException e) {
+ throw new AgentUnavailableException("Unable to plug nic for router " + vm.getName() + " in network " + network, dest.getHost().getId(), e);
+ }
+ } else {
+ String message = String.format("Unable to apply PlugNic, VM [%s] is not in the right state (\"Running\"). VM state [%s].", router.toString(), router.getState());
+ logger.warn(message);
+
+ throw new ResourceUnavailableException(message, DataCenter.class,
+ router.getDataCenterId());
+ }
+
+ return result;
+ }
+
+ @Override
+ public boolean unplugNic(final Network network, final NicTO nic, final VirtualMachineTO vm, final ReservationContext context, final DeployDestination dest)
+ throws ConcurrentOperationException, ResourceUnavailableException {
+
+ boolean result = true;
+ final VMInstanceVO router = vmDao.findById(vm.getId());
+
+ if (router.getState() == State.Running) {
+ UserVmVO userVm = userVmDao.findById(vm.getId());
+ if (userVm != null && userVm.getType() == VirtualMachine.Type.User) {
+ userVmService.collectVmNetworkStatistics(userVm);
+ }
+ try {
+ final Commands cmds = new Commands(Command.OnError.Stop);
+ final UnPlugNicCommand unplugNicCmd = new UnPlugNicCommand(nic, vm.getName());
+ Map vlanToPersistenceMap = vmVlanPersistenceMappingService.getVlanToPersistenceMapForVM(vm.getId());
+ if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
+ unplugNicCmd.setVlanToPersistenceMap(vlanToPersistenceMap);
+ }
+ cmds.addCommand("unplugnic", unplugNicCmd);
+ agentMgr.send(dest.getHost().getId(), cmds);
+
+ final UnPlugNicAnswer unplugNicAnswer = cmds.getAnswer(UnPlugNicAnswer.class);
+ if (unplugNicAnswer == null || !unplugNicAnswer.getResult()) {
+ logger.warn("Unable to unplug nic from router {}", router);
+ result = false;
+ }
+ } catch (final OperationTimedoutException e) {
+ throw new AgentUnavailableException("Unable to unplug nic from rotuer " + router + " from network " + network, dest.getHost().getId(), e);
+ }
+ } else if (router.getState() == State.Stopped || router.getState() == State.Stopping) {
+ logger.debug("Vm {} is in {}, so not sending unplug nic command to the backend", router.getInstanceName(), router.getState());
+ } else {
+ String message = String.format("Unable to apply unplug nic, VM [%s] is not in the right state (\"Running\"). VM state [%s].", router.toString(), router.getState());
+ logger.warn(message);
+
+ throw new ResourceUnavailableException(message, DataCenter.class, router.getDataCenterId());
+ }
+
+ return result;
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNicUpdateService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNicUpdateService.java
new file mode 100644
index 000000000000..89ab5f7637e6
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNicUpdateService.java
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import com.cloud.exception.ResourceUnavailableException;
+
+public interface VmNicUpdateService {
+
+ Boolean updateDefaultNicForVM(VirtualMachine vm, Nic nic, Nic defaultNic);
+
+ boolean updateVmNic(VirtualMachine vm, Nic nic, Boolean enabled) throws ResourceUnavailableException;
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmNicUpdateServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmNicUpdateServiceImpl.java
new file mode 100644
index 000000000000..5b23f6bfa50a
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmNicUpdateServiceImpl.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import jakarta.inject.Inject;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.UpdateVmNicAnswer;
+import com.cloud.agent.api.UpdateVmNicCommand;
+import com.cloud.agent.manager.Commands;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.dao.NicDao;
+
+@Component
+public class VmNicUpdateServiceImpl implements VmNicUpdateService {
+
+ private static final Logger logger = LogManager.getLogger(VmNicUpdateServiceImpl.class);
+
+ @Inject
+ protected AgentManager agentMgr;
+ @Inject
+ protected NicDao nicsDao;
+
+ @Override
+ public Boolean updateDefaultNicForVM(final VirtualMachine vm, final Nic nic, final Nic defaultNic) {
+ logger.debug("Updating default nic of vm {} from nic {} to nic {}", vm, defaultNic.getUuid(), nic.getUuid());
+ Integer chosenID = nic.getDeviceId();
+ Integer existingID = defaultNic.getDeviceId();
+ NicVO nicVO = nicsDao.findById(nic.getId());
+ NicVO defaultNicVO = nicsDao.findById(defaultNic.getId());
+
+ nicVO.setDefaultNic(true);
+ nicVO.setDeviceId(existingID);
+ defaultNicVO.setDefaultNic(false);
+ defaultNicVO.setDeviceId(chosenID);
+
+ nicsDao.persist(nicVO);
+ nicsDao.persist(defaultNicVO);
+ return true;
+ }
+
+ @Override
+ public boolean updateVmNic(final VirtualMachine vm, final Nic nic, final Boolean enabled) throws ResourceUnavailableException {
+ if (vm.getState() == State.Running) {
+ try {
+ UpdateVmNicCommand updateVmNicCmd = new UpdateVmNicCommand(nic.getMacAddress(), vm.getName(), enabled);
+ Commands cmds = new Commands(Command.OnError.Stop);
+ cmds.addCommand("updatevmnic", updateVmNicCmd);
+
+ agentMgr.send(vm.getHostId(), cmds);
+
+ UpdateVmNicAnswer updateVmNicAnswer = cmds.getAnswer(UpdateVmNicAnswer.class);
+ if (updateVmNicAnswer == null || !updateVmNicAnswer.getResult()) {
+ logger.warn("Unable to update VM {} NIC [{}].", vm.getName(), nic.getUuid());
+ return false;
+ }
+ } catch (final OperationTimedoutException e) {
+ throw new AgentUnavailableException(String.format("Unable to update NIC %s for VM %s.", nic.getUuid(), vm.getUuid()), vm.getHostId(), e);
+ }
+ }
+
+ NicVO nicVo = nicsDao.findById(nic.getId());
+ nicVo.setEnabled(enabled);
+ nicsDao.persist(nicVo);
+
+ return true;
+ }
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmOfflineStorageMigrationService.java b/engine/orchestration/src/main/java/com/cloud/vm/VmOfflineStorageMigrationService.java
new file mode 100644
index 000000000000..c12820247138
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmOfflineStorageMigrationService.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.Map;
+
+public interface VmOfflineStorageMigrationService {
+
+ void orchestrateStorageMigration(String vmUuid, Map volumeToPool);
+}
diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmOfflineStorageMigrationServiceImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VmOfflineStorageMigrationServiceImpl.java
new file mode 100644
index 000000000000..0462da163602
--- /dev/null
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VmOfflineStorageMigrationServiceImpl.java
@@ -0,0 +1,365 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.vm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import jakarta.inject.Inject;
+
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.MigrateVmToPoolAnswer;
+import com.cloud.agent.api.UnregisterVMCommand;
+import com.cloud.agent.manager.Commands;
+import com.cloud.dc.ClusterDetailsDao;
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.deploy.DataCenterDeployment;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.StorageUnavailableException;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.HypervisorGuru;
+import com.cloud.hypervisor.HypervisorGuruManager;
+import com.cloud.org.Cluster;
+import com.cloud.storage.DiskOfferingVO;
+import com.cloud.storage.StorageManager;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.Volume;
+import com.cloud.storage.Volume.Type;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.DiskOfferingDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.utils.Pair;
+import com.cloud.utils.StringUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.fsm.NoTransitionException;
+import com.cloud.vm.VirtualMachine.Event;
+import com.cloud.vm.dao.VMInstanceDao;
+
+@Component
+public class VmOfflineStorageMigrationServiceImpl implements VmOfflineStorageMigrationService {
+
+ private static final Logger logger = LogManager.getLogger(VmOfflineStorageMigrationServiceImpl.class);
+
+ @Inject
+ protected VMInstanceDao vmInstanceDao;
+ @Inject
+ protected VolumeDao volumeDao;
+ @Inject
+ protected PrimaryDataStoreDao storagePoolDao;
+ @Inject
+ protected ClusterDao clusterDao;
+ @Inject
+ protected HostDao hostDao;
+ @Inject
+ protected DiskOfferingDao diskOfferingDao;
+ @Inject
+ protected ClusterDetailsDao clusterDetailsDao;
+ @Inject
+ protected AgentManager agentMgr;
+ @Inject
+ protected HypervisorGuruManager hvGuruMgr;
+ @Inject
+ protected NetworkOrchestrationService networkMgr;
+ @Inject
+ protected VolumeOrchestrationService volumeMgr;
+ @Inject
+ protected StorageManager storageMgr;
+ @Inject
+ protected VmVolumeMigrationPlanningService vmVolumeMigrationPlanningService;
+ @Inject
+ @Lazy
+ protected VirtualMachineManager virtualMachineManager;
+
+ @Override
+ public void orchestrateStorageMigration(final String vmUuid, final Map volumeToPool) {
+ final VMInstanceVO vm = vmInstanceDao.findByUuid(vmUuid);
+
+ try {
+ Map volumeToPoolMap = prepareVmStorageMigration(vm, volumeToPool);
+
+ logger.debug("Offline migration of {} vm {} with volumes",
+ vm.getHypervisorType().toString(),
+ vm.getInstanceName());
+
+ migrateThroughHypervisorOrStorage(vm, volumeToPoolMap);
+
+ } catch (ConcurrentOperationException
+ | InsufficientCapacityException
+ | StorageUnavailableException e) {
+ String msg = String.format("Failed to migrate VM: %s", vmUuid);
+ logger.warn(msg, e);
+ throw new CloudRuntimeException(msg, e);
+ } finally {
+ try {
+ virtualMachineManager.stateTransitTo(vm, Event.AgentReportStopped, null);
+ } catch (final NoTransitionException e) {
+ String anotherMEssage = String.format("failed to change vm state of VM: %s", vmUuid);
+ logger.warn(anotherMEssage, e);
+ throw new CloudRuntimeException(anotherMEssage, e);
+ }
+ }
+ }
+
+ protected Answer[] attemptHypervisorMigration(VMInstanceVO vm, Map volumeToPool, Long hostId) {
+ if (hostId == null) {
+ return null;
+ }
+ final HypervisorGuru hvGuru = hvGuruMgr.getGuru(vm.getHypervisorType());
+
+ List commandsToSend = hvGuru.finalizeMigrate(vm, volumeToPool);
+
+ if (CollectionUtils.isNotEmpty(commandsToSend)) {
+ Commands commandsContainer = new Commands(Command.OnError.Stop);
+ commandsContainer.addCommands(commandsToSend);
+
+ try {
+ return agentMgr.send(hostId, commandsContainer);
+ } catch (AgentUnavailableException | OperationTimedoutException e) {
+ logger.warn("Hypervisor migration failed for the VM: {}", vm, e);
+ }
+ }
+ return null;
+ }
+
+ protected void afterHypervisorMigrationCleanup(VMInstanceVO vm, Map volumeToPool, Long sourceClusterId, Answer[] hypervisorMigrationResults) throws InsufficientCapacityException {
+ logger.debug("Cleaning up after hypervisor pool migration volumes for VM {}({})", vm.getInstanceName(), vm.getUuid());
+
+ StoragePool rootVolumePool = null;
+ if (MapUtils.isNotEmpty(volumeToPool)) {
+ for (Map.Entry entry : volumeToPool.entrySet()) {
+ if (Type.ROOT.equals(entry.getKey().getVolumeType())) {
+ rootVolumePool = entry.getValue();
+ break;
+ }
+ }
+ }
+ setDestinationPoolAndReallocateNetwork(rootVolumePool, vm);
+ Long destClusterId = rootVolumePool != null ? rootVolumePool.getClusterId() : null;
+ if (destClusterId != null && !destClusterId.equals(sourceClusterId)) {
+ logger.debug("Resetting lastHost for VM {}({})", vm.getInstanceName(), vm.getUuid());
+ vm.setLastHostId(null);
+ vm.setPodIdToDeployIn(rootVolumePool.getPodId());
+ }
+
+ markVolumesInPool(vm, hypervisorMigrationResults);
+ }
+
+ protected void markVolumesInPool(VMInstanceVO vm, Answer[] hypervisorMigrationResults) {
+ MigrateVmToPoolAnswer relevantAnswer = null;
+ if (hypervisorMigrationResults.length == 1 && !hypervisorMigrationResults[0].getResult()) {
+ throw new CloudRuntimeException(String.format("VM ID: %s migration failed. %s", vm.getUuid(), hypervisorMigrationResults[0].getDetails()));
+ }
+ for (Answer answer : hypervisorMigrationResults) {
+ logger.debug("Received an {}: {}", answer.getClass().getSimpleName(), answer);
+ if (answer instanceof MigrateVmToPoolAnswer) {
+ relevantAnswer = (MigrateVmToPoolAnswer) answer;
+ }
+ }
+ if (relevantAnswer == null) {
+ throw new CloudRuntimeException("No relevant migration results found");
+ }
+ List results = relevantAnswer.getVolumeTos();
+ if (results == null) {
+ results = new ArrayList<>();
+ }
+ List volumes = volumeDao.findUsableVolumesForInstance(vm.getId());
+ logger.debug("Found {} volumes for VM {}(uuid:{}, id:{})", results.size(), vm.getInstanceName(), vm.getUuid(), vm.getId());
+ for (VolumeObjectTO result : results) {
+ logger.debug("Updating volume ({}) with path '{}' on pool '{}'", result.getUuid(), result.getPath(), result.getDataStoreUuid());
+ VolumeVO volume = volumeDao.findById(result.getId());
+ StoragePool pool = storagePoolDao.findPoolByUUID(result.getDataStoreUuid());
+ if (volume == null || pool == null) {
+ continue;
+ }
+ volume.setPath(result.getPath());
+ volume.setPoolId(pool.getId());
+ volume.setPoolType(pool.getPoolType());
+ if (result.getChainInfo() != null) {
+ volume.setChainInfo(result.getChainInfo());
+ }
+ volumeDao.update(volume.getId(), volume);
+ }
+ }
+
+ protected void migrateThroughHypervisorOrStorage(VMInstanceVO vm, Map volumeToPool) throws StorageUnavailableException, InsufficientCapacityException {
+ final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+ Pair vmClusterAndHost = virtualMachineManager.findClusterAndHostIdForVm(vm, false);
+ final Long sourceClusterId = vmClusterAndHost.first();
+ final Long sourceHostId = vmClusterAndHost.second();
+ Answer[] hypervisorMigrationResults = attemptHypervisorMigration(vm, volumeToPool, sourceHostId);
+ boolean migrationResult = false;
+ if (hypervisorMigrationResults == null) {
+ migrationResult = volumeMgr.storageMigration(profile, volumeToPool);
+ if (migrationResult) {
+ postStorageMigrationCleanup(vm, volumeToPool, hostDao.findById(sourceHostId), sourceClusterId);
+ } else {
+ logger.debug("Storage migration failed");
+ }
+ } else {
+ afterHypervisorMigrationCleanup(vm, volumeToPool, sourceClusterId, hypervisorMigrationResults);
+ }
+ }
+
+ protected Map