From 7e7c5c4767c6205e48f0ca6422b34d6a29538354 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 25 Aug 2026 21:52:37 +0000 Subject: [PATCH 01/10] multikernel: add per-instance duplex IPI transport Give every parent/child pair a private duplex shared-memory link. Each direction has one producer and kernel-local serialization. Slots use a one-bit publication protocol. Carry the real endpoint identities and doorbell CPUs in the manifest. Scan local receive endpoints on doorbell delivery. Reset a link only after the previous child is confirmed parked. Keep force-halt as a persistent per-link marker. Gate incompatible images at load and early boot. Use a bzImage capability bit and a generation-specific vmlinux note. Stamp the fixed spawn-context magic field with the same generation. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 8 +- arch/x86/include/asm/multikernel.h | 5 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 19 + arch/x86/kernel/kexec-vmlinux.c | 14 +- arch/x86/kernel/platform-quirks.c | 4 +- arch/x86/multikernel/head_64.S | 3 +- arch/x86/multikernel/spawn.c | 132 +++++-- include/linux/multikernel.h | 79 ++++- include/linux/multikernel_abi.h | 9 + kernel/kexec_core.c | 57 +-- kernel/multikernel/core.c | 120 ++++--- kernel/multikernel/instance_dt.c | 150 ++++---- kernel/multikernel/internal.h | 8 + kernel/multikernel/ipi.c | 482 +++++++++++--------------- kernel/multikernel/manifest.c | 31 +- 16 files changed, 632 insertions(+), 490 deletions(-) create mode 100644 include/linux/multikernel_abi.h diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index ad6c99238d69b4..571b81296200b9 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -107,7 +107,7 @@ struct mk_spawn_context { u32 target_apic_id; /* Target CPU's APIC ID */ u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ - u32 reserved; /* Padding for alignment */ + u32 abi_magic; /* Host/spawn generation marker */ /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ /* Optional boot data belongs after boot_params, in the zeroed tail. */ @@ -175,7 +175,8 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, int mk_spawn_cpu(struct mk_instance *instance, int cpu, struct mk_spawn_context *ctx); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); /* Identity page table and trampoline setup */ diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 3a5b3fca1ab352..2be0a6d8aaddf8 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; @@ -748,6 +753,20 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { image->arch.mk_boot_params = bootparam_load_addr; + + /* + * setup_boot_parameters() copies the host subarchitecture. A + * spawn kernel must take the multikernel platform path instead. + */ + params->hdr.hardware_subarch = X86_SUBARCH_MULTIKERNEL; + + /* + * The spawn trampoline enters the compressed kernel directly, + * bypassing purgatory. The x86 boot protocol's 64-bit entry is + * 0x200 bytes from the start of the protected-mode payload. + */ + image->arch.mk_kernel_entry = kernel_load_addr + 0x200; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a7d31a722f33b7..9d8b71e717237f 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -62,7 +62,7 @@ struct elf_kernel_info { /* * Find multikernel entry point from PT_NOTE section. - * Looks for note with name "Linux" and type 0x4d4b ('MK'). + * The note type carries the generation; the descriptor remains one u64. */ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, const Elf64_Ehdr *ehdr) @@ -91,14 +91,16 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (ptr + note_size > end) break; - if (nhdr->n_type == 0x4d4b && + if (nhdr->n_type == MK_VMLINUX_NOTE_TYPE && nhdr->n_namesz == 6 && nhdr->n_descsz == sizeof(u64) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const u64 *entry; + + entry = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + pr_info("multikernel: entry=0x%llx\n", *entry); + return *entry; } ptr += note_size; } diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index feea109497efde..9209b77d1c54e3 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..54890d936e615e 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -269,7 +270,7 @@ SYM_CODE_END(multikernel_secondary_startup) .balign 4 .long 2f - 1f .long 4f - 3f - .long 0x4d4b + .long MK_VMLINUX_NOTE_TYPE 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 7ba24f38710e5d..550e4c4abb285c 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -71,6 +71,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * Spawn kernel's own trampoline for secondary CPU wakeup. @@ -84,6 +85,16 @@ static struct mk_spawn_context *mk_boot_context; */ static void *spawn_trampoline_va; static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; @@ -483,6 +494,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->arch.spawn_ctx->boot_tsc_khz = tsc_khz; instance->arch.spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->arch.spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->arch.spawn_ctx); } @@ -705,18 +717,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u32 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -733,14 +765,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -776,17 +822,25 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_boot_context; unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -794,34 +848,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - spawn_trampoline_phys = ctx->trampoline_phys; - spawn_trampoline_va = __va(spawn_trampoline_phys); - memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 5ca15323c4722d..dbb773249b4c9b 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -14,6 +14,11 @@ #include #include #include +#include +#include + +struct pci_bus; +struct mk_instance; #ifdef CONFIG_MULTIKERNEL #include @@ -84,22 +89,22 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ - unsigned int type; /* User-defined type identifier */ + unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ char buffer[MK_MAX_DATA_SIZE]; /* Actual data buffer */ }; /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -108,6 +113,40 @@ struct mk_shared_data { * CPUs the first one missed. */ u32 force_halt; + s32 parent_id; + s32 child_id; + u32 reserved; + u64 parent_doorbell_cpu; + u64 child_doorbell_cpu; +}; + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); +} + +struct mk_ipi_endpoint { + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + bool tx_enabled; + bool rx_dispatching; + bool parent_side; + bool registered; + struct list_head rx_node; }; /* Function pointer type for IPI callbacks */ @@ -152,8 +191,13 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +void mk_ipi_endpoint_close(struct mk_instance *instance); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu); +void mk_ipi_handlers_enable(void); /* * Multikernel Messaging System @@ -592,6 +636,7 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + struct mk_ipi_endpoint ipi_endpoint; /* Kexec integration */ struct kimage *kimage; /* Associated kimage object */ @@ -843,6 +888,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -857,6 +903,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -911,6 +958,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -918,6 +970,7 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 +/* Bumped whenever the shared-memory layout or message semantics change. */ #define MK_FDT_COMPATIBLE "multikernel-v1" /** @@ -947,16 +1000,6 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ int mk_manifest_add_instance_dtb(struct kimage *image, void *fdt, int mk_id); -/** - * mk_manifest_add_host_ipi() - Add the host's IPI buffer address to the manifest - * @image: Target kimage - * @fdt: The manifest FDT being built - * - * Returns: 0 on success, negative error code on failure - */ -int mk_manifest_add_host_ipi(struct kimage *image, void *fdt); - - /** * mk_instance_restore_from_manifest() - Restore this instance from the manifest * @@ -1039,6 +1082,8 @@ void mk_arch_register_cpu(mk_phys_cpu_t phys_id); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86, SDEI or diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..3d9bb5fdeee61e --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* One generation marker shared by the spawn context and image capabilities. */ +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0002 +#define MK_VMLINUX_NOTE_TYPE 0x4d4b0002 + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 01382f40aeb724..c66b68fd028577 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1702,6 +1702,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1756,10 +1762,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1774,32 +1781,34 @@ int multikernel_kexec_by_id(int mk_id) instance->ipi_data = phys_to_virt(mk_image->mk_ipi); } - /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + /* Reset only this parent/child link, after the old child is parked. */ + if (instance->ipi_data) { + mk_ipi_link_reset(instance, root_instance->id, mk_id, + mk_cpu_set_first(root_instance->cpus), + mk_cpu_set_first(instance->cpus)); + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) + goto unlock; /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * The instance is running once its CPUs leave the park loop. Publish that + * state before dropping the global kexec lock so another exec cannot race + * this boot while the readiness handshake is pending. */ - mk_ipi_ring_drop_pending(); + rc = mk_instance_set_kexec_active(mk_image->mk_id); + if (rc) { + int abort_ret = mk_instance_abort_spawn(instance); - rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc == 0) { - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) - pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); + if (abort_ret) + pr_crit("Instance %d activation abort failed: %d\n", + mk_id, abort_ret); + goto unlock; } + kexec_unlock(); + return 0; + unlock: kexec_unlock(); return rc; diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index ae94c490e405be..998cf6cf191a65 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -139,6 +139,7 @@ static void mk_instance_release(struct kref *kref) pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + mk_ipi_endpoint_unregister(instance); mk_instance_return_all_cpus(instance); mk_instance_return_pci_devices(instance); @@ -286,12 +287,25 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; /* Empty until the instance first ran, so nothing of it is executing */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + if (!instance->cpus_on_slot) + return 0; + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -299,6 +313,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1543,38 +1558,16 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** - * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI - * @mk_id: Instance ID to halt - * - * Forces a spawn kernel's CPUs to stop by arming the force-halt marker - * in the instance's shared IPI area and sending NMIs directly to each - * CPU. The NMI handler tests the marker and parks the CPU in the pool. - * - * No message is queued and no doorbell is rung: a ring message is - * consumed by the instance's ordinary interrupt path, which on a - * responsive kernel races the NMIs for it and can leave them with - * nothing to act on. The marker is host-owned and survives until the - * instance is re-executed, so the NMIs act on it regardless of timing. - * - * Use when: The spawn kernel is stuck/crashed and not responding to graceful - * shutdown, or when graceful shutdown has failed. May be repeated: an - * already-halted instance absorbs the NMIs in the park loop, so a rerun - * only rescues CPUs an earlier halt missed. - * - * Returns: 0 on success, negative error code on failure - */ -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; /* * LOADED is allowed for the retry case: a previous halt already @@ -1583,20 +1576,19 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } if (mk_cpu_set_empty(instance->cpus)) { - pr_err("Instance %d has no CPUs assigned\n", mk_id); - mk_instance_put(instance); + pr_err("Instance %d has no CPUs assigned\n", instance->id); return -EINVAL; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1608,22 +1600,70 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); + + ret = mk_instance_confirm_parked(instance); + if (ret) { + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + return ret; + } - /* - * The NMI handler parks each CPU on the instance's context. Wait - * for them to arrive before reporting the instance re-spawnable, - * exactly as the graceful path does after its shutdown ACK. - */ mk_instance_settle_halted(instance); - mk_instance_put(instance); return 0; } +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); + mk_instance_put(instance); + return ret; +} + static int __init multikernel_init(void) { int ret; + if (!root_instance) { + pr_err("Multikernel root instance is unavailable\n"); + return -ENODEV; + } + ret = mk_arch_register_force_stop(); if (ret < 0) { pr_warn("No force stop handler: %d (force halt unavailable)\n", ret); @@ -1660,6 +1700,8 @@ static int __init multikernel_init(void) return ret; } + mk_ipi_handlers_enable(); + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 56d74a11523237..a404f1dc6e8fc4 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -34,6 +35,23 @@ struct mk_instance *root_instance = NULL; EXPORT_SYMBOL_GPL(root_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + ret = mk_arch_register_force_stop(); + if (ret) + pr_emerg("multikernel: force-stop registration failed while rejecting manifest: %d\n", + ret); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /* * Collect every CPU the instance might receive through hotplug later: * the unassigned pool plus every other kernel's CPUs (the host's and @@ -156,43 +174,6 @@ int mk_manifest_add_instance_dtb(struct kimage *image, void *fdt, int mk_id) return 0; } -/** - * mk_manifest_add_host_ipi() - Add the host's IPI buffer address to the manifest - * @image: Target kimage - * @fdt: The manifest FDT being built - * - * Called during kexec preparation to pass the host's IPI receive buffer - * address to the spawn kernel so it can send messages back to the host. - * - * Returns: 0 on success, negative error code on failure - */ -int mk_manifest_add_host_ipi(struct kimage *image, void *fdt) -{ - int ret = 0; - - if (!root_instance->ipi_data) { - pr_debug("No host IPI buffer to preserve\n"); - return 0; - } - - pr_info("Preserving host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)root_instance->ipi_phys, root_instance->ipi_pages); - - ret |= fdt_begin_node(fdt, "host-ipi-buffer"); - ret |= fdt_property_u64(fdt, "phys-addr", root_instance->ipi_phys); - ret |= fdt_property_u32(fdt, "pages", root_instance->ipi_pages); - ret |= fdt_end_node(fdt); - - if (ret) { - pr_err("Failed to add host IPI buffer to manifest: %d\n", ret); - return ret; - } - - pr_info("Added host IPI buffer to manifest\n"); - return 0; -} - - /** * mk_dt_extract_instance_info() - Extract instance ID and name from DTB * @dtb_data: Device tree blob data @@ -386,6 +367,7 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char pr_err("Failed to allocate IPI buffer for instance %d\n", instance_id); goto err_free_name; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -543,6 +525,11 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan (unsigned long long)ipi_phys, ipi_pages); return 0; } + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -562,64 +549,44 @@ static int __init mk_restore_instance_ipi(const void *manifest, struct mk_instan static struct mk_instance * __init mk_restore_host_instance(const void *manifest) { struct mk_instance *host_instance; - int host_ipi_node; - const fdt64_t *phys_prop; - const fdt32_t *pages_prop; - phys_addr_t host_ipi_phys = 0; - u32 host_ipi_pages = 0; - size_t host_ipi_size = 0; + const fdt32_t *id_prop; + const fdt64_t *cpu_prop; + mk_phys_cpu_t parent_cpu; + u32 parent_id; + int ipi_node; int len; - host_ipi_node = fdt_subnode_offset(manifest, 0, "host-ipi-buffer"); - if (host_ipi_node < 0) { - pr_warn("No host-ipi-buffer node in manifest (spawn won't be able to send to host)\n"); + ipi_node = fdt_subnode_offset(manifest, 0, "ipi-buffer"); + if (ipi_node < 0 || !root_instance->ipi_data) return NULL; - } - - phys_prop = fdt_getprop(manifest, host_ipi_node, "phys-addr", &len); - if (phys_prop && len == sizeof(*phys_prop)) - host_ipi_phys = (phys_addr_t)fdt64_to_cpu(*phys_prop); - - pages_prop = fdt_getprop(manifest, host_ipi_node, "pages", &len); - if (pages_prop && len == sizeof(*pages_prop)) { - host_ipi_pages = fdt32_to_cpu(*pages_prop); - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; - } - - if (!host_ipi_phys || !host_ipi_pages) { - pr_warn("Incomplete host IPI buffer info (phys=0x%llx, pages=%u)\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); + id_prop = fdt_getprop(manifest, ipi_node, "parent-id", &len); + if (!id_prop || len != sizeof(*id_prop)) return NULL; - } + parent_id = fdt32_to_cpu(*id_prop); + cpu_prop = fdt_getprop(manifest, ipi_node, "parent-doorbell-cpu", &len); + if (!cpu_prop || len != sizeof(*cpu_prop)) + return NULL; + parent_cpu = fdt64_to_cpu(*cpu_prop); - host_instance = alloc_mk_instance(0, "", false); + host_instance = alloc_mk_instance(parent_id, "", false); if (!host_instance) return NULL; - - /* Set physical CPU 0 as default target for host IPIs */ - if (mk_cpu_set_add(host_instance->cpus, 0)) { + if (mk_cpu_set_add(host_instance->cpus, parent_cpu)) { kfree(host_instance->name); mk_cpu_set_free(host_instance->cpus); kfree(host_instance); return NULL; } - - host_instance->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!host_instance->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + host_instance->ipi_data = root_instance->ipi_data; + host_instance->ipi_phys = root_instance->ipi_phys; + host_instance->ipi_pages = root_instance->ipi_pages; + if (mk_ipi_endpoint_init(host_instance, false)) { kfree(host_instance->name); mk_cpu_set_free(host_instance->cpus); kfree(host_instance); return NULL; } - host_instance->ipi_phys = host_ipi_phys; - host_instance->ipi_pages = host_ipi_pages; - pr_info("Restored host IPI buffer: phys=0x%llx, virt=%px, pages=%u\n", - (unsigned long long)host_ipi_phys, host_instance->ipi_data, - host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); - + pr_info("Registered parent instance %u on duplex IPI link\n", parent_id); return host_instance; } @@ -645,11 +612,14 @@ int __init mk_instance_restore_from_manifest(void) const void *manifest = NULL; phys_addr_t fdt_phys; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + fdt_phys = mk_manifest_phys(); if (!fdt_phys) { pr_info("No manifest available for multikernel DTB restoration\n"); - instance = alloc_mk_instance(0, "", true); + instance = alloc_mk_instance(0, "", false); if (!instance) { pr_err("Failed to allocate root instance\n"); return -ENOMEM; @@ -682,15 +652,15 @@ int __init mk_instance_restore_from_manifest(void) int mk_node = fdt_subnode_offset(manifest, 0, "multikernel"); if (mk_node < 0) { - pr_info("No multikernel node found in manifest\n"); - ret = 0; + pr_err("No multikernel node found in supplied manifest\n"); + ret = -EINVAL; goto cleanup_fdt; } const void *dtb_data = fdt_getprop(manifest, mk_node, "dtb-data", &dtb_len); if (!dtb_data || dtb_len <= 0) { - pr_info("No dtb-data property found in multikernel node\n"); - ret = 0; + pr_err("No dtb-data property found in multikernel node\n"); + ret = -EINVAL; goto cleanup_fdt; } @@ -787,8 +757,16 @@ int __init mk_instance_restore_from_manifest(void) host_instance = mk_restore_host_instance(manifest); if (!host_instance) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); - + mk_manifest_reject_and_park(-ENODEV); + + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); + ret = mk_arch_register_force_stop(); + if (ret) + mk_manifest_reject_and_park(ret); pr_info("Successfully restored multikernel root instance %d ('%s') from manifest (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); @@ -822,6 +800,8 @@ int __init mk_instance_restore_from_manifest(void) kfree(dtb_virt); cleanup_fdt: early_memunmap((void *)manifest, PAGE_SIZE); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 2fe809f97c2281..ec086a4bebd4c9 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -13,6 +13,14 @@ extern struct idr mk_instance_idr; extern struct list_head mk_instance_list; extern struct mk_instance *root_instance; +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 749500ff1f6a2f..a0309661a43f61 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -1,375 +1,315 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved - */ -#include -#include #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include #include "internal.h" -/* Callback management */ static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); +static LIST_HEAD(mk_ipi_endpoints); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static bool mk_handlers_ready; + +static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) +{ + struct mk_shared_data *ipi_data; -static void mk_ipi_drain_ring(void); + if (instance->ipi_data) + return instance->ipi_data; + if (!instance->kimage || !instance->kimage->mk_ipi) + return NULL; + ipi_data = phys_to_virt(instance->kimage->mk_ipi); + cmpxchg(&instance->ipi_data, NULL, ipi_data); + return instance->ipi_data; +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. - */ -static inline unsigned int mk_ring_idx(unsigned int i) +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; + + if (!mk_instance_ipi_area(instance)) + return -ENODEV; + if (endpoint->registered) + return 0; + raw_spin_lock_init(&endpoint->tx_lock); + raw_spin_lock_init(&endpoint->rx_lock); + endpoint->tx = parent_side ? &instance->ipi_data->to_child : + &instance->ipi_data->to_parent; + endpoint->rx = parent_side ? &instance->ipi_data->to_parent : + &instance->ipi_data->to_child; + endpoint->tx_head = 0; + endpoint->rx_tail = 0; + endpoint->tx_enabled = true; + endpoint->rx_dispatching = false; + endpoint->parent_side = parent_side; + INIT_LIST_HEAD(&endpoint->rx_node); + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + endpoint->registered = true; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return 0; } -/** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring - * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. - */ -void mk_ipi_ring_drop_pending(void) +void mk_ipi_endpoint_close(struct mk_instance *instance) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - if (!root_instance || !root_instance->ipi_data) + if (!endpoint->registered) return; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); +} + +void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - ring = &root_instance->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); + if (!endpoint->registered) + return; + mk_ipi_endpoint_close(instance); + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + list_del_rcu(&endpoint->rx_node); + endpoint->registered = false; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + synchronize_rcu(); +} - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu) +{ + struct mk_shared_data *shared = mk_instance_ipi_area(instance); - atomic_set(&ring->tail, head); + if (!shared) + return; + mk_ipi_endpoint_unregister(instance); + mk_shared_data_reset(shared); + WRITE_ONCE(shared->parent_id, parent_id); + WRITE_ONCE(shared->child_id, child_id); + WRITE_ONCE(shared->parent_doorbell_cpu, parent_cpu); + WRITE_ONCE(shared->child_doorbell_cpu, child_cpu); + mk_ipi_endpoint_init(instance, true); } -/** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process - * - * Returns pointer to handler on success, NULL on failure - */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); handler->next = mk_handlers; mk_handlers = handler; raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); -/** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler - */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; + struct mk_ipi_handler **pp, *p = NULL; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { + for (pp = &mk_handlers; (p = *pp); pp = &p->next) { if (p == handler) { *pp = p->next; break; } - pp = &p->next; } raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - kfree(p); } EXPORT_SYMBOL(multikernel_unregister_handler); -/* - * An instance's IPI area is allocated when its image is loaded; the - * instance pointer is filled in lazily on first use. - */ -static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) -{ - struct mk_shared_data *ipi_data; - - if (instance->ipi_data) - return instance->ipi_data; - - if (!instance->kimage || !instance->kimage->mk_ipi) - return NULL; - - ipi_data = phys_to_virt(instance->kimage->mk_ipi); - if (cmpxchg(&instance->ipi_data, NULL, ipi_data) == NULL) - pr_info("Initialized IPI ring buffer for instance %d: phys=0x%llx\n", - instance->id, (unsigned long long)instance->kimage->mk_ipi); - - return instance->ipi_data; -} - -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = mk_instance_ipi_area(instance); + struct mk_shared_data *shared = mk_instance_ipi_area(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } -/** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier - * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure - */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { + struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; mk_phys_cpu_t target; + unsigned long flags; + u32 idx; + int ret = 0; - if (!instance) - return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + if (!instance || data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; + endpoint = &instance->ipi_endpoint; + if (!endpoint->registered) { + ret = mk_ipi_endpoint_init(instance, true); + if (ret) + return ret; } - - target = mk_cpu_set_first(instance->cpus); - if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPUs to receive the IPI\n", instance_id); - mk_instance_put(instance); + target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : + READ_ONCE(instance->ipi_data->parent_doorbell_cpu); + if (target == MK_PHYS_CPU_INVALID) return -ENODEV; + if (endpoint->parent_side) + WRITE_ONCE(instance->ipi_data->child_doorbell_cpu, target); + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + if (!endpoint->tx_enabled) { + ret = -ESHUTDOWN; + goto unlock; } - - if (!mk_instance_ipi_area(instance)) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->tx->entries[idx]; + /* Pair with the receiver's release when it makes the slot reusable. */ + if (smp_load_acquire(&slot->ready)) { + ret = -ENOSPC; + goto unlock; } - - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) + WRITE_ONCE(slot->sender_cpu, arch_cpu_physical_id(smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) memcpy(slot->buffer, data, data_size); + /* Publish all message fields before the receiver observes readiness. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; +unlock: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + if (!ret) + mk_arch_send_ipi(target); + return ret; +} - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); - - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance = mk_instance_find(instance_id); + int ret; + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } -static void mk_ipi_drain_ring(void) +static void mk_ipi_dispatch(struct mk_ipi_data *slot) { - struct mk_ipi_data *slot; struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; - size_t data_size; - int messages_processed = 0; + mk_ipi_callback_t callback = NULL; + void *context = NULL; + unsigned long flags; - if (!root_instance || !root_instance->ipi_data) + if (READ_ONCE(slot->data_size) > MK_MAX_DATA_SIZE) return; - - while (1) { - tail = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&root_instance->ipi_data->ring.head)); - - if (tail == head) - break; - - slot = &root_instance->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + raw_spin_lock_irqsave(&mk_handlers_lock, flags); + for (handler = mk_handlers; handler; handler = handler->next) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + callback = handler->callback; + context = handler->context; break; - - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - continue; } + } + raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); + if (callback) + callback(slot, context); +} - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; +static void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_data *slot; + unsigned long flags; + u32 idx; - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (endpoint->rx_dispatching) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; + } + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + for (;;) { + idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->rx->entries[idx]; + /* Pair with the producer's release publication of this slot. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + /* Close the empty-ring handoff race with a new publication. */ + if (smp_load_acquire(&slot->ready)) { + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + continue; } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; } - raw_spin_unlock(&mk_handlers_lock); - -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&root_instance->ipi_data->ring.tail, next_tail); - messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; + mk_ipi_dispatch(slot); + /* The callback must finish reading before the slot is reusable. */ + smp_store_release(&slot->ready, 0); + endpoint->rx_tail++; } } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static void mk_ipi_drain_all(void) { - if (!root_instance || !root_instance->ipi_data) + struct mk_ipi_endpoint *endpoint; + + if (!READ_ONCE(mk_handlers_ready)) return; + rcu_read_lock(); + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) + mk_ipi_drain_endpoint(endpoint); + rcu_read_unlock(); +} + +void mk_ipi_handlers_enable(void) +{ + WRITE_ONCE(mk_handlers_ready, true); + mk_ipi_drain_all(); +} + +void mk_poll_ipi_messages(void) +{ + unsigned long flags; - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - */ - mk_ipi_drain_ring(); + local_irq_save(flags); + mk_ipi_drain_all(); + local_irq_restore(flags); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + mk_ipi_drain_all(); } -/** - * mk_has_pending_shutdown - Check if the host demanded a forcible shutdown - * - * Tests the force-halt marker the host arms before NMIing this kernel's - * CPUs. The marker used to be a message peeked in the IPI ring, but a - * ring message is consumed by the doorbell interrupt: on a responsive - * kernel the ordinary message path could eat it before the NMIs landed, - * and every NMI then found nothing to act on. The marker is host-owned - * and stays up until the host has confirmed all CPUs parked, so it - * gives the same answer no matter when each NMI arrives. - * - * Safe to call from NMI context (a single read of shared memory). - * - * Returns: true if shutdown requested, false otherwise - */ bool mk_has_pending_shutdown(void) { - if (!root_instance || !root_instance->ipi_data) - return false; - - return READ_ONCE(root_instance->ipi_data->force_halt); + return root_instance && root_instance->ipi_data && + READ_ONCE(root_instance->ipi_data->force_halt); } diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 89a8ee0d2c59b3..271b26198e2d0e 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -23,12 +23,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -50,6 +56,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -68,14 +75,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /** @@ -123,16 +133,9 @@ int mk_manifest_finalize(struct kimage *image) return ret; } - ret = mk_manifest_add_host_ipi(image, fdt); - if (ret) { - pr_err("Failed to preserve host IPI buffer: %d\n", ret); - fdt_end_node(fdt); - fdt_finish(fdt); - return ret; - } - /* Add IPI buffer information if allocated */ if (image->mk_ipi) { + struct mk_instance *child = mk_instance_find(image->mk_id); u64 ipi_phys = (u64)image->mk_ipi; size_t ipi_buffer_size = sizeof(struct mk_shared_data); u32 ipi_pages = (u32)(PAGE_ALIGN(ipi_buffer_size) >> PAGE_SHIFT); @@ -140,7 +143,15 @@ int mk_manifest_finalize(struct kimage *image) ret = fdt_begin_node(fdt, "ipi-buffer"); ret |= fdt_property_u64(fdt, "phys-addr", ipi_phys); ret |= fdt_property_u32(fdt, "pages", ipi_pages); + ret |= fdt_property_u32(fdt, "parent-id", root_instance->id); + ret |= fdt_property_u32(fdt, "child-id", image->mk_id); + ret |= fdt_property_u64(fdt, "parent-doorbell-cpu", + mk_cpu_set_first(root_instance->cpus)); + ret |= fdt_property_u64(fdt, "child-doorbell-cpu", + child ? mk_cpu_set_first(child->cpus) : + MK_PHYS_CPU_INVALID); ret |= fdt_end_node(fdt); + mk_instance_put(child); if (ret) { pr_err("Failed to add IPI buffer to manifest: %d\n", ret); From 3fb46794454a22a04f6f4085a0ae7fbf192caa4b Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 09:16:32 +0000 Subject: [PATCH 02/10] multikernel: close IPI endpoint teardown races Preserve endpoint lifetime invariants across teardown and reset. Report legacy vmlinux images with a clear compatibility error. Signed-off-by: Nikolay Nikolaev --- arch/x86/kernel/kexec-vmlinux.c | 3 ++- kernel/kexec_core.c | 5 ++++- kernel/multikernel/ipi.c | 7 ++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index 9d8b71e717237f..a9ccbe34602594 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -163,7 +163,8 @@ static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_l */ info->multikernel_entry = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr); if (!info->multikernel_entry) { - pr_err("multikernel_startup_64 entry offset not found in PT_NOTE\n"); + pr_err("legacy or incompatible vmlinux: ABI note type 0x%x not found\n", + MK_VMLINUX_NOTE_TYPE); return -ENOEXEC; } diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index c66b68fd028577..29bf34cd6b8ef1 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -447,7 +447,6 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image, unsigned long i; cond_resched(); - if (hole_end > KEXEC_CRASH_CONTROL_MEMORY_LIMIT) break; /* See if I overlap any of the segments */ @@ -609,6 +608,10 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + /* Stop delivery before image-owned shared pages are returned. */ + if (image->mk_instance) + mk_ipi_endpoint_unregister(image->mk_instance); + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index a0309661a43f61..092aa595fe09f8 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -167,11 +167,8 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, if (!instance || data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; endpoint = &instance->ipi_endpoint; - if (!endpoint->registered) { - ret = mk_ipi_endpoint_init(instance, true); - if (ret) - return ret; - } + if (!READ_ONCE(endpoint->registered)) + return -ESHUTDOWN; target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : READ_ONCE(instance->ipi_data->parent_doorbell_cpu); if (target == MK_PHYS_CPU_INVALID) From 906eb2b9a0290c6f505c2c68b7eb313d2be83a51 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 09:47:25 +0000 Subject: [PATCH 03/10] multikernel: guard PCI probe filter when PCI is disabled Keep the multikernel build valid when PCI support is disabled. Omit the PCI-only probe filter implementation in that configuration. Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/instance_dt.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index a404f1dc6e8fc4..4962c06b2f09fa 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -819,6 +819,7 @@ early_initcall(mk_instance_restore_from_manifest); * * Returns: true if probing should proceed, false to skip entirely */ +#if IS_ENABLED(CONFIG_PCI) bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { struct mk_pci_device *pci_dev; @@ -890,6 +891,7 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn) return false; } EXPORT_SYMBOL_GPL(mk_pci_should_probe); +#endif /* CONFIG_PCI */ bool mk_platform_device_allowed(const char *name, const char *hid) { From 9c68b984ab2276d2d34b6790dd2acb4659d09259 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 09:48:39 +0000 Subject: [PATCH 04/10] multikernel: guard PCI hotplug helpers Return ENODEV for PCI hotplug requests when PCI is disabled. Preserve existing PCI-enabled behavior. Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/hotplug.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index cf4125b480f35b..e8b61b03cb6601 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -636,6 +636,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay * PCI Device Hotplug Operations */ +#if IS_ENABLED(CONFIG_PCI) static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -805,6 +806,19 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } +#else /* CONFIG_PCI */ +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -ENODEV; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -ENODEV; +} +#endif /* CONFIG_PCI */ + struct mk_device_hotplug_work { struct work_struct work; u16 domain; From 16cd35aae0ec5c5f2b4b06dbe5c09b7c999a7d13 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 11:02:44 +0000 Subject: [PATCH 05/10] multikernel: close endpoint lifetime gaps Signed-off-by: Nikolay Nikolaev --- kernel/kexec_core.c | 4 +++- kernel/multikernel/core.c | 1 + kernel/multikernel/instance_dt.c | 3 +++ kernel/multikernel/ipi.c | 5 +---- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 29bf34cd6b8ef1..c3a86d85b3a996 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1791,8 +1791,10 @@ int multikernel_kexec_by_id(int mk_id) mk_cpu_set_first(instance->cpus)); } rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc) + if (rc) { + mk_ipi_endpoint_close(instance); goto unlock; + } /* * The instance is running once its CPUs leave the park loop. Publish that diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 998cf6cf191a65..ffc08af518bd73 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1618,6 +1618,7 @@ int mk_instance_abort_spawn(struct mk_instance *instance) { int ret; + mk_ipi_endpoint_close(instance); ret = __mk_instance_force_halt(instance, true); if (ret && instance) mk_instance_set_state(instance, MK_STATE_FAILED); diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 4962c06b2f09fa..daa155ae16924c 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -381,6 +381,9 @@ static struct mk_instance * __init alloc_mk_instance(int instance_id, const char goto err_free_ipi; instance->state = MK_STATE_READY; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); kref_init(&instance->refcount); diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 092aa595fe09f8..597bc57f5aedf2 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -36,8 +36,6 @@ int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) return -ENODEV; if (endpoint->registered) return 0; - raw_spin_lock_init(&endpoint->tx_lock); - raw_spin_lock_init(&endpoint->rx_lock); endpoint->tx = parent_side ? &instance->ipi_data->to_child : &instance->ipi_data->to_parent; endpoint->rx = parent_side ? &instance->ipi_data->to_parent : @@ -47,7 +45,6 @@ int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) endpoint->tx_enabled = true; endpoint->rx_dispatching = false; endpoint->parent_side = parent_side; - INIT_LIST_HEAD(&endpoint->rx_node); raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); endpoint->registered = true; @@ -176,7 +173,7 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, if (endpoint->parent_side) WRITE_ONCE(instance->ipi_data->child_doorbell_cpu, target); raw_spin_lock_irqsave(&endpoint->tx_lock, flags); - if (!endpoint->tx_enabled) { + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { ret = -ESHUTDOWN; goto unlock; } From 96ec82e8970bc472f5d2d8d9e5f2713d47e3ca67 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 11:11:26 +0000 Subject: [PATCH 06/10] multikernel: route shutdown to linked parent Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/core.c | 2 +- kernel/multikernel/ipi.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index ffc08af518bd73..2d7f32af6239ba 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1383,7 +1383,7 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + mk_notify_down_and_park(READ_ONCE(root_instance->ipi_data->parent_id), MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 597bc57f5aedf2..7ec995f5a20e51 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -170,13 +170,13 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, READ_ONCE(instance->ipi_data->parent_doorbell_cpu); if (target == MK_PHYS_CPU_INVALID) return -ENODEV; - if (endpoint->parent_side) - WRITE_ONCE(instance->ipi_data->child_doorbell_cpu, target); raw_spin_lock_irqsave(&endpoint->tx_lock, flags); if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { ret = -ESHUTDOWN; goto unlock; } + if (endpoint->parent_side) + WRITE_ONCE(instance->ipi_data->child_doorbell_cpu, target); idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); slot = &endpoint->tx->entries[idx]; /* Pair with the receiver's release when it makes the slot reusable. */ From a44457152c133393957522455b6138e44dbf1a07 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 11:24:21 +0000 Subject: [PATCH 07/10] multikernel: synchronize CPU set access Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 20 +-- kernel/multikernel/cpuset.c | 257 +++++++++++++++++++++++++------ kernel/multikernel/instance_dt.c | 4 + 3 files changed, 217 insertions(+), 64 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index dbb773249b4c9b..39c9e9dfaafb3a 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -41,6 +41,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -56,20 +57,11 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..ceb54e7435cc20 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) { - return mk_cpu_set_index(set, id) >= 0; + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) +{ + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,195 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; - if (idx < 0) + if (!set) return false; + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); + return false; + } + memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ret = mk_cpu_set_reserve(dst, nr); + if (ret) + return ret; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > dst->cap) { + mk_cpu_set_unlock(src, src_flags); + continue; + } + + mk_cpu_set_lock(dst, &dst_flags); + memcpy(dst->ids, src->ids, src->nr * sizeof(dst->ids[0])); + dst->nr = src->nr; + mk_cpu_set_unlock(dst, dst_flags); + mk_cpu_set_unlock(src, src_flags); + return 0; + } } /** @@ -127,22 +275,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index daa155ae16924c..4424e128777406 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -584,6 +584,10 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest host_instance->ipi_phys = root_instance->ipi_phys; host_instance->ipi_pages = root_instance->ipi_pages; if (mk_ipi_endpoint_init(host_instance, false)) { + mutex_lock(&mk_instance_mutex); + idr_remove(&mk_instance_idr, host_instance->id); + list_del(&host_instance->list); + mutex_unlock(&mk_instance_mutex); kfree(host_instance->name); mk_cpu_set_free(host_instance->cpus); kfree(host_instance); From e839804111635eebe37439529a3d01249fddcd4b Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 11:25:25 +0000 Subject: [PATCH 08/10] multikernel: correct CPU set count type Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 39c9e9dfaafb3a..3827adc6c20d6e 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -57,7 +57,7 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); -int mk_cpu_set_count(const struct mk_cpu_set *set); +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); bool mk_cpu_set_empty(const struct mk_cpu_set *set); mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, From ba8d20fb95a7e4e10e06ef41ef04021a66e59dd9 Mon Sep 17 00:00:00 2001 From: "Nickolay V. Shmyrev" Date: Sat, 12 Sep 2026 11:40:25 +0000 Subject: [PATCH 09/10] multikernel: route tty output to linked parent --- drivers/tty/mktty.c | 28 +++++++++++++++++++++++++--- kernel/multikernel/ipi.c | 18 +++++++++++++----- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index cc7a3259055b72..604bf2d83e66ea 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -337,6 +337,20 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + struct mk_shared_data *shared; + + if (!root_instance || root_instance->id == 0) + return 0; + + shared = READ_ONCE(root_instance->ipi_data); + if (!shared) + return -ENODEV; + + return READ_ONCE(shared->parent_id); +} static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -377,10 +391,13 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, { struct mktty_message *msg; size_t sent = 0, chunk; - int ret; + int ret, parent_id; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -394,7 +411,7 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); - ret = multikernel_send_ipi_data(0, msg, + ret = multikernel_send_ipi_data(parent_id, msg, sizeof(*msg) - MKTTY_MAX_DATA + chunk, MKTTY_IPI_TYPE); if (ret < 0) { @@ -449,8 +466,12 @@ static void mktty_spawn_console_write(struct console *con, const char *s, { unsigned long flags; size_t chunk; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -459,12 +480,13 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); - multikernel_send_ipi_data(0, &mktty_console_msg, + multikernel_send_ipi_data(parent_id, &mktty_console_msg, sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 7ec995f5a20e51..e792c98f03c6fc 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -156,6 +156,7 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, { struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; + struct mk_shared_data *shared; mk_phys_cpu_t target; unsigned long flags; u32 idx; @@ -166,17 +167,24 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, endpoint = &instance->ipi_endpoint; if (!READ_ONCE(endpoint->registered)) return -ESHUTDOWN; - target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : - READ_ONCE(instance->ipi_data->parent_doorbell_cpu); - if (target == MK_PHYS_CPU_INVALID) - return -ENODEV; raw_spin_lock_irqsave(&endpoint->tx_lock, flags); if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { ret = -ESHUTDOWN; goto unlock; } + shared = READ_ONCE(instance->ipi_data); + if (!shared) { + ret = -ENODEV; + goto unlock; + } + target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : + READ_ONCE(shared->parent_doorbell_cpu); + if (target == MK_PHYS_CPU_INVALID) { + ret = -ENODEV; + goto unlock; + } if (endpoint->parent_side) - WRITE_ONCE(instance->ipi_data->child_doorbell_cpu, target); + WRITE_ONCE(shared->child_doorbell_cpu, target); idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); slot = &endpoint->tx->entries[idx]; /* Pair with the receiver's release when it makes the slot reusable. */ From 4a71019d1dc8940c9cbbe9af4d78078568d4814e Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 12:01:15 +0000 Subject: [PATCH 10/10] multikernel: unwind restored host on CPU setup failure --- kernel/multikernel/instance_dt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 4424e128777406..8b37adc81225e1 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -575,6 +575,10 @@ static struct mk_instance * __init mk_restore_host_instance(const void *manifest if (!host_instance) return NULL; if (mk_cpu_set_add(host_instance->cpus, parent_cpu)) { + mutex_lock(&mk_instance_mutex); + idr_remove(&mk_instance_idr, host_instance->id); + list_del(&host_instance->list); + mutex_unlock(&mk_instance_mutex); kfree(host_instance->name); mk_cpu_set_free(host_instance->cpus); kfree(host_instance);