diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index c9808a51fa371..b51c13158f99e 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -94,7 +94,7 @@ static void hv_apic_write(u32 reg, u32 val) static void hv_apic_eoi_write(void) { - struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; + struct hv_vp_assist_page *hvp = hv_get_vp_assist_page(smp_processor_id()); if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1)) return; diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index 15acbe214b794..5bad76250151d 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -397,10 +397,9 @@ int __init hv_common_init(void) hv_vp_index[i] = VP_INVAL; /* - * The VP assist page is useless to a TDX guest: the only use we - * would have for it is lazy EOI, which can not be used with TDX. + * The VP assist page cannot be trusted by hardware-isolated guests. */ - if (hv_isolation_type_tdx()) + if (hv_isolation_type_tdx() || hv_isolation_type_snp()) hv_vp_assist_page = NULL; else { hv_vp_assist_page = kcalloc(nr_cpu_ids, diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c index 8f2a6a5f926ce..b83d68ea2bc88 100644 --- a/drivers/hv/mshv_vtl_main.c +++ b/drivers/hv/mshv_vtl_main.c @@ -218,6 +218,9 @@ static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file); static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions); static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu); static DEFINE_PER_CPU(struct task_struct *, mshv_vtl_thread); +#if defined(CONFIG_SEV_GUEST) +static DEFINE_PER_CPU(bool, mshv_vtl_lower_vtl_ran); +#endif static const union hv_input_vtl input_vtl_zero; static const union hv_input_vtl input_vtl_normal = { @@ -852,10 +855,14 @@ static int mshv_vtl_get_vsm_regs(void) mshv_vsm_capabilities.as_uint64 = registers[0].value.reg64; #ifdef CONFIG_X86_64 - if (hv_isolation_type_snp()) + if (hv_isolation_type_snp()) { mshv_vsm_capabilities.dr6_shared = 0; - else if (hv_isolation_type_tdx()) { + mshv_vsm_capabilities.intercept_page_available = 0; + mshv_vsm_capabilities.return_action_available = 0; + } else if (hv_isolation_type_tdx()) { mshv_vsm_capabilities.dr6_shared = 1; + mshv_vsm_capabilities.intercept_page_available = 0; + mshv_vsm_capabilities.return_action_available = 0; } else { mshv_vsm_page_offsets.as_uint64 = registers[1].value.reg64; pr_debug("%s: VSM code page offsets: %#016llx\n", __func__, @@ -890,6 +897,9 @@ static void mshv_vtl_scan_proxy_interrupts(struct hv_per_cpu_context *per_cpu) break; if (message_type != HVMSG_X64_PROXY_INTERRUPT_INTERCEPT) { + /* SNP exit dispatch consumes non-proxy messages. */ + if (hv_isolation_type_snp()) + break; WARN_ONCE(1, "Unexpected message type: %d\n", message_type); vmbus_signal_eom(msg, message_type); continue; @@ -1871,9 +1881,13 @@ static bool mshv_vtl_process_intercept(void) { struct hv_per_cpu_context *mshv_cpu; void *synic_message_page; + struct hv_message *exit_msg; struct hv_message *msg; u32 message_type; + exit_msg = (struct hv_message *)mshv_vtl_this_run()->exit_message; + WRITE_ONCE(exit_msg->header.message_type, HVMSG_NONE); + mshv_cpu = this_cpu_ptr(hv_context.cpu_context); synic_message_page = mshv_cpu->synic_message_page; if (unlikely(!synic_message_page)) @@ -1884,7 +1898,7 @@ static bool mshv_vtl_process_intercept(void) if (message_type == HVMSG_NONE) return true; - memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg)); + memcpy(exit_msg, msg, sizeof(*msg)); vmbus_signal_eom(msg, message_type); return false; @@ -2041,7 +2055,7 @@ static void mshv_vtl_switch_to_vtl0_irqoff(void) armed = mshv_tdx_halt_timer_pre(flags & MSHV_VTL_RUN_FLAG_HALTED); - /* A VTL2 TDX kernel doesn't allocate hv_vp_assist_page at the moment */ + /* Hardware-isolated guests don't allocate hv_vp_assist_page. */ hvp = hv_vp_assist_page ? hv_vp_assist_page[smp_processor_id()] : NULL; /* @@ -2095,10 +2109,13 @@ static void mshv_vtl_switch_to_vtl0_irqoff(void) #endif } if (hv_isolation_type_snp()) { #if defined(CONFIG_SEV_GUEST) - if (unlikely(flags & MSHV_VTL_RUN_FLAG_HALTED)) + if (unlikely(flags & MSHV_VTL_RUN_FLAG_HALTED)) { native_safe_halt(); - else + } else { + this_cpu_write(mshv_vtl_lower_vtl_ran, true); snp_mshv_vtl_return(target_vtl.use_target_vtl ? target_vtl.target_vtl : 0); + this_cpu_inc(num_vtl0_transitions); + } return; #endif } else { @@ -2450,25 +2467,6 @@ static void mshv_snp_clear_exit_code(struct sev_es_save_area *vmsa, bool int_sha vmsa->vintr_ctrl &= ~V_GUEST_BUSY_MASK; } -/* - * Determine if this interrupt was handled completely in the kernel. - * - * Returns true if the exit was handled entirely in kernel, and the VMPL should be re-entered. - * Returns false if the exit must be handled by user-space. - */ -static bool mshv_snp_try_handle_interrupt_entry(struct mshv_vtl_run *run) -{ - struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; - - if (!(run->offload_flags & MSHV_VTL_OFFLOAD_FLAG_INTR_INJECT) || - READ_ONCE(hvp->vtl_entry_reason) != MSHV_ENTRY_REASON_INTERRUPT) - return false; - - mshv_snp_clear_exit_code(snp_this_vmsa(), false); - return true; -} - - /* * Try to handle the incomplete IPI SEV-SNP exit. * @@ -2501,16 +2499,27 @@ static bool mshv_snp_try_handle_incomplete_ipi(struct mshv_vtl_run *run, * Returns true if the exit was handled entirely in kernel, and the VMPL should be re-entered. * Returns false if the exit must be handled by user-space. */ -static bool mshv_snp_try_handle_exit(struct mshv_vtl_run *run) +static bool mshv_snp_try_handle_exit(struct mshv_vtl_run *run, bool lower_vtl_ran) { const bool intr_inject = MSHV_VTL_OFFLOAD_FLAG_INTR_INJECT & run->offload_flags; const bool x2apic = MSHV_VTL_OFFLOAD_FLAG_X2APIC & run->offload_flags; + union hv_input_vtl target_vtl = READ_ONCE(run->target_vtl); struct sev_es_save_area *vmsa; - if (!intr_inject) + /* Kernel exit offloads are supported only for VTL 0. */ + if (target_vtl.use_target_vtl && target_vtl.target_vtl > 0) return false; vmsa = snp_this_vmsa(); + /* + * VTL2 may wake without entering the lower VTL. In that + * case the VMSA still contains the benign SVM_EXIT_INTR sentinel left + * by the previous handled exit. + */ + if (!lower_vtl_ran) + return true; + if (!intr_inject) + return false; switch (vmsa->guest_exit_code) { case SVM_EXIT_AVIC_INCOMPLETE_IPI: @@ -2528,19 +2537,10 @@ static bool mshv_snp_try_handle_exit(struct mshv_vtl_run *run) run->offload_flags |= MSHV_VTL_OFFLOAD_FLAG_HALT_IDLE; goto handled; case SVM_EXIT_INTR: - /* - * mshv_snp_clear_exit_code() uses SVM_EXIT_INTR as a benign - * sentinel. If VTL2 was woken from native_safe_halt(), VTL0 did - * not run and the VMSA still contains that sentinel. - */ - if (run->flags & MSHV_VTL_RUN_FLAG_HALTED) - return true; - break; + goto handled; case SVM_EXIT_MSR: - if (READ_ONCE(hv_vp_assist_page[smp_processor_id()]->vtl_entry_reason) == - MSHV_ENTRY_REASON_INTERCEPT && - (u32)mshv_snp_read_vmsa_reg( - run, vmsa, offsetof(struct sev_es_save_area, rcx)) == + if ((u32)mshv_snp_read_vmsa_reg(run, vmsa, + offsetof(struct sev_es_save_area, rcx)) == HV_X64_MSR_STIMER0_COUNT && (vmsa->guest_exit_info_1 & 1)) { u64 count = (u32)mshv_snp_read_vmsa_reg( @@ -2598,39 +2598,43 @@ static void mshv_snp_release_stimer0(struct mshv_vtl_run *run) hrtimer_cancel(&this_cpu_ptr(&mshv_vtl_per_cpu)->snp_stimer0_timer); } -static bool mshv_snp_try_handle_intercept(struct mshv_vtl_run *run) +/* Proxy interrupt messages are consumed by the ISR; others need the run loop. */ +static bool mshv_snp_synic_message_pending(void) { - struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; - u32 msg_type = HVMSG_NONE; - struct hv_message *msg = NULL; - bool is_interrupt = false; - - switch (hvp->vtl_entry_reason) { - case MSHV_ENTRY_REASON_INTERRUPT: - if (!mshv_vsm_capabilities.intercept_page_available) { - struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context); - void *synic_message_page = mshv_cpu->synic_message_page; - - if (likely(synic_message_page)) { - msg = (struct hv_message *)synic_message_page + - HV_SYNIC_INTERCEPTION_SINT_INDEX; - is_interrupt = true; - } - } - break; + void *synic_message_page = this_cpu_ptr(hv_context.cpu_context)->synic_message_page; + struct hv_message *msg; + u32 msg_type; - case MSHV_ENTRY_REASON_INTERCEPT: - WARN_ON(!mshv_vsm_capabilities.intercept_page_available); - msg = (struct hv_message *)hvp->intercept_message; - break; + if (unlikely(!synic_message_page)) + return false; - default: - panic("unknown entry reason: %d", hvp->vtl_entry_reason); - } + msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX; + msg_type = READ_ONCE(msg->header.message_type); + return msg_type != HVMSG_NONE && msg_type != HVMSG_X64_PROXY_INTERRUPT_INTERCEPT; +} + +static bool mshv_snp_try_handle_synic_message(struct mshv_vtl_run *run, + bool handle_in_kernel) +{ + struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context); + union hv_input_vtl target_vtl = READ_ONCE(run->target_vtl); + void *synic_message_page = mshv_cpu->synic_message_page; + u32 msg_type = HVMSG_NONE; + struct hv_message *msg; + + WRITE_ONCE(((struct hv_message *)run->exit_message)->header.message_type, + HVMSG_NONE); - if (!msg) + if (unlikely(!synic_message_page)) return true; + + msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX; msg_type = READ_ONCE(msg->header.message_type); + if (msg_type == HVMSG_X64_PROXY_INTERRUPT_INTERCEPT) + return true; + if (msg_type != HVMSG_NONE && target_vtl.use_target_vtl && + target_vtl.target_vtl > 0) + goto to_user; switch (msg_type) { case HVMSG_NONE: @@ -2639,25 +2643,31 @@ static bool mshv_snp_try_handle_intercept(struct mshv_vtl_run *run) { struct hv_x64_exception_intercept_message *expt_msg = (struct hv_x64_exception_intercept_message *)msg->u.payload; - if (expt_msg->exception_vector != X86_TRAP_VC) - return false; + if (expt_msg->exception_vector != X86_TRAP_VC || + !handle_in_kernel) + goto to_user; } break; case HVMSG_SYNIC_SINT_DELIVERABLE: - return false; + goto to_user; case HVMSG_X64_HALT: run->flags |= MSHV_VTL_RUN_FLAG_HALTED; run->offload_flags &= ~MSHV_VTL_OFFLOAD_FLAG_HALT_IDLE; run->offload_flags |= MSHV_VTL_OFFLOAD_FLAG_HALT_HLT; + mshv_snp_clear_halt_if_irr_pending(run, snp_this_savic_page()); break; default: - return false; + goto to_user; } - if (is_interrupt) - vmbus_signal_eom(msg, msg_type); + vmbus_signal_eom(msg, msg_type); return true; + +to_user: + memcpy(run->exit_message, msg, sizeof(*msg)); + vmbus_signal_eom(msg, msg_type); + return false; } #endif /* CONFIG_SEV_GUEST */ @@ -2718,6 +2728,19 @@ static int mshv_vtl_ioctl_return_to_lower_vtl(void) preempt_enable(); return -EINTR; } +#ifdef CONFIG_SEV_GUEST + if (hv_isolation_type_snp()) { + struct mshv_vtl_run *run = mshv_vtl_this_run(); + + if (mshv_snp_synic_message_pending()) { + local_irq_restore(irq_flags); + if (!mshv_snp_try_handle_synic_message(run, true)) + goto done; + continue; + } + this_cpu_write(mshv_vtl_lower_vtl_ran, false); + } +#endif if (tick_nohz_full_enabled() || nr_cpu_ids == 1 || !enter) { mshv_vtl_switch_to_vtl0_irqoff(); @@ -2760,11 +2783,14 @@ static int mshv_vtl_ioctl_return_to_lower_vtl(void) } else if (hv_isolation_type_snp()) { #ifdef CONFIG_SEV_GUEST struct mshv_vtl_run *run = mshv_vtl_this_run(); + bool lower_vtl_ran = this_cpu_read(mshv_vtl_lower_vtl_ran); + bool vmsa_handled, synic_handled; - if (mshv_snp_try_handle_intercept(run) && - (mshv_snp_try_handle_interrupt_entry(run) || - mshv_snp_try_handle_exit(run))) + vmsa_handled = mshv_snp_try_handle_exit(run, lower_vtl_ran); + synic_handled = mshv_snp_try_handle_synic_message(run, vmsa_handled); + if (vmsa_handled && synic_handled) continue; /* Exit handled entirely in kernel */ + goto done; #endif } @@ -4460,6 +4486,9 @@ static int __init mshv_vtl_init(void) int ret; struct device *dev = mshv_dev.this_device; + if (WARN_ON_ONCE(hv_isolation_type_snp() && !IS_BUILTIN(CONFIG_SEV_GUEST))) + return -EOPNOTSUPP; + /* * This creates /dev/mshv which provides functionality to create VTLs and partitions. */