summaryrefslogtreecommitdiff
path: root/arch/x86
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/events/intel/core.c10
-rw-r--r--arch/x86/events/intel/lbr.c12
-rw-r--r--arch/x86/events/perf_event.h2
-rw-r--r--arch/x86/include/asm/kvm-x86-ops.h4
-rw-r--r--arch/x86/include/asm/kvm_host.h6
-rw-r--r--arch/x86/kvm/hyperv.c25
-rw-r--r--arch/x86/kvm/mmu/mmu.c107
-rw-r--r--arch/x86/kvm/mmu/tdp_iter.h7
-rw-r--r--arch/x86/kvm/mmu/tdp_mmu.c20
-rw-r--r--arch/x86/kvm/svm/sev.c80
-rw-r--r--arch/x86/kvm/svm/svm.c6
-rw-r--r--arch/x86/kvm/svm/svm.h2
-rw-r--r--arch/x86/kvm/vmx/nested.c75
-rw-r--r--arch/x86/kvm/x86.c17
-rw-r--r--arch/x86/kvm/x86.h2
15 files changed, 253 insertions, 122 deletions
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 2b35483e2b70..465c414f145d 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -7947,12 +7947,6 @@ __init int intel_pmu_init(void)
x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
- if (version >= 5) {
- x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated;
- if (x86_pmu.intel_cap.anythread_deprecated)
- pr_cont(" AnyThread deprecated, ");
- }
-
/* The perf side of core PMU is ready to support the mediated vPMU. */
x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_MEDIATED_VPMU;
@@ -8829,8 +8823,10 @@ __init int intel_pmu_init(void)
&x86_pmu.intel_ctrl);
/* AnyThread may be deprecated on arch perfmon v5 or later */
- if (x86_pmu.intel_cap.anythread_deprecated)
+ if (version >= 5 && edx.split.anythread_deprecated) {
x86_pmu.format_attrs = intel_arch_formats_attr;
+ pr_cont("AnyThread deprecated, ");
+ }
intel_pmu_check_event_constraints_all(NULL);
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index cae2e02fe6cc..52d14927a191 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -1213,7 +1213,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
{
u64 from, to;
int br_sel = cpuc->br_sel;
- int i, j, type, to_plm;
+ int i, j, type, from_plm, to_plm;
bool compress = false;
/* if sampling all branches, then nothing to filter */
@@ -1245,8 +1245,14 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
type |= X86_BR_NO_TX;
}
- /* if type does not correspond, then discard */
- if (type == X86_BR_NONE || (br_sel & type) != type) {
+ from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
+ /*
+ * If type does not correspond, then discard.
+ * Specifically reject entries whose from address is in
+ * kernel space when only X86_BR_USER is requested.
+ */
+ if (type == X86_BR_NONE || (br_sel & type) != type ||
+ (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) {
cpuc->lbr_entries[i].from = 0;
compress = true;
}
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index eae24bb35dc1..5902a297daa1 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -668,7 +668,7 @@ union perf_capabilities {
u64 perf_metrics:1;
u64 pebs_output_pt_available:1;
u64 pebs_timing_info:1;
- u64 anythread_deprecated:1;
+ u64 __reserved:1;
u64 rdpmc_metrics_clear:1;
};
u64 capabilities;
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 83dc5086138b..ccf23b3f0e1c 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)
KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)
KVM_X86_OP_OPTIONAL(vm_move_enc_context_from)
KVM_X86_OP_OPTIONAL(guest_memory_reclaimed)
+KVM_X86_OP_OPTIONAL(reload_vmsa)
KVM_X86_OP(get_feature_msr)
KVM_X86_OP(check_emulate_instruction)
KVM_X86_OP(apic_init_signal_blocked)
@@ -148,6 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
KVM_X86_OP_OPTIONAL_RET0(gmem_prepare)
KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level)
KVM_X86_OP_OPTIONAL(gmem_invalidate)
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+KVM_X86_OP_OPTIONAL(gmem_invalidate_range)
+#endif
#endif
#undef KVM_X86_OP
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 6db5b5f79df9..2dfd6199a10c 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -128,6 +128,8 @@
KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_HV_TLB_FLUSH \
KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
+#define KVM_REQ_VMSA_PAGE_RELOAD \
+ KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \
KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT)
@@ -1985,6 +1987,7 @@ struct kvm_x86_ops {
int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
void (*guest_memory_reclaimed)(struct kvm *kvm);
+ void (*reload_vmsa)(struct kvm_vcpu *vcpu);
int (*get_feature_msr)(u32 msr, u64 *data);
@@ -2009,6 +2012,9 @@ struct kvm_x86_ops {
void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);
int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end);
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+ void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range);
+#endif
int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
};
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index d8736b5dfcd3..5f43ae345f87 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -627,6 +627,18 @@ static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer)
}
/*
+ * Translate a stimer expiry given in 100ns reference ticks into an
+ * an absolute deadline. Saturates on overflow.
+ */
+static ktime_t stimer_add_delta(ktime_t now, u64 delta_100ns)
+{
+ if (delta_100ns >= KTIME_MAX / 100)
+ return KTIME_MAX;
+
+ return ktime_add_safe(now, 100 * delta_100ns);
+}
+
+/*
* stimer_start() assumptions:
* a) stimer->count is not equal to 0
* b) stimer->config has HV_STIMER_ENABLE flag
@@ -635,6 +647,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
{
u64 time_now;
ktime_t ktime_now;
+ ktime_t deadline;
time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm);
ktime_now = ktime_get();
@@ -657,10 +670,8 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
stimer->index,
time_now, stimer->exp_time);
- hrtimer_start(&stimer->timer,
- ktime_add_ns(ktime_now,
- 100 * (stimer->exp_time - time_now)),
- HRTIMER_MODE_ABS);
+ deadline = stimer_add_delta(ktime_now, stimer->exp_time - time_now);
+ hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS);
return 0;
}
stimer->exp_time = stimer->count;
@@ -679,9 +690,9 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
stimer->index,
time_now, stimer->count);
- hrtimer_start(&stimer->timer,
- ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)),
- HRTIMER_MODE_ABS);
+ deadline = stimer_add_delta(ktime_now, stimer->count - time_now);
+ hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS);
+
return 0;
}
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index a61750f8e1e3..51ba46aa6b7e 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -1245,18 +1245,9 @@ struct rmap_iterator {
int pos; /* index of the sptep */
};
-/*
- * Iteration must be started by this function. This should also be used after
- * removing/dropping sptes from the rmap link because in such cases the
- * information in the iterator may not be valid.
- *
- * Returns sptep if found, NULL otherwise.
- */
-static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
- struct rmap_iterator *iter)
+static u64 *__rmap_get_first(unsigned long rmap_val,
+ struct rmap_iterator *iter)
{
- unsigned long rmap_val = kvm_rmap_get(rmap_head);
-
if (!rmap_val)
return NULL;
@@ -1271,6 +1262,19 @@ static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
}
/*
+ * Iteration must be started by this function. This should also be used after
+ * removing/dropping sptes from the rmap link because in such cases the
+ * information in the iterator may not be valid.
+ *
+ * Returns sptep if found, NULL otherwise.
+ */
+static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
+ struct rmap_iterator *iter)
+{
+ return __rmap_get_first(kvm_rmap_get(rmap_head), iter);
+}
+
+/*
* Must be used with a valid iterator: e.g. after rmap_get_first().
*
* Returns sptep if found, NULL otherwise.
@@ -1304,8 +1308,9 @@ static u64 *rmap_get_next(struct rmap_iterator *iter)
__for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \
if (!WARN_ON_ONCE(!is_shadow_present_pte(*(_sptep_)))) \
-#define for_each_rmap_spte_lockless(_rmap_head_, _iter_, _sptep_, _spte_) \
- __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \
+#define for_each_rmap_spte_lockless(_rmap_val_, _iter_, _sptep_, _spte_) \
+ for (_sptep_ = __rmap_get_first(_rmap_val_, _iter_); \
+ _sptep_; _sptep_ = rmap_get_next(_iter_)) \
if (is_shadow_present_pte(_spte_ = mmu_spte_get_lockless(sptep)))
static void drop_spte(struct kvm *kvm, u64 *sptep)
@@ -1743,7 +1748,7 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
rmap_head = gfn_to_rmap(gfn, level, range->slot);
rmap_val = kvm_rmap_lock_readonly(rmap_head);
- for_each_rmap_spte_lockless(rmap_head, &iter, sptep, spte) {
+ for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) {
if (!is_accessed_spte(spte))
continue;
@@ -6896,20 +6901,11 @@ restart:
kvm_mmu_commit_zap_page(kvm, &invalid_list);
}
-/*
- * Fast invalidate all shadow pages and use lock-break technique
- * to zap obsolete pages.
- *
- * It's required when memslot is being deleted or VM is being
- * destroyed, in these cases, we should ensure that KVM MMU does
- * not use any resource of the being-deleted slot or all slots
- * after calling the function.
- */
-static void kvm_mmu_zap_all_fast(struct kvm *kvm)
+static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm)
{
lockdep_assert_held(&kvm->slots_lock);
+ lockdep_assert_held_write(&kvm->mmu_lock);
- write_lock(&kvm->mmu_lock);
trace_kvm_mmu_zap_all_fast(kvm);
/*
@@ -6946,8 +6942,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
kvm_zap_obsolete_pages(kvm);
+}
- write_unlock(&kvm->mmu_lock);
+static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm)
+{
+ lockdep_assert_held(&kvm->slots_lock);
+ lockdep_assert_not_held(&kvm->mmu_lock);
/*
* Zap the invalidated TDP MMU roots, all SPTEs must be dropped before
@@ -6961,6 +6961,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
kvm_tdp_mmu_zap_invalidated_roots(kvm, true);
}
+/*
+ * Fast invalidate all shadow pages and use lock-break technique
+ * to zap obsolete pages.
+ *
+ * It's required when memslot is being deleted or VM is being
+ * destroyed, in these cases, we should ensure that KVM MMU does
+ * not use any resource of the being-deleted slot or all slots
+ * after calling the function.
+ */
+static void kvm_mmu_zap_all_fast(struct kvm *kvm)
+{
+ write_lock(&kvm->mmu_lock);
+ __kvm_mmu_zap_all_fast_front_half(kvm);
+ write_unlock(&kvm->mmu_lock);
+
+ __kvm_mmu_zap_all_fast_back_half(kvm);
+}
+
int kvm_mmu_init_vm(struct kvm *kvm)
{
int r, i;
@@ -7535,8 +7553,8 @@ out_flush:
kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
}
-static void kvm_mmu_zap_memslot(struct kvm *kvm,
- struct kvm_memory_slot *slot)
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+ struct kvm_memory_slot *slot)
{
struct kvm_gfn_range range = {
.slot = slot,
@@ -7545,27 +7563,28 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm,
.may_block = true,
.attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED,
};
+ bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
+ kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
bool flush;
write_lock(&kvm->mmu_lock);
- flush = kvm_unmap_gfn_range(kvm, &range);
- kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
- write_unlock(&kvm->mmu_lock);
-}
-static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm)
-{
- return kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
- kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
-}
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+ if (slot->gmem.file)
+ kvm_arch_gmem_invalidate_range(kvm, &range);
+#endif
-void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
- struct kvm_memory_slot *slot)
-{
- if (kvm_memslot_flush_zap_all(kvm))
- kvm_mmu_zap_all_fast(kvm);
- else
- kvm_mmu_zap_memslot(kvm, slot);
+ if (zap_all) {
+ __kvm_mmu_zap_all_fast_front_half(kvm);
+ } else {
+ flush = kvm_unmap_gfn_range(kvm, &range);
+ kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
+ }
+
+ write_unlock(&kvm->mmu_lock);
+
+ if (zap_all)
+ __kvm_mmu_zap_all_fast_back_half(kvm);
}
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
diff --git a/arch/x86/kvm/mmu/tdp_iter.h b/arch/x86/kvm/mmu/tdp_iter.h
index 364c5da6c499..f898d8d0d93c 100644
--- a/arch/x86/kvm/mmu/tdp_iter.h
+++ b/arch/x86/kvm/mmu/tdp_iter.h
@@ -19,6 +19,13 @@ static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep)
return READ_ONCE(*rcu_dereference(sptep));
}
+/*
+ * WARNING! mmu_lock must be held for write when using the "write atomic" or
+ * "clear bits atomic" APIs, otherwise KVM could overwrite the "wrong" old SPTE
+ * value, i.e. clobber an update from a different CPU. The only exception is
+ * when KVM is freezing a leaf SPTE for removal, in which case KVM doesn't care
+ * about the exact old SPTE value (KVM will react to the actual old value).
+ */
static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte)
{
KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index c1cbae65d239..d95af13d922a 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -1334,19 +1334,17 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter)
if (WARN_ON_ONCE(is_mirror_sptep(iter->sptep)))
return;
- if (spte_ad_enabled(iter->old_spte)) {
- iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep,
- shadow_accessed_mask);
+ if (spte_ad_enabled(iter->old_spte))
new_spte = iter->old_spte & ~shadow_accessed_mask;
- } else {
+ else
new_spte = mark_spte_for_access_track(iter->old_spte);
- /*
- * It is safe for the following cmpxchg to fail. Leave the
- * Accessed bit set, as the spte is most likely young anyway.
- */
- if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte))
- return;
- }
+
+ /*
+ * Don't bother retrying if another CPU modified the SPTE, the SPTE is
+ * either being zapped or is likely still in-use, i.e. is still young.
+ */
+ if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte))
+ return;
trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level,
iter->old_spte, new_spte);
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 8b9af7d4c7f6..1ed78f770340 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -4021,19 +4021,25 @@ static int snp_begin_psc(struct vcpu_svm *svm)
return snp_do_psc(svm);
}
-static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
+static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
struct vcpu_svm *svm = to_svm(vcpu);
struct kvm_memory_slot *slot;
+ struct kvm *kvm = vcpu->kvm;
gfn_t gfn = gpa_to_gfn(gpa);
+ unsigned long mmu_seq;
struct page *page;
kvm_pfn_t pfn;
lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex);
- /* Clear use of the VMSA. */
+ /*
+ * Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly
+ * once, as it is read locklessly when responding to gfn invalidations.
+ * Pairs with the READ_ONCE() in sev_gmem_invalidate_range().
+ */
svm->vmcb->control.vmsa_pa = INVALID_PAGE;
- svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE;
+ WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE);
/*
* When replacing the VMSA during SEV-SNP AP creation,
@@ -4048,6 +4054,9 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
if (!slot)
return;
+ mmu_seq = kvm->mmu_invalidate_seq;
+ smp_rmb();
+
/*
* The new VMSA will be private memory guest memory, so retrieve the
* PFN from the gmem backend.
@@ -4066,15 +4075,20 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
*/
svm->sev_es.snp_has_guest_vmsa = true;
- /* Use the new VMSA */
- svm->sev_es.snp_guest_vmsa_gpa = gpa;
- svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
-
+ read_lock(&kvm->mmu_lock);
/*
- * gmem pages aren't currently migratable, but if this ever changes
- * then care should be taken to ensure svm->sev_es.vmsa is pinned
- * through some other means.
+ * Save the guest-provided GPA. If retry is needed, then KVM will try
+ * again with the same GPA. If the VMSA is usable, then KVM needs to
+ * track the GPA so that the VMSA can be reloaded if the backing page
+ * for the GPA is invalidated.
*/
+ svm->sev_es.snp_guest_vmsa_gpa = gpa;
+ if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
+ kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu);
+ else
+ svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
+ read_unlock(&kvm->mmu_lock);
+
kvm_release_page_clean(page);
}
@@ -4100,7 +4114,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
gpa = svm->sev_es.snp_pending_vmsa_gpa;
svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
- sev_snp_reload_vmsa(vcpu, gpa);
+ __sev_snp_reload_vmsa(vcpu, gpa);
/*
* Mark the vCPU as runnable for CREATE requests, indicated by a valid
@@ -4112,6 +4126,15 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
}
+void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu)
+{
+ struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es;
+
+ guard(mutex)(&sev_es->snp_vmsa_mutex);
+
+ __sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa);
+}
+
static int sev_snp_ap_creation(struct vcpu_svm *svm)
{
struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm);
@@ -5241,6 +5264,41 @@ next_pfn:
}
}
+void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range)
+{
+ struct kvm_vcpu *vcpu;
+ unsigned long i;
+
+ lockdep_assert_held_write(&kvm->mmu_lock);
+
+ /*
+ * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock.
+ * The vCPU's VMSA GPA is invalidated before the vCPU is made visible
+ * to other tasks, and can only become valid while holding mmu_lock,
+ * after the VM is fully committed to being an SNP VM.
+ */
+ if (!____sev_snp_guest(kvm))
+ return;
+
+ kvm_for_each_vcpu(i, vcpu, kvm) {
+ /*
+ * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex
+ * (or its generic mutex) as mmu_lock is held, i.e. this task
+ * can't sleep. The VMSA is invalidated outside of mmu_lock,
+ * but can only become valid inside of mmu_lock, i.e. the below
+ * can get false positives, but not false negatives. A false
+ * positive is benign, as a spurious request simply forces the
+ * vCPU to re-establish its VMSA.
+ */
+ gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa);
+
+ if (VALID_PAGE(gpa) &&
+ gpa_to_gfn(gpa) >= range->start &&
+ gpa_to_gfn(gpa) < range->end)
+ kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu);
+ }
+}
+
int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
{
int level, rc;
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 1158f3e28200..03fb9d7231c8 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -5463,12 +5463,14 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
.mem_enc_register_region = sev_mem_enc_register_region,
.mem_enc_unregister_region = sev_mem_enc_unregister_region,
.guest_memory_reclaimed = sev_guest_memory_reclaimed,
+ .reload_vmsa = sev_snp_reload_vmsa,
.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
.vm_move_enc_context_from = sev_vm_move_enc_context_from,
.gmem_prepare = sev_gmem_prepare,
.gmem_invalidate = sev_gmem_invalidate,
+ .gmem_invalidate_range = sev_gmem_invalidate_range,
.gmem_max_mapping_level = sev_gmem_max_mapping_level,
#endif
.check_emulate_instruction = svm_check_emulate_instruction,
@@ -5656,10 +5658,6 @@ static __init int svm_hardware_setup(void)
if (nested) {
pr_info("Nested Virtualization enabled\n");
- kvm_enable_efer_bits(EFER_SVME);
- if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ))
- kvm_enable_efer_bits(EFER_LMSLE);
-
r = nested_svm_init_msrpm_merge_offsets();
if (r)
return r;
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 344312bd1fd4..cd30740e9984 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -997,6 +997,7 @@ static inline struct page *snp_safe_alloc_page(void)
{
return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT);
}
+void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu);
int sev_vcpu_create(struct kvm_vcpu *vcpu);
void sev_free_vcpu(struct kvm_vcpu *vcpu);
@@ -1011,6 +1012,7 @@ extern unsigned int max_sev_asid;
void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code);
int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end);
+void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range);
int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu);
void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa);
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index ddf6df7bee93..ed28cf6b6efc 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -1326,6 +1326,9 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu,
* is the VPID incorporated into the MMU context. I.e. KVM must assume
* that the new vpid12 has never been used and thus represents a new
* guest ASID that cannot have entries in the TLB.
+ *
+ * Note, last_vpid is initialized as 0, so the first nested VM-Enter
+ * after VMXON will always flush the TLB to avoid using stale entries.
*/
if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
vmx->nested.last_vpid = vmcs12->virtual_processor_id;
@@ -3760,6 +3763,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
vmentry_fail_vmexit_guest_mode:
if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
+
+ /*
+ * Handle any TLB flush requests that were queued for L2 if KVM made it
+ * far enough along to switch to L2 context. Note, loading host state
+ * will generate any flushes for L1 required by VM-Exit.
+ */
+ kvm_service_local_tlb_flush_requests(vcpu);
+
leave_guest_mode(vcpu);
vmentry_fail_vmexit:
@@ -5073,8 +5084,9 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
/* trying to cancel vmlaunch/vmresume is a bug */
kvm_warn_on_nested_run_pending(vcpu);
-#ifdef CONFIG_KVM_HYPERV
+ /* Note, "checking" the request also clears the request. */
if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
+#ifdef CONFIG_KVM_HYPERV
/*
* KVM_REQ_GET_NESTED_STATE_PAGES is also used to map
* Enlightened VMCS after migration and we still need to
@@ -5082,8 +5094,8 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
* the first L2 run.
*/
(void)nested_get_evmcs_page(vcpu);
- }
#endif
+ }
/* Service pending TLB flush requests for L2 before switching to L1. */
kvm_service_local_tlb_flush_requests(vcpu);
@@ -5432,6 +5444,13 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu)
vmx->nested.vpid02 = allocate_vpid();
+ /*
+ * Clear last_vpid to ensure that the VPID is flushed on the first
+ * nested VM-Enter. Otherwise, stale TLB entries from a previous life of
+ * the VPID (e.g. different vCPU or even different VM) could be used.
+ */
+ vmx->nested.last_vpid = 0;
+
vmx->nested.vmcs02_initialized = false;
vmx->nested.vmxon = true;
@@ -6059,8 +6078,8 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
u64 vpid;
u64 gla;
} operand;
- u16 vpid02;
int r, gpr_index;
+ int cpu;
if (!(vmx->nested.msrs.secondary_ctls_high &
SECONDARY_EXEC_ENABLE_VPID) ||
@@ -6094,42 +6113,34 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
return kvm_handle_memory_failure(vcpu, r, &e);
if (operand.vpid >> 16)
- return nested_vmx_fail(vcpu,
- VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+ return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+
+ if (type != VMX_VPID_EXTENT_ALL_CONTEXT && !operand.vpid)
+ return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+
+ /* LAM doesn't apply to addresses that are inputs to TLB invalidation. */
+ if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR &&
+ is_noncanonical_invlpg_address(operand.gla, vcpu))
+ return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
/*
* Always flush the effective vpid02, i.e. never flush the current VPID
* and never explicitly flush vpid01. INVVPID targets a VPID, not a
* VMCS, and so whether or not the current vmcs12 has VPID enabled is
* irrelevant (and there may not be a loaded vmcs12).
+ *
+ * If vmcs02 was last loaded on a different pCPU, then defer the flush
+ * by invalidating the nested VPID tracking to ensure that KVM performs
+ * the invalidation on the correct pCPU.
*/
- vpid02 = nested_get_vpid02(vcpu);
- switch (type) {
- case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
- /*
- * LAM doesn't apply to addresses that are inputs to TLB
- * invalidation.
- */
- if (!operand.vpid ||
- is_noncanonical_invlpg_address(operand.gla, vcpu))
- return nested_vmx_fail(vcpu,
- VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
- vpid_sync_vcpu_addr(vpid02, operand.gla);
- break;
- case VMX_VPID_EXTENT_SINGLE_CONTEXT:
- case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
- if (!operand.vpid)
- return nested_vmx_fail(vcpu,
- VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
- vpid_sync_context(vpid02);
- break;
- case VMX_VPID_EXTENT_ALL_CONTEXT:
- vpid_sync_context(vpid02);
- break;
- default:
- WARN_ON_ONCE(1);
- return kvm_skip_emulated_instruction(vcpu);
- }
+ cpu = get_cpu();
+ if (cpu != vmx->nested.vmcs02.cpu)
+ vmx->nested.last_vpid = 0;
+ else if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR)
+ vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla);
+ else
+ vpid_sync_context(nested_get_vpid02(vcpu));
+ put_cpu();
/*
* Sync the shadow page tables if EPT is disabled, L1 is invalidating
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 47cb9eba113b..cb94b87df989 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -6736,7 +6736,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
break;
fallthrough;
case KVM_CAP_DISABLE_QUIRKS:
- kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks;
+ mutex_lock(&kvm->lock);
+ WRITE_ONCE(kvm->arch.disabled_quirks,
+ kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks));
+ mutex_unlock(&kvm->lock);
r = 0;
break;
case KVM_CAP_SPLIT_IRQCHIP: {
@@ -10012,6 +10015,12 @@ static void kvm_setup_efer_caps(void)
if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS))
kvm_enable_efer_bits(EFER_AUTOIBRS);
+
+ if (kvm_cpu_cap_has(X86_FEATURE_SVM)) {
+ kvm_enable_efer_bits(EFER_SVME);
+ if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ))
+ kvm_enable_efer_bits(EFER_LMSLE);
+ }
}
static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
@@ -11278,6 +11287,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
goto out;
}
}
+ if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu))
+ kvm_x86_call(reload_vmsa)(vcpu);
}
if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
@@ -14142,6 +14153,10 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
{
kvm_x86_call(gmem_invalidate)(start, end);
}
+void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range)
+{
+ kvm_x86_call(gmem_invalidate_range)(kvm, range);
+}
#endif
#endif
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 9de577ef9c97..ad9d2aacb87d 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -367,7 +367,7 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
{
- return !(kvm->arch.disabled_quirks & quirk);
+ return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk);
}
static __always_inline void kvm_request_l1tf_flush_l1d(void)
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-11 12:31:05 +0000