diff options
69 files changed, 734 insertions, 600 deletions
diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst index 3303499ad4c6..a36ea2bd4301 100644 --- a/Documentation/arch/x86/tdx.rst +++ b/Documentation/arch/x86/tdx.rst @@ -200,6 +200,27 @@ reflects the TCB of the currently running TDX module and therefore changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD launch time and is not affected. +Dynamic PAMT +------------ + +The Physical Address Metadata Table (PAMT) is metadata in which the TDX +module keeps data about each physical page (think struct page). Space +for it is allocated by the VMM, consumes up to about 0.4% of system +memory and needs to be supplied to the TDX module when the TDX module is +first loaded. + +Dynamic PAMT is an add-on feature that allows a VMM to dynamically +allocate the part of the PAMT which tracks 4KB pages. This reduces the +amount of memory that TDX consumes while TDs are not in use. + +When Dynamic PAMT is in use, dmesg shows it like:: + + [..] virt/tdx: Enable Dynamic PAMT + [..] virt/tdx: 10092 KB allocated for PAMT + [..] virt/tdx: TDX-Module initialized + +Dynamic PAMT is enabled automatically if supported. + TDX Interaction to Other Kernel Components ------------------------------------------ diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml index 518fbf6b2761..be4dc95e3916 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml @@ -59,6 +59,7 @@ properties: - qcom,sm8650-pdc - qcom,sm8750-pdc - qcom,x1e80100-pdc + - qcom,x1p42100-pdc - const: qcom,pdc reg: diff --git a/arch/Kconfig b/arch/Kconfig index 72890200d049..3bb2e568f5b1 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1704,44 +1704,6 @@ config HAVE_STATIC_CALL_INLINE depends on HAVE_STATIC_CALL select OBJTOOL -config HAVE_PREEMPT_DYNAMIC - bool - -config HAVE_PREEMPT_DYNAMIC_CALL - bool - depends on HAVE_STATIC_CALL - select HAVE_PREEMPT_DYNAMIC - help - An architecture should select this if it can handle the preemption - model being selected at boot time using static calls. - - Where an architecture selects HAVE_STATIC_CALL_INLINE, any call to a - preemption function will be patched directly. - - Where an architecture does not select HAVE_STATIC_CALL_INLINE, any - call to a preemption function will go through a trampoline, and the - trampoline will be patched. - - It is strongly advised to support inline static call to avoid any - overhead. - -config HAVE_PREEMPT_DYNAMIC_KEY - bool - depends on HAVE_ARCH_JUMP_LABEL - select HAVE_PREEMPT_DYNAMIC - help - An architecture should select this if it can handle the preemption - model being selected at boot time using static keys. - - Each preemption function will be given an early return based on a - static key. This should have slightly lower overhead than non-inline - static calls, as this effectively inlines each trampoline into the - start of its callee. This may avoid redundant work, and may - integrate better with CFI schemes. - - This will have greater overhead than using inline static calls as - the call to the preemption function cannot be entirely elided. - config ARCH_WANT_LD_ORPHAN_WARN bool help diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 2bbeded33da0..bd0b4aeceb21 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -216,7 +216,6 @@ config ARM64 select HAVE_PERF_EVENTS_NMI if ARM64_PSEUDO_NMI select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_POSIX_CPU_TIMERS_TASK_WORK diff --git a/arch/arm64/boot/dts/qcom/purwa.dtsi b/arch/arm64/boot/dts/qcom/purwa.dtsi index c698e6cb2543..4348dd3d1dc5 100644 --- a/arch/arm64/boot/dts/qcom/purwa.dtsi +++ b/arch/arm64/boot/dts/qcom/purwa.dtsi @@ -224,6 +224,11 @@ compatible = "qcom,x1p42100-qmp-gen4x4-pcie-phy"; }; +/* X1P42100 PDC is same as X1E80100, but without hardware register bug */ +&pdc { + compatible = "qcom,x1p42100-pdc", "qcom,pdc"; +}; + &qfprom { gpu_speed_bin: gpu-speed-bin@119 { reg = <0x119 0x2>; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 87a6c50bdd14..f0947c8176a1 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1746,8 +1746,6 @@ CONFIG_PWM_VISCONTI=m CONFIG_PWM_XILINX=m CONFIG_SL28CPLD_INTC=y CONFIG_XILINX_INTC=y -CONFIG_QCOM_PDC=y -CONFIG_QCOM_MPM=y CONFIG_TI_SCI_INTR_IRQCHIP=y CONFIG_TI_SCI_INTA_IRQCHIP=y CONFIG_RESET_GPIO=m diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h index 9ecc2766a9f2..e86e850e9b17 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -104,19 +104,9 @@ static inline bool should_resched(int preempt_offset) void preempt_schedule(void); void preempt_schedule_notrace(void); -#ifdef CONFIG_PREEMPT_DYNAMIC - -void dynamic_preempt_schedule(void); -#define __preempt_schedule() dynamic_preempt_schedule() -void dynamic_preempt_schedule_notrace(void); -#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() - -#else /* CONFIG_PREEMPT_DYNAMIC */ - #define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT_DYNAMIC */ #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c index 572efb96b23f..30bf61d031eb 100644 --- a/arch/arm64/kernel/paravirt.c +++ b/arch/arm64/kernel/paravirt.c @@ -157,9 +157,9 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, para_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); pr_info("using stolen time PV\n"); diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 1d8fb1e456d6..ef2d112d3bd3 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -169,7 +169,6 @@ config LOONGARCH select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC select HAVE_RETHOOK diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c index 10821cce554c..e8965a3f8082 100644 --- a/arch/loongarch/kernel/paravirt.c +++ b/arch/loongarch/kernel/paravirt.c @@ -308,10 +308,10 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, paravt_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); #endif if (static_key_enabled(&virt_preempt_key)) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 40c874fe2f53..7f7910f38a67 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -278,7 +278,6 @@ config PPC select HAVE_PERF_EVENTS_NMI if PPC64 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_RETHOOK if KPROBES select HAVE_REGS_AND_STACK_ACCESS_API diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index f29e7547c995..c181df4dec70 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -876,9 +876,9 @@ static void __init pSeries_setup_arch(void) static_branch_enable(&shared_processor); pv_spinlocks_init(); #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); #endif } diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5965666194b0..9a8ffca5a602 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -194,7 +194,6 @@ config RISCV select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RETHOOK select HAVE_RSEQ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 0ecc67641b09..bed39fff1f8a 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -15,6 +15,18 @@ struct seq_file; extern unsigned long boot_cpu_hartid; +enum ipi_message_type { + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CPU_STOP, + IPI_CPU_CRASH_STOP, + IPI_IRQ_WORK, + IPI_TIMER, + IPI_CPU_BACKTRACE, + IPI_KGDB_ROUNDUP, + IPI_MAX +}; + #ifdef CONFIG_SMP #include <linux/jump_label.h> diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index 5f56be79cd06..9c13a6f1ea2a 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -116,9 +116,9 @@ int __init pv_time_init(void) static_call_update(pv_steal_clock, pv_time_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); pr_info("Computing paravirt steal-time\n"); diff --git a/arch/riscv/kernel/sbi-ipi.c b/arch/riscv/kernel/sbi-ipi.c index 0cc5559c08d8..eeec178a9b95 100644 --- a/arch/riscv/kernel/sbi-ipi.c +++ b/arch/riscv/kernel/sbi-ipi.c @@ -57,7 +57,7 @@ void __init sbi_ipi_init(void) return; } - virq = ipi_mux_create(BITS_PER_BYTE, sbi_send_ipi); + virq = ipi_mux_create(IPI_MAX, sbi_send_ipi); if (virq <= 0) { pr_err("unable to create muxed IPIs\n"); irq_dispose_mapping(sbi_ipi_virq); @@ -75,7 +75,7 @@ void __init sbi_ipi_init(void) "irqchip/sbi-ipi:starting", sbi_ipi_starting_cpu, NULL); - riscv_ipi_set_virq_range(virq, BITS_PER_BYTE); + riscv_ipi_set_virq_range(virq, IPI_MAX); pr_info("providing IPIs using SBI IPI extension\n"); /* diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index fa66f9c97d74..8930b62b15e7 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -28,18 +28,6 @@ #include <asm/cacheflush.h> #include <asm/cpu_ops.h> -enum ipi_message_type { - IPI_RESCHEDULE, - IPI_CALL_FUNC, - IPI_CPU_STOP, - IPI_CPU_CRASH_STOP, - IPI_IRQ_WORK, - IPI_TIMER, - IPI_CPU_BACKTRACE, - IPI_KGDB_ROUNDUP, - IPI_MAX -}; - static const char * const ipi_names[] = { [IPI_RESCHEDULE] = "Rescheduling interrupts", [IPI_CALL_FUNC] = "Function call interrupts", diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index b78f9a1c6641..25a4400130fc 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -245,7 +245,6 @@ config S390 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_POSIX_CPU_TIMERS_TASK_WORK - select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RETHOOK diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 5560d5fca2a3..60c6f019ec45 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -155,20 +155,9 @@ static __always_inline int __preempt_count_sub_return(int val) void preempt_schedule(void); void preempt_schedule_notrace(void); -#ifdef CONFIG_PREEMPT_DYNAMIC - -void dynamic_preempt_schedule(void); -void dynamic_preempt_schedule_notrace(void); -#define __preempt_schedule() dynamic_preempt_schedule() -#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() - -#else /* CONFIG_PREEMPT_DYNAMIC */ - #define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT_DYNAMIC */ - #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index fce29c2f7ebc..f11afbbfb42c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -296,7 +296,6 @@ config X86 select HAVE_STACK_VALIDATION if HAVE_OBJTOOL select HAVE_STATIC_CALL select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL - select HAVE_PREEMPT_DYNAMIC_CALL select HAVE_RSEQ select HAVE_RUST if X86_64 select HAVE_SYSCALL_TRACEPOINTS diff --git a/arch/x86/boot/early_serial_console.c b/arch/x86/boot/early_serial_console.c index 5b83beab89e1..39fcd551fc81 100644 --- a/arch/x86/boot/early_serial_console.c +++ b/arch/x86/boot/early_serial_console.c @@ -22,6 +22,7 @@ #define DLH 1 /* Divisor latch High */ #define DEFAULT_BAUD 9600 +#define BASE_BAUD (1843200 / 16) static void early_serial_init(int port, int baud) { @@ -33,7 +34,7 @@ static void early_serial_init(int port, int baud) outb(0, port + FCR); /* no fifo */ outb(0x3, port + MCR); /* DTR + RTS */ - divisor = 115200 / baud; + divisor = BASE_BAUD / baud; c = inb(port + LCR); outb(c | DLAB, port + LCR); outb(divisor & 0xff, port + DLL); @@ -74,16 +75,13 @@ static void parse_earlyprintk(void) else pos = e - arg; } else if (!strncmp(arg + pos, "ttyS", 4)) { - static const int bases[] = { 0x3f8, 0x2f8 }; - int idx = 0; - /* += strlen("ttyS"); */ pos += 4; if (arg[pos++] == '1') - idx = 1; - - port = bases[idx]; + port = 0x2f8; /* ttyS1 */ + else + port = DEFAULT_SERIAL_PORT; } if (arg[pos] == ',') @@ -98,7 +96,6 @@ static void parse_earlyprintk(void) early_serial_init(port, baud); } -#define BASE_BAUD (1843200/16) static unsigned int probe_baud(int port) { unsigned char lcr, dll, dlh; diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f70ee74b5f92..ee7066ca3f2c 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -430,6 +430,7 @@ #define X86_FEATURE_SUCCOR (17*32+ 1) /* "succor" Uncorrectable error containment and recovery */ #define X86_FEATURE_CPPC_PERF_PRIO (17*32+ 2) /* CPPC Floor Perf support */ #define X86_FEATURE_SMCA (17*32+ 3) /* "smca" Scalable MCA */ +#define X86_FEATURE_BTB_CTX_ISOLATION (17*32+ 4) /* AMD: Branch predictions contexts isolated */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EDX), word 18 */ #define X86_FEATURE_AVX512_4VNNIW (18*32+ 2) /* "avx512_4vnniw" AVX-512 Neural Network Instructions */ diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e213c9ae3e30..5c358c40eae8 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -99,6 +99,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_OPTIONAL_RET0(set_external_spte) KVM_X86_OP_OPTIONAL(free_external_spt) +KVM_X86_OP_OPTIONAL_RET0(topup_external_cache) KVM_X86_OP(has_wbinvd_exit) KVM_X86_OP(get_l2_tsc_offset) KVM_X86_OP(get_l2_tsc_multiplier) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 683bb8bf43a9..57d37491c7c0 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1644,6 +1644,7 @@ struct kvm_x86_ops { /* Update external page tables for page table about to be freed. */ void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp); + int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts); bool (*has_wbinvd_exit)(void); diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index fafb6f8cdac3..d16d9c2f7b08 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -137,43 +137,15 @@ static __always_inline bool should_resched(int preempt_offset) extern asmlinkage void preempt_schedule(void); extern asmlinkage void preempt_schedule_thunk(void); -#define preempt_schedule_dynamic_enabled preempt_schedule_thunk -#define preempt_schedule_dynamic_disabled NULL - extern asmlinkage void preempt_schedule_notrace(void); extern asmlinkage void preempt_schedule_notrace_thunk(void); -#define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace_thunk -#define preempt_schedule_notrace_dynamic_disabled NULL - -#ifdef CONFIG_PREEMPT_DYNAMIC - -DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); - -#define __preempt_schedule() \ -do { \ - __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \ - asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \ -} while (0) - -DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); - -#define __preempt_schedule_notrace() \ -do { \ - __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \ - asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \ -} while (0) - -#else /* PREEMPT_DYNAMIC */ - #define __preempt_schedule() \ asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT); #define __preempt_schedule_notrace() \ asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT); -#endif /* PREEMPT_DYNAMIC */ - #endif /* PREEMPTION */ #undef __pc_op diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 89e97d5761d8..e186dfe5bf88 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -36,6 +36,7 @@ /* Bit definitions of TDX_FEATURES0 metadata field */ #define TDX_FEATURES0_TD_PRESERVING BIT_ULL(1) #define TDX_FEATURES0_NO_RBP_MOD BIT_ULL(18) +#define TDX_FEATURES0_DYNAMIC_PAMT BIT_ULL(36) #ifndef __ASSEMBLER__ @@ -118,12 +119,34 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinf return sysinfo->features.tdx_features0 & TDX_FEATURES0_TD_PRESERVING; } +bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo); + +/* Simple structure for pre-allocating DPAMT pages outside of spinlocks. */ +struct tdx_pamt_cache { + struct list_head page_list; + int cnt; +}; + +static inline void tdx_init_pamt_cache(struct tdx_pamt_cache *cache) +{ + INIT_LIST_HEAD(&cache->page_list); + cache->cnt = 0; +} + +void tdx_free_pamt_cache(struct tdx_pamt_cache *cache); +int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages); +int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache); +void tdx_pamt_put(kvm_pfn_t pfn); + int tdx_guest_keyid_alloc(void); u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); void tdx_quirk_reset_paddr(unsigned long base, unsigned long size); +struct page *tdx_alloc_control_page(void); +void tdx_free_control_page(struct page *page); + struct tdx_td { /* TD root structure: */ struct page *tdr_page; diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h index 41150d546589..8a3cc1a2a41e 100644 --- a/arch/x86/include/asm/tdx_global_metadata.h +++ b/arch/x86/include/asm/tdx_global_metadata.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* Automatically generated TDX global metadata structures. */ -#ifndef _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H -#define _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H +/* TDX global metadata structures. */ +#ifndef _X86_VIRT_TDX_TDX_GLOBAL_METADATA_H +#define _X86_VIRT_TDX_TDX_GLOBAL_METADATA_H #include <linux/types.h> @@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr { u16 pamt_4k_entry_size; u16 pamt_2m_entry_size; u16 pamt_1g_entry_size; + + /* Optional metadata, if DPAMT is supported */ + u8 pamt_page_bitmap_entry_bits; }; struct tdx_sys_info_td_ctrl { diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 56eac5611c31..1b2381da4d83 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -1175,6 +1175,7 @@ enum srso_mitigation { SRSO_MITIGATION_IBPB, SRSO_MITIGATION_IBPB_ON_VMEXIT, SRSO_MITIGATION_BP_SPEC_REDUCE, + SRSO_MITIGATION_USER_IBPB, }; static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO; @@ -2908,7 +2909,8 @@ static const char * const srso_strings[] = { [SRSO_MITIGATION_SAFE_RET] = "Mitigation: Safe RET", [SRSO_MITIGATION_IBPB] = "Mitigation: IBPB", [SRSO_MITIGATION_IBPB_ON_VMEXIT] = "Mitigation: IBPB on VMEXIT only", - [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation" + [SRSO_MITIGATION_BP_SPEC_REDUCE] = "Mitigation: Reduced Speculation", + [SRSO_MITIGATION_USER_IBPB] = "Mitigation: IBPB on context switch", }; static int __init srso_parse_cmdline(char *str) @@ -2948,7 +2950,9 @@ static void __init srso_select_mitigation(void) * required. Otherwise the 'microcode' mitigation is sufficient * to protect the user->user and guest->guest vectors. */ - if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) || + if ((cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) && + !boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION)) + || (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) && !boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))) { srso_mitigation = SRSO_MITIGATION_SAFE_RET; @@ -3024,6 +3028,16 @@ static void __init srso_update_mitigation(void) boot_cpu_has(X86_FEATURE_IBPB_BRTYPE)) srso_mitigation = SRSO_MITIGATION_IBPB; + /* + * See if IBPB on context switch is the only thing needed to address + * GUEST/GUEST and USER/USER vectors. + */ + if (srso_mitigation == SRSO_MITIGATION_MICROCODE && + boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO) && + boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION) && + spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE) + srso_mitigation = SRSO_MITIGATION_USER_IBPB; + pr_info("%s\n", srso_strings[srso_mitigation]); } diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 8665a6474806..41b4880be7ed 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -64,6 +64,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_AMD_WORKLOAD_CLASS, CPUID_EAX, 22, 0x80000021, 0 }, { X86_FEATURE_TSA_SQ_NO, CPUID_ECX, 1, 0x80000021, 0 }, { X86_FEATURE_TSA_L1_NO, CPUID_ECX, 2, 0x80000021, 0 }, + { X86_FEATURE_BTB_CTX_ISOLATION, CPUID_ECX, 8, 0x80000021, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { X86_FEATURE_AMD_LBR_PMC_FREEZE, CPUID_EAX, 2, 0x80000022, 0 }, diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 4505f808af5e..a5f2aabb2da1 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -106,7 +106,13 @@ static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list) left_dirty++; } - cond_resched(); + /* + * cond_resched() only schedules when TIF_NEED_RESCHED is set. + * During this boot-time loop that condition may not happen for a + * long time, so report an RCU-Tasks quiescent state explicitly. + * Therefore, change cond_resched() to cond_resched_tasks_rcu_qs(). + */ + cond_resched_tasks_rcu_qs(); } list_splice(&dirty, dirty_page_list); diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 34b73573b108..f7ab9e7902cf 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -328,9 +328,9 @@ static int vmware_cpu_down_prepare(unsigned int cpu) static __init int activate_jump_labels(void) { if (has_steal_clock) { - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c index e681ec9cf1dc..e6f23933a6df 100644 --- a/arch/x86/kernel/crash.c +++ b/arch/x86/kernel/crash.c @@ -369,9 +369,9 @@ int crash_load_segments(struct kimage *image) * maximum CPUs and maximum memory ranges. */ if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) - pnum = 2 + CONFIG_NR_CPUS_DEFAULT + CONFIG_CRASH_MAX_MEMORY_RANGES; + pnum = 2 + CONFIG_NR_CPUS + CONFIG_CRASH_MAX_MEMORY_RANGES; else - pnum += 2 + CONFIG_NR_CPUS_DEFAULT; + pnum += 2 + CONFIG_NR_CPUS; if (pnum < (unsigned long)PN_XNUM) { kbuf.memsz = pnum * sizeof(Elf64_Phdr); @@ -430,7 +430,7 @@ unsigned int arch_crash_get_elfcorehdr_size(void) unsigned int sz; /* kernel_map, VMCOREINFO and maximum CPUs */ - sz = 2 + CONFIG_NR_CPUS_DEFAULT; + sz = 2 + CONFIG_NR_CPUS; if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) sz += CONFIG_CRASH_MAX_MEMORY_RANGES; sz *= sizeof(Elf64_Phdr); diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 6b0a5861ccb8..2ce01fdb8b4b 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -1054,9 +1054,9 @@ const __initconst struct hypervisor_x86 x86_hyper_kvm = { static __init int activate_jump_labels(void) { if (has_steal_clock) { - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (steal_acc) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } return 0; diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c index 0ca64900192f..eb690ba90180 100644 --- a/arch/x86/kernel/shstk.c +++ b/arch/x86/kernel/shstk.c @@ -490,7 +490,7 @@ static int wrss_control(bool enable) * when disabling. */ if (!features_enabled(ARCH_SHSTK_SHSTK)) - return -EPERM; + return -EINVAL; /* Already enabled/disabled? */ if (features_enabled(ARCH_SHSTK_WRSS) == enable) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..2816861ca279 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -617,6 +617,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect) PT64_ROOT_MAX_LEVEL); if (r) return r; + + r = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL); + if (r) + return r; } r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache, PT64_ROOT_MAX_LEVEL); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..7c79f4d4bf99 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -362,7 +362,7 @@ static void tdx_reclaim_control_page(struct page *ctrl_page) if (tdx_reclaim_page(ctrl_page)) return; - __free_page(ctrl_page); + tdx_free_control_page(ctrl_page); } struct tdx_flush_vp_arg { @@ -589,7 +589,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm) tdx_quirk_reset_paddr(page_to_phys(kvm_tdx->td.tdr_page), PAGE_SIZE); - __free_page(kvm_tdx->td.tdr_page); + tdx_free_control_page(kvm_tdx->td.tdr_page); kvm_tdx->td.tdr_page = NULL; } @@ -681,6 +681,8 @@ int tdx_vcpu_create(struct kvm_vcpu *vcpu) if (!irqchip_split(vcpu->kvm)) return -EINVAL; + tdx_init_pamt_cache(&tdx->pamt_cache); + fpstate_set_confidential(&vcpu->arch.guest_fpu); vcpu->arch.apic->guest_apic_protected = true; INIT_LIST_HEAD(&tdx->vt.pi_wakeup_list); @@ -866,6 +868,8 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu) struct vcpu_tdx *tdx = to_tdx(vcpu); int i; + tdx_free_pamt_cache(&tdx->pamt_cache); + if (vcpu->cpu != -1) { KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm); tdx_flush_vp_on_cpu(vcpu); @@ -1618,6 +1622,17 @@ void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); } +static int tdx_topup_external_pamt_cache(struct kvm_vcpu *vcpu, int min_nr_spts) +{ + /* + * Minus one page to exclude the root SPT, but plus one page for a + * possible 4KB private mapping. + */ + min_nr_spts += -1 + 1; + + return tdx_topup_pamt_cache(&to_tdx(vcpu)->pamt_cache, min_nr_spts); +} + static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level, kvm_pfn_t pfn) { @@ -1676,16 +1691,28 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn, static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 new_spte) { + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); gpa_t gpa = gfn_to_gpa(gfn); u64 err, entry, level_state; struct page *sept_pt; + int ret; + + if (KVM_BUG_ON(!vcpu, kvm)) + return -EIO; sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level); if (!sept_pt) return -EIO; + ret = tdx_pamt_get(page_to_pfn(sept_pt), &to_tdx(vcpu)->pamt_cache); + if (KVM_BUG_ON(ret, kvm)) + return ret; + err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt, &entry, &level_state); + if (err) + tdx_pamt_put(page_to_pfn(sept_pt)); + if (unlikely(tdx_operand_busy(err))) return -EBUSY; @@ -1698,8 +1725,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn, static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level, u64 new_spte) { + struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); kvm_pfn_t pfn = spte_to_pfn(new_spte); + int ret; + + if (KVM_BUG_ON(!vcpu, kvm)) + return -EIO; /* TODO: handle large pages. */ if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm)) @@ -1707,6 +1739,10 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK); + ret = tdx_pamt_get(pfn, &to_tdx(vcpu)->pamt_cache); + if (KVM_BUG_ON(ret, kvm)) + return ret; + /* * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() * before kvm_tdx->state. Userspace must not be allowed to pre-fault @@ -1719,10 +1755,15 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve * If the TD isn't finalized/runnable, then userspace is initializing * the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD. */ - if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE)) - return tdx_mem_page_add(kvm, gfn, level, pfn); + if (likely(kvm_tdx->state == TD_STATE_RUNNABLE)) + ret = tdx_mem_page_aug(kvm, gfn, level, pfn); + else + ret = tdx_mem_page_add(kvm, gfn, level, pfn); + + if (ret) + tdx_pamt_put(pfn); - return tdx_mem_page_aug(kvm, gfn, level, pfn); + return ret; } /* @@ -1819,6 +1860,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn, return -EIO; tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE); + tdx_pamt_put(pfn); return 0; } @@ -1862,6 +1904,8 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte, */ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp) { + struct page *sept_pt = virt_to_page(sp->external_spt); + /* * KVM doesn't (yet) zap page table pages in mirror page table while * TD is active, though guest pages mapped in mirror page table could be @@ -1875,15 +1919,15 @@ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp) * the page to prevent the kernel from accessing the encrypted page. */ if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) || - tdx_reclaim_page(virt_to_page(sp->external_spt))) + tdx_reclaim_page(sept_pt)) goto out; /* - * Immediately free the S-EPT page because RCU-time free is unnecessary - * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding - * readers. + * Immediately free the S-EPT page as the TDX subsystem doesn't support + * freeing pages from RCU callbacks, and more importantly because + * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers. */ - free_page((unsigned long)sp->external_spt); + tdx_free_control_page(sept_pt); out: sp->external_spt = NULL; } @@ -2456,7 +2500,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params, ret = -ENOMEM; - tdr_page = alloc_page(GFP_KERNEL_ACCOUNT); + tdr_page = tdx_alloc_control_page(); if (!tdr_page) goto free_hkid; @@ -2469,7 +2513,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params, goto free_tdr; for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) { - tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT); + tdcs_pages[i] = tdx_alloc_control_page(); if (!tdcs_pages[i]) goto free_tdcs; } @@ -2587,10 +2631,8 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params, teardown: /* Only free pages not yet added, so start at 'i' */ for (; i < kvm_tdx->td.tdcs_nr_pages; i++) { - if (tdcs_pages[i]) { - __free_page(tdcs_pages[i]); - tdcs_pages[i] = NULL; - } + tdx_free_control_page(tdcs_pages[i]); + tdcs_pages[i] = NULL; } if (!kvm_tdx->td.tdcs_pages) kfree(tdcs_pages); @@ -2605,16 +2647,13 @@ free_packages: free_cpumask_var(packages); free_tdcs: - for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) { - if (tdcs_pages[i]) - __free_page(tdcs_pages[i]); - } + for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) + tdx_free_control_page(tdcs_pages[i]); kfree(tdcs_pages); kvm_tdx->td.tdcs_pages = NULL; free_tdr: - if (tdr_page) - __free_page(tdr_page); + tdx_free_control_page(tdr_page); kvm_tdx->td.tdr_page = NULL; free_hkid: @@ -2948,7 +2987,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) int ret, i; u64 err; - page = alloc_page(GFP_KERNEL_ACCOUNT); + page = tdx_alloc_control_page(); if (!page) return -ENOMEM; tdx->vp.tdvpr_page = page; @@ -2968,7 +3007,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) } for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) { - page = alloc_page(GFP_KERNEL_ACCOUNT); + page = tdx_alloc_control_page(); if (!page) { ret = -ENOMEM; goto free_tdcx; @@ -2990,7 +3029,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) * method, but the rest are freed here. */ for (; i < kvm_tdx->td.tdcx_nr_pages; i++) { - __free_page(tdx->vp.tdcx_pages[i]); + tdx_free_control_page(tdx->vp.tdcx_pages[i]); tdx->vp.tdcx_pages[i] = NULL; } return -EIO; @@ -3018,16 +3057,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx) free_tdcx: for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) { - if (tdx->vp.tdcx_pages[i]) - __free_page(tdx->vp.tdcx_pages[i]); + tdx_free_control_page(tdx->vp.tdcx_pages[i]); tdx->vp.tdcx_pages[i] = NULL; } kfree(tdx->vp.tdcx_pages); tdx->vp.tdcx_pages = NULL; free_tdvpr: - if (tdx->vp.tdvpr_page) - __free_page(tdx->vp.tdvpr_page); + tdx_free_control_page(tdx->vp.tdvpr_page); tdx->vp.tdvpr_page = NULL; tdx->vp.tdvpr_pa = 0; @@ -3487,6 +3524,10 @@ int __init tdx_hardware_setup(void) vt_x86_ops.set_external_spte = tdx_sept_set_private_spte; vt_x86_ops.free_external_spt = tdx_sept_free_private_spt; + + if (tdx_supports_dynamic_pamt(tdx_sysinfo)) + vt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache; + vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt; return 0; diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h index ac8323a68b16..fd368e3ee060 100644 --- a/arch/x86/kvm/vmx/tdx.h +++ b/arch/x86/kvm/vmx/tdx.h @@ -72,6 +72,8 @@ struct vcpu_tdx { u64 map_gpa_next; u64 map_gpa_end; + + struct tdx_pamt_cache pamt_cache; }; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err); diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 1b9ff749dd8e..063574ed8625 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -30,6 +30,7 @@ #include <linux/suspend.h> #include <linux/syscore_ops.h> #include <linux/idr.h> +#include <linux/vmalloc.h> #include <asm/page.h> #include <asm/special_insns.h> #include <asm/msr-index.h> @@ -46,6 +47,9 @@ #include "seamcall_internal.h" #include "tdx.h" +/* Number of DPAMT pages to be provided to TDX module per 2MB region of PA */ +#define TDX_DPAMT_ENTRY_PAGE_CNT 2 + struct tdx_module_state { bool initialized; bool sysinit_done; @@ -63,6 +67,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized); static struct tdmr_info_list tdx_tdmr_list; +/* + * On a machine with DPAMT, the kernel maintains a reference counter + * for every 2MB range. The counter indicates how many users there are for + * the DPAMT at the 2MB range. The kernel allocates DPAMT refcounts at + * initialization. + */ +static atomic_t *dpamt_refcounts; + /* All TDX-usable memory regions. Protected by mem_hotplug_lock. */ static LIST_HEAD(tdx_memlist); @@ -253,6 +265,42 @@ static struct syscore tdx_syscore = { }; /* + * Allocate DPAMT reference counters for all physical memory. + * + * It consumes 2MB for every 1TB of physical memory. + */ +static __init int init_dpamt_refcounts(void) +{ + size_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) * sizeof(*dpamt_refcounts); + + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return 0; + + dpamt_refcounts = vzalloc(size); + if (!dpamt_refcounts) + return -ENOMEM; + + return 0; +} + +static __init void free_dpamt_refcounts(void) +{ + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return; + + vfree(dpamt_refcounts); + dpamt_refcounts = NULL; +} + +static atomic_t *tdx_find_dpamt_refcount(unsigned long pfn) +{ + /* Find which PMD a PFN is in. */ + unsigned long index = pfn >> (PMD_SHIFT - PAGE_SHIFT); + + return &dpamt_refcounts[index]; +} + +/* * Add a memory region as a TDX memory block. The caller must make sure * all memory regions are added in address ascending order and don't * overlap. @@ -510,35 +558,37 @@ static __init int fill_out_tdmrs(struct list_head *tmb_list, return 0; } +static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info *tdmr) +{ + unsigned long pamt_sz, nr_pamt_entries; + int bits_per_entry; + + bits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits; + nr_pamt_entries = tdmr->size >> PAGE_SHIFT; + pamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE); + + return PAGE_ALIGN(pamt_sz); +} + /* * Calculate PAMT size given a TDMR and a page size. The returned * PAMT size is always aligned up to 4K page boundary. */ -static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz, - u16 pamt_entry_size) +static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz) { unsigned long pamt_sz, nr_pamt_entries; + const int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT }; + const u16 pamt_entry_size[TDX_PS_NR] = { + tdx_sysinfo.tdmr.pamt_4k_entry_size, + tdx_sysinfo.tdmr.pamt_2m_entry_size, + tdx_sysinfo.tdmr.pamt_1g_entry_size, + }; - switch (pgsz) { - case TDX_PS_4K: - nr_pamt_entries = tdmr->size >> PAGE_SHIFT; - break; - case TDX_PS_2M: - nr_pamt_entries = tdmr->size >> PMD_SHIFT; - break; - case TDX_PS_1G: - nr_pamt_entries = tdmr->size >> PUD_SHIFT; - break; - default: - WARN_ON_ONCE(1); - return 0; - } + nr_pamt_entries = tdmr->size >> tdx_pg_size_shift[pgsz]; + pamt_sz = nr_pamt_entries * pamt_entry_size[pgsz]; - pamt_sz = nr_pamt_entries * pamt_entry_size; /* TDX requires PAMT size must be 4K aligned */ - pamt_sz = ALIGN(pamt_sz, PAGE_SIZE); - - return pamt_sz; + return PAGE_ALIGN(pamt_sz); } /* @@ -576,15 +626,11 @@ static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_lis * within @tdmr, and set up PAMTs for @tdmr. */ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr, - struct list_head *tmb_list, - u16 pamt_entry_size[]) + struct list_head *tmb_list) { - unsigned long pamt_base[TDX_PS_NR]; - unsigned long pamt_size[TDX_PS_NR]; - unsigned long tdmr_pamt_base; unsigned long tdmr_pamt_size; struct page *pamt; - int pgsz, nid; + int nid; nid = tdmr_get_nid(tdmr, tmb_list); @@ -592,13 +638,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr, * Calculate the PAMT size for each TDX supported page size * and the total PAMT size. */ - tdmr_pamt_size = 0; - for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) { - pamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz, - pamt_entry_size[pgsz]); - tdmr_pamt_size += pamt_size[pgsz]; + tdmr->pamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G); + tdmr->pamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M); + + if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) { + /* With DPAMT, PAMT_4K is replaced with a bitmap */ + tdmr->pamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr); + } else { + tdmr->pamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K); } + tdmr_pamt_size = tdmr->pamt_4k_size + tdmr->pamt_2m_size + tdmr->pamt_1g_size; + /* * Allocate one chunk of physically contiguous memory for all * PAMTs. This helps minimize the PAMT's use of reserved areas @@ -606,25 +657,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr, */ pamt = alloc_contig_pages(tdmr_pamt_size >> PAGE_SHIFT, GFP_KERNEL, nid, &node_online_map); - if (!pamt) - return -ENOMEM; /* - * Break the contiguous allocation back up into the - * individual PAMTs for each page size. + * tdmr->pamt_4k_base is still zero so the error + * path of the caller will skip freeing the PAMT. */ - tdmr_pamt_base = page_to_pfn(pamt) << PAGE_SHIFT; - for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) { - pamt_base[pgsz] = tdmr_pamt_base; - tdmr_pamt_base += pamt_size[pgsz]; - } + if (!pamt) + return -ENOMEM; - tdmr->pamt_4k_base = pamt_base[TDX_PS_4K]; - tdmr->pamt_4k_size = pamt_size[TDX_PS_4K]; - tdmr->pamt_2m_base = pamt_base[TDX_PS_2M]; - tdmr->pamt_2m_size = pamt_size[TDX_PS_2M]; - tdmr->pamt_1g_base = pamt_base[TDX_PS_1G]; - tdmr->pamt_1g_size = pamt_size[TDX_PS_1G]; + tdmr->pamt_4k_base = page_to_phys(pamt); + tdmr->pamt_2m_base = tdmr->pamt_4k_base + tdmr->pamt_4k_size; + tdmr->pamt_1g_base = tdmr->pamt_2m_base + tdmr->pamt_2m_size; return 0; } @@ -655,10 +698,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr, tdmr_get_pamt(tdmr, &pamt_base, &pamt_size); /* Do nothing if PAMT hasn't been allocated for this TDMR */ - if (!pamt_size) - return; - - if (WARN_ON_ONCE(!pamt_base)) + if (!pamt_base) return; pamt_func(pamt_base, pamt_size); @@ -684,14 +724,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list) /* Allocate and set up PAMTs for all TDMRs */ static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, - struct list_head *tmb_list, - u16 pamt_entry_size[]) + struct list_head *tmb_list) { int i, ret = 0; for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { - ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list, - pamt_entry_size); + ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list); if (ret) goto err; } @@ -968,18 +1006,13 @@ static __init int construct_tdmrs(struct list_head *tmb_list, struct tdmr_info_list *tdmr_list, struct tdx_sys_info_tdmr *sysinfo_tdmr) { - u16 pamt_entry_size[TDX_PS_NR] = { - sysinfo_tdmr->pamt_4k_entry_size, - sysinfo_tdmr->pamt_2m_entry_size, - sysinfo_tdmr->pamt_1g_entry_size, - }; int ret; ret = fill_out_tdmrs(tmb_list, tdmr_list); if (ret) return ret; - ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size); + ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list); if (ret) return ret; @@ -998,6 +1031,8 @@ static __init int construct_tdmrs(struct list_head *tmb_list, return ret; } +#define TDX_SYS_CONFIG_DYNAMIC_PAMT BIT(16) + static __init int config_tdx_module(struct tdmr_info_list *tdmr_list, u64 global_keyid) { @@ -1026,6 +1061,12 @@ static __init int config_tdx_module(struct tdmr_info_list *tdmr_list, args.rcx = __pa(tdmr_pa_array); args.rdx = tdmr_list->nr_consumed_tdmrs; args.r8 = global_keyid; + + if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) { + pr_info("Enable Dynamic PAMT\n"); + args.r8 |= TDX_SYS_CONFIG_DYNAMIC_PAMT; + } + ret = seamcall_prerr(TDH_SYS_CONFIG, &args); /* Free the array as it is not required anymore. */ @@ -1167,10 +1208,14 @@ static __init int init_tdx_module(void) */ get_online_mems(); - ret = build_tdx_memlist(&tdx_memlist); + ret = init_dpamt_refcounts(); if (ret) goto out_put_tdxmem; + ret = build_tdx_memlist(&tdx_memlist); + if (ret) + goto err_free_dpamt_refcounts; + /* Allocate enough space for constructing TDMRs */ ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr); if (ret) @@ -1220,6 +1265,8 @@ err_free_tdmrs: free_tdmr_list(&tdx_tdmr_list); err_free_tdxmem: free_tdx_memlist(&tdx_memlist); +err_free_dpamt_refcounts: + free_dpamt_refcounts(); goto out_put_tdxmem; } @@ -2005,6 +2052,269 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn) } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid); +bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo) +{ + return sysinfo->features.tdx_features0 & TDX_FEATURES0_DYNAMIC_PAMT; +} +EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt); + +static struct page *tdx_alloc_page_pamt_cache(struct tdx_pamt_cache *cache) +{ + struct page *page; + + page = list_first_entry_or_null(&cache->page_list, struct page, lru); + if (page) { + list_del(&page->lru); + cache->cnt--; + } + + return page; +} + +static struct page *alloc_dpamt_page(struct tdx_pamt_cache *cache) +{ + if (cache) + return tdx_alloc_page_pamt_cache(cache); + + return alloc_page(GFP_KERNEL_ACCOUNT); +} + +static int alloc_pamt_array(struct page **pamt_pages, struct tdx_pamt_cache *cache) +{ + int i, j; + + for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) { + pamt_pages[i] = alloc_dpamt_page(cache); + if (!pamt_pages[i]) + goto err; + } + + return 0; + +err: + for (j = 0; j < i; j++) + __free_page(pamt_pages[j]); + + return -ENOMEM; +} + +static void free_pamt_array(struct page **pamt_pages) +{ + int i; + + for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) { + /* + * Reset pages unconditionally to cover cases + * where they were passed to the TDX module. + */ + tdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE); + + __free_page(pamt_pages[i]); + } +} + +/* Helper for building DPAMT seamcall() arguments. */ +static u64 pamt_2mb_arg(kvm_pfn_t pfn) +{ + /* Find the 2MB-wide DPAMT region for 'pfn': */ + unsigned long hpa_2mb = ALIGN_DOWN(pfn << PAGE_SHIFT, PMD_SIZE); + + /* + * TDX ABI requires specifying the page level the installed DPAMT + * backing will cover, even though today only 2MB is supported. + */ + return hpa_2mb | TDX_PS_2M; +} + +/* Add DPAMT backing for the 2MB region surrounding the given pfn. */ +static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page **pamt_pages) +{ + struct tdx_module_args args = { + .rcx = pamt_2mb_arg(pfn), + .rdx = page_to_phys(pamt_pages[0]), + .r8 = page_to_phys(pamt_pages[1]), + }; + + return seamcall(TDH_PHYMEM_PAMT_ADD, &args); +} + +/* Remove DPAMT backing for the 2MB region surrounding the given pfn. */ +static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page **pamt_pages) +{ + struct tdx_module_args args = { + .rcx = pamt_2mb_arg(pfn), + }; + u64 ret; + + ret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, &args); + if (ret) + return ret; + + /* Copy PAMT pages out of the struct per the TDX ABI */ + pamt_pages[0] = phys_to_page(args.rdx); + pamt_pages[1] = phys_to_page(args.r8); + + return 0; +} + +/* Serializes adding/removing DPAMT memory */ +static DEFINE_SPINLOCK(dpamt_lock); + +/* Bump DPAMT refcount for the given pfn and allocate DPAMT backing if needed. */ +int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache) +{ + struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT]; + atomic_t *dpamt_refcount; + u64 tdx_status; + int ret; + + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return 0; + + ret = alloc_pamt_array(pamt_pages, cache); + if (ret) + return ret; + + dpamt_refcount = tdx_find_dpamt_refcount(pfn); + + spin_lock(&dpamt_lock); + + /* + * If the DPAMT entry is already added (i.e. refcount >= 1), + * then just increment the refcount. + */ + if (atomic_inc_not_zero(dpamt_refcount)) + goto out_free; + + /* Try to add the PAMT page and take the refcount 0->1. */ + tdx_status = tdh_phymem_pamt_add(pfn, pamt_pages); + if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) { + ret = -EIO; + goto out_free; + } + + atomic_set(dpamt_refcount, 1); + spin_unlock(&dpamt_lock); + return 0; + +out_free: + spin_unlock(&dpamt_lock); + free_pamt_array(pamt_pages); + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get); + +/* Drop DPAMT refcount for the given pfn and free DPAMT backing if needed. */ +void tdx_pamt_put(kvm_pfn_t pfn) +{ + struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {}; + atomic_t *dpamt_refcount; + u64 tdx_status; + + if (!tdx_supports_dynamic_pamt(&tdx_sysinfo)) + return; + + dpamt_refcount = tdx_find_dpamt_refcount(pfn); + + spin_lock(&dpamt_lock); + /* + * If there is more than 1 reference on the DPAMT entry, don't + * remove it yet. Just decrement the refcount. + */ + if (atomic_read(dpamt_refcount) > 1) { + atomic_dec(dpamt_refcount); + goto out_unlock; + } + + /* Try to remove the pamt page and take the refcount 1->0. */ + tdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages); + + /* + * Don't free pamt_pages as it could hold garbage when + * tdh_phymem_pamt_remove() fails. Don't panic/BUG_ON(), as + * there is no risk of data corruption, but do yell loudly as + * failure indicates a kernel bug, memory is being leaked, and + * the dangling DPAMT entry may cause future operations to fail. + */ + if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) + goto out_unlock; + + atomic_set(dpamt_refcount, 0); + spin_unlock(&dpamt_lock); + free_pamt_array(pamt_pages); + return; +out_unlock: + spin_unlock(&dpamt_lock); +} +EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put); + +void tdx_free_pamt_cache(struct tdx_pamt_cache *cache) +{ + struct page *page; + + while ((page = tdx_alloc_page_pamt_cache(cache))) + __free_page(page); +} +EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache); + +int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages) +{ + if (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(&tdx_sysinfo))) + return 0; + + npages *= TDX_DPAMT_ENTRY_PAGE_CNT; + + while (cache->cnt < npages) { + struct page *page = alloc_page(GFP_KERNEL_ACCOUNT); + + if (!page) + return -ENOMEM; + + list_add(&page->lru, &cache->page_list); + cache->cnt++; + } + + return 0; +} +EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache); + +/* + * Return a page that can be gifted to the TDX module for use as a "control" + * page, i.e. pages that are used for control structures for a given TDX + * guest, and thus obtain TDX protections, including DPAMT tracking. + */ +struct page *tdx_alloc_control_page(void) +{ + struct page *page; + + page = alloc_page(GFP_KERNEL_ACCOUNT); + if (!page) + return NULL; + + if (tdx_pamt_get(page_to_pfn(page), NULL)) { + __free_page(page); + return NULL; + } + + return page; +} +EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page); + +/* + * Free a page that was gifted to the TDX module for use as a control + * page. After this, the page is no longer protected by TDX. + */ +void tdx_free_control_page(struct page *page) +{ + if (!page) + return; + + tdx_pamt_put(page_to_pfn(page)); + __free_page(page); +} +EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page); + void tdx_sys_disable(void) { struct tdx_module_args args = {}; diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index bdfd0e1e337a..a886c54decaa 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -48,6 +48,8 @@ #define TDH_SYS_CONFIG 45 #define TDH_SYS_SHUTDOWN 52 #define TDH_SYS_UPDATE 53 +#define TDH_PHYMEM_PAMT_ADD 58 +#define TDH_PHYMEM_PAMT_REMOVE 59 #define TDH_SYS_DISABLE 69 /* diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c index e49c300f23d4..98ebf17aab1c 100644 --- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c +++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Automatically generated functions to read TDX global metadata. + * Functions to read TDX global metadata. * * This file doesn't compile on its own as it lacks of inclusion * of SEAMCALL wrapper primitive which reads global metadata. @@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinf return ret; } +static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr *sysinfo_tdmr) +{ + int ret; + u64 val; + + ret = read_sys_metadata_field(0x9100000000000013, &val); + if (!ret) + sysinfo_tdmr->pamt_page_bitmap_entry_bits = val; + + return ret; +} + static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr) { int ret = 0; @@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo) ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl); ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf); + /* + * The kernel supports using TDX without DPAMT, so + * avoid reporting failure if it's not supported. Don't + * try to support buggy TDX modules that advertise + * DPAMT but don't expose the metadata. + */ + if (!ret && tdx_supports_dynamic_pamt(sysinfo)) + ret = get_tdx_sys_info_tdmr_dpamt(&sysinfo->tdmr); + return ret; } diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c index 0bdd9d7ec545..e56eee7e3781 100644 --- a/drivers/clocksource/timer-clint.c +++ b/drivers/clocksource/timer-clint.c @@ -243,7 +243,7 @@ static int __init clint_timer_init_dt(struct device_node *np) } #ifdef CONFIG_SMP - rc = ipi_mux_create(BITS_PER_BYTE, clint_send_ipi); + rc = ipi_mux_create(IPI_MAX, clint_send_ipi); if (rc <= 0) { pr_err("unable to create muxed IPIs\n"); rc = (rc < 0) ? rc : -ENODEV; @@ -251,7 +251,7 @@ static int __init clint_timer_init_dt(struct device_node *np) } irq_set_chained_handler(clint_ipi_irq, clint_ipi_interrupt); - riscv_ipi_set_virq_range(rc, BITS_PER_BYTE); + riscv_ipi_set_virq_range(rc, IPI_MAX); clint_clear_ipi(); #endif diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 20b77fbc51ee..a8f8c423b795 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -522,17 +522,19 @@ config GOLDFISH_PIC config QCOM_PDC tristate "Qualcomm PDC" - depends on ARCH_QCOM + depends on ARCH_QCOM || COMPILE_TEST select IRQ_DOMAIN_HIERARCHY + default ARCH_QCOM help Power Domain Controller driver to manage and configure wakeup IRQs for Qualcomm Technologies Inc (QTI) mobile chips. config QCOM_MPM tristate "Qualcomm MPM" - depends on ARCH_QCOM + depends on ARCH_QCOM || COMPILE_TEST depends on MAILBOX select IRQ_DOMAIN_HIERARCHY + default ARCH_QCOM if ARM64 help MSM Power Manager driver to manage and configure wakeup IRQs for Qualcomm Technologies Inc (QTI) mobile chips. diff --git a/drivers/irqchip/irq-aclint-sswi.c b/drivers/irqchip/irq-aclint-sswi.c index ca06efd86fa1..5e010fc401f7 100644 --- a/drivers/irqchip/irq-aclint-sswi.c +++ b/drivers/irqchip/irq-aclint-sswi.c @@ -138,7 +138,7 @@ static int __init aclint_sswi_probe(struct fwnode_handle *fwnode) } /* Register SSWI irq and handler */ - virq = ipi_mux_create(BITS_PER_BYTE, aclint_sswi_ipi_send); + virq = ipi_mux_create(IPI_MAX, aclint_sswi_ipi_send); if (virq <= 0) { pr_err("unable to create muxed IPIs\n"); irq_dispose_mapping(sswi_ipi_virq); @@ -152,7 +152,7 @@ static int __init aclint_sswi_probe(struct fwnode_handle *fwnode) aclint_sswi_starting_cpu, aclint_sswi_dying_cpu); - riscv_ipi_set_virq_range(virq, BITS_PER_BYTE); + riscv_ipi_set_virq_range(virq, IPI_MAX); return 0; } diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 6e1fa5b247fc..b0fca8f01cd4 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -2276,7 +2276,6 @@ static struct bool single_redist; int enabled_rdists; u32 maint_irq; - int maint_irq_mode; phys_addr_t vcpu_base; } acpi_data __initdata; @@ -2454,21 +2453,19 @@ static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *hea { struct acpi_madt_generic_interrupt *gicc = (struct acpi_madt_generic_interrupt *)header; - int maint_irq_mode; static int first_madt = true; if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) return 0; - maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? - ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; + if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) + pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n"); if (first_madt) { first_madt = false; acpi_data.maint_irq = gicc->vgic_interrupt; - acpi_data.maint_irq_mode = maint_irq_mode; acpi_data.vcpu_base = gicc->gicv_base_address; return 0; @@ -2478,7 +2475,6 @@ static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *hea * The maintenance interrupt and GICV should be the same for every CPU */ if ((acpi_data.maint_irq != gicc->vgic_interrupt) || - (acpi_data.maint_irq_mode != maint_irq_mode) || (acpi_data.vcpu_base != gicc->gicv_base_address)) return -EINVAL; @@ -2511,7 +2507,7 @@ static void __init gic_acpi_setup_kvm_info(void) gic_v3_kvm_info.type = GIC_V3; irq = acpi_register_gsi(NULL, acpi_data.maint_irq, - acpi_data.maint_irq_mode, + ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_HIGH); if (irq <= 0) return; diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index ac2d423b1723..c365f8150300 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -1166,21 +1166,18 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain) if (ret) goto out_int; - ret = set_handle_irq(gicv5_handle_irq); + ret = gicv5_irs_enable(); if (ret) goto out_int; - ret = gicv5_irs_enable(); - if (ret) - goto out_handle; + if (set_handle_irq(gicv5_handle_irq)) + panic("GICv5: unable to install root IRQ handler\n"); gicv5_smp_init(); gicv5_irs_its_probe(); return 0; -out_handle: - set_handle_irq(NULL); out_int: gicv5_cpu_disable_interrupts(); gicv5_free_domains(); diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index f6bc29f515fb..b2926a3ddaf1 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -1527,7 +1527,6 @@ static struct { phys_addr_t cpu_phys_base; u32 maint_irq; - int maint_irq_mode; phys_addr_t vctrl_base; phys_addr_t vcpu_base; } acpi_data __initdata; @@ -1553,10 +1552,11 @@ gic_acpi_parse_madt_cpu(union acpi_subtable_headers *header, if (cpu_base_assigned && gic_cpu_base != acpi_data.cpu_phys_base) return -EINVAL; + if (processor->flags & ACPI_MADT_VGIC_IRQ_MODE) + pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n"); + acpi_data.cpu_phys_base = gic_cpu_base; acpi_data.maint_irq = processor->vgic_interrupt; - acpi_data.maint_irq_mode = (processor->flags & ACPI_MADT_VGIC_IRQ_MODE) ? - ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; acpi_data.vctrl_base = processor->gich_base_address; acpi_data.vcpu_base = processor->gicv_base_address; @@ -1616,7 +1616,7 @@ static void __init gic_acpi_setup_kvm_info(void) vcpu_res->end = vcpu_res->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; irq = acpi_register_gsi(NULL, acpi_data.maint_irq, - acpi_data.maint_irq_mode, + ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_HIGH); if (irq <= 0) return; diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index 12efd241ce88..823f5f2ecb3d 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -67,12 +67,12 @@ static int __init imsic_ipi_domain_init(void) return 0; /* Create IMSIC IPI multiplexing */ - virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send); + virq = ipi_mux_create(IPI_MAX, imsic_ipi_send); if (virq <= 0) return virq < 0 ? virq : -ENOMEM; /* Set vIRQ range */ - riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI); + riscv_ipi_set_virq_range(virq, IPI_MAX); /* Announce that IMSIC is providing IPIs */ pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID); diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c index b8d1bbbf42f7..9505ddbd9eec 100644 --- a/drivers/irqchip/irq-riscv-imsic-state.c +++ b/drivers/irqchip/irq-riscv-imsic-state.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "riscv-imsic: " fmt #include <linux/acpi.h> #include <linux/cpu.h> +#include <linux/bits.h> #include <linux/bitmap.h> #include <linux/interrupt.h> #include <linux/irq.h> @@ -769,9 +770,9 @@ static int __init imsic_parse_fwnode(struct fwnode_handle *fwnode, return -EINVAL; } global->base_addr = res.start; - global->base_addr &= ~(BIT(global->guest_index_bits + - global->hart_index_bits + - IMSIC_MMIO_PAGE_SHIFT) - 1); + global->base_addr &= ~GENMASK(global->guest_index_bits + + global->hart_index_bits + + IMSIC_MMIO_PAGE_SHIFT - 1, 0); global->base_addr &= ~((BIT(global->group_index_bits) - 1) << global->group_index_shift); @@ -850,9 +851,9 @@ int __init imsic_setup_state(struct fwnode_handle *fwnode, void *opaque) } base_addr = mmios[i].start; - base_addr &= ~(BIT(global->guest_index_bits + - global->hart_index_bits + - IMSIC_MMIO_PAGE_SHIFT) - 1); + base_addr &= ~GENMASK(global->guest_index_bits + + global->hart_index_bits + + IMSIC_MMIO_PAGE_SHIFT - 1, 0); base_addr &= ~((BIT(global->group_index_bits) - 1) << global->group_index_shift); if (base_addr != global->base_addr) { diff --git a/drivers/irqchip/irq-riscv-imsic-state.h b/drivers/irqchip/irq-riscv-imsic-state.h index c42ee180b305..878cc192ccec 100644 --- a/drivers/irqchip/irq-riscv-imsic-state.h +++ b/drivers/irqchip/irq-riscv-imsic-state.h @@ -13,7 +13,6 @@ #include <linux/timer.h> #define IMSIC_IPI_ID 1 -#define IMSIC_NR_IPI 8 struct imsic_vector { /* Fixed details of the vector */ diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c index ce6d80c7f17a..29025a212ece 100644 --- a/drivers/irqchip/qcom-pdc.c +++ b/drivers/irqchip/qcom-pdc.c @@ -715,7 +715,10 @@ static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *pare } pdc->x1e_quirk = true; + } + if (of_device_is_compatible(node, "qcom,x1e80100-pdc") || + of_device_is_compatible(node, "qcom,x1p42100-pdc")) { if (!qcom_scm_is_available()) return -EPROBE_DEFER; diff --git a/drivers/xen/time.c b/drivers/xen/time.c index a2be0a4d45b0..a02d48a2aa68 100644 --- a/drivers/xen/time.c +++ b/drivers/xen/time.c @@ -169,7 +169,7 @@ void __init xen_time_setup_guest(void) static_call_update(pv_steal_clock, xen_steal_clock); - static_key_slow_inc(¶virt_steal_enabled); + static_branch_inc(¶virt_steal_enabled); if (xen_runstate_remote) - static_key_slow_inc(¶virt_steal_rq_enabled); + static_branch_inc(¶virt_steal_rq_enabled); } diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index c8683c046615..1adddeab8545 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -96,19 +96,9 @@ static __always_inline bool should_resched(int preempt_offset) extern asmlinkage void preempt_schedule(void); extern asmlinkage void preempt_schedule_notrace(void); -#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -void dynamic_preempt_schedule(void); -void dynamic_preempt_schedule_notrace(void); -#define __preempt_schedule() dynamic_preempt_schedule() -#define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() - -#else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/ - #define __preempt_schedule() preempt_schedule() #define __preempt_schedule_notrace() preempt_schedule_notrace() -#endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/ #endif /* CONFIG_PREEMPTION */ #endif /* __ASM_PREEMPT_H */ diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 0bb6c03481fa..2be273bb68f0 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -346,22 +346,7 @@ typedef struct irqentry_state { * * Conditional reschedule with additional sanity checks. */ -void raw_irqentry_exit_cond_resched(void); - -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched -#define irqentry_exit_cond_resched_dynamic_disabled NULL -DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void); -#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() -#endif -#else /* CONFIG_PREEMPT_DYNAMIC */ -#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() -#endif /* CONFIG_PREEMPT_DYNAMIC */ +void irqentry_exit_cond_resched(void); /** * irqentry_enter_from_kernel_mode - Establish state before invoking the irq handler diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 24414c79e59a..4b11d1dc0a67 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -43,30 +43,10 @@ struct completion; struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD - extern int __cond_resched(void); # define might_resched() __cond_resched() - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -extern int __cond_resched(void); - -DECLARE_STATIC_CALL(might_resched, __cond_resched); - -static __always_inline void might_resched(void) -{ - static_call_mod(might_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_might_resched(void); -# define might_resched() dynamic_might_resched() - #else - # define might_resched() do { } while (0) - #endif /* CONFIG_PREEMPT_* */ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 2e689de7b29a..06ff44a4b8b6 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -498,21 +498,11 @@ DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_n #ifdef CONFIG_PREEMPT_DYNAMIC -extern bool preempt_model_none(void); -extern bool preempt_model_voluntary(void); extern bool preempt_model_full(void); extern bool preempt_model_lazy(void); #else -static inline bool preempt_model_none(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_NONE); -} -static inline bool preempt_model_voluntary(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); -} static inline bool preempt_model_full(void) { return IS_ENABLED(CONFIG_PREEMPT); @@ -525,6 +515,16 @@ static inline bool preempt_model_lazy(void) #endif +static inline bool preempt_model_none(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_NONE); +} + +static inline bool preempt_model_voluntary(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); +} + static inline bool preempt_model_rt(void) { return IS_ENABLED(CONFIG_PREEMPT_RT); diff --git a/include/linux/sched.h b/include/linux/sched.h index 284963d52c90..5ce350a616e4 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2138,44 +2138,19 @@ static inline void set_need_resched_current(void) * value indicates whether a reschedule was done in fact. * cond_resched_lock() will drop the spinlock before scheduling, */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) extern int __cond_resched(void); -#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -DECLARE_STATIC_CALL(cond_resched, __cond_resched); - -static __always_inline int _cond_resched(void) -{ - return static_call_mod(cond_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_cond_resched(void); - -static __always_inline int _cond_resched(void) -{ - return dynamic_cond_resched(); -} - -#else /* !CONFIG_PREEMPTION */ - static inline int _cond_resched(void) { return __cond_resched(); } - -#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ - -#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ - +#else static inline int _cond_resched(void) { return 0; } - -#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ +#endif #define cond_resched() ({ \ __might_resched(__FILE__, __LINE__, 0); \ diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index e90efaf6d26e..694126411dfe 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -182,9 +182,9 @@ extern unsigned long long task_sched_runtime(struct task_struct *task); #ifdef CONFIG_PARAVIRT -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_FALSE(paravirt_steal_enabled); +DECLARE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN u64 dummy_steal_clock(int cpu); diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec index 15632358bcf7..a97ed9605602 100644 --- a/kernel/Kconfig.kexec +++ b/kernel/Kconfig.kexec @@ -167,7 +167,7 @@ config CRASH_MAX_MEMORY_RANGES memory regions that the elfcorehdr buffer/segment can accommodate. These regions are obtained via walk_system_ram_res(); eg. the 'System RAM' entries in /proc/iomem. - This value is combined with NR_CPUS_DEFAULT and multiplied by + This value is combined with NR_CPUS and multiplied by sizeof(Elf64_Phdr) to determine the final elfcorehdr memory buffer/ segment size. The value 8192, for example, covers a (sparsely populated) 1TiB system diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index f294dad43bd7..985aea617cfe 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -132,10 +132,9 @@ config PREEMPTION config PREEMPT_DYNAMIC bool "Preemption behaviour defined on boot" - depends on HAVE_PREEMPT_DYNAMIC - select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY + depends on ARCH_HAS_PREEMPT_LAZY select PREEMPT_BUILD - default y if HAVE_PREEMPT_DYNAMIC_CALL + default y help This option allows to define the preemption model on the kernel command line parameter and thus override the default preemption @@ -145,9 +144,7 @@ config PREEMPT_DYNAMIC provide a pre-built kernel binary to reduce the number of kernel flavors they offer while still offering different usecases. - The runtime overhead is negligible with HAVE_STATIC_CALL_INLINE enabled - but if runtime patching is not available for the specific architecture - then the potential overhead should be considered. + The runtime overhead is negligible. Interesting if you want the same pre-built kernel should be used for both Server and Desktop workloads. diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 2b36aa9fade0..d0bd2d0cf899 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -648,7 +648,7 @@ int crash_check_hotplug_support(void) * new list of CPUs and memory. To make changes to the elfcorehdr, it * should be large enough to permit a growing number of CPU and Memory * resources. One can estimate the elfcorehdr memory size based on - * NR_CPUS_DEFAULT and CRASH_MAX_MEMORY_RANGES. The elfcorehdr is + * NR_CPUS and CRASH_MAX_MEMORY_RANGES. The elfcorehdr is * excluded from SHA verification by default if the architecture * supports crash hotplug. */ diff --git a/kernel/entry/common.c b/kernel/entry/common.c index e3d381fd3d25..e234b04373fe 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -123,7 +123,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) /** * arch_irqentry_exit_need_resched - Architecture specific need resched function * - * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed. + * Invoked from irqentry_exit_cond_resched() to check if resched is needed. * Defaults return true. * * The main purpose is to permit arch to avoid preemption of a task from an IRQ. @@ -134,7 +134,7 @@ static inline bool arch_irqentry_exit_need_resched(void); static inline bool arch_irqentry_exit_need_resched(void) { return true; } #endif -void raw_irqentry_exit_cond_resched(void) +void irqentry_exit_cond_resched(void) { if (!preempt_count()) { /* Sanity check RCU and thread stack */ @@ -145,19 +145,6 @@ void raw_irqentry_exit_cond_resched(void) preempt_schedule_irq(); } } -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) - return; - raw_irqentry_exit_cond_resched(); -} -#endif -#endif noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) { diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 3a818f07a101..7cbcc3fb9e6d 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -28,7 +28,15 @@ static struct lock_class_key irq_desc_lock_class; static int __init irq_affinity_setup(char *str) { alloc_bootmem_cpumask_var(&irq_default_affinity); - cpulist_parse(str, irq_default_affinity); + if (cpulist_parse(str, irq_default_affinity) < 0) { + /* + * Clear the mask so that init_irq_default_affinity() + * defaults it later. + */ + cpumask_clear(irq_default_affinity); + pr_warn("irqaffinity: incorrect CPU range, using default\n"); + return 1; + } /* * Set at least the boot cpu. We don't want to end up with * bugreports caused by random commandline masks diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b998ef6b87af..853afc869715 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -805,7 +805,7 @@ struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf) /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_rq_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #endif static void update_rq_clock_task(struct rq *rq, s64 delta) @@ -844,7 +844,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) } #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING - if (static_key_false((¶virt_steal_rq_enabled))) { + if (static_branch_unlikely(¶virt_steal_rq_enabled)) { u64 prev_steal; steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); @@ -3766,6 +3766,7 @@ static inline void proxy_reset_donor(struct rq *rq) WARN_ON_ONCE(rq->donor == rq->curr); put_prev_set_next_task(rq, rq->donor, rq->curr); + rq->next_class = rq->curr->sched_class; rq_set_donor(rq, rq->curr); zap_balance_callbacks(rq); resched_curr(rq); @@ -5142,7 +5143,7 @@ static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) lockdep_assert_rq_held(rq); while (head) { - func = (void (*)(struct rq *))head->func; + func = head->func; next = head->next; head->next = NULL; head = next; @@ -6509,7 +6510,10 @@ static bool try_steal_cookie(int this, int that) return false; do { - if (p == src->core_pick || p == src->curr) + if (p == src->core_pick || p == src->curr || p == src->donor) + goto next; + + if (task_is_blocked(p)) goto next; if (!is_cpu_allowed(p, this)) @@ -6857,9 +6861,9 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, __must_hold(__rq_lockp(rq)) { struct rq *target_rq = cpu_rq(target_cpu); + LIST_HEAD(migrate_list); lockdep_assert_rq_held(rq); - WARN_ON(p == rq->curr); /* * Since we are migrating a blocked donor, it could be rq->donor, * and we want to make sure there aren't any references from this @@ -6872,13 +6876,20 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, * before we release the lock. */ proxy_resched_idle(rq); - - deactivate_task(rq, p, DEQUEUE_NOCLOCK); - proxy_set_task_cpu(p, target_cpu); - + for (; p; p = p->blocked_donor) { + WARN_ON(p == rq->curr); + deactivate_task(rq, p, DEQUEUE_NOCLOCK); + proxy_set_task_cpu(p, target_cpu); + /* + * We can re-use se.group_node to migrate the thing, + * because @p is deactivated (won't be balanced) and + * we hold the rq_lock. + */ + list_add(&p->se.group_node, &migrate_list); + } proxy_release_rq_lock(rq, rf); - attach_one_task(target_rq, p); + __attach_tasks(target_rq, &migrate_list); proxy_reacquire_rq_lock(rq, rf); } @@ -7043,6 +7054,14 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) owner->blocked_donor = p; } WARN_ON_ONCE(owner && !owner->on_rq); + + if (owner && !sched_cpu_cookie_match(rq, owner)) { + if (curr_in_chain) + return proxy_resched_idle(rq); + p = donor; /* Deactivate the donor, not the runnable owner */ + clear_task_blocked_on(p, NULL); + goto deactivate; + } return owner; deactivate: @@ -7481,27 +7500,6 @@ asmlinkage __visible void __sched notrace preempt_schedule(void) NOKPROBE_SYMBOL(preempt_schedule); EXPORT_SYMBOL(preempt_schedule); -#ifdef CONFIG_PREEMPT_DYNAMIC -# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL -# ifndef preempt_schedule_dynamic_enabled -# define preempt_schedule_dynamic_enabled preempt_schedule -# define preempt_schedule_dynamic_disabled NULL -# endif -DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); -EXPORT_STATIC_CALL_TRAMP(preempt_schedule); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); -void __sched notrace dynamic_preempt_schedule(void) -{ - if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) - return; - preempt_schedule(); -} -NOKPROBE_SYMBOL(dynamic_preempt_schedule); -EXPORT_SYMBOL(dynamic_preempt_schedule); -# endif -#endif /* CONFIG_PREEMPT_DYNAMIC */ - /** * preempt_schedule_notrace - preempt_schedule called by tracing * @@ -7554,27 +7552,6 @@ asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) } EXPORT_SYMBOL_GPL(preempt_schedule_notrace); -#ifdef CONFIG_PREEMPT_DYNAMIC -# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -# ifndef preempt_schedule_notrace_dynamic_enabled -# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace -# define preempt_schedule_notrace_dynamic_disabled NULL -# endif -DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); -EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); -void __sched notrace dynamic_preempt_schedule_notrace(void) -{ - if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) - return; - preempt_schedule_notrace(); -} -NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); -EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); -# endif -#endif - #endif /* CONFIG_PREEMPTION */ /* @@ -7791,7 +7768,7 @@ out_unlock: } #endif /* CONFIG_RT_MUTEXES */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) int __sched __cond_resched(void) { if (should_resched(0) && !irqs_disabled()) { @@ -7819,38 +7796,6 @@ int __sched __cond_resched(void) EXPORT_SYMBOL(__cond_resched); #endif -#ifdef CONFIG_PREEMPT_DYNAMIC -# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL -# define cond_resched_dynamic_enabled __cond_resched -# define cond_resched_dynamic_disabled ((void *)&__static_call_return0) -DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); -EXPORT_STATIC_CALL_TRAMP(cond_resched); - -# define might_resched_dynamic_enabled __cond_resched -# define might_resched_dynamic_disabled ((void *)&__static_call_return0) -DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); -EXPORT_STATIC_CALL_TRAMP(might_resched); -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); -int __sched dynamic_cond_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_cond_resched)) - return 0; - return __cond_resched(); -} -EXPORT_SYMBOL(dynamic_cond_resched); - -static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); -int __sched dynamic_might_resched(void) -{ - if (!static_branch_unlikely(&sk_dynamic_might_resched)) - return 0; - return __cond_resched(); -} -EXPORT_SYMBOL(dynamic_might_resched); -# endif -#endif /* CONFIG_PREEMPT_DYNAMIC */ - /* * __cond_resched_lock() - if a reschedule is pending, drop the given lock, * call schedule, and on return reacquire the lock. @@ -7920,50 +7865,21 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write); # endif /* - * SC:cond_resched - * SC:might_resched - * SC:preempt_schedule - * SC:preempt_schedule_notrace - * SC:irqentry_exit_cond_resched - * - * * NONE: - * cond_resched <- __cond_resched - * might_resched <- RET0 - * preempt_schedule <- NOP - * preempt_schedule_notrace <- NOP - * irqentry_exit_cond_resched <- NOP - * dynamic_preempt_lazy <- false + * (unselectable) * * VOLUNTARY: - * cond_resched <- __cond_resched - * might_resched <- __cond_resched - * preempt_schedule <- NOP - * preempt_schedule_notrace <- NOP - * irqentry_exit_cond_resched <- NOP - * dynamic_preempt_lazy <- false + * (unselectable) * * FULL: - * cond_resched <- RET0 - * might_resched <- RET0 - * preempt_schedule <- preempt_schedule - * preempt_schedule_notrace <- preempt_schedule_notrace - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- false * * LAZY: - * cond_resched <- RET0 - * might_resched <- RET0 - * preempt_schedule <- preempt_schedule - * preempt_schedule_notrace <- preempt_schedule_notrace - * irqentry_exit_cond_resched <- irqentry_exit_cond_resched * dynamic_preempt_lazy <- true */ enum { preempt_dynamic_undefined = -1, - preempt_dynamic_none, - preempt_dynamic_voluntary, preempt_dynamic_full, preempt_dynamic_lazy, }; @@ -7972,21 +7888,11 @@ int preempt_dynamic_mode = preempt_dynamic_undefined; int sched_dynamic_mode(const char *str) { -# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY)) - if (!strcmp(str, "none")) - return preempt_dynamic_none; - - if (!strcmp(str, "voluntary")) - return preempt_dynamic_voluntary; -# endif - if (!strcmp(str, "full")) return preempt_dynamic_full; -# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY if (!strcmp(str, "lazy")) return preempt_dynamic_lazy; -# endif return -EINVAL; } @@ -7994,71 +7900,18 @@ int sched_dynamic_mode(const char *str) # define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key) # define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key) -# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) -# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) -# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f) -# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f) -# else -# error "Unsupported PREEMPT_DYNAMIC mechanism" -# endif - static DEFINE_MUTEX(sched_dynamic_mutex); static void __sched_dynamic_update(int mode) { - /* - * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in - * the ZERO state, which is invalid. - */ - preempt_dynamic_enable(cond_resched); - preempt_dynamic_enable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - switch (mode) { - case preempt_dynamic_none: - preempt_dynamic_enable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_disable(preempt_schedule); - preempt_dynamic_disable(preempt_schedule_notrace); - preempt_dynamic_disable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - if (mode != preempt_dynamic_mode) - pr_info("Dynamic Preempt: none\n"); - break; - - case preempt_dynamic_voluntary: - preempt_dynamic_enable(cond_resched); - preempt_dynamic_enable(might_resched); - preempt_dynamic_disable(preempt_schedule); - preempt_dynamic_disable(preempt_schedule_notrace); - preempt_dynamic_disable(irqentry_exit_cond_resched); - preempt_dynamic_key_disable(preempt_lazy); - if (mode != preempt_dynamic_mode) - pr_info("Dynamic Preempt: voluntary\n"); - break; - case preempt_dynamic_full: - preempt_dynamic_disable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_disable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: full\n"); break; case preempt_dynamic_lazy: - preempt_dynamic_disable(cond_resched); - preempt_dynamic_disable(might_resched); - preempt_dynamic_enable(preempt_schedule); - preempt_dynamic_enable(preempt_schedule_notrace); - preempt_dynamic_enable(irqentry_exit_cond_resched); preempt_dynamic_key_enable(preempt_lazy); if (mode != preempt_dynamic_mode) pr_info("Dynamic Preempt: lazy\n"); @@ -8091,11 +7944,7 @@ __setup("preempt=", setup_preempt_mode); static void __init preempt_dynamic_init(void) { if (preempt_dynamic_mode == preempt_dynamic_undefined) { - if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { - sched_dynamic_update(preempt_dynamic_none); - } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { - sched_dynamic_update(preempt_dynamic_voluntary); - } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { + if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { sched_dynamic_update(preempt_dynamic_lazy); } else { /* Default static call setting, nothing to do */ @@ -8115,8 +7964,6 @@ static void __init preempt_dynamic_init(void) } \ EXPORT_SYMBOL_GPL(preempt_model_##mode) -PREEMPT_MODEL_ACCESSOR(none); -PREEMPT_MODEL_ACCESSOR(voluntary); PREEMPT_MODEL_ACCESSOR(full); PREEMPT_MODEL_ACCESSOR(lazy); @@ -8129,7 +7976,7 @@ static inline void preempt_dynamic_init(void) { } #endif /* CONFIG_PREEMPT_DYNAMIC */ const char *preempt_modes[] = { - "none", "voluntary", "full", "lazy", NULL, + "full", "lazy", NULL, }; const char *preempt_model_str(void) diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 06bddaa738e5..f16970ca81d0 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -255,7 +255,7 @@ void __account_forceidle_time(struct task_struct *p, u64 delta) * occasion account more time than the calling functions think elapsed. */ #ifdef CONFIG_PARAVIRT -struct static_key paravirt_steal_enabled; +DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN static u64 native_steal_clock(int cpu) @@ -270,7 +270,7 @@ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); static __always_inline u64 steal_account_process_time(u64 maxtime) { #ifdef CONFIG_PARAVIRT - if (static_key_false(¶virt_steal_enabled)) { + if (static_branch_unlikely(¶virt_steal_enabled)) { u64 steal; steal = paravirt_steal_clock(smp_processor_id()); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0663c00c41c0..de6a361a87c7 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se) * chosen as the deadline is too small, don't even try to * start the timer in the past! */ - if (ktime_us_delta(act, now) < 0) + if (ktime_before(act, now)) return 0; /* diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 72236db67983..fda10b3fa249 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -73,13 +73,13 @@ static int sched_feat_show(struct seq_file *m, void *v) #ifdef CONFIG_JUMP_LABEL -#define jump_label_key__true STATIC_KEY_INIT_TRUE -#define jump_label_key__false STATIC_KEY_INIT_FALSE +#define jump_label_key__true { .key_true = STATIC_KEY_TRUE_INIT } +#define jump_label_key__false { .key_false = STATIC_KEY_FALSE_INIT } #define SCHED_FEAT(name, enabled) \ jump_label_key__##enabled , -struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { +union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR] = { #include "features.h" }; @@ -87,12 +87,12 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { static void sched_feat_disable(int i) { - static_key_disable_cpuslocked(&sched_feat_keys[i]); + static_branch_disable_cpuslocked(&sched_feat_keys[i].key_true); } static void sched_feat_enable(int i) { - static_key_enable_cpuslocked(&sched_feat_keys[i]); + static_branch_enable_cpuslocked(&sched_feat_keys[i].key_false); } #else /* !CONFIG_JUMP_LABEL: */ static void sched_feat_disable(int i) { }; @@ -280,16 +280,10 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, static int sched_dynamic_show(struct seq_file *m, void *v) { - int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2; int mode = READ_ONCE(preempt_dynamic_mode); - int j; - /* Count entries in NULL terminated preempt_modes */ - for (j = 0; preempt_modes[j]; j++) - ; - j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY); - - for (; i < j; i++) { + /* Stop at NULL terminator */ + for (int i = 0; preempt_modes[i]; i++) { if (mode == i) seq_puts(m, "("); seq_puts(m, preempt_modes[i]); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ade1eceb39b8..b8bd308c2d5b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11194,21 +11194,7 @@ next: */ static void attach_tasks(struct lb_env *env) { - struct list_head *tasks = &env->tasks; - struct task_struct *p; - struct rq_flags rf; - - rq_lock(env->dst_rq, &rf); - update_rq_clock(env->dst_rq); - - while (!list_empty(tasks)) { - p = list_first_entry(tasks, struct task_struct, se.group_node); - list_del_init(&p->se.group_node); - - attach_task(env->dst_rq, p); - } - - rq_unlock(env->dst_rq, &rf); + __attach_tasks(env->dst_rq, &env->tasks); } #ifdef CONFIG_NO_HZ_COMMON diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e656c7059bf8..6c3ad70e58b8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2447,16 +2447,25 @@ extern __read_mostly unsigned int sysctl_sched_features; #ifdef CONFIG_JUMP_LABEL -#define SCHED_FEAT(name, enabled) \ -static __always_inline bool static_branch_##name(struct static_key *key) \ -{ \ - return static_key_##enabled(key); \ +union sched_feat_key { + struct static_key_true key_true; + struct static_key_false key_false; +}; + +#define sched_feat_branch_true(key) static_branch_likely(&(key)->key_true) +#define sched_feat_branch_false(key) static_branch_unlikely(&(key)->key_false) + +#define SCHED_FEAT(name, enabled) \ +static __always_inline bool \ +static_branch_##name(union sched_feat_key *key) \ +{ \ + return sched_feat_branch_##enabled(key); \ } #include "features.h" #undef SCHED_FEAT -extern struct static_key sched_feat_keys[__SCHED_FEAT_NR]; +extern union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR]; #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x])) #else /* !CONFIG_JUMP_LABEL: */ @@ -3139,6 +3148,25 @@ static inline void attach_one_task(struct rq *rq, struct task_struct *p) attach_task(rq, p); } +/* + * __attach_tasks() - attaches a list of tasks (using se.group_node) to + * the new rq + */ +static inline void __attach_tasks(struct rq *rq, struct list_head *tasks) +{ + guard(rq_lock)(rq); + update_rq_clock(rq); + + while (!list_empty(tasks)) { + struct task_struct *p; + + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + attach_task(rq, p); + } +} + #ifdef CONFIG_PREEMPT_RT # define SCHED_NR_MIGRATE_BREAK 8 #else diff --git a/kernel/softirq.c b/kernel/softirq.c index 5d02c36c40e3..c3729c5b284b 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -645,8 +645,8 @@ restart: h->action(); trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { - pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", - vec_nr, softirq_to_name[vec_nr], h->action, + pr_err("huh, entered softirq %u %s with preempt_count %08x, exited with %08x?\n", + vec_nr, softirq_to_name[vec_nr], prev_count, preempt_count()); preempt_count_set(prev_count); } |