From fd420f4d2d7ec638d4d258b3432ee79ac22dbdc0 Mon Sep 17 00:00:00 2001 From: ferreo Date: Fri, 3 Jan 2025 16:42:16 +0100 Subject: [PATCH] Update patches/0001-cachyos-base-all.patch --- patches/0001-cachyos-base-all.patch | 1250 +++++++++++++++++++++++++-- 1 file changed, 1161 insertions(+), 89 deletions(-) diff --git a/patches/0001-cachyos-base-all.patch b/patches/0001-cachyos-base-all.patch index 79e033c..8d5ce94 100644 --- a/patches/0001-cachyos-base-all.patch +++ b/patches/0001-cachyos-base-all.patch @@ -1,7 +1,7 @@ -From 68fd401540c8011e3210751d8727074bcbb60150 Mon Sep 17 00:00:00 2001 +From 720d41e042e68b79484fdac3f913f58b6eac6a14 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:49:54 +0100 -Subject: [PATCH 01/12] amd-cache-optimizer +Date: Thu, 2 Jan 2025 12:32:56 +0100 +Subject: [PATCH 01/13] amd-cache-optimizer Signed-off-by: Peter Jung --- @@ -270,12 +270,12 @@ index 000000000000..0f6d3c54d879 +MODULE_DESCRIPTION("AMD 3D V-Cache Performance Optimizer Driver"); +MODULE_LICENSE("GPL"); -- -2.47.1 +2.48.0.rc1 -From fae888443fc1f6cf82450cc88c9fc14a4157d759 Mon Sep 17 00:00:00 2001 +From 95a158ca8276de49ce922ff8558d11ab4d207d8c Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:50:07 +0100 -Subject: [PATCH 02/12] amd-pstate +Date: Thu, 2 Jan 2025 12:33:09 +0100 +Subject: [PATCH 02/13] amd-pstate Signed-off-by: Peter Jung --- @@ -1172,12 +1172,1101 @@ index dd4682857c12..23698d0f4bb4 100644 /* * BUG word(s) -- -2.47.1 +2.48.0.rc1 -From 63aefa7ec768a3b856b41412903f2b8f94c25e95 Mon Sep 17 00:00:00 2001 +From a781a95178b415da9b016670497607c029f9f73b Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:50:18 +0100 -Subject: [PATCH 03/12] autofdo +Date: Thu, 2 Jan 2025 12:46:56 +0100 +Subject: [PATCH 03/13] amd-tlb-broadcast + +Signed-off-by: Peter Jung +--- + arch/x86/Kconfig | 2 +- + arch/x86/hyperv/mmu.c | 1 - + arch/x86/include/asm/cpufeatures.h | 1 + + arch/x86/include/asm/invlpgb.h | 93 ++++++ + arch/x86/include/asm/mmu.h | 6 + + arch/x86/include/asm/mmu_context.h | 12 + + arch/x86/include/asm/paravirt.h | 5 - + arch/x86/include/asm/paravirt_types.h | 2 - + arch/x86/include/asm/tlbbatch.h | 1 + + arch/x86/include/asm/tlbflush.h | 31 +- + arch/x86/kernel/cpu/amd.c | 16 ++ + arch/x86/kernel/kvm.c | 1 - + arch/x86/kernel/paravirt.c | 6 - + arch/x86/kernel/setup.c | 4 + + arch/x86/mm/pgtable.c | 16 +- + arch/x86/mm/tlb.c | 393 +++++++++++++++++++++++++- + arch/x86/xen/mmu_pv.c | 1 - + mm/memory.c | 1 - + mm/mmap.c | 2 - + mm/swap_state.c | 1 - + mm/vma.c | 2 - + 21 files changed, 541 insertions(+), 56 deletions(-) + create mode 100644 arch/x86/include/asm/invlpgb.h + +diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig +index 171be04eca1f..76f9e6d11872 100644 +--- a/arch/x86/Kconfig ++++ b/arch/x86/Kconfig +@@ -270,7 +270,7 @@ config X86 + select HAVE_PCI + select HAVE_PERF_REGS + select HAVE_PERF_USER_STACK_DUMP +- select MMU_GATHER_RCU_TABLE_FREE if PARAVIRT ++ select MMU_GATHER_RCU_TABLE_FREE + select MMU_GATHER_MERGE_VMAS + select HAVE_POSIX_CPU_TIMERS_TASK_WORK + select HAVE_REGS_AND_STACK_ACCESS_API +diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c +index 1cc113200ff5..cbe6c71e17c1 100644 +--- a/arch/x86/hyperv/mmu.c ++++ b/arch/x86/hyperv/mmu.c +@@ -240,5 +240,4 @@ void hyperv_setup_mmu_ops(void) + + pr_info("Using hypercall for remote TLB flush\n"); + pv_ops.mmu.flush_tlb_multi = hyperv_flush_tlb_multi; +- pv_ops.mmu.tlb_remove_table = tlb_remove_table; + } +diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h +index a7c93191b7c6..19892faf43d5 100644 +--- a/arch/x86/include/asm/cpufeatures.h ++++ b/arch/x86/include/asm/cpufeatures.h +@@ -335,6 +335,7 @@ + #define X86_FEATURE_CLZERO (13*32+ 0) /* "clzero" CLZERO instruction */ + #define X86_FEATURE_IRPERF (13*32+ 1) /* "irperf" Instructions Retired Count */ + #define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* "xsaveerptr" Always save/restore FP error pointers */ ++#define X86_FEATURE_INVLPGB (13*32+ 3) /* "invlpgb" INVLPGB instruction */ + #define X86_FEATURE_RDPRU (13*32+ 4) /* "rdpru" Read processor register at user level */ + #define X86_FEATURE_WBNOINVD (13*32+ 9) /* "wbnoinvd" WBNOINVD instruction */ + #define X86_FEATURE_AMD_IBPB (13*32+12) /* Indirect Branch Prediction Barrier */ +diff --git a/arch/x86/include/asm/invlpgb.h b/arch/x86/include/asm/invlpgb.h +new file mode 100644 +index 000000000000..2669ebfffe81 +--- /dev/null ++++ b/arch/x86/include/asm/invlpgb.h +@@ -0,0 +1,93 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _ASM_X86_INVLPGB ++#define _ASM_X86_INVLPGB ++ ++#include ++ ++/* ++ * INVLPGB does broadcast TLB invalidation across all the CPUs in the system. ++ * ++ * The INVLPGB instruction is weakly ordered, and a batch of invalidations can ++ * be done in a parallel fashion. ++ * ++ * TLBSYNC is used to ensure that pending INVLPGB invalidations initiated from ++ * this CPU have completed. ++ */ ++static inline void __invlpgb(unsigned long asid, unsigned long pcid, unsigned long addr, ++ int extra_count, bool pmd_stride, unsigned long flags) ++{ ++ u64 rax = addr | flags; ++ u32 ecx = (pmd_stride << 31) | extra_count; ++ u32 edx = (pcid << 16) | asid; ++ ++ asm volatile("invlpgb" : : "a" (rax), "c" (ecx), "d" (edx)); ++} ++ ++/* ++ * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination ++ * of the three. For example: ++ * - INVLPGB_VA | INVLPGB_INCLUDE_GLOBAL: invalidate all TLB entries at the address ++ * - INVLPGB_PCID: invalidate all TLB entries matching the PCID ++ * ++ * The first can be used to invalidate (kernel) mappings at a particular ++ * address across all processes. ++ * ++ * The latter invalidates all TLB entries matching a PCID. ++ */ ++#define INVLPGB_VA BIT(0) ++#define INVLPGB_PCID BIT(1) ++#define INVLPGB_ASID BIT(2) ++#define INVLPGB_INCLUDE_GLOBAL BIT(3) ++#define INVLPGB_FINAL_ONLY BIT(4) ++#define INVLPGB_INCLUDE_NESTED BIT(5) ++ ++/* Flush all mappings for a given pcid and addr, not including globals. */ ++static inline void invlpgb_flush_user(unsigned long pcid, ++ unsigned long addr) ++{ ++ __invlpgb(0, pcid, addr, 0, 0, INVLPGB_PCID | INVLPGB_VA); ++} ++ ++static inline void invlpgb_flush_user_nr(unsigned long pcid, unsigned long addr, ++ int nr, bool pmd_stride) ++{ ++ __invlpgb(0, pcid, addr, nr - 1, pmd_stride, INVLPGB_PCID | INVLPGB_VA | INVLPGB_FINAL_ONLY); ++} ++ ++/* Flush all mappings for a given ASID, not including globals. */ ++static inline void invlpgb_flush_single_asid(unsigned long asid) ++{ ++ __invlpgb(asid, 0, 0, 0, 0, INVLPGB_ASID); ++} ++ ++/* Flush all mappings for a given PCID, not including globals. */ ++static inline void invlpgb_flush_single_pcid(unsigned long pcid) ++{ ++ __invlpgb(0, pcid, 0, 0, 0, INVLPGB_PCID); ++} ++ ++/* Flush all mappings, including globals, for all PCIDs. */ ++static inline void invlpgb_flush_all(void) ++{ ++ __invlpgb(0, 0, 0, 0, 0, INVLPGB_INCLUDE_GLOBAL); ++} ++ ++/* Flush addr, including globals, for all PCIDs. */ ++static inline void invlpgb_flush_addr(unsigned long addr, int nr) ++{ ++ __invlpgb(0, 0, addr, nr - 1, 0, INVLPGB_INCLUDE_GLOBAL); ++} ++ ++/* Flush all mappings for all PCIDs except globals. */ ++static inline void invlpgb_flush_all_nonglobals(void) ++{ ++ __invlpgb(0, 0, 0, 0, 0, 0); ++} ++ ++/* Wait for INVLPGB originated by this CPU to complete. */ ++static inline void tlbsync(void) ++{ ++ asm volatile("tlbsync"); ++} ++ ++#endif /* _ASM_X86_INVLPGB */ +diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h +index ce4677b8b735..83d0986295d3 100644 +--- a/arch/x86/include/asm/mmu.h ++++ b/arch/x86/include/asm/mmu.h +@@ -46,6 +46,12 @@ typedef struct { + unsigned long flags; + #endif + ++#ifdef CONFIG_CPU_SUP_AMD ++ struct list_head broadcast_asid_list; ++ u16 broadcast_asid; ++ bool asid_transition; ++#endif ++ + #ifdef CONFIG_ADDRESS_MASKING + /* Active LAM mode: X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */ + unsigned long lam_cr3_mask; +diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h +index 2886cb668d7f..2c347b51d9b9 100644 +--- a/arch/x86/include/asm/mmu_context.h ++++ b/arch/x86/include/asm/mmu_context.h +@@ -139,6 +139,8 @@ static inline void mm_reset_untag_mask(struct mm_struct *mm) + #define enter_lazy_tlb enter_lazy_tlb + extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk); + ++extern void destroy_context_free_broadcast_asid(struct mm_struct *mm); ++ + /* + * Init a new mm. Used on mm copies, like at fork() + * and on mm's that are brand-new, like at execve(). +@@ -160,6 +162,13 @@ static inline int init_new_context(struct task_struct *tsk, + mm->context.execute_only_pkey = -1; + } + #endif ++ ++#ifdef CONFIG_CPU_SUP_AMD ++ INIT_LIST_HEAD(&mm->context.broadcast_asid_list); ++ mm->context.broadcast_asid = 0; ++ mm->context.asid_transition = false; ++#endif ++ + mm_reset_untag_mask(mm); + init_new_context_ldt(mm); + return 0; +@@ -169,6 +178,9 @@ static inline int init_new_context(struct task_struct *tsk, + static inline void destroy_context(struct mm_struct *mm) + { + destroy_context_ldt(mm); ++#ifdef CONFIG_CPU_SUP_AMD ++ destroy_context_free_broadcast_asid(mm); ++#endif + } + + extern void switch_mm(struct mm_struct *prev, struct mm_struct *next, +diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h +index d4eb9e1d61b8..794ba3647c6c 100644 +--- a/arch/x86/include/asm/paravirt.h ++++ b/arch/x86/include/asm/paravirt.h +@@ -91,11 +91,6 @@ static inline void __flush_tlb_multi(const struct cpumask *cpumask, + PVOP_VCALL2(mmu.flush_tlb_multi, cpumask, info); + } + +-static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) +-{ +- PVOP_VCALL2(mmu.tlb_remove_table, tlb, table); +-} +- + static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) + { + PVOP_VCALL1(mmu.exit_mmap, mm); +diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h +index 8d4fbe1be489..13405959e4db 100644 +--- a/arch/x86/include/asm/paravirt_types.h ++++ b/arch/x86/include/asm/paravirt_types.h +@@ -136,8 +136,6 @@ struct pv_mmu_ops { + void (*flush_tlb_multi)(const struct cpumask *cpus, + const struct flush_tlb_info *info); + +- void (*tlb_remove_table)(struct mmu_gather *tlb, void *table); +- + /* Hook for intercepting the destruction of an mm_struct. */ + void (*exit_mmap)(struct mm_struct *mm); + void (*notify_page_enc_status_changed)(unsigned long pfn, int npages, bool enc); +diff --git a/arch/x86/include/asm/tlbbatch.h b/arch/x86/include/asm/tlbbatch.h +index 1ad56eb3e8a8..f9a17edf63ad 100644 +--- a/arch/x86/include/asm/tlbbatch.h ++++ b/arch/x86/include/asm/tlbbatch.h +@@ -10,6 +10,7 @@ struct arch_tlbflush_unmap_batch { + * the PFNs being flushed.. + */ + struct cpumask cpumask; ++ bool used_invlpgb; + }; + + #endif /* _ARCH_X86_TLBBATCH_H */ +diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h +index 69e79fff41b8..a2f9b7370717 100644 +--- a/arch/x86/include/asm/tlbflush.h ++++ b/arch/x86/include/asm/tlbflush.h +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -64,6 +65,23 @@ static inline void cr4_clear_bits(unsigned long mask) + */ + #define TLB_NR_DYN_ASIDS 6 + ++#ifdef CONFIG_CPU_SUP_AMD ++#define is_dyn_asid(asid) (asid) < TLB_NR_DYN_ASIDS ++#define is_broadcast_asid(asid) (asid) >= TLB_NR_DYN_ASIDS ++#define in_asid_transition(info) (info->mm && info->mm->context.asid_transition) ++#define mm_broadcast_asid(mm) (mm->context.broadcast_asid) ++#else ++#define is_dyn_asid(asid) true ++#define is_broadcast_asid(asid) false ++#define in_asid_transition(info) false ++#define mm_broadcast_asid(mm) 0 ++ ++inline bool needs_broadcast_asid_reload(struct mm_struct *next, u16 prev_asid) ++{ ++ return false; ++} ++#endif ++ + struct tlb_context { + u64 ctx_id; + u64 tlb_gen; +@@ -182,6 +200,7 @@ static inline void cr4_init_shadow(void) + + extern unsigned long mmu_cr4_features; + extern u32 *trampoline_cr4_features; ++extern u16 invlpgb_count_max; + + extern void initialize_tlbstate_and_flush(void); + +@@ -277,21 +296,15 @@ static inline u64 inc_mm_tlb_gen(struct mm_struct *mm) + return atomic64_inc_return(&mm->context.tlb_gen); + } + +-static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, +- struct mm_struct *mm, +- unsigned long uaddr) +-{ +- inc_mm_tlb_gen(mm); +- cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); +- mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); +-} +- + static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm) + { + flush_tlb_mm(mm); + } + + extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); ++extern void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, ++ struct mm_struct *mm, ++ unsigned long uaddr); + + static inline bool pte_flags_need_flush(unsigned long oldflags, + unsigned long newflags, +diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c +index 79d2e17f6582..4dc42705aaca 100644 +--- a/arch/x86/kernel/cpu/amd.c ++++ b/arch/x86/kernel/cpu/amd.c +@@ -1135,6 +1135,22 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) + tlb_lli_2m[ENTRIES] = eax & mask; + + tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1; ++ ++ if (c->extended_cpuid_level < 0x80000008) ++ return; ++ ++ cpuid(0x80000008, &eax, &ebx, &ecx, &edx); ++ ++ /* Max number of pages INVLPGB can invalidate in one shot */ ++ invlpgb_count_max = (edx & 0xffff) + 1; ++ ++ /* If supported, enable translation cache extensions (TCE) */ ++ cpuid(0x80000001, &eax, &ebx, &ecx, &edx); ++ if (ecx & BIT(17)) { ++ u64 msr = native_read_msr(MSR_EFER);; ++ msr |= BIT(15); ++ wrmsrl(MSR_EFER, msr); ++ } + } + + static const struct cpu_dev amd_cpu_dev = { +diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c +index 21e9e4845354..83b7679658b1 100644 +--- a/arch/x86/kernel/kvm.c ++++ b/arch/x86/kernel/kvm.c +@@ -838,7 +838,6 @@ static void __init kvm_guest_init(void) + #ifdef CONFIG_SMP + if (pv_tlb_flush_supported()) { + pv_ops.mmu.flush_tlb_multi = kvm_flush_tlb_multi; +- pv_ops.mmu.tlb_remove_table = tlb_remove_table; + pr_info("KVM setup pv remote TLB flush\n"); + } + +diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c +index fec381533555..c019771e0123 100644 +--- a/arch/x86/kernel/paravirt.c ++++ b/arch/x86/kernel/paravirt.c +@@ -59,11 +59,6 @@ void __init native_pv_lock_init(void) + static_branch_enable(&virt_spin_lock_key); + } + +-static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) +-{ +- tlb_remove_page(tlb, table); +-} +- + struct static_key paravirt_steal_enabled; + struct static_key paravirt_steal_rq_enabled; + +@@ -191,7 +186,6 @@ struct paravirt_patch_template pv_ops = { + .mmu.flush_tlb_kernel = native_flush_tlb_global, + .mmu.flush_tlb_one_user = native_flush_tlb_one_user, + .mmu.flush_tlb_multi = native_flush_tlb_multi, +- .mmu.tlb_remove_table = native_tlb_remove_table, + + .mmu.exit_mmap = paravirt_nop, + .mmu.notify_page_enc_status_changed = paravirt_nop, +diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c +index f1fea506e20f..6c4d08f8f7b1 100644 +--- a/arch/x86/kernel/setup.c ++++ b/arch/x86/kernel/setup.c +@@ -138,6 +138,10 @@ __visible unsigned long mmu_cr4_features __ro_after_init; + __visible unsigned long mmu_cr4_features __ro_after_init = X86_CR4_PAE; + #endif + ++#ifdef CONFIG_CPU_SUP_AMD ++u16 invlpgb_count_max __ro_after_init; ++#endif ++ + #ifdef CONFIG_IMA + static phys_addr_t ima_kexec_buffer_phys; + static size_t ima_kexec_buffer_size; +diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c +index 5745a354a241..3dc4af1f7868 100644 +--- a/arch/x86/mm/pgtable.c ++++ b/arch/x86/mm/pgtable.c +@@ -18,14 +18,6 @@ EXPORT_SYMBOL(physical_mask); + #define PGTABLE_HIGHMEM 0 + #endif + +-#ifndef CONFIG_PARAVIRT +-static inline +-void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) +-{ +- tlb_remove_page(tlb, table); +-} +-#endif +- + gfp_t __userpte_alloc_gfp = GFP_PGTABLE_USER | PGTABLE_HIGHMEM; + + pgtable_t pte_alloc_one(struct mm_struct *mm) +@@ -54,7 +46,7 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) + { + pagetable_pte_dtor(page_ptdesc(pte)); + paravirt_release_pte(page_to_pfn(pte)); +- paravirt_tlb_remove_table(tlb, pte); ++ tlb_remove_table(tlb, pte); + } + + #if CONFIG_PGTABLE_LEVELS > 2 +@@ -70,7 +62,7 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) + tlb->need_flush_all = 1; + #endif + pagetable_pmd_dtor(ptdesc); +- paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc)); ++ tlb_remove_table(tlb, ptdesc_page(ptdesc)); + } + + #if CONFIG_PGTABLE_LEVELS > 3 +@@ -80,14 +72,14 @@ void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) + + pagetable_pud_dtor(ptdesc); + paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); +- paravirt_tlb_remove_table(tlb, virt_to_page(pud)); ++ tlb_remove_table(tlb, virt_to_page(pud)); + } + + #if CONFIG_PGTABLE_LEVELS > 4 + void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) + { + paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); +- paravirt_tlb_remove_table(tlb, virt_to_page(p4d)); ++ tlb_remove_table(tlb, virt_to_page(p4d)); + } + #endif /* CONFIG_PGTABLE_LEVELS > 4 */ + #endif /* CONFIG_PGTABLE_LEVELS > 3 */ +diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c +index b0678d59ebdb..0ea9d1c077f6 100644 +--- a/arch/x86/mm/tlb.c ++++ b/arch/x86/mm/tlb.c +@@ -74,13 +74,15 @@ + * use different names for each of them: + * + * ASID - [0, TLB_NR_DYN_ASIDS-1] +- * the canonical identifier for an mm ++ * the canonical identifier for an mm, dynamically allocated on each CPU ++ * [TLB_NR_DYN_ASIDS, MAX_ASID_AVAILABLE-1] ++ * the canonical, global identifier for an mm, identical across all CPUs + * +- * kPCID - [1, TLB_NR_DYN_ASIDS] ++ * kPCID - [1, MAX_ASID_AVAILABLE] + * the value we write into the PCID part of CR3; corresponds to the + * ASID+1, because PCID 0 is special. + * +- * uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS] ++ * uPCID - [2048 + 1, 2048 + MAX_ASID_AVAILABLE] + * for KPTI each mm has two address spaces and thus needs two + * PCID values, but we can still do with a single ASID denomination + * for each mm. Corresponds to kPCID + 2048. +@@ -225,6 +227,18 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, + return; + } + ++ /* ++ * TLB consistency for this ASID is maintained with INVLPGB; ++ * TLB flushes happen even while the process isn't running. ++ */ ++#ifdef CONFIG_CPU_SUP_AMD ++ if (static_cpu_has(X86_FEATURE_INVLPGB) && mm_broadcast_asid(next)) { ++ *new_asid = mm_broadcast_asid(next); ++ *need_flush = false; ++ return; ++ } ++#endif ++ + if (this_cpu_read(cpu_tlbstate.invalidate_other)) + clear_asid_other(); + +@@ -251,6 +265,245 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen, + *need_flush = true; + } + ++#ifdef CONFIG_CPU_SUP_AMD ++/* ++ * Logic for AMD INVLPGB support. ++ */ ++static DEFINE_RAW_SPINLOCK(broadcast_asid_lock); ++static u16 last_broadcast_asid = TLB_NR_DYN_ASIDS; ++static DECLARE_BITMAP(broadcast_asid_used, MAX_ASID_AVAILABLE) = { 0 }; ++static LIST_HEAD(broadcast_asid_list); ++static int broadcast_asid_available = MAX_ASID_AVAILABLE - TLB_NR_DYN_ASIDS - 1; ++ ++static void reset_broadcast_asid_space(void) ++{ ++ mm_context_t *context; ++ ++ lockdep_assert_held(&broadcast_asid_lock); ++ ++ /* ++ * Flush once when we wrap around the ASID space, so we won't need ++ * to flush every time we allocate an ASID for boradcast flushing. ++ */ ++ invlpgb_flush_all_nonglobals(); ++ tlbsync(); ++ ++ /* ++ * Leave the currently used broadcast ASIDs set in the bitmap, since ++ * those cannot be reused before the next wraparound and flush.. ++ */ ++ bitmap_clear(broadcast_asid_used, 0, MAX_ASID_AVAILABLE); ++ list_for_each_entry(context, &broadcast_asid_list, broadcast_asid_list) ++ __set_bit(context->broadcast_asid, broadcast_asid_used); ++ ++ last_broadcast_asid = TLB_NR_DYN_ASIDS; ++} ++ ++static u16 get_broadcast_asid(void) ++{ ++ lockdep_assert_held(&broadcast_asid_lock); ++ ++ do { ++ u16 start = last_broadcast_asid; ++ u16 asid = find_next_zero_bit(broadcast_asid_used, MAX_ASID_AVAILABLE, start); ++ ++ if (asid >= MAX_ASID_AVAILABLE) { ++ reset_broadcast_asid_space(); ++ continue; ++ } ++ ++ /* Try claiming this broadcast ASID. */ ++ if (!test_and_set_bit(asid, broadcast_asid_used)) { ++ last_broadcast_asid = asid; ++ return asid; ++ } ++ } while (1); ++} ++ ++/* ++ * Returns true if the mm is transitioning from a CPU-local ASID to a broadcast ++ * (INVLPGB) ASID, or the other way around. ++ */ ++static bool needs_broadcast_asid_reload(struct mm_struct *next, u16 prev_asid) ++{ ++ u16 broadcast_asid = mm_broadcast_asid(next); ++ ++ if (broadcast_asid && prev_asid != broadcast_asid) ++ return true; ++ ++ if (!broadcast_asid && is_broadcast_asid(prev_asid)) ++ return true; ++ ++ return false; ++} ++ ++void destroy_context_free_broadcast_asid(struct mm_struct *mm) ++{ ++ if (!mm->context.broadcast_asid) ++ return; ++ ++ guard(raw_spinlock_irqsave)(&broadcast_asid_lock); ++ mm->context.broadcast_asid = 0; ++ list_del(&mm->context.broadcast_asid_list); ++ broadcast_asid_available++; ++} ++ ++static bool mm_active_cpus_exceeds(struct mm_struct *mm, int threshold) ++{ ++ int count = 0; ++ int cpu; ++ ++ if (cpumask_weight(mm_cpumask(mm)) <= threshold) ++ return false; ++ ++ for_each_cpu(cpu, mm_cpumask(mm)) { ++ /* Skip the CPUs that aren't really running this process. */ ++ if (per_cpu(cpu_tlbstate.loaded_mm, cpu) != mm) ++ continue; ++ ++ if (per_cpu(cpu_tlbstate_shared.is_lazy, cpu)) ++ continue; ++ ++ if (++count > threshold) ++ return true; ++ } ++ return false; ++} ++ ++/* ++ * Assign a broadcast ASID to the current process, protecting against ++ * races between multiple threads in the process. ++ */ ++static void use_broadcast_asid(struct mm_struct *mm) ++{ ++ guard(raw_spinlock_irqsave)(&broadcast_asid_lock); ++ ++ /* This process is already using broadcast TLB invalidation. */ ++ if (mm->context.broadcast_asid) ++ return; ++ ++ mm->context.broadcast_asid = get_broadcast_asid(); ++ mm->context.asid_transition = true; ++ list_add(&mm->context.broadcast_asid_list, &broadcast_asid_list); ++ broadcast_asid_available--; ++} ++ ++/* ++ * Figure out whether to assign a broadcast (global) ASID to a process. ++ * We vary the threshold by how empty or full broadcast ASID space is. ++ * 1/4 full: >= 4 active threads ++ * 1/2 full: >= 8 active threads ++ * 3/4 full: >= 16 active threads ++ * 7/8 full: >= 32 active threads ++ * etc ++ * ++ * This way we should never exhaust the broadcast ASID space, even on very ++ * large systems, and the processes with the largest number of active ++ * threads should be able to use broadcast TLB invalidation. ++ */ ++#define HALFFULL_THRESHOLD 8 ++static bool meets_broadcast_asid_threshold(struct mm_struct *mm) ++{ ++ int avail = broadcast_asid_available; ++ int threshold = HALFFULL_THRESHOLD; ++ ++ if (!avail) ++ return false; ++ ++ if (avail > MAX_ASID_AVAILABLE * 3 / 4) { ++ threshold = HALFFULL_THRESHOLD / 4; ++ } else if (avail > MAX_ASID_AVAILABLE / 2) { ++ threshold = HALFFULL_THRESHOLD / 2; ++ } else if (avail < MAX_ASID_AVAILABLE / 3) { ++ do { ++ avail *= 2; ++ threshold *= 2; ++ } while ((avail + threshold) < MAX_ASID_AVAILABLE / 2); ++ } ++ ++ return mm_active_cpus_exceeds(mm, threshold); ++} ++ ++static void count_tlb_flush(struct mm_struct *mm) ++{ ++ if (!static_cpu_has(X86_FEATURE_INVLPGB)) ++ return; ++ ++ /* Check every once in a while. */ ++ if ((current->pid & 0x1f) != (jiffies & 0x1f)) ++ return; ++ ++ if (meets_broadcast_asid_threshold(mm)) ++ use_broadcast_asid(mm); ++} ++ ++static void finish_asid_transition(struct flush_tlb_info *info) ++{ ++ struct mm_struct *mm = info->mm; ++ int bc_asid = mm_broadcast_asid(mm); ++ int cpu; ++ ++ if (!mm->context.asid_transition) ++ return; ++ ++ for_each_cpu(cpu, mm_cpumask(mm)) { ++ if (READ_ONCE(per_cpu(cpu_tlbstate.loaded_mm, cpu)) != mm) ++ continue; ++ ++ /* ++ * If at least one CPU is not using the broadcast ASID yet, ++ * send a TLB flush IPI. The IPI should cause stragglers ++ * to transition soon. ++ */ ++ if (per_cpu(cpu_tlbstate.loaded_mm_asid, cpu) != bc_asid) { ++ flush_tlb_multi(mm_cpumask(info->mm), info); ++ return; ++ } ++ } ++ ++ /* All the CPUs running this process are using the broadcast ASID. */ ++ mm->context.asid_transition = 0; ++} ++ ++static void broadcast_tlb_flush(struct flush_tlb_info *info) ++{ ++ bool pmd = info->stride_shift == PMD_SHIFT; ++ unsigned long maxnr = invlpgb_count_max; ++ unsigned long asid = info->mm->context.broadcast_asid; ++ unsigned long addr = info->start; ++ unsigned long nr; ++ ++ /* Flushing multiple pages at once is not supported with 1GB pages. */ ++ if (info->stride_shift > PMD_SHIFT) ++ maxnr = 1; ++ ++ if (info->end == TLB_FLUSH_ALL || info->freed_tables) { ++ invlpgb_flush_single_pcid(kern_pcid(asid)); ++ /* Do any CPUs supporting INVLPGB need PTI? */ ++ if (static_cpu_has(X86_FEATURE_PTI)) ++ invlpgb_flush_single_pcid(user_pcid(asid)); ++ } else do { ++ /* ++ * Calculate how many pages can be flushed at once; if the ++ * remainder of the range is less than one page, flush one. ++ */ ++ nr = min(maxnr, (info->end - addr) >> info->stride_shift); ++ nr = max(nr, 1); ++ ++ invlpgb_flush_user_nr(kern_pcid(asid), addr, nr, pmd); ++ /* Do any CPUs supporting INVLPGB need PTI? */ ++ if (static_cpu_has(X86_FEATURE_PTI)) ++ invlpgb_flush_user_nr(user_pcid(asid), addr, nr, pmd); ++ addr += nr << info->stride_shift; ++ } while (addr < info->end); ++ ++ finish_asid_transition(info); ++ ++ /* Wait for the INVLPGBs kicked off above to finish. */ ++ tlbsync(); ++} ++#endif /* CONFIG_CPU_SUP_AMD */ ++ + /* + * Given an ASID, flush the corresponding user ASID. We can delay this + * until the next time we switch to it. +@@ -556,8 +809,9 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, + */ + if (prev == next) { + /* Not actually switching mm's */ +- VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) != +- next->context.ctx_id); ++ if (is_dyn_asid(prev_asid)) ++ VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) != ++ next->context.ctx_id); + + /* + * If this races with another thread that enables lam, 'new_lam' +@@ -573,6 +827,23 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, + !cpumask_test_cpu(cpu, mm_cpumask(next)))) + cpumask_set_cpu(cpu, mm_cpumask(next)); + ++ /* ++ * Check if the current mm is transitioning to a new ASID. ++ */ ++ if (needs_broadcast_asid_reload(next, prev_asid)) { ++ next_tlb_gen = atomic64_read(&next->context.tlb_gen); ++ ++ choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush); ++ goto reload_tlb; ++ } ++ ++ /* ++ * Broadcast TLB invalidation keeps this PCID up to date ++ * all the time. ++ */ ++ if (is_broadcast_asid(prev_asid)) ++ return; ++ + /* + * If the CPU is not in lazy TLB mode, we are just switching + * from one thread in a process to another thread in the same +@@ -629,8 +900,10 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, + barrier(); + } + ++reload_tlb: + new_lam = mm_lam_cr3_mask(next); + if (need_flush) { ++ VM_BUG_ON(is_broadcast_asid(new_asid)); + this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); + this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); + load_new_mm_cr3(next->pgd, new_asid, new_lam, true); +@@ -749,7 +1022,7 @@ static void flush_tlb_func(void *info) + const struct flush_tlb_info *f = info; + struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm); + u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); +- u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen); ++ u64 local_tlb_gen; + bool local = smp_processor_id() == f->initiating_cpu; + unsigned long nr_invalidate = 0; + u64 mm_tlb_gen; +@@ -769,6 +1042,16 @@ static void flush_tlb_func(void *info) + if (unlikely(loaded_mm == &init_mm)) + return; + ++ /* Reload the ASID if transitioning into or out of a broadcast ASID */ ++ if (needs_broadcast_asid_reload(loaded_mm, loaded_mm_asid)) { ++ switch_mm_irqs_off(NULL, loaded_mm, NULL); ++ loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); ++ } ++ ++ /* Broadcast ASIDs are always kept up to date with INVLPGB. */ ++ if (is_broadcast_asid(loaded_mm_asid)) ++ return; ++ + VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].ctx_id) != + loaded_mm->context.ctx_id); + +@@ -786,6 +1069,8 @@ static void flush_tlb_func(void *info) + return; + } + ++ local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen); ++ + if (unlikely(f->new_tlb_gen != TLB_GENERATION_INVALID && + f->new_tlb_gen <= local_tlb_gen)) { + /* +@@ -825,7 +1110,7 @@ static void flush_tlb_func(void *info) + * + * The only question is whether to do a full or partial flush. + * +- * We do a partial flush if requested and two extra conditions ++ * We do a partial flush if requested and three extra conditions + * are met: + * + * 1. f->new_tlb_gen == local_tlb_gen + 1. We have an invariant that +@@ -852,10 +1137,14 @@ static void flush_tlb_func(void *info) + * date. By doing a full flush instead, we can increase + * local_tlb_gen all the way to mm_tlb_gen and we can probably + * avoid another flush in the very near future. ++ * ++ * 3. No page tables were freed. If page tables were freed, a full ++ * flush ensures intermediate translations in the TLB get flushed. + */ + if (f->end != TLB_FLUSH_ALL && + f->new_tlb_gen == local_tlb_gen + 1 && +- f->new_tlb_gen == mm_tlb_gen) { ++ f->new_tlb_gen == mm_tlb_gen && ++ !f->freed_tables) { + /* Partial flush */ + unsigned long addr = f->start; + +@@ -926,7 +1215,7 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask, + * up on the new contents of what used to be page tables, while + * doing a speculative memory access. + */ +- if (info->freed_tables) ++ if (info->freed_tables || in_asid_transition(info)) + on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true); + else + on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func, +@@ -998,14 +1287,18 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + bool freed_tables) + { + struct flush_tlb_info *info; ++ unsigned long threshold = tlb_single_page_flush_ceiling; + u64 new_tlb_gen; + int cpu; + ++ if (static_cpu_has(X86_FEATURE_INVLPGB)) ++ threshold *= invlpgb_count_max; ++ + cpu = get_cpu(); + + /* Should we flush just the requested range? */ + if ((end == TLB_FLUSH_ALL) || +- ((end - start) >> stride_shift) > tlb_single_page_flush_ceiling) { ++ ((end - start) >> stride_shift) > threshold) { + start = 0; + end = TLB_FLUSH_ALL; + } +@@ -1021,8 +1314,11 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + * a local TLB flush is needed. Optimize this use-case by calling + * flush_tlb_func_local() directly in this case. + */ +- if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { ++ if (IS_ENABLED(CONFIG_CPU_SUP_AMD) && mm_broadcast_asid(mm)) { ++ broadcast_tlb_flush(info); ++ } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { + flush_tlb_multi(mm_cpumask(mm), info); ++ count_tlb_flush(mm); + } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { + lockdep_assert_irqs_enabled(); + local_irq_disable(); +@@ -1045,9 +1341,41 @@ static void do_flush_tlb_all(void *info) + void flush_tlb_all(void) + { + count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); ++ if (cpu_feature_enabled(X86_FEATURE_INVLPGB)) { ++ guard(preempt)(); ++ invlpgb_flush_all(); ++ tlbsync(); ++ return; ++ } + on_each_cpu(do_flush_tlb_all, NULL, 1); + } + ++static void broadcast_kernel_range_flush(unsigned long start, unsigned long end) ++{ ++ unsigned long addr; ++ unsigned long maxnr = invlpgb_count_max; ++ unsigned long threshold = tlb_single_page_flush_ceiling * maxnr; ++ ++ /* ++ * TLBSYNC only waits for flushes originating on the same CPU. ++ * Disabling migration allows us to wait on all flushes. ++ */ ++ guard(preempt)(); ++ ++ if (end == TLB_FLUSH_ALL || ++ (end - start) > threshold << PAGE_SHIFT) { ++ invlpgb_flush_all(); ++ } else { ++ unsigned long nr; ++ for (addr = start; addr < end; addr += nr << PAGE_SHIFT) { ++ nr = min((end - addr) >> PAGE_SHIFT, maxnr); ++ invlpgb_flush_addr(addr, nr); ++ } ++ } ++ ++ tlbsync(); ++} ++ + static void do_kernel_range_flush(void *info) + { + struct flush_tlb_info *f = info; +@@ -1060,6 +1388,11 @@ static void do_kernel_range_flush(void *info) + + void flush_tlb_kernel_range(unsigned long start, unsigned long end) + { ++ if (cpu_feature_enabled(X86_FEATURE_INVLPGB)) { ++ broadcast_kernel_range_flush(start, end); ++ return; ++ } ++ + /* Balance as user space task's flush, a bit conservative */ + if (end == TLB_FLUSH_ALL || + (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) { +@@ -1244,7 +1577,6 @@ EXPORT_SYMBOL_GPL(__flush_tlb_all); + void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) + { + struct flush_tlb_info *info; +- + int cpu = get_cpu(); + + info = get_flush_tlb_info(NULL, 0, TLB_FLUSH_ALL, 0, false, +@@ -1263,12 +1595,49 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) + local_irq_enable(); + } + ++ /* ++ * If we issued (asynchronous) INVLPGB flushes, wait for them here. ++ * The cpumask above contains only CPUs that were running tasks ++ * not using broadcast TLB flushing. ++ */ ++ if (cpu_feature_enabled(X86_FEATURE_INVLPGB) && batch->used_invlpgb) { ++ tlbsync(); ++ migrate_enable(); ++ batch->used_invlpgb = false; ++ } ++ + cpumask_clear(&batch->cpumask); + + put_flush_tlb_info(); + put_cpu(); + } + ++void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, ++ struct mm_struct *mm, ++ unsigned long uaddr) ++{ ++ if (static_cpu_has(X86_FEATURE_INVLPGB) && mm_broadcast_asid(mm)) { ++ u16 asid = mm_broadcast_asid(mm); ++ /* ++ * Queue up an asynchronous invalidation. The corresponding ++ * TLBSYNC is done in arch_tlbbatch_flush(), and must be done ++ * on the same CPU. ++ */ ++ if (!batch->used_invlpgb) { ++ batch->used_invlpgb = true; ++ migrate_disable(); ++ } ++ invlpgb_flush_user_nr(kern_pcid(asid), uaddr, 1, 0); ++ /* Do any CPUs supporting INVLPGB need PTI? */ ++ if (static_cpu_has(X86_FEATURE_PTI)) ++ invlpgb_flush_user_nr(user_pcid(asid), uaddr, 1, 0); ++ } else { ++ inc_mm_tlb_gen(mm); ++ cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); ++ } ++ mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); ++} ++ + /* + * Blindly accessing user memory from NMI context can be dangerous + * if we're in the middle of switching the current user task or +diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c +index 55a4996d0c04..041e17282af0 100644 +--- a/arch/x86/xen/mmu_pv.c ++++ b/arch/x86/xen/mmu_pv.c +@@ -2137,7 +2137,6 @@ static const typeof(pv_ops) xen_mmu_ops __initconst = { + .flush_tlb_kernel = xen_flush_tlb, + .flush_tlb_one_user = xen_flush_tlb_one_user, + .flush_tlb_multi = xen_flush_tlb_multi, +- .tlb_remove_table = tlb_remove_table, + + .pgd_alloc = xen_pgd_alloc, + .pgd_free = xen_pgd_free, +diff --git a/mm/memory.c b/mm/memory.c +index d322ddfe6791..d9ecd1ad789f 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -1921,7 +1921,6 @@ void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, + struct mmu_notifier_range range; + struct mmu_gather tlb; + +- lru_add_drain(); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, + address, end); + hugetlb_zap_begin(vma, &range.start, &range.end); +diff --git a/mm/mmap.c b/mm/mmap.c +index 7fb4c1e97175..8dad3bfb7b9b 100644 +--- a/mm/mmap.c ++++ b/mm/mmap.c +@@ -1927,7 +1927,6 @@ void exit_mmap(struct mm_struct *mm) + goto destroy; + } + +- lru_add_drain(); + flush_cache_mm(mm); + tlb_gather_mmu_fullmm(&tlb, mm); + /* update_hiwater_rss(mm) here? but nobody should be looking */ +@@ -2370,7 +2369,6 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) + vma, new_start, length, false, true)) + return -ENOMEM; + +- lru_add_drain(); + tlb_gather_mmu(&tlb, mm); + next = vma_next(&vmi); + if (new_end > old_start) { +diff --git a/mm/swap_state.c b/mm/swap_state.c +index 4669f29cf555..ffbdfc8a46ef 100644 +--- a/mm/swap_state.c ++++ b/mm/swap_state.c +@@ -317,7 +317,6 @@ void free_pages_and_swap_cache(struct encoded_page **pages, int nr) + struct folio_batch folios; + unsigned int refs[PAGEVEC_SIZE]; + +- lru_add_drain(); + folio_batch_init(&folios); + for (int i = 0; i < nr; i++) { + struct folio *folio = page_folio(encoded_page_ptr(pages[i])); +diff --git a/mm/vma.c b/mm/vma.c +index 7621384d64cf..c7461e21ef70 100644 +--- a/mm/vma.c ++++ b/mm/vma.c +@@ -347,7 +347,6 @@ void unmap_region(struct ma_state *mas, struct vm_area_struct *vma, + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather tlb; + +- lru_add_drain(); + tlb_gather_mmu(&tlb, mm); + update_hiwater_rss(mm); + unmap_vmas(&tlb, mas, vma, vma->vm_start, vma->vm_end, vma->vm_end, +@@ -1089,7 +1088,6 @@ static inline void vms_clear_ptes(struct vma_munmap_struct *vms, + * were isolated before we downgraded mmap_lock. + */ + mas_set(mas_detach, 1); +- lru_add_drain(); + tlb_gather_mmu(&tlb, vms->vma->vm_mm); + update_hiwater_rss(vms->vma->vm_mm); + unmap_vmas(&tlb, mas_detach, vms->vma, vms->start, vms->end, +-- +2.48.0.rc1 + +From 56cd1a9140f7307c0e7122fbd3402af8623c9ef5 Mon Sep 17 00:00:00 2001 +From: Peter Jung +Date: Thu, 2 Jan 2025 12:33:42 +0100 +Subject: [PATCH 04/13] autofdo Signed-off-by: Peter Jung --- @@ -1590,7 +2679,7 @@ index a578178468f1..a2d251917629 100644 M: Petr Mladek R: Steven Rostedt diff --git a/Makefile b/Makefile -index 685a57f6c8d2..9eb33adc33e7 100644 +index 8a10105c2539..7eb3671440fc 100644 --- a/Makefile +++ b/Makefile @@ -1019,6 +1019,8 @@ include-$(CONFIG_KMSAN) += scripts/Makefile.kmsan @@ -1669,7 +2758,7 @@ index d317a843f7ea..f1b86eb30340 100644 SCHED_TEXT LOCK_TEXT diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig -index 171be04eca1f..f691eafefc42 100644 +index 76f9e6d11872..512b148b011a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -126,6 +126,8 @@ config X86 @@ -1918,12 +3007,12 @@ index 3d27983dc908..6f64d611faea 100644 return -1; /* until end of address space */ -- -2.47.1 +2.48.0.rc1 -From 0914fb7e5b7a07ab07317160b090c1e8bbdd5ca9 Mon Sep 17 00:00:00 2001 +From e72d469d61a4320a06e56466efad4158b13f5e69 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:50:32 +0100 -Subject: [PATCH 04/12] bbr3 +Date: Thu, 2 Jan 2025 12:33:53 +0100 +Subject: [PATCH 05/13] bbr3 Signed-off-by: Peter Jung --- @@ -5304,12 +6393,12 @@ index 79064580c8c0..697270ce1ea6 100644 event = icsk->icsk_pending; -- -2.47.1 +2.48.0.rc1 -From e344a34773ac1f24697490d3d5790f10f3f81c5c Mon Sep 17 00:00:00 2001 +From bfc08381752914883a9b653d57e9abc41565e7bf Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:50:43 +0100 -Subject: [PATCH 05/12] cachy +Date: Thu, 2 Jan 2025 12:34:03 +0100 +Subject: [PATCH 06/13] cachy Signed-off-by: Peter Jung --- @@ -5538,7 +6627,7 @@ index 6d942b5c58f0..1768a106aab1 100644 ====== diff --git a/Makefile b/Makefile -index 9eb33adc33e7..802baad237e2 100644 +index 7eb3671440fc..3f7d6b08f74d 100644 --- a/Makefile +++ b/Makefile @@ -802,11 +802,19 @@ KBUILD_CFLAGS += -fno-delete-null-pointer-checks @@ -12891,7 +13980,7 @@ index 38ef6d06888e..0f78364efd4f 100644 config SCHED_HRTICK diff --git a/kernel/fork.c b/kernel/fork.c -index ce8be55e5e04..e97e527cec69 100644 +index e192bdbc9ade..d27b8f5582df 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -107,6 +107,10 @@ @@ -12905,7 +13994,7 @@ index ce8be55e5e04..e97e527cec69 100644 #include #include #include -@@ -2158,6 +2162,10 @@ __latent_entropy struct task_struct *copy_process( +@@ -2157,6 +2161,10 @@ __latent_entropy struct task_struct *copy_process( if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) return ERR_PTR(-EINVAL); @@ -12916,7 +14005,7 @@ index ce8be55e5e04..e97e527cec69 100644 /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. -@@ -3311,6 +3319,12 @@ int ksys_unshare(unsigned long unshare_flags) +@@ -3310,6 +3318,12 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; @@ -13568,12 +14657,12 @@ index fe7947f77406..99e138cfdd95 100644 release_sock(sk); if (reqsk_queue_empty(&icsk->icsk_accept_queue)) -- -2.47.1 +2.48.0.rc1 -From 7e83775ac56854885575c57878201bb8a61a1377 Mon Sep 17 00:00:00 2001 +From aba6b6426a05c3f2c08c1059bf13d7bd5b653dbe Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:51:01 +0100 -Subject: [PATCH 06/12] crypto +Date: Thu, 2 Jan 2025 12:34:15 +0100 +Subject: [PATCH 07/13] crypto Signed-off-by: Peter Jung --- @@ -15174,12 +16263,12 @@ index bbcff1fb78cb..752812bc4991 100644 ## PCLMULQDQ tables ## Table is 128 entries x 2 words (8 bytes) each -- -2.47.1 +2.48.0.rc1 -From c6aea0c46173f8802aba928df945ef92dcc20b64 Mon Sep 17 00:00:00 2001 +From 12373a066ea908fb7d9d9e335e328239bf32ea79 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:51:12 +0100 -Subject: [PATCH 07/12] fixes +Date: Thu, 2 Jan 2025 12:35:55 +0100 +Subject: [PATCH 08/13] fixes Signed-off-by: Peter Jung --- @@ -15188,7 +16277,7 @@ Signed-off-by: Peter Jung arch/x86/mm/tlb.c | 2 +- drivers/bluetooth/btmtk.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + - drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 50 +++++++++++++++-- + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 51 ++++++++++++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 6 ++- drivers/gpu/drm/drm_edid.c | 47 ++++++++++++++-- drivers/hid/hid-ids.h | 1 + @@ -15204,7 +16293,7 @@ Signed-off-by: Peter Jung kernel/workqueue.c | 22 ++++++-- scripts/package/PKGBUILD | 5 ++ sound/pci/hda/patch_realtek.c | 2 + - 21 files changed, 256 insertions(+), 119 deletions(-) + 21 files changed, 257 insertions(+), 119 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 00551f340dbe..833b2344ce79 100644 @@ -15255,10 +16344,10 @@ index 99d345b686fa..6e2458088800 100644 asm volatile("\n" "1:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c -index b0678d59ebdb..a2becb85bea7 100644 +index 0ea9d1c077f6..0080175153ef 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c -@@ -569,7 +569,7 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, +@@ -823,7 +823,7 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next, * mm_cpumask. The TLB shootdown code can figure out from * cpu_tlbstate_shared.is_lazy whether or not to send an IPI. */ @@ -15302,7 +16391,7 @@ index 7617963901fa..03933b2c5ebc 100644 struct debugfs_blob_wrapper debugfs_vbios_blob; struct debugfs_blob_wrapper debugfs_discovery_blob; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c -index 51904906545e..d5d3391cc788 100644 +index 51904906545e..2ba12047a95f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -145,6 +145,8 @@ const char *amdgpu_asic_name[] = { @@ -15314,7 +16403,15 @@ index 51904906545e..d5d3391cc788 100644 /** * DOC: pcie_replay_count -@@ -4507,6 +4509,11 @@ int amdgpu_device_init(struct amdgpu_device *adev, +@@ -3723,6 +3725,7 @@ static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev) + r = adev->ip_blocks[i].version->funcs->resume(adev); + if (r) + return r; ++ adev->ip_blocks[i].status.hw = true; + } + } + +@@ -4507,6 +4510,11 @@ int amdgpu_device_init(struct amdgpu_device *adev, amdgpu_device_check_iommu_direct_map(adev); @@ -15326,7 +16423,7 @@ index 51904906545e..d5d3391cc788 100644 return 0; release_ras_con: -@@ -4571,6 +4578,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) +@@ -4571,6 +4579,8 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) drain_workqueue(adev->mman.bdev.wq); adev->shutdown = true; @@ -15335,7 +16432,7 @@ index 51904906545e..d5d3391cc788 100644 /* make sure IB test finished before entering exclusive mode * to avoid preemption on IB test */ -@@ -4688,8 +4697,8 @@ static int amdgpu_device_evict_resources(struct amdgpu_device *adev) +@@ -4688,8 +4698,8 @@ static int amdgpu_device_evict_resources(struct amdgpu_device *adev) { int ret; @@ -15346,7 +16443,7 @@ index 51904906545e..d5d3391cc788 100644 return 0; ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); -@@ -4701,6 +4710,41 @@ static int amdgpu_device_evict_resources(struct amdgpu_device *adev) +@@ -4701,6 +4711,41 @@ static int amdgpu_device_evict_resources(struct amdgpu_device *adev) /* * Suspend & resume. */ @@ -15388,7 +16485,7 @@ index 51904906545e..d5d3391cc788 100644 /** * amdgpu_device_prepare - prepare for device suspend * -@@ -4740,7 +4784,7 @@ int amdgpu_device_prepare(struct drm_device *dev) +@@ -4740,7 +4785,7 @@ int amdgpu_device_prepare(struct drm_device *dev) return 0; unprepare: @@ -16129,37 +17226,12 @@ index 192fc75b51e6..d88fc0ca893d 100644 SND_PCI_QUIRK(0x1043, 0x1eb3, "ASUS Ally RCLA72", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x1043, 0x1ed3, "ASUS HN7306W", ALC287_FIXUP_CS35L41_I2C_2), -- -2.47.1 +2.48.0.rc1 - -From 50e5354d1e4a5ce9d71bf956e78ab0167b46ae0c Mon Sep 17 00:00:00 2001 -From: "Jan Alexander Steffens (heftig)" -Date: Fri, 27 Dec 2024 15:08:09 +0100 -Subject: [PATCH] drm/amdgpu: Add missing statement in resume_phase3 - -Fixes: 73dae652dcac776296890da215ee7dec357a1032 -See: https://gitlab.freedesktop.org/drm/amd/-/issues/3853#note_2714815 -For: https://gitlab.archlinux.org/archlinux/packaging/packages/linux/-/issues/101 ---- - drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c -index 51904906545e59..ad4cd84e40f28d 100644 ---- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c -+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c -@@ -3723,6 +3723,7 @@ static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev) - r = adev->ip_blocks[i].version->funcs->resume(adev); - if (r) - return r; -+ adev->ip_blocks[i].status.hw = true; - } - } - -From f520a57069c8c9dab1231a6254a1af4e05a4bebc Mon Sep 17 00:00:00 2001 +From 012daedc3acd973c050ae9497e892f884759d0c1 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:51:57 +0100 -Subject: [PATCH 08/12] ntsync +Date: Thu, 2 Jan 2025 12:36:13 +0100 +Subject: [PATCH 09/13] ntsync Signed-off-by: Peter Jung --- @@ -19184,12 +20256,12 @@ index 000000000000..3aad311574c4 + +TEST_HARNESS_MAIN -- -2.47.1 +2.48.0.rc1 -From 0cc2e53ebfcfab4ce050b958795508582d11c4cb Mon Sep 17 00:00:00 2001 +From eedb315d53e038424a8d1a87683aede04a00a1c6 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:52:26 +0100 -Subject: [PATCH 09/12] perf-per-core +Date: Thu, 2 Jan 2025 12:36:25 +0100 +Subject: [PATCH 10/13] perf-per-core Signed-off-by: Peter Jung --- @@ -20181,12 +21253,12 @@ index 2361ed4d2b15..37a9afffb59e 100644 CPUHP_AP_PERF_S390_SF_ONLINE, CPUHP_AP_PERF_ARM_CCI_ONLINE, -- -2.47.1 +2.48.0.rc1 -From 4a1c243cf572b60fce8e804577a69e308942a433 Mon Sep 17 00:00:00 2001 +From 04af767f7fead4b41ab46b1a60abb813a78b4734 Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:52:41 +0100 -Subject: [PATCH 10/12] pksm +Date: Thu, 2 Jan 2025 12:36:37 +0100 +Subject: [PATCH 11/13] pksm Signed-off-by: Peter Jung --- @@ -20614,12 +21686,12 @@ index 01071182763e..7394bad8178e 100644 +464 common process_ksm_disable sys_process_ksm_disable sys_process_ksm_disable +465 common process_ksm_status sys_process_ksm_status sys_process_ksm_status -- -2.47.1 +2.48.0.rc1 -From 379c5b54fcac5fca5eefd8fe5f3f5d3342b257cf Mon Sep 17 00:00:00 2001 +From a95cb1707ef01550e8564667856d28f1bd1e23cb Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:53:09 +0100 -Subject: [PATCH 11/12] t2 +Date: Thu, 2 Jan 2025 12:36:48 +0100 +Subject: [PATCH 12/13] t2 Signed-off-by: Peter Jung --- @@ -30863,12 +31935,12 @@ index b03d526e4c45..66d09cbec5a8 100755 last; } -- -2.47.1 +2.48.0.rc1 -From ffe6e261d3ee197a08c96468e5f55eab115ae7b9 Mon Sep 17 00:00:00 2001 +From 0bfd166c7a3a5bda18934375b06d80c63fea950c Mon Sep 17 00:00:00 2001 From: Peter Jung -Date: Thu, 19 Dec 2024 18:53:34 +0100 -Subject: [PATCH 12/12] zstd +Date: Thu, 2 Jan 2025 12:36:58 +0100 +Subject: [PATCH 13/13] zstd Signed-off-by: Peter Jung --- @@ -49515,4 +50587,4 @@ index 469fc3059be0..0ae819f0c927 100644 EXPORT_SYMBOL(zstd_reset_dstream); -- -2.47.1 +2.48.0.rc1