diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6a7d7bf4b35a77f8247b3d67ca5d0e9c185d0a08..b3eb384c4cd828016ce78ba9ed7d01f7e23e11ea 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -175,6 +175,7 @@ config ARM64 select HAVE_KRETPROBES select HAVE_GENERIC_VDSO select IOMMU_DMA if IOMMU_SUPPORT + select HOTPLUG_SMT if (SMP && HOTPLUG_CPU) select IRQ_DOMAIN select IRQ_FORCED_THREADING select MODULES_USE_ELF_RELA diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 4ecf361cb562acfaa2d527cf488db563d943f43d..b3a4e1d01ae891cc26e98735b499f21b26060b32 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -42,11 +43,16 @@ static bool __init acpi_cpu_is_threaded(int cpu) */ int __init parse_acpi_topology(void) { + int thread_num, max_smt_thread_num = 1; + struct xarray core_threads; int cpu, topology_id; + void *entry; if (acpi_disabled) return 0; + xa_init(&core_threads); + for_each_possible_cpu(cpu) { int i, cache_id; @@ -58,6 +64,20 @@ int __init parse_acpi_topology(void) cpu_topology[cpu].thread_id = topology_id; topology_id = find_acpi_cpu_topology(cpu, 1); cpu_topology[cpu].core_id = topology_id; + + entry = xa_load(&core_threads, topology_id); + if (!entry) { + xa_store(&core_threads, topology_id, + xa_mk_value(1), GFP_KERNEL); + } else { + thread_num = xa_to_value(entry); + thread_num++; + xa_store(&core_threads, topology_id, + xa_mk_value(thread_num), GFP_KERNEL); + + if (thread_num > max_smt_thread_num) + max_smt_thread_num = thread_num; + } } else { cpu_topology[cpu].thread_id = -1; cpu_topology[cpu].core_id = topology_id; @@ -80,6 +100,9 @@ int __init parse_acpi_topology(void) } } + topology_smt_set_num_threads(max_smt_thread_num); + + xa_destroy(&core_threads); return 0; } #endif diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index f3459df117aa8bdd2c051949966dca511d69c0f2..a6e38a29e95a9f44bddd49e37c9556a16fdbe074 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -132,6 +132,8 @@ static inline int topology_max_smt_threads(void) return __max_smt_threads; } +#include + int topology_update_package_map(unsigned int apicid, unsigned int cpu); int topology_update_die_map(unsigned int dieid, unsigned int cpu); int topology_phys_to_logical_pkg(unsigned int pkg); diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 1e4466ae95a41bfe6e85cca96180bc8fa49cfc20..71be1bf011be766c4e975e74f98f3e9feb98e6a9 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -123,7 +123,7 @@ void __init check_bugs(void) * identify_boot_cpu() initialized SMT support information, let the * core code know. */ - cpu_smt_check_topology(); + cpu_smt_set_num_threads(smp_num_siblings, smp_num_siblings); if (!IS_ENABLED(CONFIG_SMP)) { pr_info("CPU: "); diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 80343cf0f9ad951c93dddfba74428928f23f6ebd..cd0f09443f97001e0508335218e49877b9c30b4d 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -305,6 +305,13 @@ static int __init parse_core(struct device_node *core, int package_id, i++; } while (t); + /* + * We've already gotten threads number in this core, update the SMT + * threads number when necessary. + */ + if (i > topology_smt_get_num_threads()) + topology_smt_set_num_threads(i); + cpu = get_cpu_for_node(core); if (cpu >= 0) { if (!leaf) { @@ -468,6 +475,36 @@ const struct cpumask *cpu_clustergroup_mask(int cpu) return &cpu_topology[cpu].cluster_sibling; } +#ifdef CONFIG_HOTPLUG_SMT + +/* Maximum threads number per-Core */ +static unsigned int topology_smt_num_threads = 1; + +void __init topology_smt_set_num_threads(unsigned int num_threads) +{ + topology_smt_num_threads = num_threads; +} + +unsigned int __init topology_smt_get_num_threads(void) +{ + return topology_smt_num_threads; +} + +/* + * On SMT Hotplug the primary thread of the SMT won't be disabled. For x86 they + * seem to have a primary thread for special purpose. For other arthitectures + * like arm64 there's no such restriction for a primary thread, so make the + * first thread in the SMT as the primary thread. + */ +bool topology_is_primary_thread(unsigned int cpu) +{ + if (cpu == cpumask_first(topology_sibling_cpumask(cpu))) + return true; + + return false; +} +#endif + void update_siblings_masks(unsigned int cpuid) { struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; @@ -568,6 +605,16 @@ void __init init_cpu_topology(void) reset_cpu_topology(); else if (of_have_populated_dt() && parse_dt_topology()) reset_cpu_topology(); + + /* + * By this stage we get to know whether we support SMT or not, update + * the information for the core. We don't support + * CONFIG_SMT_NUM_THREADS_DYNAMIC so make the + * max_threads == num_threads. + */ + cpu_smt_set_num_threads(topology_smt_get_num_threads(), + topology_smt_get_num_threads()); + } void store_cpu_topology(unsigned int cpuid) diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 9d7a80debe064a006f55e33cac3282728eb3d0d2..d4bd67d01d0f12802d7380d4ab8fb3699f240414 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -62,6 +62,23 @@ const struct cpumask *cpu_clustergroup_mask(int cpu); void update_siblings_masks(unsigned int cpu); void remove_cpu_topology(unsigned int cpuid); void reset_cpu_topology(void); + +#ifdef CONFIG_HOTPLUG_SMT +bool topology_is_primary_thread(unsigned int cpu); +void topology_smt_set_num_threads(unsigned int num_threads); +unsigned int topology_smt_get_num_threads(void); +#else +static inline bool topology_is_primary_thread(unsigned int cpu) +{ + return false; +} +static inline void topology_smt_set_num_threads(unsigned int num_threads) { } +static inline unsigned int topology_smt_get_num_threads(void) +{ + return 1; +} +#endif + #endif #endif /* _LINUX_ARCH_TOPOLOGY_H_ */ diff --git a/include/linux/cpu.h b/include/linux/cpu.h index b42e9c4134475af58a1c30d9e9bb7e32a2cf123c..509d277c1cc97547c8d0f8b4a416515fa54a2842 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -18,6 +18,7 @@ #include #include #include +#include struct device; struct device_node; @@ -206,30 +207,6 @@ void cpuhp_report_idle_dead(void); static inline void cpuhp_report_idle_dead(void) { } #endif /* #ifdef CONFIG_HOTPLUG_CPU */ -enum cpuhp_smt_control { - CPU_SMT_ENABLED, - CPU_SMT_DISABLED, - CPU_SMT_FORCE_DISABLED, - CPU_SMT_NOT_SUPPORTED, - CPU_SMT_NOT_IMPLEMENTED, -}; - -#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) -extern enum cpuhp_smt_control cpu_smt_control; -extern void cpu_smt_disable(bool force); -extern void cpu_smt_check_topology(void); -extern bool cpu_smt_possible(void); -extern int cpuhp_smt_enable(void); -extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); -#else -# define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) -static inline void cpu_smt_disable(bool force) { } -static inline void cpu_smt_check_topology(void) { } -static inline bool cpu_smt_possible(void) { return false; } -static inline int cpuhp_smt_enable(void) { return 0; } -static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } -#endif - extern bool cpu_mitigations_off(void); extern bool cpu_mitigations_auto_nosmt(void); diff --git a/include/linux/cpu_smt.h b/include/linux/cpu_smt.h new file mode 100644 index 0000000000000000000000000000000000000000..0c1664294b579529fbd9690abdfad4e64d43aa9d --- /dev/null +++ b/include/linux/cpu_smt.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CPU_SMT_H_ +#define _LINUX_CPU_SMT_H_ + +enum cpuhp_smt_control { + CPU_SMT_ENABLED, + CPU_SMT_DISABLED, + CPU_SMT_FORCE_DISABLED, + CPU_SMT_NOT_SUPPORTED, + CPU_SMT_NOT_IMPLEMENTED, +}; + +#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) +extern enum cpuhp_smt_control cpu_smt_control; +extern unsigned int cpu_smt_num_threads; +extern void cpu_smt_disable(bool force); +extern void cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads); +extern bool cpu_smt_possible(void); +extern int cpuhp_smt_enable(void); +extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); +#else +# define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) +# define cpu_smt_num_threads 1 +static inline void cpu_smt_disable(bool force) { } +static inline void cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads) { } +static inline bool cpu_smt_possible(void) { return false; } +static inline int cpuhp_smt_enable(void) { return 0; } +static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } +#endif + +#endif /* _LINUX_CPU_SMT_H_ */ diff --git a/kernel/cpu.c b/kernel/cpu.c index c08456af0c7fe31719a57f83e3d3d773cdf77f89..173f69d7b88e634457b0d1e11b04f55708e9c331 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -391,7 +391,10 @@ static void lockdep_release_cpus_lock(void) void __weak arch_smt_update(void) { } #ifdef CONFIG_HOTPLUG_SMT + enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; +static unsigned int cpu_smt_max_threads __ro_after_init; +unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX; void __init cpu_smt_disable(bool force) { @@ -405,16 +408,33 @@ void __init cpu_smt_disable(bool force) pr_info("SMT: disabled\n"); cpu_smt_control = CPU_SMT_DISABLED; } + cpu_smt_num_threads = 1; } /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code. */ -void __init cpu_smt_check_topology(void) +void __init cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads) { - if (!topology_smt_supported()) + WARN_ON(!num_threads || (num_threads > max_threads)); + + if (max_threads == 1) cpu_smt_control = CPU_SMT_NOT_SUPPORTED; + + cpu_smt_max_threads = max_threads; + + /* + * If SMT has been disabled via the kernel command line or SMT is + * not supported, set cpu_smt_num_threads to 1 for consistency. + * If enabled, take the architecture requested number of threads + * to bring up into account. + */ + if (cpu_smt_control != CPU_SMT_ENABLED) + cpu_smt_num_threads = 1; + else if (num_threads < cpu_smt_num_threads) + cpu_smt_num_threads = num_threads; } static int __init smt_cmdline_disable(char *str)