From 27872b081683aa8a1512293036b1e807ac13ba8e Mon Sep 17 00:00:00 2001 From: "Ben Dooks (Codethink)" Date: Thu, 17 Oct 2019 12:33:41 +0100 Subject: [PATCH 01/94] irqchip/gic-v3: Fix __iomem warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The __iomem attribute should go before the * in the prototype. Move to silence the following sparse warnings: ./arch/arm/include/asm/arch_gicv3.h:340:15: warning: incorrect type in argument 1 (different address spaces) ./arch/arm/include/asm/arch_gicv3.h:340:15: expected void const volatile [noderef] *addr ./arch/arm/include/asm/arch_gicv3.h:340:15: got void * ./arch/arm/include/asm/arch_gicv3.h:343:17: warning: incorrect type in argument 2 (different address spaces) ./arch/arm/include/asm/arch_gicv3.h:343:17: expected void volatile [noderef] *addr ./arch/arm/include/asm/arch_gicv3.h:343:17: got void * ./arch/arm/include/asm/arch_gicv3.h:350:37: warning: incorrect type in argument 2 (different address spaces) ./arch/arm/include/asm/arch_gicv3.h:350:37: expected void volatile [noderef] *addr ./arch/arm/include/asm/arch_gicv3.h:350:37: got void *[noderef] addr drivers/irqchip/irq-gic-v3-its.c:2832:46: warning: incorrect type in argument 2 (different address spaces) drivers/irqchip/irq-gic-v3-its.c:2832:46: expected void *[noderef] addr drivers/irqchip/irq-gic-v3-its.c:2832:46: got void [noderef] * ./arch/arm/include/asm/arch_gicv3.h:340:15: warning: incorrect type in argument 1 (different address spaces) ./arch/arm/include/asm/arch_gicv3.h:340:15: expected void const volatile [noderef] *addr ./arch/arm/include/asm/arch_gicv3.h:340:15: got void * ./arch/arm/include/asm/arch_gicv3.h:343:17: warning: incorrect type in argument 2 (different address spaces) ./arch/arm/include/asm/arch_gicv3.h:343:17: expected void volatile [noderef] *addr ./arch/arm/include/asm/arch_gicv3.h:343:17: got void * ./arch/arm/include/asm/arch_gicv3.h:350:37: warning: incorrect type in argument 2 (different address spaces) ./arch/arm/include/asm/arch_gicv3.h:350:37: expected void volatile [noderef] *addr ./arch/arm/include/asm/arch_gicv3.h:350:37: got void *[noderef] addr Signed-off-by: Ben Dooks Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191017113341.13778-1-ben.dooks@codethink.co.uk Signed-off-by: 温家明 --- arch/arm/include/asm/arch_gicv3.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index 0555f14cc8be..fa50bb04f580 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -333,7 +333,7 @@ static inline u64 __gic_readq_nonatomic(const volatile void __iomem *addr) * GITS_VPENDBASER - the Valid bit must be cleared before changing * anything else. */ -static inline void gits_write_vpendbaser(u64 val, void * __iomem addr) +static inline void gits_write_vpendbaser(u64 val, void __iomem *addr) { u32 tmp; -- Gitee From aa2f46c4990d4afca8e7c96881d6d7e03a771d75 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 27 Oct 2019 14:41:59 +0000 Subject: [PATCH 02/94] KVM: arm64: vgic-v4: Move the GICv4 residency flow to be driven by vcpu_load/put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the VHE code was reworked, a lot of the vgic stuff was moved around, but the GICv4 residency code did stay untouched, meaning that we come in and out of residency on each flush/sync, which is obviously suboptimal. To address this, let's move things around a bit: - Residency entry (flush) moves to vcpu_load - Residency exit (sync) moves to vcpu_put - On blocking (entry to WFI), we "put" - On unblocking (exit from WFI), we "load" Because these can nest (load/block/put/load/unblock/put, for example), we now have per-VPE tracking of the residency state. Additionally, vgic_v4_put gains a "need doorbell" parameter, which only gets set to true when blocking because of a WFI. This allows a finer control of the doorbell, which now also gets disabled as soon as it gets signaled. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191027144234.8395-2-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v4.c | 7 +++- include/kvm/arm_vgic.h | 4 +-- include/linux/irqchip/arm-gic-v4.h | 2 ++ virt/kvm/arm/arm.c | 12 ++++--- virt/kvm/arm/vgic/vgic-v3.c | 4 +++ virt/kvm/arm/vgic/vgic-v4.c | 55 ++++++++++++++---------------- virt/kvm/arm/vgic/vgic.c | 4 --- virt/kvm/arm/vgic/vgic.h | 2 -- 8 files changed, 48 insertions(+), 42 deletions(-) diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 563e87ed0766..45969927cc81 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -141,12 +141,17 @@ static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info) int its_schedule_vpe(struct its_vpe *vpe, bool on) { struct its_cmd_info info; + int ret; WARN_ON(preemptible()); info.cmd_type = on ? SCHEDULE_VPE : DESCHEDULE_VPE; - return its_send_vpe_cmd(vpe, &info); + ret = its_send_vpe_cmd(vpe, &info); + if (!ret) + vpe->resident = on; + + return ret; } int its_invall_vpe(struct its_vpe *vpe) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index af4f09c02bf1..4dc58d7a0010 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -396,7 +396,7 @@ int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq, int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int irq, struct kvm_kernel_irq_routing_entry *irq_entry); -void kvm_vgic_v4_enable_doorbell(struct kvm_vcpu *vcpu); -void kvm_vgic_v4_disable_doorbell(struct kvm_vcpu *vcpu); +int vgic_v4_load(struct kvm_vcpu *vcpu); +int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db); #endif /* __KVM_ARM_VGIC_H */ diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index e6b155713b47..ab1396afe08a 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -35,6 +35,8 @@ struct its_vpe { /* Doorbell interrupt */ int irq; irq_hw_number_t vpe_db_lpi; + /* VPE resident */ + bool resident; /* VPE proxy mapping */ int vpe_proxy_event; /* diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index f7150fbeeb55..9238a9ba9643 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -322,20 +322,24 @@ void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) /* * If we're about to block (most likely because we've just hit a * WFI), we need to sync back the state of the GIC CPU interface - * so that we have the lastest PMR and group enables. This ensures + * so that we have the latest PMR and group enables. This ensures * that kvm_arch_vcpu_runnable has up-to-date data to decide * whether we have pending interrupts. + * + * For the same reason, we want to tell GICv4 that we need + * doorbells to be signalled, should an interrupt become pending. */ preempt_disable(); kvm_vgic_vmcr_sync(vcpu); + vgic_v4_put(vcpu, true); preempt_enable(); - - kvm_vgic_v4_enable_doorbell(vcpu); } void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) { - kvm_vgic_v4_disable_doorbell(vcpu); + preempt_disable(); + vgic_v4_load(vcpu); + preempt_enable(); } int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index b3a97dcaa30d..affc561b689a 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -664,6 +664,8 @@ void vgic_v3_load(struct kvm_vcpu *vcpu) if (has_vhe()) __vgic_v3_activate_traps(vcpu); + + WARN_ON(vgic_v4_load(vcpu)); } void vgic_v3_vmcr_sync(struct kvm_vcpu *vcpu) @@ -676,6 +678,8 @@ void vgic_v3_vmcr_sync(struct kvm_vcpu *vcpu) void vgic_v3_put(struct kvm_vcpu *vcpu) { + WARN_ON(vgic_v4_put(vcpu, false)); + vgic_v3_vmcr_sync(vcpu); kvm_call_hyp(__vgic_v3_save_aprs, vcpu); diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 477af6aebb97..7e1f3202968a 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -85,6 +85,10 @@ static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info) { struct kvm_vcpu *vcpu = info; + /* We got the message, no need to fire again */ + if (!irqd_irq_disabled(&irq_to_desc(irq)->irq_data)) + disable_irq_nosync(irq); + vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true; kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); kvm_vcpu_kick(vcpu); @@ -192,20 +196,30 @@ void vgic_v4_teardown(struct kvm *kvm) its_vm->vpes = NULL; } -int vgic_v4_sync_hwstate(struct kvm_vcpu *vcpu) +int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db) { - if (!vgic_supports_direct_msis(vcpu->kvm)) + struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + struct irq_desc *desc = irq_to_desc(vpe->irq); + + if (!vgic_supports_direct_msis(vcpu->kvm) || !vpe->resident) return 0; - return its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, false); + /* + * If blocking, a doorbell is required. Undo the nested + * disable_irq() calls... + */ + while (need_db && irqd_irq_disabled(&desc->irq_data)) + enable_irq(vpe->irq); + + return its_schedule_vpe(vpe, false); } -int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu) +int vgic_v4_load(struct kvm_vcpu *vcpu) { - int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq; + struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; int err; - if (!vgic_supports_direct_msis(vcpu->kvm)) + if (!vgic_supports_direct_msis(vcpu->kvm) || vpe->resident) return 0; /* @@ -214,11 +228,14 @@ int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu) * doc in drivers/irqchip/irq-gic-v4.c to understand how this * turns into a VMOVP command at the ITS level. */ - err = irq_set_affinity(irq, cpumask_of(smp_processor_id())); + err = irq_set_affinity(vpe->irq, cpumask_of(smp_processor_id())); if (err) return err; - err = its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, true); + /* Disabled the doorbell, as we're about to enter the guest */ + disable_irq_nosync(vpe->irq); + + err = its_schedule_vpe(vpe, true); if (err) return err; @@ -226,9 +243,7 @@ int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu) * Now that the VPE is resident, let's get rid of a potential * doorbell interrupt that would still be pending. */ - err = irq_set_irqchip_state(irq, IRQCHIP_STATE_PENDING, false); - - return err; + return irq_set_irqchip_state(vpe->irq, IRQCHIP_STATE_PENDING, false); } static struct vgic_its *vgic_get_its(struct kvm *kvm, @@ -335,21 +350,3 @@ int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int virq, mutex_unlock(&its->its_lock); return ret; } - -void kvm_vgic_v4_enable_doorbell(struct kvm_vcpu *vcpu) -{ - if (vgic_supports_direct_msis(vcpu->kvm)) { - int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq; - if (irq) - enable_irq(irq); - } -} - -void kvm_vgic_v4_disable_doorbell(struct kvm_vcpu *vcpu) -{ - if (vgic_supports_direct_msis(vcpu->kvm)) { - int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq; - if (irq) - disable_irq(irq); - } -} diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c index 45a870cb63f5..99b02ca730a8 100644 --- a/virt/kvm/arm/vgic/vgic.c +++ b/virt/kvm/arm/vgic/vgic.c @@ -857,8 +857,6 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) { struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - WARN_ON(vgic_v4_sync_hwstate(vcpu)); - /* An empty ap_list_head implies used_lrs == 0 */ if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head)) return; @@ -882,8 +880,6 @@ static inline void vgic_restore_state(struct kvm_vcpu *vcpu) /* Flush our emulation state into the GIC hardware before entering the guest. */ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) { - WARN_ON(vgic_v4_flush_hwstate(vcpu)); - /* * If there are no virtual interrupts active or pending for this * VCPU, then there is no work to do and we can bail out without diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index 83066a81b16a..c7fefd6b1c80 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -316,7 +316,5 @@ void vgic_its_invalidate_cache(struct kvm *kvm); bool vgic_supports_direct_msis(struct kvm *kvm); int vgic_v4_init(struct kvm *kvm); void vgic_v4_teardown(struct kvm *kvm); -int vgic_v4_sync_hwstate(struct kvm_vcpu *vcpu); -int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu); #endif -- Gitee From dd476c0f41bcf116327e837b3e1079852c5c82e5 Mon Sep 17 00:00:00 2001 From: Christoffer Dall Date: Mon, 28 Oct 2019 14:05:41 +0100 Subject: [PATCH 03/94] KVM: arm64: Don't set HCR_EL2.TVM when S2FWB is supported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On CPUs that support S2FWB (Armv8.4+), KVM configures the stage 2 page tables to override the memory attributes of memory accesses, regardless of the stage 1 page table configurations, and also when the stage 1 MMU is turned off. This results in all memory accesses to RAM being cacheable, including during early boot of the guest. On CPUs without this feature, memory accesses were non-cacheable during boot until the guest turned on the stage 1 MMU, and we had to detect when the guest turned on the MMU, such that we could invalidate all cache entries and ensure a consistent view of memory with the MMU turned on. When the guest turned on the caches, we would call stage2_flush_vm() from kvm_toggle_cache(). However, stage2_flush_vm() walks all the stage 2 tables, and calls __kvm_flush-dcache_pte, which on a system with S2FWB does ... absolutely nothing. We can avoid that whole song and dance, and simply not set TVM when creating a VM on a system that has S2FWB. Signed-off-by: Christoffer Dall Signed-off-by: Marc Zyngier Reviewed-by: Mark Rutland Link: https://lore.kernel.org/r/20191028130541.30536-1-christoffer.dall@arm.com Signed-off-by: 温家明 --- arch/arm64/include/asm/kvm_arm.h | 1 - arch/arm64/include/asm/kvm_emulate.h | 12 +++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 9be64c0ad31f..60f6d2c3c2c5 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -61,7 +61,6 @@ * RW: 64bit by default, can be overridden for 32bit VMs * TAC: Trap ACTLR * TSC: Trap SMC - * TVM: Trap VM ops (until M+C set in SCTLR_EL1) * TSW: Trap cache operations by set/way * TWE: Trap WFE * TWI: Trap WFI diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 4a4c20a1bf26..4f74ee3ce7d0 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -53,8 +53,18 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) /* trap error record accesses */ vcpu->arch.hcr_el2 |= HCR_TERR; } - if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) + + if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) { vcpu->arch.hcr_el2 |= HCR_FWB; + } else { + /* + * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C + * get set in SCTLR_EL1 such that we can detect when the guest + * MMU gets turned on and do the necessary cache maintenance + * then. + */ + vcpu->arch.hcr_el2 |= HCR_TVM; + } if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) vcpu->arch.hcr_el2 &= ~HCR_RW; -- Gitee From 06aded4cd119d82e56c965fe424fbdf39643974e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Fri, 6 Dec 2024 15:20:30 +0800 Subject: [PATCH 04/94] resolve patch Don't set HCR_EL2.TVM conflict --- arch/arm64/include/asm/kvm_arm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 60f6d2c3c2c5..c847d76ef569 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -74,7 +74,7 @@ * PTW: Take a stage2 fault if a stage1 walk steps in device memory */ #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ - HCR_TVM | HCR_BSU_IS | HCR_FB | HCR_TAC | \ + HCR_BSU_IS | HCR_FB | HCR_TAC | \ HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ HCR_FMO | HCR_IMO | HCR_PTW ) #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) -- Gitee From 2ae41ae4faa4f790908ba5916bc08567f4ec3ccf Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Tue, 29 Oct 2019 15:19:18 +0800 Subject: [PATCH 05/94] KVM: arm/arm64: vgic: Fix some comments typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix various comments, including wrong function names, grammar mistakes and specification references. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191029071919.177-3-yuzenghui@huawei.com Signed-off-by: 温家明 --- include/kvm/arm_vgic.h | 2 +- virt/kvm/arm/vgic/vgic-v3.c | 2 +- virt/kvm/arm/vgic/vgic-v4.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 4dc58d7a0010..1ecd69c65ba6 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -240,7 +240,7 @@ struct vgic_dist { * Contains the attributes and gpa of the LPI configuration table. * Since we report GICR_TYPER.CommonLPIAff as 0b00, we can share * one address across all redistributors. - * GICv3 spec: 6.1.2 "LPI Configuration tables" + * GICv3 spec: IHI 0069E 6.1.1 "LPI Configuration tables" */ u64 propbaser; diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index affc561b689a..f45635a6f0ec 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -357,7 +357,7 @@ int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq) } /** - * vgic_its_save_pending_tables - Save the pending tables into guest RAM + * vgic_v3_save_pending_tables - Save the pending tables into guest RAM * kvm lock and all vcpu lock must be held */ int vgic_v3_save_pending_tables(struct kvm *kvm) diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 7e1f3202968a..0965fb0c427a 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -281,7 +281,7 @@ int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq, mutex_lock(&its->its_lock); - /* Perform then actual DevID/EventID -> LPI translation. */ + /* Perform the actual DevID/EventID -> LPI translation. */ ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid, irq_entry->msi.data, &irq); if (ret) -- Gitee From 3c6d40432722aee21b5255c673fef2b668c57dd1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 7 Nov 2019 16:04:11 +0000 Subject: [PATCH 06/94] KVM: vgic-v4: Track the number of VLPIs per vcpu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to find out whether a vcpu is likely to be the target of VLPIs (and to further optimize the way we deal with those), let's track the number of VLPIs a vcpu can receive. This gets implemented with an atomic variable that gets incremented or decremented on map, unmap and move of a VLPI. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Christoffer Dall Link: https://lore.kernel.org/r/20191107160412.30301-2-maz@kernel.org Signed-off-by: 温家明 --- include/linux/irqchip/arm-gic-v4.h | 2 ++ virt/kvm/arm/vgic/vgic-init.c | 1 + virt/kvm/arm/vgic/vgic-its.c | 3 +++ virt/kvm/arm/vgic/vgic-v4.c | 2 ++ 4 files changed, 8 insertions(+) diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index ab1396afe08a..5dbcfc65f21e 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -32,6 +32,8 @@ struct its_vm { struct its_vpe { struct page *vpt_page; struct its_vm *its_vm; + /* per-vPE VLPI tracking */ + atomic_t vlpi_count; /* Doorbell interrupt */ int irq; irq_hw_number_t vpe_db_lpi; diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c index 689910153889..9127bf433253 100644 --- a/virt/kvm/arm/vgic/vgic-init.c +++ b/virt/kvm/arm/vgic/vgic-init.c @@ -204,6 +204,7 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) INIT_LIST_HEAD(&vgic_cpu->ap_list_head); raw_spin_lock_init(&vgic_cpu->ap_list_lock); + atomic_set(&vgic_cpu->vgic_v3.its_vpe.vlpi_count, 0); /* * Enable and configure all SGIs to be edge-triggered and diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c index e06bb8ed7688..5c3eea13e4c0 100644 --- a/virt/kvm/arm/vgic/vgic-its.c +++ b/virt/kvm/arm/vgic/vgic-its.c @@ -367,7 +367,10 @@ static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu) if (ret) return ret; + if (map.vpe) + atomic_dec(&map.vpe->vlpi_count); map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + atomic_inc(&map.vpe->vlpi_count); ret = its_map_vlpi(irq->host_irq, &map); } diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 0965fb0c427a..46f875589c47 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -309,6 +309,7 @@ int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq, irq->hw = true; irq->host_irq = virq; + atomic_inc(&map.vpe->vlpi_count); out: mutex_unlock(&its->its_lock); @@ -342,6 +343,7 @@ int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int virq, WARN_ON(!(irq->hw && irq->host_irq == virq)); if (irq->hw) { + atomic_dec(&irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count); irq->hw = false; ret = its_unmap_vlpi(virq); } -- Gitee From 079aef8de51dc893fb6f182e2d9d93608efca74c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 7 Nov 2019 16:04:12 +0000 Subject: [PATCH 07/94] KVM: arm64: Opportunistically turn off WFI trapping when using direct LPI injection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just like we do for WFE trapping, it can be useful to turn off WFI trapping when the physical CPU is not oversubscribed (that is, the vcpu is the only runnable process on this CPU) *and* that we're using direct injection of interrupts. The conditions are reevaluated on each vcpu_load(), ensuring that we don't switch to this mode on a busy system. On a GICv4 system, this has the effect of reducing the generation of doorbell interrupts to zero when the right conditions are met, which is a huge improvement over the current situation (where the doorbells are screaming if the CPU ever hits a blocking WFI). Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Christoffer Dall Link: https://lore.kernel.org/r/20191107160412.30301-3-maz@kernel.org Signed-off-by: 温家明 --- arch/arm/include/asm/kvm_emulate.h | 4 ++-- arch/arm64/include/asm/kvm_emulate.h | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h index c1747fcb86d3..a0f122bffce7 100644 --- a/arch/arm/include/asm/kvm_emulate.h +++ b/arch/arm/include/asm/kvm_emulate.h @@ -112,12 +112,12 @@ static inline unsigned long *vcpu_hcr(const struct kvm_vcpu *vcpu) return (unsigned long *)&vcpu->arch.hcr; } -static inline void vcpu_clear_wfe_traps(struct kvm_vcpu *vcpu) +static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr &= ~HCR_TWE; } -static inline void vcpu_set_wfe_traps(struct kvm_vcpu *vcpu) +static inline void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr |= HCR_TWE; } diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 4f74ee3ce7d0..226339e1ed5b 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -87,14 +87,19 @@ static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) return (unsigned long *)&vcpu->arch.hcr_el2; } -static inline void vcpu_clear_wfe_traps(struct kvm_vcpu *vcpu) +static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 &= ~HCR_TWE; + if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count)) + vcpu->arch.hcr_el2 &= ~HCR_TWI; + else + vcpu->arch.hcr_el2 |= HCR_TWI; } -static inline void vcpu_set_wfe_traps(struct kvm_vcpu *vcpu) +static inline void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 |= HCR_TWE; + vcpu->arch.hcr_el2 |= HCR_TWI; } static inline void vcpu_ptrauth_enable(struct kvm_vcpu *vcpu) -- Gitee From 11ec1f7bf8a65ea4488cc03ca1cea07c514507c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Fri, 6 Dec 2024 15:36:27 +0800 Subject: [PATCH 08/94] resolve patch Opportunistically turn off WFI trapping conflict --- virt/kvm/arm/arm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index 9238a9ba9643..3e3d95ea393c 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -402,9 +402,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_vcpu_pmu_restore_guest(vcpu); if (single_task_running()) - vcpu_clear_wfe_traps(vcpu); + vcpu_clear_wfx_traps(vcpu); else - vcpu_set_wfe_traps(vcpu); + vcpu_set_wfx_traps(vcpu); if (vcpu_has_ptrauth(vcpu)) { struct kvm_cpu_context __maybe_unused *ctxt = vcpu->arch.host_cpu_context; -- Gitee From d5400305d2541a60d0c22a1abada943e181882f1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:57:55 +0000 Subject: [PATCH 09/94] irqchip/gic-v3-its: Free collection mapping on device teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We allocate the collection mapping on device creation, but somehow free it on the irqdomain free path, which is pretty inconsistent and has led to bugs in the past. Move it to the point where we teardown the device, making the alloc/free symetric. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191108165805.3071-2-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 151eaa84e075..e2fe461f6c59 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2608,6 +2608,7 @@ static void its_free_device(struct its_device *its_dev) raw_spin_lock_irqsave(&its_dev->its->lock, flags); list_del(&its_dev->entry); raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); + kfree(its_dev->event_map.col_map); kfree(its_dev->itt); kfree(its_dev); } @@ -2811,7 +2812,6 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, its_lpi_free(its_dev->event_map.lpi_map, its_dev->event_map.lpi_base, its_dev->event_map.nr_lpis); - kfree(its_dev->event_map.col_map); /* Unmap device/itt */ its_send_mapd(its_dev, 0); -- Gitee From 9a3daefd46e9fa4e581f3b04479bebce9a15eb51 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:57:56 +0000 Subject: [PATCH 10/94] irqchip/gic-v3-its: Factor out wait_for_syncr primitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Waiting for a redistributor to have performed an operation is a common thing to do, and the idiom is already spread around. As we're going to make even more use of this, let's have a primitive that does just that. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191027144234.8395-3-maz@kernel.org Link: https://lore.kernel.org/r/20191108165805.3071-3-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e2fe461f6c59..fd88174830d9 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1100,6 +1100,12 @@ static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) dsb(ishst); } +static void wait_for_syncr(void __iomem *rdbase) +{ + while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) + cpu_relax(); +} + static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -2904,8 +2910,7 @@ static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); - while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) - cpu_relax(); + wait_for_syncr(rdbase); return; } @@ -3061,8 +3066,7 @@ static void its_vpe_send_inv(struct irq_data *d) rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR); - while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) - cpu_relax(); + wait_for_syncr(rdbase); } else { its_vpe_send_cmd(vpe, its_send_inv); } @@ -3104,8 +3108,7 @@ static int its_vpe_set_irqchip_state(struct irq_data *d, gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); } else { gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); - while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) - cpu_relax(); + wait_for_syncr(rdbase); } } else { if (state) -- Gitee From dd7a7b858700f3be95f369e10f35a969ce18a6be Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:57:57 +0000 Subject: [PATCH 11/94] irqchip/gic-v3-its: Allow LPI invalidation via the DirectLPI interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We currently don't make much use of the DirectLPI feature, and it would be beneficial to do this more, if only because it becomes a mandatory feature for GICv4.1. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191027144234.8395-4-maz@kernel.org Link: https://lore.kernel.org/r/20191108165805.3071-4-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 40 +++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index fd88174830d9..2dd16459a8bc 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -201,6 +201,12 @@ static u16 get_its_list(struct its_vm *vm) return (u16)its_list; } +static inline u32 its_get_event_id(struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + return d->hwirq - its_dev->event_map.lpi_base; +} + static struct its_collection *dev_event_to_col(struct its_device *its_dev, u32 event) { @@ -209,6 +215,13 @@ static struct its_collection *dev_event_to_col(struct its_device *its_dev, return its->collections + its_dev->event_map.col_map[event]; } +static struct its_collection *irq_to_col(struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + + return dev_event_to_col(its_dev, its_get_event_id(d)); +} + static struct its_collection *valid_col(struct its_collection *col) { if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0))) @@ -1056,12 +1069,6 @@ static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) * irqchip functions - assumes MSI, mostly. */ -static inline u32 its_get_event_id(struct irq_data *d) -{ - struct its_device *its_dev = irq_data_get_irq_chip_data(d); - return d->hwirq - its_dev->event_map.lpi_base; -} - static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) { irq_hw_number_t hwirq; @@ -1106,12 +1113,28 @@ static void wait_for_syncr(void __iomem *rdbase) cpu_relax(); } +static void direct_lpi_inv(struct irq_data *d) +{ + struct its_collection *col; + void __iomem *rdbase; + + /* Target the redistributor this LPI is currently routed to */ + col = irq_to_col(d); + rdbase = per_cpu_ptr(gic_rdists->rdist, col->col_id)->rd_base; + gic_write_lpir(d->hwirq, rdbase + GICR_INVLPIR); + + wait_for_syncr(rdbase); +} + static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); lpi_write_config(d, clr, set); - its_send_inv(its_dev, its_get_event_id(d)); + if (gic_rdists->has_direct_lpi && !irqd_is_forwarded_to_vcpu(d)) + direct_lpi_inv(d); + else + its_send_inv(its_dev, its_get_event_id(d)); } static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) @@ -3064,8 +3087,9 @@ static void its_vpe_send_inv(struct irq_data *d) if (gic_rdists->has_direct_lpi) { void __iomem *rdbase; + /* Target the redistributor this VPE is currently known on */ rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; - gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR); + gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR); wait_for_syncr(rdbase); } else { its_vpe_send_cmd(vpe, its_send_inv); -- Gitee From ec0b6c64a71b57e57c7d1d50283128fd33ff1636 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:57:58 +0000 Subject: [PATCH 12/94] irqchip/gic-v3-its: Make is_v4 use a TYPER copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of caching the GICv4 compatibility in a discrete way, cache the TYPER register instead, which can then be used to implement the same functionnality. This will get used more extensively in subsequent patches. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191027144234.8395-5-maz@kernel.org Link: https://lore.kernel.org/r/20191108165805.3071-5-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2dd16459a8bc..69c14bd0b9cf 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -105,6 +105,7 @@ struct its_node { struct its_collection *collections; struct fwnode_handle *fwnode_handle; u64 (*get_msi_base)(struct its_device *its_dev); + u64 typer; u64 cbaser_save; u32 ctlr_save; struct list_head its_device_list; @@ -115,10 +116,11 @@ struct its_node { int numa_node; unsigned int msi_domain_flags; u32 pre_its_base; /* for Socionext Synquacer */ - bool is_v4; int vlpi_redist_offset; }; +#define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) + #define ITS_ITT_ALIGN SZ_256 /* The maximum number of VPEID bits supported by VLPI commands */ @@ -191,7 +193,7 @@ static u16 get_its_list(struct its_vm *vm) unsigned long its_list = 0; list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) + if (!is_v4(its)) continue; if (vm->vlpi_count[its->list_nr]) @@ -1044,7 +1046,7 @@ static void its_send_vmovp(struct its_vpe *vpe) /* Emit VMOVPs */ list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) + if (!is_v4(its)) continue; if (!vpe->its_vm->vlpi_count[its->list_nr]) @@ -1577,7 +1579,7 @@ static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) struct its_cmd_info *info = vcpu_info; /* Need a v4 ITS */ - if (!its_dev->its->is_v4) + if (!is_v4(its_dev->its)) return -EINVAL; /* Unmap request? */ @@ -2546,7 +2548,7 @@ static bool its_alloc_vpe_table(u32 vpe_id) list_for_each_entry(its, &its_nodes, entry) { struct its_baser *baser; - if (!its->is_v4) + if (!is_v4(its)) continue; baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); @@ -3029,7 +3031,7 @@ static void its_vpe_invall(struct its_vpe *vpe) struct its_node *its; list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) + if (!is_v4(its)) continue; if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) @@ -3298,7 +3300,7 @@ static int its_vpe_irq_domain_activate(struct irq_domain *domain, vpe->col_idx = cpumask_first(cpu_online_mask); list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) + if (!is_v4(its)) continue; its_send_vmapp(its, vpe, true); @@ -3324,7 +3326,7 @@ static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, return; list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) + if (!is_v4(its)) continue; its_send_vmapp(its, vpe, false); @@ -3756,12 +3758,12 @@ static int __init its_probe_one(struct resource *res, INIT_LIST_HEAD(&its->entry); INIT_LIST_HEAD(&its->its_device_list); typer = gic_read_typer(its_base + GITS_TYPER); + its->typer = typer; its->base = its_base; its->phys_base = res->start; its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer); its->device_ids = GITS_TYPER_DEVBITS(typer); - its->is_v4 = !!(typer & GITS_TYPER_VLPIS); - if (its->is_v4) { + if (is_v4(its)) { if (!(typer & GITS_TYPER_VMOVP)) { err = its_compute_its_list_map(res, its_base); if (err < 0) @@ -3828,7 +3830,7 @@ static int __init its_probe_one(struct resource *res, gits_write_cwriter(0, its->base + GITS_CWRITER); ctlr = readl_relaxed(its->base + GITS_CTLR); ctlr |= GITS_CTLR_ENABLE; - if (its->is_v4) + if (is_v4(its)) ctlr |= GITS_CTLR_ImDe; writel_relaxed(ctlr, its->base + GITS_CTLR); @@ -4150,7 +4152,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, return err; list_for_each_entry(its, &its_nodes, entry) - has_v4 |= its->is_v4; + has_v4 |= is_v4(its); if (has_v4 & rdists->has_vlpis) { if (its_init_vpe_domain() || -- Gitee From 69c6dc845233f1e4f285e57407993925eab024ec Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:57:59 +0000 Subject: [PATCH 13/94] irqchip/gic-v3-its: Kill its->ite_size and use TYPER copy instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we have a copy of TYPER in the ITS structure, rely on this to provide the same service as its->ite_size, which gets axed. Errata workarounds are now updating the cached fields instead of requiring a separate field in the ITS structure. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191027144234.8395-6-maz@kernel.org Link: https://lore.kernel.org/r/20191108165805.3071-6-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 8 ++++---- include/linux/irqchip/arm-gic-v3.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 69c14bd0b9cf..470166d0084b 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -111,7 +112,6 @@ struct its_node { struct list_head its_device_list; u64 flags; unsigned long list_nr; - u32 ite_size; u32 device_ids; int numa_node; unsigned int msi_domain_flags; @@ -2587,7 +2587,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, * sized as a power of two (and you need at least one bit...). */ nr_ites = max(2, nvecs); - sz = nr_ites * its->ite_size; + sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); if (alloc_lpis) { @@ -3398,7 +3398,8 @@ static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data) struct its_node *its = data; /* On QDF2400, the size of the ITE is 16Bytes */ - its->ite_size = 16; + its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE; + its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1); return true; } @@ -3761,7 +3762,6 @@ static int __init its_probe_one(struct resource *res, its->typer = typer; its->base = its_base; its->phys_base = res->start; - its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer); its->device_ids = GITS_TYPER_DEVBITS(typer); if (is_v4(its)) { if (!(typer & GITS_TYPER_VMOVP)) { diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 5cc10cf7cb3e..4bce7a904075 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -334,7 +334,7 @@ #define GITS_TYPER_PLPIS (1UL << 0) #define GITS_TYPER_VLPIS (1UL << 1) #define GITS_TYPER_ITT_ENTRY_SIZE_SHIFT 4 -#define GITS_TYPER_ITT_ENTRY_SIZE(r) ((((r) >> GITS_TYPER_ITT_ENTRY_SIZE_SHIFT) & 0xf) + 1) +#define GITS_TYPER_ITT_ENTRY_SIZE GENMASK_ULL(7, 4) #define GITS_TYPER_IDBITS_SHIFT 8 #define GITS_TYPER_DEVBITS_SHIFT 13 #define GITS_TYPER_DEVBITS(r) ((((r) >> GITS_TYPER_DEVBITS_SHIFT) & 0x1f) + 1) -- Gitee From 738f846b82406f41363e28ae99dabb5c12bc13ac Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:58:00 +0000 Subject: [PATCH 14/94] irqchip/gic-v3-its: Kill its->device_ids and use TYPER copy instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we have a copy of TYPER in the ITS structure, rely on this to provide the same service as its->device_ids, which gets axed. Errata workarounds are now updating the cached fields instead of requiring a separate field in the ITS structure. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191027144234.8395-7-maz@kernel.org Link: https://lore.kernel.org/r/20191108165805.3071-7-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 24 +++++++++++++----------- include/linux/irqchip/arm-gic-v3.h | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 470166d0084b..3180208ba385 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -112,7 +112,6 @@ struct its_node { struct list_head its_device_list; u64 flags; unsigned long list_nr; - u32 device_ids; int numa_node; unsigned int msi_domain_flags; u32 pre_its_base; /* for Socionext Synquacer */ @@ -120,6 +119,7 @@ struct its_node { }; #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) +#define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) #define ITS_ITT_ALIGN SZ_256 @@ -2085,9 +2085,9 @@ static bool its_parse_indirect_baser(struct its_node *its, if (new_order >= MAX_ORDER) { new_order = MAX_ORDER - 1; ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); - pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n", + pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", &its->phys_base, its_base_type_string[type], - its->device_ids, ids); + device_ids(its), ids); } *order = new_order; @@ -2133,7 +2133,7 @@ static int its_alloc_tables(struct its_node *its) case GITS_BASER_TYPE_DEVICE: indirect = its_parse_indirect_baser(its, baser, psz, &order, - its->device_ids); + device_ids(its)); break; case GITS_BASER_TYPE_VCPU: @@ -2529,7 +2529,7 @@ static bool its_alloc_device_table(struct its_node *its, u32 dev_id) /* Don't allow device id that exceeds ITS hardware limit */ if (!baser) - return (ilog2(dev_id) < its->device_ids); + return (ilog2(dev_id) < device_ids(its)); return its_alloc_table_entry(its, baser, dev_id); } @@ -3377,8 +3377,9 @@ static bool __maybe_unused its_enable_quirk_cavium_22375(void *data) { struct its_node *its = data; - /* erratum 22375: only alloc 8MB table size */ - its->device_ids = 0x14; /* 20 bits, 8MB */ + /* erratum 22375: only alloc 8MB table size (20 bits) */ + its->typer &= ~GITS_TYPER_DEVBITS; + its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1); its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; return true; @@ -3433,8 +3434,10 @@ static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data) its->get_msi_base = its_irq_get_msi_base_pre_its; ids = ilog2(pre_its_window[1]) - 2; - if (its->device_ids > ids) - its->device_ids = ids; + if (device_ids(its) > ids) { + its->typer &= ~GITS_TYPER_DEVBITS; + its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1); + } /* the pre-ITS breaks isolation, so disable MSI remapping */ its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP; @@ -3661,7 +3664,7 @@ static int its_init_vpe_domain(void) } /* Use the last possible DevID */ - devid = GENMASK(its->device_ids - 1, 0); + devid = GENMASK(device_ids(its) - 1, 0); vpe_proxy.dev = its_create_device(its, devid, entries, false); if (!vpe_proxy.dev) { kfree(vpe_proxy.vpes); @@ -3762,7 +3765,6 @@ static int __init its_probe_one(struct resource *res, its->typer = typer; its->base = its_base; its->phys_base = res->start; - its->device_ids = GITS_TYPER_DEVBITS(typer); if (is_v4(its)) { if (!(typer & GITS_TYPER_VMOVP)) { err = its_compute_its_list_map(res, its_base); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 4bce7a904075..b6514e8893bf 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -337,7 +337,7 @@ #define GITS_TYPER_ITT_ENTRY_SIZE GENMASK_ULL(7, 4) #define GITS_TYPER_IDBITS_SHIFT 8 #define GITS_TYPER_DEVBITS_SHIFT 13 -#define GITS_TYPER_DEVBITS(r) ((((r) >> GITS_TYPER_DEVBITS_SHIFT) & 0x1f) + 1) +#define GITS_TYPER_DEVBITS GENMASK_ULL(17, 13) #define GITS_TYPER_PTA (1UL << 19) #define GITS_TYPER_HCC_SHIFT 24 #define GITS_TYPER_HCC(r) (((r) >> GITS_TYPER_HCC_SHIFT) & 0xff) -- Gitee From 9cb53fa0d792280f58452a2551c18e6d7efee6a8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:58:01 +0000 Subject: [PATCH 15/94] irqchip/gic-v3-its: Add its_vlpi_map helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obtaining the mapping information for a VLPI is something quite common, and the GICv4.1 code is going to make even more use of it. Expose it as a separate set of helpers. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191027144234.8395-8-maz@kernel.org Link: https://lore.kernel.org/r/20191108165805.3071-8-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 47 ++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 3180208ba385..11b2c5447cc6 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -217,6 +217,15 @@ static struct its_collection *dev_event_to_col(struct its_device *its_dev, return its->collections + its_dev->event_map.col_map[event]; } +static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev, + u32 event) +{ + if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis)) + return NULL; + + return &its_dev->event_map.vlpi_maps[event]; +} + static struct its_collection *irq_to_col(struct irq_data *d) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -978,7 +987,7 @@ static void its_send_invall(struct its_node *its, struct its_collection *col) static void its_send_vmapti(struct its_device *dev, u32 id) { - struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; + struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id); struct its_cmd_desc desc; desc.its_vmapti_cmd.vpe = map->vpe; @@ -992,7 +1001,7 @@ static void its_send_vmapti(struct its_device *dev, u32 id) static void its_send_vmovi(struct its_device *dev, u32 id) { - struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; + struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id); struct its_cmd_desc desc; desc.its_vmovi_cmd.vpe = map->vpe; @@ -1070,20 +1079,26 @@ static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) /* * irqchip functions - assumes MSI, mostly. */ +static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + if (!irqd_is_forwarded_to_vcpu(d)) + return NULL; + + return dev_event_to_vlpi_map(its_dev, event); +} static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) { + struct its_vlpi_map *map = get_vlpi_map(d); irq_hw_number_t hwirq; void *va; u8 *cfg; - if (irqd_is_forwarded_to_vcpu(d)) { - struct its_device *its_dev = irq_data_get_irq_chip_data(d); - u32 event = its_get_event_id(d); - struct its_vlpi_map *map; - - va = page_address(its_dev->event_map.vm->vprop_page); - map = &its_dev->event_map.vlpi_maps[event]; + if (map) { + va = page_address(map->vm->vprop_page); hwirq = map->vintid; /* Remember the updated property */ @@ -1143,11 +1158,14 @@ static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); u32 event = its_get_event_id(d); + struct its_vlpi_map *map; - if (its_dev->event_map.vlpi_maps[event].db_enabled == enable) + map = dev_event_to_vlpi_map(its_dev, event); + + if (map->db_enabled == enable) return; - its_dev->event_map.vlpi_maps[event].db_enabled = enable; + map->db_enabled = enable; /* * More fun with the architecture: @@ -1498,19 +1516,18 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); - u32 event = its_get_event_id(d); + struct its_vlpi_map *map = get_vlpi_map(d); int ret = 0; mutex_lock(&its_dev->event_map.vlpi_lock); - if (!its_dev->event_map.vm || - !its_dev->event_map.vlpi_maps[event].vm) { + if (!its_dev->event_map.vm || !map->vm) { ret = -EINVAL; goto out; } /* Copy our mapping information to the incoming request */ - *info->map = its_dev->event_map.vlpi_maps[event]; + *info->map = *map; out: mutex_unlock(&its_dev->event_map.vlpi_lock); -- Gitee From 393c566c3dccebdbeb8c9956439f79b48761c575 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 22 Jan 2020 13:53:44 +0000 Subject: [PATCH 16/94] irqchip/gic-v3-its: Fix get_vlpi_map() breakage with doorbells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When updating an LPI configuration, get_vlpi_map() may be passed a irq_data structure relative to an ITS domain (the normal case) or one that is relative to the core GICv3 domain in the case of a GICv4 doorbell. In the latter case, special care must be take not to dereference the irq_chip data as an its_dev structure, as that isn't what is stored there. Instead, check *first* whether the IRQ is forwarded to a vcpu, and only then try to obtain the vlpi mapping. Fixes: c1d4d5cd203c ("irqchip/gic-v3-its: Add its_vlpi_map helpers") Signed-off-by: Marc Zyngier Reported-by: Zenghui Yu Link: https://lore.kernel.org/r/20200122085609.658-1-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 11b2c5447cc6..b5f862eb4eb2 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1081,13 +1081,14 @@ static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) */ static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) { - struct its_device *its_dev = irq_data_get_irq_chip_data(d); - u32 event = its_get_event_id(d); + if (irqd_is_forwarded_to_vcpu(d)) { + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); - if (!irqd_is_forwarded_to_vcpu(d)) - return NULL; + return dev_event_to_vlpi_map(its_dev, event); + } - return dev_event_to_vlpi_map(its_dev, event); + return NULL; } static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) -- Gitee From 3c10cceb1d79f3282e021ea2f9ddf184bece44d4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:58:02 +0000 Subject: [PATCH 17/94] irqchip/gic-v3-its: Synchronise INV command targetting a VLPI using VSYNC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have so far alwways invalidated VLPIs usinc an INV+SYNC sequence, but that's pretty wrong for two reasons: - SYNC only synchronises physical LPIs - The collection ID that for the associated LPI doesn't match the redistributor the vPE is associated with Instead, send an INV+VSYNC for forwarded LPIs, ensuring that the ITS can properly synchronise the invalidation of VLPIs. Fixes: 015ec0386ab6 ("irqchip/gic-v3-its: Add VLPI configuration handling") Reported-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191108165805.3071-9-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 36 +++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index b5f862eb4eb2..462259dd6da6 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -710,6 +710,24 @@ static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, return valid_vpe(its, desc->its_vmovp_cmd.vpe); } +static struct its_vpe *its_build_vinv_cmd(struct its_node *its, + struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_vlpi_map *map; + + map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev, + desc->its_inv_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_INV); + its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_inv_cmd.event_id); + + its_fixup_cmd(cmd); + + return valid_vpe(its, map->vpe); +} + static u64 its_cmd_ptr_to_offset(struct its_node *its, struct its_cmd_block *ptr) { @@ -1076,6 +1094,20 @@ static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); } +static void its_send_vinv(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; + + /* + * There is no real VINV command. This is just a normal INV, + * with a VSYNC instead of a SYNC. + */ + desc.its_inv_cmd.dev = dev; + desc.its_inv_cmd.event_id = event_id; + + its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); +} + /* * irqchip functions - assumes MSI, mostly. */ @@ -1151,8 +1183,10 @@ static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) lpi_write_config(d, clr, set); if (gic_rdists->has_direct_lpi && !irqd_is_forwarded_to_vcpu(d)) direct_lpi_inv(d); - else + else if (!irqd_is_forwarded_to_vcpu(d)) its_send_inv(its_dev, its_get_event_id(d)); + else + its_send_vinv(its_dev, its_get_event_id(d)); } static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) -- Gitee From 93e824d25d482d3dce354b1b7cbe0964fcfca764 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:58:03 +0000 Subject: [PATCH 18/94] irqchip/gic-v3-its: Synchronise INT/CLEAR commands targetting a VLPI using VSYNC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have so far always injected/cleared VLPIs using either INT+SYNC or CLEAR+SYNC sequences, but that's pretty wrong for two reasons: - SYNC only synchronises physical LPIs - The collection ID that for the associated LPI doesn't match the redistributor the vPE is associated with Instead, send an {INT,CLEAR}+VSYNC for forwarded LPIs, ensuring that the ITS synchronises against the virtual pending table. Reported-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191108165805.3071-10-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 79 ++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 462259dd6da6..e6986427dac0 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -728,6 +728,42 @@ static struct its_vpe *its_build_vinv_cmd(struct its_node *its, return valid_vpe(its, map->vpe); } +static struct its_vpe *its_build_vint_cmd(struct its_node *its, + struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_vlpi_map *map; + + map = dev_event_to_vlpi_map(desc->its_int_cmd.dev, + desc->its_int_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_INT); + its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_int_cmd.event_id); + + its_fixup_cmd(cmd); + + return valid_vpe(its, map->vpe); +} + +static struct its_vpe *its_build_vclear_cmd(struct its_node *its, + struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_vlpi_map *map; + + map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev, + desc->its_clear_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_CLEAR); + its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_clear_cmd.event_id); + + its_fixup_cmd(cmd); + + return valid_vpe(its, map->vpe); +} + static u64 its_cmd_ptr_to_offset(struct its_node *its, struct its_cmd_block *ptr) { @@ -1108,6 +1144,34 @@ static void its_send_vinv(struct its_device *dev, u32 event_id) its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); } +static void its_send_vint(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; + + /* + * There is no real VINT command. This is just a normal INT, + * with a VSYNC instead of a SYNC. + */ + desc.its_int_cmd.dev = dev; + desc.its_int_cmd.event_id = event_id; + + its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc); +} + +static void its_send_vclear(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; + + /* + * There is no real VCLEAR command. This is just a normal CLEAR, + * with a VSYNC instead of a SYNC. + */ + desc.its_clear_cmd.dev = dev; + desc.its_clear_cmd.event_id = event_id; + + its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); +} + /* * irqchip functions - assumes MSI, mostly. */ @@ -1424,10 +1488,17 @@ static int its_irq_set_irqchip_state(struct irq_data *d, if (which != IRQCHIP_STATE_PENDING) return -EINVAL; - if (state) - its_send_int(its_dev, event); - else - its_send_clear(its_dev, event); + if (irqd_is_forwarded_to_vcpu(d)) { + if (state) + its_send_vint(its_dev, event); + else + its_send_vclear(its_dev, event); + } else { + if (state) + its_send_int(its_dev, event); + else + its_send_clear(its_dev, event); + } return 0; } -- Gitee From da0d88ac07524c8fd2eeca748ec7799ec83a1a51 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:58:04 +0000 Subject: [PATCH 19/94] irqchip/gic-v3-its: Lock VLPI map array before translating it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obtaining the mapping ivformation for a VLPI should always be done with the vlpi_lock for this device held. Otherwise, we expose ourselves to races against a concurrent unmap. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191108165805.3071-11-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e6986427dac0..f36579b2626b 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1622,12 +1622,14 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); - struct its_vlpi_map *map = get_vlpi_map(d); + struct its_vlpi_map *map; int ret = 0; mutex_lock(&its_dev->event_map.vlpi_lock); - if (!its_dev->event_map.vm || !map->vm) { + map = get_vlpi_map(d); + + if (!its_dev->event_map.vm || !map) { ret = -EINVAL; goto out; } -- Gitee From 23a6c545e4e5b2f7d964411eb9122b55ef73ca20 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 8 Nov 2019 16:58:05 +0000 Subject: [PATCH 20/94] irqchip/gic-v3-its: Make vlpi_lock a spinlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The VLPI map is currently a mutex, and that's a bad idea as this lock can be taken in non-preemptible contexts. Convert it to a raw spinlock, and turn the memory allocation of the VLPI map to be atomic. Reported-by: Heyi Guo Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20191108165805.3071-12-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index f36579b2626b..46aad1f87b3b 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -135,7 +135,7 @@ struct event_lpi_map { u16 *col_map; irq_hw_number_t lpi_base; int nr_lpis; - struct mutex vlpi_lock; + raw_spinlock_t vlpi_lock; struct its_vm *vm; struct its_vlpi_map *vlpi_maps; int nr_vlpis; @@ -1566,13 +1566,13 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) if (!info->map) return -EINVAL; - mutex_lock(&its_dev->event_map.vlpi_lock); + raw_spin_lock(&its_dev->event_map.vlpi_lock); if (!its_dev->event_map.vm) { struct its_vlpi_map *maps; maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), - GFP_KERNEL); + GFP_ATOMIC); if (!maps) { ret = -ENOMEM; goto out; @@ -1615,7 +1615,7 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) } out: - mutex_unlock(&its_dev->event_map.vlpi_lock); + raw_spin_unlock(&its_dev->event_map.vlpi_lock); return ret; } @@ -1625,7 +1625,7 @@ static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) struct its_vlpi_map *map; int ret = 0; - mutex_lock(&its_dev->event_map.vlpi_lock); + raw_spin_lock(&its_dev->event_map.vlpi_lock); map = get_vlpi_map(d); @@ -1638,7 +1638,7 @@ static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) *info->map = *map; out: - mutex_unlock(&its_dev->event_map.vlpi_lock); + raw_spin_unlock(&its_dev->event_map.vlpi_lock); return ret; } @@ -1648,7 +1648,7 @@ static int its_vlpi_unmap(struct irq_data *d) u32 event = its_get_event_id(d); int ret = 0; - mutex_lock(&its_dev->event_map.vlpi_lock); + raw_spin_lock(&its_dev->event_map.vlpi_lock); if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { ret = -EINVAL; @@ -1678,7 +1678,7 @@ static int its_vlpi_unmap(struct irq_data *d) } out: - mutex_unlock(&its_dev->event_map.vlpi_lock); + raw_spin_unlock(&its_dev->event_map.vlpi_lock); return ret; } @@ -2743,7 +2743,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, dev->event_map.col_map = col_map; dev->event_map.lpi_base = lpi_base; dev->event_map.nr_lpis = nr_lpis; - mutex_init(&dev->event_map.vlpi_lock); + raw_spin_lock_init(&dev->event_map.vlpi_lock); dev->device_id = dev_id; INIT_LIST_HEAD(&dev->entry); -- Gitee From 284f2d1597812fd1b46b6058600213a595ba081e Mon Sep 17 00:00:00 2001 From: Heyi Guo Date: Sat, 30 Nov 2019 15:38:49 +0800 Subject: [PATCH 21/94] irqchip/gic-v4: Use Inner-Shareable attributes for virtual pending tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is no special reason to set virtual LPI pending table as non-shareable. If we choose to hard code the shareability without probing, Inner-Shareable is likely to be a better choice, as the VPEs can move around and benefit from having the redistributors snooping each other's cache, if that's something they can do. Furthermore, Hisilicon hip08 ends up with unspecified errors when mixing shareability attributes. So let's move to IS attributes for the VPT. This has also been tested on D05 and didn't show any regression. Signed-off-by: Heyi Guo [maz: rewrote commit message] Signed-off-by: Marc Zyngier Tested-by: Marc Zyngier Link: https://lore.kernel.org/r/20191130073849.38378-1-guoheyi@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 2 +- include/linux/irqchip/arm-gic-v3.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 46aad1f87b3b..22e36928e38b 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3118,7 +3118,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) val = virt_to_phys(page_address(vpe->vpt_page)) & GENMASK_ULL(51, 16); val |= GICR_VPENDBASER_RaWaWb; - val |= GICR_VPENDBASER_NonShareable; + val |= GICR_VPENDBASER_InnerShareable; /* * There is no good way of finding out if the pending table is * empty as we can race against the doorbell interrupt very diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index b6514e8893bf..fd6e44a30b70 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -289,6 +289,9 @@ #define GICR_VPENDBASER_NonShareable \ GIC_BASER_SHAREABILITY(GICR_VPENDBASER, NonShareable) +#define GICR_VPENDBASER_InnerShareable \ + GIC_BASER_SHAREABILITY(GICR_VPENDBASER, InnerShareable) + #define GICR_VPENDBASER_nCnB GIC_BASER_CACHEABILITY(GICR_VPENDBASER, INNER, nCnB) #define GICR_VPENDBASER_nC GIC_BASER_CACHEABILITY(GICR_VPENDBASER, INNER, nC) #define GICR_VPENDBASER_RaWt GIC_BASER_CACHEABILITY(GICR_VPENDBASER, INNER, RaWt) -- Gitee From 67b0c1f9c1554152a55f58605a149facf9a8f7ce Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:25 +0000 Subject: [PATCH 22/94] irqchip/gic-v3: Add GICv4.1 VPEID size discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While GICv4.0 mandates 16 bit worth of VPEIDs, GICv4.1 allows smaller implementations to be built. Add the required glue to dynamically compute the limit. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-3-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 11 ++++++++++- drivers/irqchip/irq-gic-v3.c | 3 +++ include/linux/irqchip/arm-gic-v3.h | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 22e36928e38b..243b5ec322b8 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -124,7 +124,16 @@ struct its_node { #define ITS_ITT_ALIGN SZ_256 /* The maximum number of VPEID bits supported by VLPI commands */ -#define ITS_MAX_VPEID_BITS (16) +#define ITS_MAX_VPEID_BITS \ + ({ \ + int nvpeid = 16; \ + if (gic_rdists->has_rvpeid && \ + gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \ + nvpeid = 1 + (gic_rdists->gicd_typer2 & \ + GICD_TYPER2_VID); \ + \ + nvpeid; \ + }) #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS)) /* Convert page order to size in bytes */ diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 77a130c03223..04374ce99589 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1542,6 +1542,9 @@ static int __init gic_init_bases(void __iomem *dist_base, pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); + + gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); + gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, &gic_data); irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index fd6e44a30b70..98094d775cb4 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -13,6 +13,7 @@ #define GICD_CTLR 0x0000 #define GICD_TYPER 0x0004 #define GICD_IIDR 0x0008 +#define GICD_TYPER2 0x000C #define GICD_STATUSR 0x0010 #define GICD_SETSPI_NSR 0x0040 #define GICD_CLRSPI_NSR 0x0048 @@ -89,6 +90,9 @@ #define GICD_TYPER_ESPIS(typer) \ (((typer) & GICD_TYPER_ESPI) ? GICD_TYPER_SPIS((typer) >> 27) : 0) +#define GICD_TYPER2_VIL (1U << 7) +#define GICD_TYPER2_VID GENMASK(4, 0) + #define GICD_IROUTER_SPI_MODE_ONE (0U << 31) #define GICD_IROUTER_SPI_MODE_ANY (1U << 31) -- Gitee From 4e218b268635599ce5a7743dd6eed0fadcd16adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Fri, 6 Dec 2024 15:53:19 +0800 Subject: [PATCH 23/94] resolve patch Add GICv4.1 VPEID size discovery conflict --- include/linux/irqchip/arm-gic-v3.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 98094d775cb4..024d55053f02 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -619,6 +619,7 @@ struct rdists { void *prop_table_va; u64 flags; u32 gicd_typer; + u32 gicd_typer2; bool has_vlpis; bool has_direct_lpi; }; -- Gitee From 8aa40603e2c7ba2f84f2e84e58212ced24717c5d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:24 +0000 Subject: [PATCH 24/94] irqchip/gic-v3: Detect GICv4.1 supporting RVPEID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GICv4.1 supports the RVPEID ("Residency per vPE ID"), which allows for a much efficient way of making virtual CPUs resident (to allow direct injection of interrupts). The functionnality needs to be discovered on each and every redistributor in the system, and disabled if the settings are inconsistent. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-2-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3.c | 21 ++++++++++++++++++--- include/linux/irqchip/arm-gic-v3.h | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 04374ce99589..dab249c54c7b 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -849,8 +849,21 @@ static int __gic_update_rdist_properties(struct redist_region *region, void __iomem *ptr) { u64 typer = gic_read_typer(ptr + GICR_TYPER); + gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); - gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS); + + /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ + gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); + gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | + gic_data.rdists.has_rvpeid); + + /* Detect non-sensical configurations */ + if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { + gic_data.rdists.has_direct_lpi = false; + gic_data.rdists.has_vlpis = false; + gic_data.rdists.has_rvpeid = false; + } + gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); return 1; @@ -863,9 +876,10 @@ static void gic_update_rdist_properties(void) if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) gic_data.ppi_nr = 0; pr_info("%d PPIs implemented\n", gic_data.ppi_nr); - pr_info("%sVLPI support, %sdirect LPI support\n", + pr_info("%sVLPI support, %sdirect LPI support, %sRVPEID support\n", !gic_data.rdists.has_vlpis ? "no " : "", - !gic_data.rdists.has_direct_lpi ? "no " : ""); + !gic_data.rdists.has_direct_lpi ? "no " : "", + !gic_data.rdists.has_rvpeid ? "no " : ""); } /* Check whether it's single security state view */ @@ -1549,6 +1563,7 @@ static int __init gic_init_bases(void __iomem *dist_base, &gic_data); irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); + gic_data.rdists.has_rvpeid = true; gic_data.rdists.has_vlpis = true; gic_data.rdists.has_direct_lpi = true; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 024d55053f02..d67e57cb128c 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -238,6 +238,7 @@ #define GICR_TYPER_VLPIS (1U << 1) #define GICR_TYPER_DirectLPIS (1U << 3) #define GICR_TYPER_LAST (1U << 4) +#define GICR_TYPER_RVPEID (1U << 7) #define GIC_V3_REDIST_SIZE 0x20000 -- Gitee From cbe1b7545ebfe375edf34d106a8db4bb3a22c34d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Fri, 6 Dec 2024 15:59:23 +0800 Subject: [PATCH 25/94] resolve patch Detect GICv4.1 supporting RVPEID conflict --- include/linux/irqchip/arm-gic-v3.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index d67e57cb128c..69beb4cfab58 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -622,6 +622,7 @@ struct rdists { u32 gicd_typer; u32 gicd_typer2; bool has_vlpis; + bool has_rvpeid; bool has_direct_lpi; }; -- Gitee From e6627f85e4fe100ac6969e6e871caf798161b4d1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:28 +0000 Subject: [PATCH 26/94] irqchip/gic-v4.1: VPE table (aka GICR_VPROPBASER) allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GICv4.1 defines a new VPE table that is potentially shared between both the ITSs and the redistributors, following complicated affinity rules. To make things more confusing, the programming of this table at the redistributor level is reusing the GICv4.0 GICR_VPROPBASER register for something completely different. The code flow is somewhat complexified by the need to respect the affinities required by the HW, meaning that tables can either be inherited from a previously discovered ITS or redistributor. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-6-maz@kernel.org Signed-off-by: 温家明 --- arch/arm/include/asm/arch_gicv3.h | 2 + arch/arm64/include/asm/arch_gicv3.h | 1 + drivers/irqchip/irq-gic-v3-its.c | 310 +++++++++++++++++++++++++++- include/linux/irqchip/arm-gic-v3.h | 33 ++- 4 files changed, 339 insertions(+), 7 deletions(-) diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index fa50bb04f580..b5752f0e8936 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -10,6 +10,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #include #include @@ -327,6 +328,7 @@ static inline u64 __gic_readq_nonatomic(const volatile void __iomem *addr) /* * GITS_VPROPBASER - hi and lo bits may be accessed independently. */ +#define gits_read_vpropbaser(c) __gic_readq_nonatomic(c) #define gits_write_vpropbaser(v, c) __gic_writeq_nonatomic(v, c) /* diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index ee9bdaa40532..43362ce1ad3d 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -141,6 +141,7 @@ static inline u32 gic_read_rpr(void) #define gicr_read_pendbaser(c) readq_relaxed(c) #define gits_write_vpropbaser(v, c) writeq_relaxed(v, c) +#define gits_read_vpropbaser(c) readq_relaxed(c) #define gits_write_vpendbaser(v, c) writeq_relaxed(v, c) #define gits_read_vpendbaser(c) readq_relaxed(c) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 243b5ec322b8..d8bf93d36100 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -109,6 +109,7 @@ struct its_node { u64 typer; u64 cbaser_save; u32 ctlr_save; + u32 mpidr; struct list_head its_device_list; u64 flags; unsigned long list_nr; @@ -119,6 +120,7 @@ struct its_node { }; #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) +#define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) #define ITS_ITT_ALIGN SZ_256 @@ -2229,6 +2231,65 @@ static bool its_parse_indirect_baser(struct its_node *its, return indirect; } +static u32 compute_common_aff(u64 val) +{ + u32 aff, clpiaff; + + aff = FIELD_GET(GICR_TYPER_AFFINITY, val); + clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val); + + return aff & ~(GENMASK(31, 0) >> (clpiaff * 8)); +} + +static u32 compute_its_aff(struct its_node *its) +{ + u64 val; + u32 svpet; + + /* + * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute + * the resulting affinity. We then use that to see if this match + * our own affinity. + */ + svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); + val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet); + val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); + return compute_common_aff(val); +} + +static struct its_node *find_sibling_its(struct its_node *cur_its) +{ + struct its_node *its; + u32 aff; + + if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer)) + return NULL; + + aff = compute_its_aff(cur_its); + + list_for_each_entry(its, &its_nodes, entry) { + u64 baser; + + if (!is_v4_1(its) || its == cur_its) + continue; + + if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) + continue; + + if (aff != compute_its_aff(its)) + continue; + + /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ + baser = its->tables[2].val; + if (!(baser & GITS_BASER_VALID)) + continue; + + return its; + } + + return NULL; +} + static void its_free_tables(struct its_node *its) { int i; @@ -2271,6 +2332,17 @@ static int its_alloc_tables(struct its_node *its) break; case GITS_BASER_TYPE_VCPU: + if (is_v4_1(its)) { + struct its_node *sibling; + + WARN_ON(i != 2); + if ((sibling = find_sibling_its(its))) { + *baser = sibling->tables[2]; + its_write_baser(its, baser, baser->val); + continue; + } + } + indirect = its_parse_indirect_baser(its, baser, psz, &order, ITS_MAX_VPEID_BITS); @@ -2297,6 +2369,220 @@ static int its_alloc_tables(struct its_node *its) return 0; } +static u64 inherit_vpe_l1_table_from_its(void) +{ + struct its_node *its; + u64 val; + u32 aff; + + val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); + aff = compute_common_aff(val); + + list_for_each_entry(its, &its_nodes, entry) { + u64 baser, addr; + + if (!is_v4_1(its)) + continue; + + if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) + continue; + + if (aff != compute_its_aff(its)) + continue; + + /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ + baser = its->tables[2].val; + if (!(baser & GITS_BASER_VALID)) + continue; + + /* We have a winner! */ + val = GICR_VPROPBASER_4_1_VALID; + if (baser & GITS_BASER_INDIRECT) + val |= GICR_VPROPBASER_4_1_INDIRECT; + val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, + FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)); + switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) { + case GIC_PAGE_SIZE_64K: + addr = GITS_BASER_ADDR_48_to_52(baser); + break; + default: + addr = baser & GENMASK_ULL(47, 12); + break; + } + val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12); + val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK, + FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser)); + val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK, + FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser)); + val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); + + return val; + } + + return 0; +} + +static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) +{ + u32 aff; + u64 val; + int cpu; + + val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); + aff = compute_common_aff(val); + + for_each_possible_cpu(cpu) { + void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; + u32 tmp; + + if (!base || cpu == smp_processor_id()) + continue; + + val = gic_read_typer(base + GICR_TYPER); + tmp = compute_common_aff(val); + if (tmp != aff) + continue; + + /* + * At this point, we have a victim. This particular CPU + * has already booted, and has an affinity that matches + * ours wrt CommonLPIAff. Let's use its own VPROPBASER. + * Make sure we don't write the Z bit in that case. + */ + val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); + val &= ~GICR_VPROPBASER_4_1_Z; + + *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; + + return val; + } + + return 0; +} + +static int allocate_vpe_l1_table(void) +{ + void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); + u64 val, gpsz, npg, pa; + unsigned int psz = SZ_64K; + unsigned int np, epp, esz; + struct page *page; + + if (!gic_rdists->has_rvpeid) + return 0; + + /* + * if VPENDBASER.Valid is set, disable any previously programmed + * VPE by setting PendingLast while clearing Valid. This has the + * effect of making sure no doorbell will be generated and we can + * then safely clear VPROPBASER.Valid. + */ + if (gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) + gits_write_vpendbaser(GICR_VPENDBASER_PendingLast, + vlpi_base + GICR_VPENDBASER); + + /* + * If we can inherit the configuration from another RD, let's do + * so. Otherwise, we have to go through the allocation process. We + * assume that all RDs have the exact same requirements, as + * nothing will work otherwise. + */ + val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask); + if (val & GICR_VPROPBASER_4_1_VALID) + goto out; + + gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_KERNEL); + if (!gic_data_rdist()->vpe_table_mask) + return -ENOMEM; + + val = inherit_vpe_l1_table_from_its(); + if (val & GICR_VPROPBASER_4_1_VALID) + goto out; + + /* First probe the page size */ + val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); + gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + val = gits_read_vpropbaser(vlpi_base + GICR_VPROPBASER); + gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); + esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); + + switch (gpsz) { + default: + gpsz = GIC_PAGE_SIZE_4K; + /* fall through */ + case GIC_PAGE_SIZE_4K: + psz = SZ_4K; + break; + case GIC_PAGE_SIZE_16K: + psz = SZ_16K; + break; + case GIC_PAGE_SIZE_64K: + psz = SZ_64K; + break; + } + + /* + * Start populating the register from scratch, including RO fields + * (which we want to print in debug cases...) + */ + val = 0; + val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz); + val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz); + + /* How many entries per GIC page? */ + esz++; + epp = psz / (esz * SZ_8); + + /* + * If we need more than just a single L1 page, flag the table + * as indirect and compute the number of required L1 pages. + */ + if (epp < ITS_MAX_VPEID) { + int nl2; + + val |= GICR_VPROPBASER_4_1_INDIRECT; + + /* Number of L2 pages required to cover the VPEID space */ + nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp); + + /* Number of L1 pages to point to the L2 pages */ + npg = DIV_ROUND_UP(nl2 * SZ_8, psz); + } else { + npg = 1; + } + + val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg); + + /* Right, that's the number of CPU pages we need for L1 */ + np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); + + pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", + np, npg, psz, epp, esz); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(np * PAGE_SIZE)); + if (!page) + return -ENOMEM; + + gic_data_rdist()->vpe_l1_page = page; + pa = virt_to_phys(page_address(page)); + WARN_ON(!IS_ALIGNED(pa, psz)); + + val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12); + val |= GICR_VPROPBASER_RaWb; + val |= GICR_VPROPBASER_InnerShareable; + val |= GICR_VPROPBASER_4_1_Z; + val |= GICR_VPROPBASER_4_1_VALID; + +out: + gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); + + pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", + smp_processor_id(), val, + cpumask_pr_args(gic_data_rdist()->vpe_table_mask)); + + return 0; +} + static int its_alloc_collections(struct its_node *its) { int i; @@ -2496,7 +2782,7 @@ static void its_cpu_init_lpis(void) val |= GICR_CTLR_ENABLE_LPIS; writel_relaxed(val, rbase + GICR_CTLR); - if (gic_rdists->has_vlpis) { + if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); /* @@ -2520,6 +2806,16 @@ static void its_cpu_init_lpis(void) WARN_ON(val & GICR_VPENDBASER_Dirty); } + if (allocate_vpe_l1_table()) { + /* + * If the allocation has failed, we're in massive trouble. + * Disable direct injection, and pray that no VM was + * already running... + */ + gic_rdists->has_rvpeid = false; + gic_rdists->has_vlpis = false; + } + /* Make sure the GIC has seen the above */ dsb(sy); out: @@ -3912,6 +4208,14 @@ static int __init its_probe_one(struct resource *res, } else { pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); } + + if (is_v4_1(its)) { + u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer); + its->mpidr = readl_relaxed(its_base + GITS_MPIDR); + + pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", + &res->start, its->mpidr, svpet); + } } its->numa_node = numa_node; @@ -4269,6 +4573,8 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, bool has_v4 = false; int err; + gic_rdists = rdists; + its_parent = parent_domain; of_node = to_of_node(handle); if (of_node) @@ -4281,8 +4587,6 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, return -ENXIO; } - gic_rdists = rdists; - err = allocate_lpi_tables(); if (err) return err; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 69beb4cfab58..e56b4331ddfe 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -102,6 +102,11 @@ #define GIC_V3_DIST_SIZE 0x10000 +#define GIC_PAGE_SIZE_4K 0ULL +#define GIC_PAGE_SIZE_16K 1ULL +#define GIC_PAGE_SIZE_64K 2ULL +#define GIC_PAGE_SIZE_MASK 3ULL + /* * Re-Distributor registers, offsets from RD_base */ @@ -239,6 +244,8 @@ #define GICR_TYPER_DirectLPIS (1U << 3) #define GICR_TYPER_LAST (1U << 4) #define GICR_TYPER_RVPEID (1U << 7) +#define GICR_TYPER_COMMON_LPI_AFF GENMASK_ULL(25, 24) +#define GICR_TYPER_AFFINITY GENMASK_ULL(63, 32) #define GIC_V3_REDIST_SIZE 0x20000 @@ -277,6 +284,18 @@ #define GICR_VPROPBASER_RaWaWt GIC_BASER_CACHEABILITY(GICR_VPROPBASER, INNER, RaWaWt) #define GICR_VPROPBASER_RaWaWb GIC_BASER_CACHEABILITY(GICR_VPROPBASER, INNER, RaWaWb) +/* + * GICv4.1 VPROPBASER reinvention. A subtle mix between the old + * VPROPBASER and ITS_BASER. Just not quite any of the two. + */ +#define GICR_VPROPBASER_4_1_VALID (1ULL << 63) +#define GICR_VPROPBASER_4_1_ENTRY_SIZE GENMASK_ULL(61, 59) +#define GICR_VPROPBASER_4_1_INDIRECT (1ULL << 55) +#define GICR_VPROPBASER_4_1_PAGE_SIZE GENMASK_ULL(54, 53) +#define GICR_VPROPBASER_4_1_Z (1ULL << 52) +#define GICR_VPROPBASER_4_1_ADDR GENMASK_ULL(51, 12) +#define GICR_VPROPBASER_4_1_SIZE GENMASK_ULL(6, 0) + #define GICR_VPENDBASER 0x0078 #define GICR_VPENDBASER_SHAREABILITY_SHIFT (10) @@ -317,6 +336,7 @@ #define GITS_CTLR 0x0000 #define GITS_IIDR 0x0004 #define GITS_TYPER 0x0008 +#define GITS_MPIDR 0x0018 #define GITS_CBASER 0x0080 #define GITS_CWRITER 0x0088 #define GITS_CREADR 0x0090 @@ -350,6 +370,8 @@ #define GITS_TYPER_HCC_SHIFT 24 #define GITS_TYPER_HCC(r) (((r) >> GITS_TYPER_HCC_SHIFT) & 0xff) #define GITS_TYPER_VMOVP (1ULL << 37) +#define GITS_TYPER_VMAPP (1ULL << 40) +#define GITS_TYPER_SVPET GENMASK_ULL(42, 41) #define GITS_IIDR_REV_SHIFT 12 #define GITS_IIDR_REV_MASK (0xf << GITS_IIDR_REV_SHIFT) @@ -420,10 +442,11 @@ #define GITS_BASER_InnerShareable \ GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable) #define GITS_BASER_PAGE_SIZE_SHIFT (8) -#define GITS_BASER_PAGE_SIZE_4K (0ULL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_16K (1ULL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_64K (2ULL << GITS_BASER_PAGE_SIZE_SHIFT) -#define GITS_BASER_PAGE_SIZE_MASK (3ULL << GITS_BASER_PAGE_SIZE_SHIFT) +#define __GITS_BASER_PSZ(sz) (GIC_PAGE_SIZE_ ## sz << GITS_BASER_PAGE_SIZE_SHIFT) +#define GITS_BASER_PAGE_SIZE_4K __GITS_BASER_PSZ(4K) +#define GITS_BASER_PAGE_SIZE_16K __GITS_BASER_PSZ(16K) +#define GITS_BASER_PAGE_SIZE_64K __GITS_BASER_PSZ(64K) +#define GITS_BASER_PAGE_SIZE_MASK __GITS_BASER_PSZ(MASK) #define GITS_BASER_PAGES_MAX 256 #define GITS_BASER_PAGES_SHIFT (0) #define GITS_BASER_NR_PAGES(r) (((r) & 0xff) + 1) @@ -613,8 +636,10 @@ struct rdists { struct { void __iomem *rd_base; struct page *pend_page; + struct page *vpe_l1_page; phys_addr_t phys_base; bool lpi_enabled; + cpumask_t *vpe_table_mask; } __percpu *rdist; phys_addr_t prop_table_pa; void *prop_table_va; -- Gitee From 31c4fd8d927ff6f8ea3c12d267cecc581daf6de2 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Tue, 30 Jun 2020 21:37:46 +0800 Subject: [PATCH 27/94] irqchip/gic-v4.1: Use GFP_ATOMIC flag in allocate_vpe_l1_table() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Booting the latest kernel with DEBUG_ATOMIC_SLEEP=y on a GICv4.1 enabled box, I get the following kernel splat: [ 0.053766] BUG: sleeping function called from invalid context at mm/slab.h:567 [ 0.053767] in_atomic(): 1, irqs_disabled(): 128, non_block: 0, pid: 0, name: swapper/1 [ 0.053769] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 5.8.0-rc3+ #23 [ 0.053770] Call trace: [ 0.053774] dump_backtrace+0x0/0x218 [ 0.053775] show_stack+0x2c/0x38 [ 0.053777] dump_stack+0xc4/0x10c [ 0.053779] ___might_sleep+0xfc/0x140 [ 0.053780] __might_sleep+0x58/0x90 [ 0.053782] slab_pre_alloc_hook+0x7c/0x90 [ 0.053783] kmem_cache_alloc_trace+0x60/0x2f0 [ 0.053785] its_cpu_init+0x6f4/0xe40 [ 0.053786] gic_starting_cpu+0x24/0x38 [ 0.053788] cpuhp_invoke_callback+0xa0/0x710 [ 0.053789] notify_cpu_starting+0xcc/0xd8 [ 0.053790] secondary_start_kernel+0x148/0x200 # ./scripts/faddr2line vmlinux its_cpu_init+0x6f4/0xe40 its_cpu_init+0x6f4/0xe40: allocate_vpe_l1_table at drivers/irqchip/irq-gic-v3-its.c:2818 (inlined by) its_cpu_init_lpis at drivers/irqchip/irq-gic-v3-its.c:3138 (inlined by) its_cpu_init at drivers/irqchip/irq-gic-v3-its.c:5166 It turned out that we're allocating memory using GFP_KERNEL (may sleep) within the CPU hotplug notifier, which is indeed an atomic context. Bad thing may happen if we're playing on a system with more than a single CommonLPIAff group. Avoid it by turning this into an atomic allocation. Fixes: 5e5168461c22 ("irqchip/gic-v4.1: VPE table (aka GICR_VPROPBASER) allocation") Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200630133746.816-1-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index d8bf93d36100..684ca668f5a7 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2491,7 +2491,7 @@ static int allocate_vpe_l1_table(void) if (val & GICR_VPROPBASER_4_1_VALID) goto out; - gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_KERNEL); + gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC); if (!gic_data_rdist()->vpe_table_mask) return -ENOMEM; @@ -2558,7 +2558,7 @@ static int allocate_vpe_l1_table(void) pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", np, npg, psz, epp, esz); - page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(np * PAGE_SIZE)); + page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); if (!page) return -ENOMEM; -- Gitee From 93fc4d0d8d50f419b0431e23e3a53f8771332ab4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:29 +0000 Subject: [PATCH 28/94] irqchip/gic-v4.1: Implement the v4.1 flavour of VMAPP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ITS VMAPP command gains some new fields with GICv4.1: - a default doorbell, which allows a single doorbell to be used for all the VLPIs routed to a given VPE - a pointer to the configuration table (instead of having it in a register that gets context switched) - a flag indicating whether this is the first map or the last unmap for this particular VPE - a flag indicating whether the pending table is known to be zeroed, or not Plumb in the new fields in the VMAPP builder, and add the map/unmap refcounting so that the ITS can do the right thing. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-7-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 60 +++++++++++++++++++++++++++--- include/linux/irqchip/arm-gic-v4.h | 18 +++++++-- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 684ca668f5a7..6784dcee0ea5 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -456,6 +456,27 @@ static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); } +static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa) +{ + its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16); +} + +static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc) +{ + its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8); +} + +static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz) +{ + its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9); +} + +static void its_encode_vmapp_default_db(struct its_cmd_block *cmd, + u32 vpe_db_lpi) +{ + its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); +} + static inline void its_fixup_cmd(struct its_cmd_block *cmd) { /* Let's fixup BE commands */ @@ -639,19 +660,45 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, struct its_cmd_block *cmd, struct its_cmd_desc *desc) { - unsigned long vpt_addr; + unsigned long vpt_addr, vconf_addr; u64 target; - - vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); - target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; + bool alloc; its_encode_cmd(cmd, GITS_CMD_VMAPP); its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); its_encode_valid(cmd, desc->its_vmapp_cmd.valid); + + if (!desc->its_vmapp_cmd.valid) { + if (is_v4_1(its)) { + alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); + its_encode_alloc(cmd, alloc); + } + + goto out; + } + + vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); + target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; + its_encode_target(cmd, target); its_encode_vpt_addr(cmd, vpt_addr); its_encode_vpt_size(cmd, LPI_NRBITS - 1); + if (!is_v4_1(its)) + goto out; + + vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); + + alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); + + its_encode_alloc(cmd, alloc); + + /* We can only signal PTZ when alloc==1. Why do we have two bits? */ + its_encode_ptz(cmd, alloc); + its_encode_vconf_addr(cmd, vconf_addr); + its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi); + +out: its_fixup_cmd(cmd); return valid_vpe(its, desc->its_vmapp_cmd.vpe); @@ -3627,7 +3674,10 @@ static int its_vpe_init(struct its_vpe *vpe) vpe->vpe_id = vpe_id; vpe->vpt_page = vpt_page; - vpe->vpe_proxy_event = -1; + if (gic_rdists->has_rvpeid) + atomic_set(&vpe->vmapp_count, 0); + else + vpe->vpe_proxy_event = -1; return 0; } diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 5dbcfc65f21e..498e523085a7 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -39,8 +39,20 @@ struct its_vpe { irq_hw_number_t vpe_db_lpi; /* VPE resident */ bool resident; - /* VPE proxy mapping */ - int vpe_proxy_event; + union { + /* GICv4.0 implementations */ + struct { + /* VPE proxy mapping */ + int vpe_proxy_event; + /* Implementation Defined Area Invalid */ + bool idai; + }; + /* GICv4.1 implementations */ + struct { + atomic_t vmapp_count; + }; + }; + /* * This collection ID is used to indirect the target * redistributor for this VPE. The ID itself isn't involved in @@ -49,8 +61,6 @@ struct its_vpe { u16 col_idx; /* Unique (system-wide) VPE identifier */ u16 vpe_id; - /* Implementation Defined Area Invalid */ - bool idai; /* Pending VLPIs on schedule out? */ bool pending_last; }; -- Gitee From e16b598de5392c55bff6a7ff5fc724be64e6e950 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:30 +0000 Subject: [PATCH 29/94] irqchip/gic-v4.1: Don't use the VPE proxy if RVPEID is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The infamous VPE proxy device isn't used with GICv4.1 because: - we can invalidate any LPI from the DirectLPI MMIO interface - the ITS and redistributors understand the life cycle of the doorbell, so we don't need to enable/disable it all the time So let's escape early from the proxy related functions. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-8-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 6784dcee0ea5..3b4a975449fd 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3341,7 +3341,7 @@ static const struct irq_domain_ops its_domain_ops = { /* * This is insane. * - * If a GICv4 doesn't implement Direct LPIs (which is extremely + * If a GICv4.0 doesn't implement Direct LPIs (which is extremely * likely), the only way to perform an invalidate is to use a fake * device to issue an INV command, implying that the LPI has first * been mapped to some event on that device. Since this is not exactly @@ -3349,9 +3349,20 @@ static const struct irq_domain_ops its_domain_ops = { * only issue an UNMAP if we're short on available slots. * * Broken by design(tm). + * + * GICv4.1, on the other hand, mandates that we're able to invalidate + * by writing to a MMIO register. It doesn't implement the whole of + * DirectLPI, but that's good enough. And most of the time, we don't + * even have to invalidate anything, as the redistributor can be told + * whether to generate a doorbell or not (we thus leave it enabled, + * always). */ static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe) { + /* GICv4.1 doesn't use a proxy, so nothing to do here */ + if (gic_rdists->has_rvpeid) + return; + /* Already unmapped? */ if (vpe->vpe_proxy_event == -1) return; @@ -3374,6 +3385,10 @@ static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe) static void its_vpe_db_proxy_unmap(struct its_vpe *vpe) { + /* GICv4.1 doesn't use a proxy, so nothing to do here */ + if (gic_rdists->has_rvpeid) + return; + if (!gic_rdists->has_direct_lpi) { unsigned long flags; @@ -3385,6 +3400,10 @@ static void its_vpe_db_proxy_unmap(struct its_vpe *vpe) static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe) { + /* GICv4.1 doesn't use a proxy, so nothing to do here */ + if (gic_rdists->has_rvpeid) + return; + /* Already mapped? */ if (vpe->vpe_proxy_event != -1) return; @@ -3407,6 +3426,10 @@ static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) unsigned long flags; struct its_collection *target_col; + /* GICv4.1 doesn't use a proxy, so nothing to do here */ + if (gic_rdists->has_rvpeid) + return; + if (gic_rdists->has_direct_lpi) { void __iomem *rdbase; -- Gitee From 953b5a87e3ff3f909d9769ffbe95edfe4834bed2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:31 +0000 Subject: [PATCH 30/94] irqchip/gic-v4.1: Implement the v4.1 flavour of VMOVP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With GICv4.1, VMOVP is extended to allow a default doorbell to be specified, as well as a validity bit for this doorbell. As an added bonus, VMOVP isn't required anymore of moving a VPE between redistributors that share the same affinity. Let's add this support to the VMOVP builder, and make sure we don't issue the command if we don't really need to. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-9-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 40 ++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 3b4a975449fd..e0e4f1b15b70 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -477,6 +477,17 @@ static void its_encode_vmapp_default_db(struct its_cmd_block *cmd, its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); } +static void its_encode_vmovp_default_db(struct its_cmd_block *cmd, + u32 vpe_db_lpi) +{ + its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0); +} + +static void its_encode_db(struct its_cmd_block *cmd, bool db) +{ + its_mask_encode(&cmd->raw_cmd[2], db, 63, 63); +} + static inline void its_fixup_cmd(struct its_cmd_block *cmd) { /* Let's fixup BE commands */ @@ -763,6 +774,11 @@ static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); its_encode_target(cmd, target); + if (is_v4_1(its)) { + its_encode_db(cmd, true); + its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi); + } + its_fixup_cmd(cmd); return valid_vpe(its, desc->its_vmovp_cmd.vpe); @@ -3456,7 +3472,7 @@ static int its_vpe_set_affinity(struct irq_data *d, bool force) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); - int cpu = cpumask_first(mask_val); + int from, cpu = cpumask_first(mask_val); /* * Changing affinity is mega expensive, so let's be as lazy as @@ -3464,14 +3480,24 @@ static int its_vpe_set_affinity(struct irq_data *d, * into the proxy device, we need to move the doorbell * interrupt to its new location. */ - if (vpe->col_idx != cpu) { - int from = vpe->col_idx; + if (vpe->col_idx == cpu) + goto out; - vpe->col_idx = cpu; - its_send_vmovp(vpe); - its_vpe_db_proxy_move(vpe, from, cpu); - } + from = vpe->col_idx; + vpe->col_idx = cpu; + + /* + * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD + * is sharing its VPE table with the current one. + */ + if (gic_data_rdist_cpu(cpu)->vpe_table_mask && + cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask)) + goto out; + its_send_vmovp(vpe); + its_vpe_db_proxy_move(vpe, from, cpu); + +out: irq_data_update_effective_affinity(d, cpumask_of(cpu)); return IRQ_SET_MASK_OK_DONE; -- Gitee From 8bbfb463fd22861611304a36ce181e19ff9d3bdc Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:32 +0000 Subject: [PATCH 31/94] irqchip/gic-v4.1: Plumb skeletal VPE irqchip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just like for GICv4.0, each VPE has its own doorbell interrupt, and thus an irqchip that manages them. Since the doorbell management is quite different on GICv4.1, let's introduce an almost empty irqchip the will get populated over the next new patches. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-10-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e0e4f1b15b70..810a9c81b6cb 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3688,6 +3688,32 @@ static struct irq_chip its_vpe_irq_chip = { .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, }; +static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) +{ + struct its_cmd_info *info = vcpu_info; + + switch (info->cmd_type) { + case SCHEDULE_VPE: + return 0; + + case DESCHEDULE_VPE: + return 0; + + case INVALL_VPE: + return 0; + + default: + return -EINVAL; + } +} + +static struct irq_chip its_vpe_4_1_irq_chip = { + .name = "GICv4.1-vpe", + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = its_vpe_set_affinity, + .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, +}; + static int its_vpe_id_alloc(void) { return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); @@ -3768,6 +3794,7 @@ static void its_vpe_irq_domain_free(struct irq_domain *domain, static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *args) { + struct irq_chip *irqchip = &its_vpe_irq_chip; struct its_vm *vm = args; unsigned long *bitmap; struct page *vprop_page; @@ -3795,6 +3822,9 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq vm->nr_db_lpis = nr_ids; vm->vprop_page = vprop_page; + if (gic_rdists->has_rvpeid) + irqchip = &its_vpe_4_1_irq_chip; + for (i = 0; i < nr_irqs; i++) { vm->vpes[i]->vpe_db_lpi = base + i; err = its_vpe_init(vm->vpes[i]); @@ -3805,7 +3835,7 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq if (err) break; irq_domain_set_hwirq_and_chip(domain, virq + i, i, - &its_vpe_irq_chip, vm->vpes[i]); + irqchip, vm->vpes[i]); set_bit(i, bitmap); } -- Gitee From 7b0eb8eb3f3e0764f627441285c2a33836ec6665 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:33 +0000 Subject: [PATCH 32/94] irqchip/gic-v4.1: Add mask/unmask doorbell callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit masking/unmasking doorbells on GICv4.1 relies on a new INVDB command, which broadcasts the invalidation to all RDs. Implement the new command as well as the masking callbacks, and plug the whole thing into the v4.1 VPE irqchip. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-11-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 73 ++++++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 3 +- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 810a9c81b6cb..df6ea579a639 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -343,6 +343,10 @@ struct its_cmd_desc { u16 seq_num; u16 its_list; } its_vmovp_cmd; + + struct { + struct its_vpe *vpe; + } its_invdb_cmd; }; }; @@ -838,6 +842,21 @@ static struct its_vpe *its_build_vclear_cmd(struct its_node *its, return valid_vpe(its, map->vpe); } +static struct its_vpe *its_build_invdb_cmd(struct its_node *its, + struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + if (WARN_ON(!is_v4_1(its))) + return NULL; + + its_encode_cmd(cmd, GITS_CMD_INVDB); + its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id); + + its_fixup_cmd(cmd); + + return valid_vpe(its, desc->its_invdb_cmd.vpe); +} + static u64 its_cmd_ptr_to_offset(struct its_node *its, struct its_cmd_block *ptr) { @@ -1246,6 +1265,14 @@ static void its_send_vclear(struct its_device *dev, u32 event_id) its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); } +static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) +{ + struct its_cmd_desc desc; + + desc.its_invdb_cmd.vpe = vpe; + its_send_single_vcommand(its, its_build_invdb_cmd, &desc); +} + /* * irqchip functions - assumes MSI, mostly. */ @@ -3688,6 +3715,50 @@ static struct irq_chip its_vpe_irq_chip = { .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, }; +static struct its_node *find_4_1_its(void) +{ + static struct its_node *its = NULL; + + if (!its) { + list_for_each_entry(its, &its_nodes, entry) { + if (is_v4_1(its)) + return its; + } + + /* Oops? */ + its = NULL; + } + + return its; +} + +static void its_vpe_4_1_send_inv(struct irq_data *d) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + struct its_node *its; + + /* + * GICv4.1 wants doorbells to be invalidated using the + * INVDB command in order to be broadcast to all RDs. Send + * it to the first valid ITS, and let the HW do its magic. + */ + its = find_4_1_its(); + if (its) + its_send_invdb(its, vpe); +} + +static void its_vpe_4_1_mask_irq(struct irq_data *d) +{ + lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); + its_vpe_4_1_send_inv(d); +} + +static void its_vpe_4_1_unmask_irq(struct irq_data *d) +{ + lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); + its_vpe_4_1_send_inv(d); +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { struct its_cmd_info *info = vcpu_info; @@ -3709,6 +3780,8 @@ static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) static struct irq_chip its_vpe_4_1_irq_chip = { .name = "GICv4.1-vpe", + .irq_mask = its_vpe_4_1_mask_irq, + .irq_unmask = its_vpe_4_1_unmask_irq, .irq_eoi = irq_chip_eoi_parent, .irq_set_affinity = its_vpe_set_affinity, .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index e56b4331ddfe..e43a8fa75eba 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -487,8 +487,9 @@ #define GITS_CMD_VMAPTI GITS_CMD_GICv4(GITS_CMD_MAPTI) #define GITS_CMD_VMOVI GITS_CMD_GICv4(GITS_CMD_MOVI) #define GITS_CMD_VSYNC GITS_CMD_GICv4(GITS_CMD_SYNC) -/* VMOVP is the odd one, as it doesn't have a physical counterpart */ +/* VMOVP and INVDB are the odd ones, as they dont have a physical counterpart */ #define GITS_CMD_VMOVP GITS_CMD_GICv4(2) +#define GITS_CMD_INVDB GITS_CMD_GICv4(0xe) /* * ITS error numbers -- Gitee From 9958489086b347968c0d608d64c7829e479950d7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:34 +0000 Subject: [PATCH 33/94] irqchip/gic-v4.1: Add VPE residency callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Making a VPE resident on GICv4.1 is pretty simple, as it is just a single write to the local redistributor. We just need extra information about which groups to enable, which the KVM code will have to provide. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-12-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 17 +++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 9 +++++++++ include/linux/irqchip/arm-gic-v4.h | 5 +++++ 3 files changed, 31 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index df6ea579a639..a03d3601d016 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3759,12 +3759,29 @@ static void its_vpe_4_1_unmask_irq(struct irq_data *d) its_vpe_4_1_send_inv(d); } +static void its_vpe_4_1_schedule(struct its_vpe *vpe, + struct its_cmd_info *info) +{ + void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); + u64 val = 0; + + /* Schedule the VPE */ + val |= GICR_VPENDBASER_Valid; + val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0; + val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; + val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); + + gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); struct its_cmd_info *info = vcpu_info; switch (info->cmd_type) { case SCHEDULE_VPE: + its_vpe_4_1_schedule(vpe, info); return 0; case DESCHEDULE_VPE: diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index e43a8fa75eba..1f2c542e2848 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -330,6 +330,15 @@ #define GICR_VPENDBASER_IDAI (1ULL << 62) #define GICR_VPENDBASER_Valid (1ULL << 63) +/* + * GICv4.1 VPENDBASER, used for VPE residency. On top of these fields, + * also use the above Valid, PendingLast and Dirty. + */ +#define GICR_VPENDBASER_4_1_DB (1ULL << 62) +#define GICR_VPENDBASER_4_1_VGRP0EN (1ULL << 59) +#define GICR_VPENDBASER_4_1_VGRP1EN (1ULL << 58) +#define GICR_VPENDBASER_4_1_VPEID GENMASK_ULL(15, 0) + /* * ITS registers, offsets from ITS_base */ diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 498e523085a7..d9c34968467a 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -100,6 +100,11 @@ struct its_cmd_info { union { struct its_vlpi_map *map; u8 config; + bool req_db; + struct { + bool g0en; + bool g1en; + }; }; }; -- Gitee From 67164a1f1bb686f12a9a412a7aa46ba84814efdd Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:35 +0000 Subject: [PATCH 34/94] irqchip/gic-v4.1: Add VPE eviction callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When descheduling a VPE, special care must be taken to tell the GIC about whether we want to receive a doorbell or not. This is a major improvement on GICv4.0, where the doorbell had to be separately enabled/disabled. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-13-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 53 +++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index a03d3601d016..9987995b061f 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2764,7 +2764,7 @@ static int __init allocate_lpi_tables(void) return 0; } -static u64 its_clear_vpend_valid(void __iomem *vlpi_base) +static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) { u32 count = 1000000; /* 1s! */ bool clean; @@ -2772,6 +2772,8 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base) val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); val &= ~GICR_VPENDBASER_Valid; + val &= ~clr; + val |= set; gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); do { @@ -2784,6 +2786,11 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base) } } while (!clean && count); + if (unlikely(val & GICR_VPENDBASER_Dirty)) { + pr_err_ratelimited("ITS virtual pending table not cleaning\n"); + val |= GICR_VPENDBASER_PendingLast; + } + return val; } @@ -2892,7 +2899,7 @@ static void its_cpu_init_lpis(void) * ancient programming gets left in and has possibility of * corrupting memory. */ - val = its_clear_vpend_valid(vlpi_base); + val = its_clear_vpend_valid(vlpi_base, 0, 0); WARN_ON(val & GICR_VPENDBASER_Dirty); } @@ -3567,16 +3574,10 @@ static void its_vpe_deschedule(struct its_vpe *vpe) void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); u64 val; - val = its_clear_vpend_valid(vlpi_base); + val = its_clear_vpend_valid(vlpi_base, 0, 0); - if (unlikely(val & GICR_VPENDBASER_Dirty)) { - pr_err_ratelimited("ITS virtual pending table not cleaning\n"); - vpe->idai = false; - vpe->pending_last = true; - } else { - vpe->idai = !!(val & GICR_VPENDBASER_IDAI); - vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); - } + vpe->idai = !!(val & GICR_VPENDBASER_IDAI); + vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); } static void its_vpe_invall(struct its_vpe *vpe) @@ -3774,6 +3775,35 @@ static void its_vpe_4_1_schedule(struct its_vpe *vpe, gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); } +static void its_vpe_4_1_deschedule(struct its_vpe *vpe, + struct its_cmd_info *info) +{ + void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); + u64 val; + + if (info->req_db) { + /* + * vPE is going to block: make the vPE non-resident with + * PendingLast clear and DB set. The GIC guarantees that if + * we read-back PendingLast clear, then a doorbell will be + * delivered when an interrupt comes. + */ + val = its_clear_vpend_valid(vlpi_base, + GICR_VPENDBASER_PendingLast, + GICR_VPENDBASER_4_1_DB); + vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); + } else { + /* + * We're not blocking, so just make the vPE non-resident + * with PendingLast set, indicating that we'll be back. + */ + val = its_clear_vpend_valid(vlpi_base, + 0, + GICR_VPENDBASER_PendingLast); + vpe->pending_last = true; + } +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); @@ -3785,6 +3815,7 @@ static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) return 0; case DESCHEDULE_VPE: + its_vpe_4_1_deschedule(vpe, info); return 0; case INVALL_VPE: -- Gitee From 6d1bd58d34d43bb6f576a61199b4c58edfb44df1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:36 +0000 Subject: [PATCH 35/94] irqchip/gic-v4.1: Add VPE INVALL callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GICv4.1 redistributors have a VPE-aware INVALL register. Progress! We can now emulate a guest-requested INVALL without emiting a VINVALL command. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-14-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 14 ++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 9987995b061f..7c24fd7152ff 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3804,6 +3804,19 @@ static void its_vpe_4_1_deschedule(struct its_vpe *vpe, } } +static void its_vpe_4_1_invall(struct its_vpe *vpe) +{ + void __iomem *rdbase; + u64 val; + + val = GICR_INVALLR_V; + val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); + + /* Target the redistributor this vPE is currently known on */ + rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; + gic_write_lpir(val, rdbase + GICR_INVALLR); +} + static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); @@ -3819,6 +3832,7 @@ static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) return 0; case INVALL_VPE: + its_vpe_4_1_invall(vpe); return 0; default: diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 1f2c542e2848..d7b22d07316b 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -247,6 +247,12 @@ #define GICR_TYPER_COMMON_LPI_AFF GENMASK_ULL(25, 24) #define GICR_TYPER_AFFINITY GENMASK_ULL(63, 32) +#define GICR_INVLPIR_VPEID GENMASK_ULL(47, 32) +#define GICR_INVLPIR_V GENMASK_ULL(63, 63) + +#define GICR_INVALLR_VPEID GICR_INVLPIR_VPEID +#define GICR_INVALLR_V GICR_INVLPIR_V + #define GIC_V3_REDIST_SIZE 0x20000 #define LPI_PROP_GROUP1 (1 << 1) -- Gitee From 9110af3f0dbd273e074b30c0299bf66139f1a7bb Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:37 +0000 Subject: [PATCH 36/94] irqchip/gic-v4.1: Suppress per-VLPI doorbell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since GICv4.1 gives us a per-VPE doorbell, avoid programming anything else on VMOVI/VMAPI/VMAPTI and on any other action that would have otherwise resulted in a per-VLPI doorbell to be programmed. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-15-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 7c24fd7152ff..20f648ada50e 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -725,7 +725,7 @@ static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, { u32 db; - if (desc->its_vmapti_cmd.db_enabled) + if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled) db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; else db = 1023; @@ -748,7 +748,7 @@ static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, { u32 db; - if (desc->its_vmovi_cmd.db_enabled) + if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled) db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; else db = 1023; @@ -1360,6 +1360,13 @@ static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) u32 event = its_get_event_id(d); struct its_vlpi_map *map; + /* + * GICv4.1 does away with the per-LPI nonsense, nothing to do + * here. + */ + if (is_v4_1(its_dev->its)) + return; + map = dev_event_to_vlpi_map(its_dev, event); if (map->db_enabled == enable) -- Gitee From 9c9f74b1dfb6a5ffe0d73d03655e284eef6531c4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 24 Dec 2019 11:10:38 +0000 Subject: [PATCH 37/94] irqchip/gic-v4.1: Allow direct invalidation of VLPIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just like for INVALL, GICv4.1 has grown a VPE-aware INVLPI register. Let's plumb it in and make use of the DirectLPI code in that case. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20191224111055.11836-16-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 55 +++++++++++++++++++----------- include/linux/irqchip/arm-gic-v3.h | 1 + 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 20f648ada50e..3b0485058b03 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -237,11 +237,27 @@ static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev, return &its_dev->event_map.vlpi_maps[event]; } -static struct its_collection *irq_to_col(struct irq_data *d) +static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) +{ + if (irqd_is_forwarded_to_vcpu(d)) { + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + return dev_event_to_vlpi_map(its_dev, event); + } + + return NULL; +} + +static int irq_to_cpuid(struct irq_data *d) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_vlpi_map *map = get_vlpi_map(d); + + if (map) + return map->vpe->col_idx; - return dev_event_to_col(its_dev, its_get_event_id(d)); + return its_dev->event_map.col_map[its_get_event_id(d)]; } static struct its_collection *valid_col(struct its_collection *col) @@ -1276,18 +1292,6 @@ static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) /* * irqchip functions - assumes MSI, mostly. */ -static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) -{ - if (irqd_is_forwarded_to_vcpu(d)) { - struct its_device *its_dev = irq_data_get_irq_chip_data(d); - u32 event = its_get_event_id(d); - - return dev_event_to_vlpi_map(its_dev, event); - } - - return NULL; -} - static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) { struct its_vlpi_map *map = get_vlpi_map(d); @@ -1330,13 +1334,25 @@ static void wait_for_syncr(void __iomem *rdbase) static void direct_lpi_inv(struct irq_data *d) { - struct its_collection *col; + struct its_vlpi_map *map = get_vlpi_map(d); void __iomem *rdbase; + u64 val; + + if (map) { + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + + WARN_ON(!is_v4_1(its_dev->its)); + + val = GICR_INVLPIR_V; + val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id); + val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid); + } else { + val = d->hwirq; + } /* Target the redistributor this LPI is currently routed to */ - col = irq_to_col(d); - rdbase = per_cpu_ptr(gic_rdists->rdist, col->col_id)->rd_base; - gic_write_lpir(d->hwirq, rdbase + GICR_INVLPIR); + rdbase = per_cpu_ptr(gic_rdists->rdist, irq_to_cpuid(d))->rd_base; + gic_write_lpir(val, rdbase + GICR_INVLPIR); wait_for_syncr(rdbase); } @@ -1346,7 +1362,8 @@ static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) struct its_device *its_dev = irq_data_get_irq_chip_data(d); lpi_write_config(d, clr, set); - if (gic_rdists->has_direct_lpi && !irqd_is_forwarded_to_vcpu(d)) + if (gic_rdists->has_direct_lpi && + (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) direct_lpi_inv(d); else if (!irqd_is_forwarded_to_vcpu(d)) its_send_inv(its_dev, its_get_event_id(d)); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index d7b22d07316b..45ecf7316ac1 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -247,6 +247,7 @@ #define GICR_TYPER_COMMON_LPI_AFF GENMASK_ULL(25, 24) #define GICR_TYPER_AFFINITY GENMASK_ULL(63, 32) +#define GICR_INVLPIR_INTID GENMASK_ULL(31, 0) #define GICR_INVLPIR_VPEID GENMASK_ULL(47, 32) #define GICR_INVLPIR_V GENMASK_ULL(63, 63) -- Gitee From e1ac71b0d1f9237551c667e867734c6221d42e9a Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Tue, 14 Jan 2020 19:22:12 +0800 Subject: [PATCH 38/94] KVM: arm/arm64: vgic-its: Properly check the unmapped coll in DISCARD handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discard is supposed to fail if the collection is not mapped to any target redistributor. We currently check if the collection is mapped by "ite->collection" but this is incomplete (e.g., mapping a LPI to an unmapped collection also results in a non NULL ite->collection). What actually needs to be checked is its_is_collection_mapped(), let's turn to it. Also take this chance to remove an extra blank line. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200114112212.1411-1-yuzenghui@huawei.com Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-its.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c index 5c3eea13e4c0..a72f584254d0 100644 --- a/virt/kvm/arm/vgic/vgic-its.c +++ b/virt/kvm/arm/vgic/vgic-its.c @@ -846,9 +846,8 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its, u32 event_id = its_cmd_get_id(its_cmd); struct its_ite *ite; - ite = find_ite(its, device_id, event_id); - if (ite && ite->collection) { + if (ite && its_is_collection_mapped(ite->collection)) { /* * Though the spec talks about removing the pending state, we * don't bother here since we clear the ITTE anyway and the -- Gitee From 6e174ae01a2056638e9d156b157b35a86aea4d90 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Thu, 6 Feb 2020 15:57:06 +0800 Subject: [PATCH 39/94] irqchip/gic-v4.1: Fix programming of GICR_VPROPBASER_4_1_SIZE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Size field of GICv4.1 VPROPBASER register indicates number of pages minus one and together Page_Size and Size control the vPEID width. Let's respect this requirement of the architecture. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200206075711.1275-2-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 3b0485058b03..02c36c6fedb9 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2665,7 +2665,7 @@ static int allocate_vpe_l1_table(void) npg = 1; } - val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg); + val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1); /* Right, that's the number of CPU pages we need for L1 */ np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); -- Gitee From ab58512bf7800e72b8c44efee09de8913229a75a Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Thu, 6 Feb 2020 15:57:07 +0800 Subject: [PATCH 40/94] irqchip/gic-v4.1: Set vpe_l1_base for all redistributors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, we will not set vpe_l1_page for the current RD if we can inherit the vPE configuration table from another RD (or ITS), which results in an inconsistency between RDs within the same CommonLPIAff group. Let's rename it to vpe_l1_base to indicate the base address of the vPE configuration table of this RD, and set it properly for *all* v4.1 redistributors. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200206075711.1275-3-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 5 ++++- include/linux/irqchip/arm-gic-v3.h | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 02c36c6fedb9..099d9bb665ff 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2510,6 +2510,8 @@ static u64 inherit_vpe_l1_table_from_its(void) continue; /* We have a winner! */ + gic_data_rdist()->vpe_l1_base = its->tables[2].base; + val = GICR_VPROPBASER_4_1_VALID; if (baser & GITS_BASER_INDIRECT) val |= GICR_VPROPBASER_4_1_INDIRECT; @@ -2566,6 +2568,7 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); val &= ~GICR_VPROPBASER_4_1_Z; + gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; return val; @@ -2676,7 +2679,7 @@ static int allocate_vpe_l1_table(void) if (!page) return -ENOMEM; - gic_data_rdist()->vpe_l1_page = page; + gic_data_rdist()->vpe_l1_base = page_address(page); pa = virt_to_phys(page_address(page)); WARN_ON(!IS_ALIGNED(pa, psz)); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 45ecf7316ac1..db30051b4273 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -653,10 +653,10 @@ struct rdists { struct { void __iomem *rd_base; struct page *pend_page; - struct page *vpe_l1_page; phys_addr_t phys_base; bool lpi_enabled; cpumask_t *vpe_table_mask; + void *vpe_l1_base; } __percpu *rdist; phys_addr_t prop_table_pa; void *prop_table_va; -- Gitee From 61e4d347fd74d85ae00586c56a04db32978f5714 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Thu, 6 Feb 2020 15:57:08 +0800 Subject: [PATCH 41/94] irqchip/gic-v4.1: Ensure L2 vPE table is allocated at RD level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In GICv4, we will ensure that level2 vPE table memory is allocated for the specified vpe_id on all v4 ITS, in its_alloc_vpe_table(). This still works well for the typical GICv4.1 implementation, where the new vPE table is shared between the ITSs and the RDs. To make it explicit, let us introduce allocate_vpe_l2_table() to make sure that the L2 tables are allocated on all v4.1 RDs. We're likely not need to allocate memory in it because the vPE table is shared and (L2 table is) already allocated at ITS level, except for the case where the ITS doesn't share anything (say SVPET == 0, practically unlikely but architecturally allowed). The implementation of allocate_vpe_l2_table() is mostly copied from its_alloc_table_entry(). Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200206075711.1275-4-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 099d9bb665ff..ff9d0c60e12e 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2577,6 +2577,72 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) return 0; } +static bool allocate_vpe_l2_table(int cpu, u32 id) +{ + void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; + u64 val, gpsz, npg; + unsigned int psz, esz, idx; + struct page *page; + __le64 *table; + + if (!gic_rdists->has_rvpeid) + return true; + + val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); + + esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1; + gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); + npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1; + + switch (gpsz) { + default: + WARN_ON(1); + /* fall through */ + case GIC_PAGE_SIZE_4K: + psz = SZ_4K; + break; + case GIC_PAGE_SIZE_16K: + psz = SZ_16K; + break; + case GIC_PAGE_SIZE_64K: + psz = SZ_64K; + break; + } + + /* Don't allow vpe_id that exceeds single, flat table limit */ + if (!(val & GICR_VPROPBASER_4_1_INDIRECT)) + return (id < (npg * psz / (esz * SZ_8))); + + /* Compute 1st level table index & check if that exceeds table limit */ + idx = id >> ilog2(psz / (esz * SZ_8)); + if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE)) + return false; + + table = gic_data_rdist_cpu(cpu)->vpe_l1_base; + + /* Allocate memory for 2nd level table */ + if (!table[idx]) { + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); + if (!page) + return false; + + /* Flush Lvl2 table to PoC if hw doesn't support coherency */ + if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) + gic_flush_dcache_to_poc(page_address(page), psz); + + table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); + + /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ + if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) + gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); + + /* Ensure updated table contents are visible to RD hardware */ + dsb(sy); + } + + return true; +} + static int allocate_vpe_l1_table(void) { void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); @@ -3091,6 +3157,7 @@ static bool its_alloc_device_table(struct its_node *its, u32 dev_id) static bool its_alloc_vpe_table(u32 vpe_id) { struct its_node *its; + int cpu; /* * Make sure the L2 tables are allocated on *all* v4 ITSs. We @@ -3113,6 +3180,19 @@ static bool its_alloc_vpe_table(u32 vpe_id) return false; } + /* Non v4.1? No need to iterate RDs and go back early. */ + if (!gic_rdists->has_rvpeid) + return true; + + /* + * Make sure the L2 tables are allocated for all copies of + * the L1 table on *all* v4.1 RDs. + */ + for_each_possible_cpu(cpu) { + if (!allocate_vpe_l2_table(cpu, vpe_id)) + return false; + } + return true; } -- Gitee From dd44474439fb54994b8b5e630e77f1bf10ee524a Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Thu, 6 Feb 2020 15:57:09 +0800 Subject: [PATCH 42/94] irqchip/gic-v4.1: Drop 'tmp' in inherit_vpe_l1_table_from_rd() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The variable 'tmp' in inherit_vpe_l1_table_from_rd() is actually not needed, drop it. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200206075711.1275-5-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index ff9d0c60e12e..40db4b98e7b1 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2549,14 +2549,12 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) for_each_possible_cpu(cpu) { void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; - u32 tmp; if (!base || cpu == smp_processor_id()) continue; val = gic_read_typer(base + GICR_TYPER); - tmp = compute_common_aff(val); - if (tmp != aff) + if (aff != compute_common_aff(val)) continue; /* -- Gitee From acee1b74ae5748feb727954c160e7516e167f709 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Thu, 6 Feb 2020 15:57:10 +0800 Subject: [PATCH 43/94] irqchip/gic-v3-its: Remove superfluous WARN_ON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "ITS virtual pending table not cleaning" is already complained inside its_clear_vpend_valid(), there's no need to trigger a WARN_ON again. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200206075711.1275-6-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 40db4b98e7b1..c73b720c3806 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2991,7 +2991,6 @@ static void its_cpu_init_lpis(void) * corrupting memory. */ val = its_clear_vpend_valid(vlpi_base, 0, 0); - WARN_ON(val & GICR_VPENDBASER_Dirty); } if (allocate_vpe_l1_table()) { -- Gitee From 0eba9e4721205a3a428fca6a22377a1804dfac6b Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Thu, 6 Feb 2020 15:57:11 +0800 Subject: [PATCH 44/94] irqchip/gic-v3-its: Rename VPENDBASER/VPROPBASER accessors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V{PEND,PROP}BASER registers are actually located in VLPI_base frame of the *redistributor*. Rename their accessors to reflect this fact. No functional changes. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200206075711.1275-7-yuzenghui@huawei.com Signed-off-by: 温家明 --- arch/arm/include/asm/arch_gicv3.h | 12 ++++++------ arch/arm64/include/asm/arch_gicv3.h | 8 ++++---- drivers/irqchip/irq-gic-v3-its.c | 28 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index b5752f0e8936..c815477b4303 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -326,16 +326,16 @@ static inline u64 __gic_readq_nonatomic(const volatile void __iomem *addr) #define gits_write_cwriter(v, c) __gic_writeq_nonatomic(v, c) /* - * GITS_VPROPBASER - hi and lo bits may be accessed independently. + * GICR_VPROPBASER - hi and lo bits may be accessed independently. */ -#define gits_read_vpropbaser(c) __gic_readq_nonatomic(c) -#define gits_write_vpropbaser(v, c) __gic_writeq_nonatomic(v, c) +#define gicr_read_vpropbaser(c) __gic_readq_nonatomic(c) +#define gicr_write_vpropbaser(v, c) __gic_writeq_nonatomic(v, c) /* - * GITS_VPENDBASER - the Valid bit must be cleared before changing + * GICR_VPENDBASER - the Valid bit must be cleared before changing * anything else. */ -static inline void gits_write_vpendbaser(u64 val, void __iomem *addr) +static inline void gicr_write_vpendbaser(u64 val, void __iomem *addr) { u32 tmp; @@ -352,7 +352,7 @@ static inline void gits_write_vpendbaser(u64 val, void __iomem *addr) __gic_writeq_nonatomic(val, addr); } -#define gits_read_vpendbaser(c) __gic_readq_nonatomic(c) +#define gicr_read_vpendbaser(c) __gic_readq_nonatomic(c) static inline bool gic_prio_masking_enabled(void) { diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index 43362ce1ad3d..89f11411c923 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -140,11 +140,11 @@ static inline u32 gic_read_rpr(void) #define gicr_write_pendbaser(v, c) writeq_relaxed(v, c) #define gicr_read_pendbaser(c) readq_relaxed(c) -#define gits_write_vpropbaser(v, c) writeq_relaxed(v, c) -#define gits_read_vpropbaser(c) readq_relaxed(c) +#define gicr_write_vpropbaser(v, c) writeq_relaxed(v, c) +#define gicr_read_vpropbaser(c) readq_relaxed(c) -#define gits_write_vpendbaser(v, c) writeq_relaxed(v, c) -#define gits_read_vpendbaser(c) readq_relaxed(c) +#define gicr_write_vpendbaser(v, c) writeq_relaxed(v, c) +#define gicr_read_vpendbaser(c) readq_relaxed(c) static inline bool gic_prio_masking_enabled(void) { diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index c73b720c3806..53937801679d 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2563,7 +2563,7 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) * ours wrt CommonLPIAff. Let's use its own VPROPBASER. * Make sure we don't write the Z bit in that case. */ - val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); + val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); val &= ~GICR_VPROPBASER_4_1_Z; gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; @@ -2586,7 +2586,7 @@ static bool allocate_vpe_l2_table(int cpu, u32 id) if (!gic_rdists->has_rvpeid) return true; - val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); + val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1; gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); @@ -2658,8 +2658,8 @@ static int allocate_vpe_l1_table(void) * effect of making sure no doorbell will be generated and we can * then safely clear VPROPBASER.Valid. */ - if (gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) - gits_write_vpendbaser(GICR_VPENDBASER_PendingLast, + if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) + gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, vlpi_base + GICR_VPENDBASER); /* @@ -2682,8 +2682,8 @@ static int allocate_vpe_l1_table(void) /* First probe the page size */ val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); - gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); - val = gits_read_vpropbaser(vlpi_base + GICR_VPROPBASER); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER); gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); @@ -2754,7 +2754,7 @@ static int allocate_vpe_l1_table(void) val |= GICR_VPROPBASER_4_1_VALID; out: - gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", @@ -2861,14 +2861,14 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) bool clean; u64 val; - val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); + val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); val &= ~GICR_VPENDBASER_Valid; val &= ~clr; val |= set; - gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); do { - val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); + val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); clean = !(val & GICR_VPENDBASER_Dirty); if (!clean) { count--; @@ -2983,7 +2983,7 @@ static void its_cpu_init_lpis(void) val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", smp_processor_id(), val); - gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); /* * Also clear Valid bit of GICR_VPENDBASER, in case some @@ -3652,7 +3652,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; val |= GICR_VPROPBASER_RaWb; val |= GICR_VPROPBASER_InnerShareable; - gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); val = virt_to_phys(page_address(vpe->vpt_page)) & GENMASK_ULL(51, 16); @@ -3670,7 +3670,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) val |= GICR_VPENDBASER_PendingLast; val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; val |= GICR_VPENDBASER_Valid; - gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); } static void its_vpe_deschedule(struct its_vpe *vpe) @@ -3876,7 +3876,7 @@ static void its_vpe_4_1_schedule(struct its_vpe *vpe, val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); - gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); } static void its_vpe_4_1_deschedule(struct its_vpe *vpe, -- Gitee From 386666a9c208a63bb8912c01d50f7760462c2a01 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 9 Feb 2020 22:48:50 +0000 Subject: [PATCH 45/94] irqchip/gic-v4.1: Avoid 64bit division for the sake of 32bit ARM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to allow the GICv4 code to link properly on 32bit ARM, make sure we don't use 64bit divisions when it isn't strictly necessary. Fixes: 4e6437f12d6e ("irqchip/gic-v4.1: Ensure L2 vPE table is allocated at RD level") Reported-by: Stephen Rothwell Cc: Zenghui Yu Signed-off-by: Marc Zyngier Signed-off-by: Linus Torvalds Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 53937801679d..2afe14611f96 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2578,8 +2578,8 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) static bool allocate_vpe_l2_table(int cpu, u32 id) { void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; - u64 val, gpsz, npg; - unsigned int psz, esz, idx; + unsigned int psz, esz, idx, npg, gpsz; + u64 val; struct page *page; __le64 *table; -- Gitee From 6772256e67d3e5268db7f75c7509cac9121e0806 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 18 Feb 2020 13:07:16 -0800 Subject: [PATCH 46/94] KVM: Reinstall old memslots if arch preparation fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reinstall the old memslots if preparing the new memory region fails after invalidating a to-be-{re}moved memslot. Remove the superfluous 'old_memslots' variable so that it's somewhat clear that the error handling path needs to free the unused memslots, not simply the 'old' memslots. Fixes: bc6678a33d9b9 ("KVM: introduce kvm->srcu and convert kvm_set_memory_region to SRCU update") Reviewed-by: Christoffer Dall Reviewed-by: Peter Xu Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini Signed-off-by: 温家明 --- virt/kvm/kvm_main.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f654f8855d05..ac74b31efb81 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1053,7 +1053,7 @@ int __kvm_set_memory_region(struct kvm *kvm, unsigned long npages; struct kvm_memory_slot *slot; struct kvm_memory_slot old, new; - struct kvm_memslots *slots = NULL, *old_memslots; + struct kvm_memslots *slots; int as_id, id; enum kvm_mr_change change; @@ -1162,7 +1162,13 @@ int __kvm_set_memory_region(struct kvm *kvm, slot = id_to_memslot(slots, id); slot->flags |= KVM_MEMSLOT_INVALID; - old_memslots = install_new_memslots(kvm, as_id, slots); + /* + * We can re-use the old memslots, the only difference from the + * newly installed memslots is the invalid flag, which will get + * dropped by update_memslots anyway. We'll also revert to the + * old memslots if preparing the new memory region fails. + */ + slots = install_new_memslots(kvm, as_id, slots); /* From this point no new shadow pages pointing to a deleted, * or moved, memslot will be created. @@ -1172,13 +1178,6 @@ int __kvm_set_memory_region(struct kvm *kvm, * - kvm_is_visible_gfn (mmu_check_roots) */ kvm_arch_flush_shadow_memslot(kvm, slot); - - /* - * We can re-use the old_memslots from above, the only difference - * from the currently installed memslots is the invalid flag. This - * will get overwritten by update_memslots anyway. - */ - slots = old_memslots; } r = kvm_arch_prepare_memory_region(kvm, &new, mem, change); @@ -1192,15 +1191,17 @@ int __kvm_set_memory_region(struct kvm *kvm, } update_memslots(slots, &new, change); - old_memslots = install_new_memslots(kvm, as_id, slots); + slots = install_new_memslots(kvm, as_id, slots); kvm_arch_commit_memory_region(kvm, mem, &old, &new, change); kvm_free_memslot(kvm, &old, &new); - kvfree(old_memslots); + kvfree(slots); return 0; out_slots: + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) + slots = install_new_memslots(kvm, as_id, slots); kvfree(slots); out_free: kvm_free_memslot(kvm, &new, &old); -- Gitee From 964b99bd84aa072c6f20ef83a2d31bfe849ea884 Mon Sep 17 00:00:00 2001 From: Heyi Guo Date: Tue, 25 Feb 2020 17:00:23 +0800 Subject: [PATCH 47/94] irqchip/gic-v3-its: Fix access width for gicr_syncr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GICR_SYNCR is a 32bit register, so it is better to access it with 32bit access width, though we have not seen any real problem. Signed-off-by: Heyi Guo Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200225090023.28020-1-guoheyi@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2afe14611f96..e27f386a1ab6 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1328,7 +1328,7 @@ static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) static void wait_for_syncr(void __iomem *rdbase) { - while (gic_read_lpir(rdbase + GICR_SYNCR) & 1) + while (readl_relaxed(rdbase + GICR_SYNCR) & 1) cpu_relax(); } -- Gitee From 474c9a40e6ddfb1ce3d42c9b0d0a3e424a9af781 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 13 Mar 2020 11:01:15 +0000 Subject: [PATCH 48/94] irqchip/gic-v3-its: Probe ITS page size for all GITS_BASERn registers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GICv3 ITS driver assumes that once it has latched on a page size for a given BASER register, it can use the same page size as the maximum page size for all subsequent BASER registers. Although it worked so far, nothing in the architecture guarantees this, and Nianyao Tang hit this problem on some undisclosed implementation. Let's bite the bullet and probe the the supported page size on all BASER registers before starting to populate the tables. This simplifies the setup a bit, at the expense of a few additional MMIO accesses. Signed-off-by: Marc Zyngier Reported-by: Nianyao Tang Tested-by: Nianyao Tang Link: https://lore.kernel.org/r/1584089195-63897-1-git-send-email-zhangshaokun@hisilicon.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 94 ++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e27f386a1ab6..f9710a9f574a 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2165,18 +2165,17 @@ static void its_write_baser(struct its_node *its, struct its_baser *baser, } static int its_setup_baser(struct its_node *its, struct its_baser *baser, - u64 cache, u64 shr, u32 psz, u32 order, - bool indirect) + u64 cache, u64 shr, u32 order, bool indirect) { u64 val = its_read_baser(its, baser); u64 esz = GITS_BASER_ENTRY_SIZE(val); u64 type = GITS_BASER_TYPE(val); u64 baser_phys, tmp; - u32 alloc_pages; + u32 alloc_pages, psz; struct page *page; void *base; -retry_alloc_baser: + psz = baser->psz; alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); if (alloc_pages > GITS_BASER_PAGES_MAX) { pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", @@ -2249,25 +2248,6 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, goto retry_baser; } - if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { - /* - * Page size didn't stick. Let's try a smaller - * size and retry. If we reach 4K, then - * something is horribly wrong... - */ - free_pages((unsigned long)base, order); - baser->base = NULL; - - switch (psz) { - case SZ_16K: - psz = SZ_4K; - goto retry_alloc_baser; - case SZ_64K: - psz = SZ_16K; - goto retry_alloc_baser; - } - } - if (val != tmp) { pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", &its->phys_base, its_base_type_string[type], @@ -2293,13 +2273,14 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, static bool its_parse_indirect_baser(struct its_node *its, struct its_baser *baser, - u32 psz, u32 *order, u32 ids) + u32 *order, u32 ids) { u64 tmp = its_read_baser(its, baser); u64 type = GITS_BASER_TYPE(tmp); u64 esz = GITS_BASER_ENTRY_SIZE(tmp); u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb; u32 new_order = *order; + u32 psz = baser->psz; bool indirect = false; /* No need to enable Indirection if memory requirement < (psz*2)bytes */ @@ -2417,11 +2398,58 @@ static void its_free_tables(struct its_node *its) } } +static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser) +{ + u64 psz = SZ_64K; + + while (psz) { + u64 val, gpsz; + + val = its_read_baser(its, baser); + val &= ~GITS_BASER_PAGE_SIZE_MASK; + + switch (psz) { + case SZ_64K: + gpsz = GITS_BASER_PAGE_SIZE_64K; + break; + case SZ_16K: + gpsz = GITS_BASER_PAGE_SIZE_16K; + break; + case SZ_4K: + default: + gpsz = GITS_BASER_PAGE_SIZE_4K; + break; + } + + gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT; + + val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz); + its_write_baser(its, baser, val); + + if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz) + break; + + switch (psz) { + case SZ_64K: + psz = SZ_16K; + break; + case SZ_16K: + psz = SZ_4K; + break; + case SZ_4K: + default: + return -1; + } + } + + baser->psz = psz; + return 0; +} + static int its_alloc_tables(struct its_node *its) { u64 shr = GITS_BASER_InnerShareable; u64 cache = GITS_BASER_RaWaWb; - u32 psz = SZ_64K; int err, i; if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) @@ -2432,16 +2460,22 @@ static int its_alloc_tables(struct its_node *its) struct its_baser *baser = its->tables + i; u64 val = its_read_baser(its, baser); u64 type = GITS_BASER_TYPE(val); - u32 order = get_order(psz); bool indirect = false; + u32 order; - switch (type) { - case GITS_BASER_TYPE_NONE: + if (type == GITS_BASER_TYPE_NONE) continue; + if (its_probe_baser_psz(its, baser)) { + its_free_tables(its); + return -ENXIO; + } + + order = get_order(baser->psz); + + switch (type) { case GITS_BASER_TYPE_DEVICE: - indirect = its_parse_indirect_baser(its, baser, - psz, &order, + indirect = its_parse_indirect_baser(its, baser, &order, device_ids(its)); break; -- Gitee From 2de054843459c2c890f1a10c4f1990c30378631a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Fri, 6 Dec 2024 16:18:04 +0800 Subject: [PATCH 49/94] resolve patch Probe ITS page size conflict --- drivers/irqchip/irq-gic-v3-its.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index f9710a9f574a..43a660a4c4e3 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2491,25 +2491,22 @@ static int its_alloc_tables(struct its_node *its) } } - indirect = its_parse_indirect_baser(its, baser, - psz, &order, + indirect = its_parse_indirect_baser(its, baser, &order, ITS_MAX_VPEID_BITS); break; case GITS_BASER_TYPE_COLLECTION: - indirect = its_parse_indirect_baser(its, baser, - psz, &order, + indirect = its_parse_indirect_baser(its, baser, &order, order_base_2(num_possible_cpus())); break; } - err = its_setup_baser(its, baser, cache, shr, psz, order, indirect); + err = its_setup_baser(its, baser, cache, shr, order, indirect); if (err < 0) { its_free_tables(its); return err; } /* Update settings which will be used for next BASERn */ - psz = baser->psz; cache = baser->val & GITS_BASER_CACHEABILITY_MASK; shr = baser->val & GITS_BASER_SHAREABILITY_MASK; } -- Gitee From 0a5df0c96fe8e43e61a1fb9ed8f77c29a3fd0dc3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:08 +0000 Subject: [PATCH 50/94] irqchip/gic-v3: Use SGIs without active state if offered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To allow the direct injection of SGIs into a guest, the GICv4.1 architecture has to sacrifice the Active state so that SGIs look a lot like LPIs (they are injected by the same mechanism). In order not to break existing software, the architecture gives offers guests OSs the choice: SGIs with or without an active state. It is the hypervisors duty to honor the guest's choice. For this, the architecture offers a discovery bit indicating whether the GIC supports GICv4.1 SGIs (GICD_TYPER2.nASSGIcap), and another bit indicating whether the guest wants Active-less SGIs or not (controlled by GICD_CTLR.nASSGIreq). A hypervisor not supporting GICv4.1 SGIs would leave nASSGIcap clear, and a guest not knowing about GICv4.1 SGIs (or definitely wanting an Active state) would leave nASSGIreq clear (both being thankfully backward compatible with older revisions of the GIC). Since Linux is perfectly happy without an active state on SGIs, inform the hypervisor that we'll use that if offered. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-2-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3.c | 10 ++++++++-- include/linux/irqchip/arm-gic-v3.h | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index dab249c54c7b..6e49173bae34 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -714,6 +714,7 @@ static void __init gic_dist_init(void) unsigned int i; u64 affinity; void __iomem *base = gic_data.dist_base; + u32 val; /* Disable the distributor */ writel_relaxed(0, base + GICD_CTLR); @@ -746,9 +747,14 @@ static void __init gic_dist_init(void) /* Now do the common stuff, and wait for the distributor to drain */ gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp); + val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; + if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { + pr_info("Enabling SGIs without active state\n"); + val |= GICD_CTLR_nASSGIreq; + } + /* Enable distributor with ARE, Group1 */ - writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, - base + GICD_CTLR); + writel_relaxed(val, base + GICD_CTLR); /* * Set all global interrupts to the boot CPU only. ARE must be diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index db30051b4273..a3e27fa3cce9 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -57,6 +57,7 @@ #define GICD_SPENDSGIR 0x0F20 #define GICD_CTLR_RWP (1U << 31) +#define GICD_CTLR_nASSGIreq (1U << 8) #define GICD_CTLR_DS (1U << 6) #define GICD_CTLR_ARE_NS (1U << 4) #define GICD_CTLR_ENABLE_G1A (1U << 1) @@ -90,6 +91,7 @@ #define GICD_TYPER_ESPIS(typer) \ (((typer) & GICD_TYPER_ESPI) ? GICD_TYPER_SPIS((typer) >> 27) : 0) +#define GICD_TYPER2_nASSGIcap (1U << 8) #define GICD_TYPER2_VIL (1U << 7) #define GICD_TYPER2_VID GENMASK(4, 0) -- Gitee From 98710a20783d6d9bcaa04c2b768ca43a96f401b7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:09 +0000 Subject: [PATCH 51/94] irqchip/gic-v4.1: Skip absent CPUs while iterating over redistributors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a system that is only sparsly populated with CPUs, we can end-up with redistributors structures that are not initialized. Let's make sure we don't try and access those when iterating over them (in this case when checking we have a L2 VPE table). Fixes: 4e6437f12d6e ("irqchip/gic-v4.1: Ensure L2 vPE table is allocated at RD level") Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-3-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 43a660a4c4e3..2a38e44e0424 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -2617,6 +2617,10 @@ static bool allocate_vpe_l2_table(int cpu, u32 id) if (!gic_rdists->has_rvpeid) return true; + /* Skip non-present CPUs */ + if (!base) + return true; + val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1; -- Gitee From 2f73f3e0408669bf50310ee9e1ddfb494b77a8e8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:10 +0000 Subject: [PATCH 52/94] irqchip/gic-v4.1: Ensure mutual exclusion between vPE affinity change and RD access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before GICv4.1, all operations would be serialized with the affinity changes by virtue of using the same ITS command queue. With v4.1, things change, as invalidations (and a number of other operations) are issued using the redistributor MMIO frame. We must thus make sure that these redistributor accesses cannot race against aginst the affinity change, or we may end-up talking to the wrong redistributor. To ensure this, we expand the irq_to_cpuid() helper to take a spinlock when the LPI is mapped to a vLPI (a new per-VPE lock) on each operation that requires mutual exclusion. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-4-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 56 +++++++++++++++++++++++++----- include/linux/irqchip/arm-gic-v4.h | 5 +++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2a38e44e0424..2814863b61dc 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -249,15 +249,41 @@ static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) return NULL; } -static int irq_to_cpuid(struct irq_data *d) +static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags) +{ + raw_spin_lock_irqsave(&vpe->vpe_lock, *flags); + return vpe->col_idx; +} + +static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags) +{ + raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); +} + +static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags) { - struct its_device *its_dev = irq_data_get_irq_chip_data(d); struct its_vlpi_map *map = get_vlpi_map(d); + int cpu; - if (map) - return map->vpe->col_idx; + if (map) { + cpu = vpe_to_cpuid_lock(map->vpe, flags); + } else { + /* Physical LPIs are already locked via the irq_desc lock */ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + cpu = its_dev->event_map.col_map[its_get_event_id(d)]; + /* Keep GCC quiet... */ + *flags = 0; + } + + return cpu; +} + +static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags) +{ + struct its_vlpi_map *map = get_vlpi_map(d); - return its_dev->event_map.col_map[its_get_event_id(d)]; + if (map) + vpe_to_cpuid_unlock(map->vpe, flags); } static struct its_collection *valid_col(struct its_collection *col) @@ -1336,7 +1362,9 @@ static void direct_lpi_inv(struct irq_data *d) { struct its_vlpi_map *map = get_vlpi_map(d); void __iomem *rdbase; + unsigned long flags; u64 val; + int cpu; if (map) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -1351,10 +1379,12 @@ static void direct_lpi_inv(struct irq_data *d) } /* Target the redistributor this LPI is currently routed to */ - rdbase = per_cpu_ptr(gic_rdists->rdist, irq_to_cpuid(d))->rd_base; + cpu = irq_to_cpuid_lock(d, &flags); + rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; gic_write_lpir(val, rdbase + GICR_INVLPIR); wait_for_syncr(rdbase); + irq_to_cpuid_unlock(d, flags); } static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) @@ -3646,17 +3676,25 @@ static int its_vpe_set_affinity(struct irq_data *d, { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); int from, cpu = cpumask_first(mask_val); + unsigned long flags; /* * Changing affinity is mega expensive, so let's be as lazy as * we can and only do it if we really have to. Also, if mapped * into the proxy device, we need to move the doorbell * interrupt to its new location. + * + * Another thing is that changing the affinity of a vPE affects + * *other interrupts* such as all the vLPIs that are routed to + * this vPE. This means that the irq_desc lock is not enough to + * protect us, and that we must ensure nobody samples vpe->col_idx + * during the update, hence the lock below which must also be + * taken on any vLPI handling path that evaluates vpe->col_idx. */ - if (vpe->col_idx == cpu) + from = vpe_to_cpuid_lock(vpe, &flags); + if (from == cpu) goto out; - from = vpe->col_idx; vpe->col_idx = cpu; /* @@ -3672,6 +3710,7 @@ static int its_vpe_set_affinity(struct irq_data *d, out: irq_data_update_effective_affinity(d, cpumask_of(cpu)); + vpe_to_cpuid_unlock(vpe, flags); return IRQ_SET_MASK_OK_DONE; } @@ -4021,6 +4060,7 @@ static int its_vpe_init(struct its_vpe *vpe) return -ENOMEM; } + raw_spin_lock_init(&vpe->vpe_lock); vpe->vpe_id = vpe_id; vpe->vpt_page = vpt_page; if (gic_rdists->has_rvpeid) diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index d9c34968467a..439963f4c66a 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -53,6 +53,11 @@ struct its_vpe { }; }; + /* + * Ensures mutual exclusion between affinity setting of the + * vPE and vLPI operations using vpe->col_idx. + */ + raw_spinlock_t vpe_lock; /* * This collection ID is used to indirect the target * redistributor for this VPE. The ID itself isn't involved in -- Gitee From aa81ff1308aff0a8e63c96e79b1adde810ac9355 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 13 Feb 2024 10:12:06 +0000 Subject: [PATCH 53/94] irqchip/gic-v3-its: Fix GICv4.1 VPE affinity update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When updating the affinity of a VPE, the VMOVP command is currently skipped if the two CPUs are part of the same VPE affinity. But this is wrong, as the doorbell corresponding to this VPE is still delivered on the 'old' CPU, which screws up the balancing. Furthermore, offlining that 'old' CPU results in doorbell interrupts generated for this VPE being discarded. The harsh reality is that VMOVP cannot be elided when a set_affinity() request occurs. It needs to be obeyed, and if an optimisation is to be made, it is at the point where the affinity change request is made (such as in KVM). Drop the VMOVP elision altogether, and only use the vpe_table_mask to try and stay within the same ITS affinity group if at all possible. Fixes: dd3f050a216e (irqchip/gic-v4.1: Implement the v4.1 flavour of VMOVP) Reported-by: Kunkun Jiang Signed-off-by: Marc Zyngier Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20240213101206.2137483-4-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 2814863b61dc..48f0707fa2c3 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3675,8 +3675,9 @@ static int its_vpe_set_affinity(struct irq_data *d, bool force) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); - int from, cpu = cpumask_first(mask_val); + struct cpumask common, *table_mask; unsigned long flags; + int from, cpu; /* * Changing affinity is mega expensive, so let's be as lazy as @@ -3692,19 +3693,22 @@ static int its_vpe_set_affinity(struct irq_data *d, * taken on any vLPI handling path that evaluates vpe->col_idx. */ from = vpe_to_cpuid_lock(vpe, &flags); - if (from == cpu) - goto out; - - vpe->col_idx = cpu; + table_mask = gic_data_rdist_cpu(from)->vpe_table_mask; /* - * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD - * is sharing its VPE table with the current one. + * If we are offered another CPU in the same GICv4.1 ITS + * affinity, pick this one. Otherwise, any CPU will do. */ - if (gic_data_rdist_cpu(cpu)->vpe_table_mask && - cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask)) + if (table_mask && cpumask_and(&common, mask_val, table_mask)) + cpu = cpumask_test_cpu(from, &common) ? from : cpumask_first(&common); + else + cpu = cpumask_first(mask_val); + + if (from == cpu) goto out; + vpe->col_idx = cpu; + its_send_vmovp(vpe); its_vpe_db_proxy_move(vpe, from, cpu); -- Gitee From 1c1fa769a83cf2048eeb58cf9235d5be2a5926ad Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Wed, 4 Mar 2020 20:33:11 +0000 Subject: [PATCH 54/94] irqchip/gic-v4.1: Wait for completion of redistributor's INVALL operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In GICv4.1, we emulate a guest-issued INVALL command by a direct write to GICR_INVALLR. Before we finish the emulation and go back to guest, let's make sure the physical invalidate operation is actually completed and no stale data will be left in redistributor. Per the specification, this can be achieved by polling the GICR_SYNCR.Busy bit (to zero). Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200302092145.899-1-yuzenghui@huawei.com Link: https://lore.kernel.org/r/20200304203330.4967-5-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 48f0707fa2c3..84d79ce367fa 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3997,6 +3997,8 @@ static void its_vpe_4_1_invall(struct its_vpe *vpe) /* Target the redistributor this vPE is currently known on */ rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; gic_write_lpir(val, rdbase + GICR_INVALLR); + + wait_for_syncr(rdbase); } static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) -- Gitee From 1e9080f535ec135e35903891ced1e92d5fbe5635 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:12 +0000 Subject: [PATCH 55/94] irqchip/gic-v4.1: Ensure mutual exclusion betwen invalidations on the same RD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GICv4.1 spec says that it is CONTRAINED UNPREDICTABLE to write to any of the GICR_INV{LPI,ALL}R registers if GICR_SYNCR.Busy == 1. To deal with it, we must ensure that only a single invalidation can happen at a time for a given redistributor. Add a per-RD lock to that effect and take it around the invalidation/syncr-read to deal with this. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-6-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 6 ++++++ drivers/irqchip/irq-gic-v3.c | 1 + include/linux/irqchip/arm-gic-v3.h | 1 + 3 files changed, 8 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 84d79ce367fa..7916e2f2b4d8 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1380,10 +1380,12 @@ static void direct_lpi_inv(struct irq_data *d) /* Target the redistributor this LPI is currently routed to */ cpu = irq_to_cpuid_lock(d, &flags); + raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; gic_write_lpir(val, rdbase + GICR_INVLPIR); wait_for_syncr(rdbase); + raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); irq_to_cpuid_unlock(d, flags); } @@ -3826,9 +3828,11 @@ static void its_vpe_send_inv(struct irq_data *d) void __iomem *rdbase; /* Target the redistributor this VPE is currently known on */ + raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR); wait_for_syncr(rdbase); + raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); } else { its_vpe_send_cmd(vpe, its_send_inv); } @@ -3995,10 +3999,12 @@ static void its_vpe_4_1_invall(struct its_vpe *vpe) val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); /* Target the redistributor this vPE is currently known on */ + raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; gic_write_lpir(val, rdbase + GICR_INVALLR); wait_for_syncr(rdbase); + raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); } static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 6e49173bae34..5f5586c8759c 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -825,6 +825,7 @@ static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) typer = gic_read_typer(ptr + GICR_TYPER); if ((typer >> 32) == aff) { u64 offset = ptr - region->redist_base; + raw_spin_lock_init(&gic_data_rdist()->rd_lock); gic_data_rdist_rd_base() = ptr; gic_data_rdist()->phys_base = region->phys_base + offset; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index a3e27fa3cce9..359d35562135 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -653,6 +653,7 @@ struct rdists { struct { + raw_spinlock_t rd_lock; void __iomem *rd_base; struct page *pend_page; phys_addr_t phys_base; -- Gitee From 3755d5352f748da49d0d3c0d1e933eb1c9dcca39 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Mon, 20 Jul 2020 17:23:28 +0800 Subject: [PATCH 56/94] irqchip/gic-v4.1: Ensure accessing the correct RD when writing INVALLR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GICv4.1 spec tells us that it's CONSTRAINED UNPREDICTABLE to issue a register-based invalidation operation for a vPEID not mapped to that RD, or another RD within the same CommonLPIAff group. To follow this rule, commit f3a059219bc7 ("irqchip/gic-v4.1: Ensure mutual exclusion between vPE affinity change and RD access") tried to address the race between the RD accesses and the vPE affinity change, but somehow forgot to take GICR_INVALLR into account. Let's take the vpe_lock before evaluating vpe->col_idx to fix it. Fixes: f3a059219bc7 ("irqchip/gic-v4.1: Ensure mutual exclusion between vPE affinity change and RD access") Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200720092328.708-1-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 7916e2f2b4d8..043683ad9eb7 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3993,18 +3993,22 @@ static void its_vpe_4_1_deschedule(struct its_vpe *vpe, static void its_vpe_4_1_invall(struct its_vpe *vpe) { void __iomem *rdbase; + unsigned long flags; u64 val; + int cpu; val = GICR_INVALLR_V; val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); /* Target the redistributor this vPE is currently known on */ - raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); - rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; + cpu = vpe_to_cpuid_lock(vpe, &flags); + raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); + rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; gic_write_lpir(val, rdbase + GICR_INVALLR); wait_for_syncr(rdbase); - raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); + raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); + vpe_to_cpuid_unlock(vpe, flags); } static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) -- Gitee From 9f9f8ed2094da0dd23477344dbed3d650a4869a4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 17 Jun 2023 08:32:42 +0100 Subject: [PATCH 57/94] irqchip/gic-v4.1: Properly lock VPEs when doing a directLPI invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We normally rely on the irq_to_cpuid_[un]lock() primitives to make sure nothing will change col->idx while performing a LPI invalidation. However, these primitives do not cover VPE doorbells, and we have some open-coded locking for that. Unfortunately, this locking is pretty bogus. Instead, extend the above primitives to cover VPE doorbells and convert the whole thing to it. Fixes: f3a059219bc7 ("irqchip/gic-v4.1: Ensure mutual exclusion between vPE affinity change and RD access") Reported-by: Kunkun Jiang Signed-off-by: Marc Zyngier Cc: Zenghui Yu Cc: wanghaibin.wang@huawei.com Tested-by: Kunkun Jiang Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20230617073242.3199746-1-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 75 ++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 043683ad9eb7..dd7a860761ce 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -260,13 +260,23 @@ static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags) raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); } +static struct irq_chip its_vpe_irq_chip; + static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags) { - struct its_vlpi_map *map = get_vlpi_map(d); + struct its_vpe *vpe = NULL; int cpu; - if (map) { - cpu = vpe_to_cpuid_lock(map->vpe, flags); + if (d->chip == &its_vpe_irq_chip) { + vpe = irq_data_get_irq_chip_data(d); + } else { + struct its_vlpi_map *map = get_vlpi_map(d); + if (map) + vpe = map->vpe; + } + + if (vpe) { + cpu = vpe_to_cpuid_lock(vpe, flags); } else { /* Physical LPIs are already locked via the irq_desc lock */ struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -280,10 +290,18 @@ static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags) static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags) { - struct its_vlpi_map *map = get_vlpi_map(d); + struct its_vpe *vpe = NULL; + + if (d->chip == &its_vpe_irq_chip) { + vpe = irq_data_get_irq_chip_data(d); + } else { + struct its_vlpi_map *map = get_vlpi_map(d); + if (map) + vpe = map->vpe; + } - if (map) - vpe_to_cpuid_unlock(map->vpe, flags); + if (vpe) + vpe_to_cpuid_unlock(vpe, flags); } static struct its_collection *valid_col(struct its_collection *col) @@ -1358,14 +1376,29 @@ static void wait_for_syncr(void __iomem *rdbase) cpu_relax(); } -static void direct_lpi_inv(struct irq_data *d) +static void __direct_lpi_inv(struct irq_data *d, u64 val) { - struct its_vlpi_map *map = get_vlpi_map(d); void __iomem *rdbase; unsigned long flags; - u64 val; int cpu; + /* Target the redistributor this LPI is currently routed to */ + cpu = irq_to_cpuid_lock(d, &flags); + raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); + + rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; + gic_write_lpir(val, rdbase + GICR_INVLPIR); + wait_for_syncr(rdbase); + + raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); + irq_to_cpuid_unlock(d, flags); +} + +static void direct_lpi_inv(struct irq_data *d) +{ + struct its_vlpi_map *map = get_vlpi_map(d); + u64 val; + if (map) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -1378,15 +1411,7 @@ static void direct_lpi_inv(struct irq_data *d) val = d->hwirq; } - /* Target the redistributor this LPI is currently routed to */ - cpu = irq_to_cpuid_lock(d, &flags); - raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); - rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; - gic_write_lpir(val, rdbase + GICR_INVLPIR); - - wait_for_syncr(rdbase); - raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); - irq_to_cpuid_unlock(d, flags); + __direct_lpi_inv(d, val); } static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) @@ -3824,18 +3849,10 @@ static void its_vpe_send_inv(struct irq_data *d) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); - if (gic_rdists->has_direct_lpi) { - void __iomem *rdbase; - - /* Target the redistributor this VPE is currently known on */ - raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); - rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; - gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR); - wait_for_syncr(rdbase); - raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); - } else { + if (gic_rdists->has_direct_lpi) + __direct_lpi_inv(d, d->parent_data->hwirq); + else its_vpe_send_cmd(vpe, its_send_inv); - } } static void its_vpe_mask_irq(struct irq_data *d) -- Gitee From 5400c6547fb8ede1cee1d916216f50f00bbc6caa Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:13 +0000 Subject: [PATCH 58/94] irqchip/gic-v4.1: Advertise support v4.1 to KVM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tell KVM that we support v4.1. Nothing uses this information so far. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-7-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 9 ++++++++- drivers/irqchip/irq-gic-v3.c | 2 ++ include/linux/irqchip/arm-gic-common.h | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index dd7a860761ce..36c983b291c5 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -5047,6 +5047,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, struct device_node *of_node; struct its_node *its; bool has_v4 = false; + bool has_v4_1 = false; int err; gic_rdists = rdists; @@ -5067,8 +5068,14 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, if (err) return err; - list_for_each_entry(its, &its_nodes, entry) + list_for_each_entry(its, &its_nodes, entry) { has_v4 |= is_v4(its); + has_v4_1 |= is_v4_1(its); + } + + /* Don't bother with inconsistent systems */ + if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) + rdists->has_rvpeid = false; if (has_v4 & rdists->has_vlpis) { if (its_init_vpe_domain() || diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 5f5586c8759c..a8dd158bea40 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1747,6 +1747,7 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) gic_v3_kvm_info.vcpu = r; gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; + gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; gic_set_kvm_info(&gic_v3_kvm_info); } @@ -2062,6 +2063,7 @@ static void __init gic_acpi_setup_kvm_info(void) } gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; + gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; gic_set_kvm_info(&gic_v3_kvm_info); } diff --git a/include/linux/irqchip/arm-gic-common.h b/include/linux/irqchip/arm-gic-common.h index b9850f5f1906..fa8c0455c352 100644 --- a/include/linux/irqchip/arm-gic-common.h +++ b/include/linux/irqchip/arm-gic-common.h @@ -32,6 +32,8 @@ struct gic_kvm_info { struct resource vctrl; /* vlpi support */ bool has_v4; + /* rvpeid support */ + bool has_v4_1; }; const struct gic_kvm_info *gic_get_kvm_info(void); -- Gitee From eff5ebd548b55b7d15e5ad043221227116fc0513 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:14 +0000 Subject: [PATCH 59/94] irqchip/gic-v4.1: Map the ITS SGIR register page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One of the new features of GICv4.1 is to allow virtual SGIs to be directly signaled to a VPE. For that, the ITS has grown a new 64kB page containing only a single register that is used to signal a SGI to a given VPE. Add a second mapping covering this new 64kB range, and take this opportunity to limit the original mapping to 64kB, which is enough to cover the span of the ITS registers. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-8-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 36c983b291c5..de8cb374d225 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -99,6 +99,7 @@ struct its_node { struct mutex dev_alloc_lock; struct list_head entry; void __iomem *base; + void __iomem *sgir_base; phys_addr_t phys_base; struct its_cmd_block *cmd_base; struct its_cmd_block *cmd_write; @@ -4636,7 +4637,7 @@ static int __init its_probe_one(struct resource *res, struct page *page; int err; - its_base = ioremap(res->start, resource_size(res)); + its_base = ioremap(res->start, SZ_64K); if (!its_base) { pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); return -ENOMEM; @@ -4687,6 +4688,13 @@ static int __init its_probe_one(struct resource *res, if (is_v4_1(its)) { u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer); + + its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K); + if (!its->sgir_base) { + err = -ENOMEM; + goto out_free_its; + } + its->mpidr = readl_relaxed(its_base + GITS_MPIDR); pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", @@ -4700,7 +4708,7 @@ static int __init its_probe_one(struct resource *res, get_order(ITS_CMD_QUEUE_SZ)); if (!page) { err = -ENOMEM; - goto out_free_its; + goto out_unmap_sgir; } its->cmd_base = (void *)page_address(page); its->cmd_write = its->cmd_base; @@ -4764,6 +4772,9 @@ static int __init its_probe_one(struct resource *res, its_free_tables(its); out_free_cmd: free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); +out_unmap_sgir: + if (its->sgir_base) + iounmap(its->sgir_base); out_free_its: kfree(its); out_unmap: -- Gitee From beaa26ca9dd784ee378fb06ddac9a51a7115eb9b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:15 +0000 Subject: [PATCH 60/94] irqchip/gic-v4.1: Plumb skeletal VSGI irqchip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since GICv4.1 has the capability to inject 16 SGIs into each VPE, and that I'm keen not to invent too many specific interfaces to manipulate these interrupts, let's pretend that each of these SGIs is an actual Linux interrupt. For that matter, let's introduce a minimal irqchip and irqdomain setup that will get fleshed up in the following patches. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-9-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 75 +++++++++++++++++++++++++++++- drivers/irqchip/irq-gic-v4.c | 8 +++- include/linux/irqchip/arm-gic-v4.h | 9 +++- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index de8cb374d225..3dce438d1e9e 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4061,6 +4061,72 @@ static struct irq_chip its_vpe_4_1_irq_chip = { .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, }; +static int its_sgi_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, + bool force) +{ + /* + * There is no notion of affinity for virtual SGIs, at least + * not on the host (since they can only be targetting a vPE). + * Tell the kernel we've done whatever it asked for. + */ + return IRQ_SET_MASK_OK; +} + +static struct irq_chip its_sgi_irq_chip = { + .name = "GICv4.1-sgi", + .irq_set_affinity = its_sgi_set_affinity, +}; + +static int its_sgi_irq_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + struct its_vpe *vpe = args; + int i; + + /* Yes, we do want 16 SGIs */ + WARN_ON(nr_irqs != 16); + + for (i = 0; i < 16; i++) { + vpe->sgi_config[i].priority = 0; + vpe->sgi_config[i].enabled = false; + vpe->sgi_config[i].group = false; + + irq_domain_set_hwirq_and_chip(domain, virq + i, i, + &its_sgi_irq_chip, vpe); + irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY); + } + + return 0; +} + +static void its_sgi_irq_domain_free(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs) +{ + /* Nothing to do */ +} + +static int its_sgi_irq_domain_activate(struct irq_domain *domain, + struct irq_data *d, bool reserve) +{ + return 0; +} + +static void its_sgi_irq_domain_deactivate(struct irq_domain *domain, + struct irq_data *d) +{ + /* Nothing to do */ +} + +static const struct irq_domain_ops its_sgi_domain_ops = { + .alloc = its_sgi_irq_domain_alloc, + .free = its_sgi_irq_domain_free, + .activate = its_sgi_irq_domain_activate, + .deactivate = its_sgi_irq_domain_deactivate, +}; + static int its_vpe_id_alloc(void) { return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); @@ -5089,8 +5155,15 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, rdists->has_rvpeid = false; if (has_v4 & rdists->has_vlpis) { + const struct irq_domain_ops *sgi_ops; + + if (has_v4_1) + sgi_ops = &its_sgi_domain_ops; + else + sgi_ops = NULL; + if (its_init_vpe_domain() || - its_init_v4(parent_domain, &its_vpe_domain_ops)) { + its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) { rdists->has_vlpis = false; pr_err("ITS: Disabling GICv4 support\n"); } diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 45969927cc81..c01910d53f9e 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -85,6 +85,7 @@ static struct irq_domain *gic_domain; static const struct irq_domain_ops *vpe_domain_ops; +static const struct irq_domain_ops *sgi_domain_ops; int its_alloc_vcpu_irqs(struct its_vm *vm) { @@ -216,12 +217,15 @@ int its_prop_update_vlpi(int irq, u8 config, bool inv) return irq_set_vcpu_affinity(irq, &info); } -int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *ops) +int its_init_v4(struct irq_domain *domain, + const struct irq_domain_ops *vpe_ops, + const struct irq_domain_ops *sgi_ops) { if (domain) { pr_info("ITS: Enabling GICv4 support\n"); gic_domain = domain; - vpe_domain_ops = ops; + vpe_domain_ops = vpe_ops; + sgi_domain_ops = sgi_ops; return 0; } diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 439963f4c66a..44e8c19e3d56 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -49,6 +49,11 @@ struct its_vpe { }; /* GICv4.1 implementations */ struct { + struct { + u8 priority; + bool enabled; + bool group; + } sgi_config[16]; atomic_t vmapp_count; }; }; @@ -123,6 +128,8 @@ int its_unmap_vlpi(int irq); int its_prop_update_vlpi(int irq, u8 config, bool inv); struct irq_domain_ops; -int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *ops); +int its_init_v4(struct irq_domain *domain, + const struct irq_domain_ops *vpe_ops, + const struct irq_domain_ops *sgi_ops); #endif -- Gitee From ac0f40a2190006ed43e3a6d66e5a8907ac98f756 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:16 +0000 Subject: [PATCH 61/94] irqchip/gic-v4.1: Add initial SGI configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GICv4.1 ITS has yet another new command (VSGI) which allows a VPE-targeted SGI to be configured (or have its pending state cleared). Add support for this command and plumb it into the activate irqdomain callback so that it is ready to be used. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-10-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 92 +++++++++++++++++++++++++++++- include/linux/irqchip/arm-gic-v3.h | 3 +- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 3dce438d1e9e..e72480d05102 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -408,6 +408,15 @@ struct its_cmd_desc { struct { struct its_vpe *vpe; } its_invdb_cmd; + + struct { + struct its_vpe *vpe; + u8 sgi; + u8 priority; + bool enable; + bool group; + bool clear; + } its_vsgi_cmd; }; }; @@ -553,6 +562,31 @@ static void its_encode_db(struct its_cmd_block *cmd, bool db) its_mask_encode(&cmd->raw_cmd[2], db, 63, 63); } +static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi) +{ + its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32); +} + +static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio) +{ + its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20); +} + +static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp) +{ + its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10); +} + +static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr) +{ + its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9); +} + +static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en) +{ + its_mask_encode(&cmd->raw_cmd[0], en, 8, 8); +} + static inline void its_fixup_cmd(struct its_cmd_block *cmd) { /* Let's fixup BE commands */ @@ -918,6 +952,26 @@ static struct its_vpe *its_build_invdb_cmd(struct its_node *its, return valid_vpe(its, desc->its_invdb_cmd.vpe); } +static struct its_vpe *its_build_vsgi_cmd(struct its_node *its, + struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + if (WARN_ON(!is_v4_1(its))) + return NULL; + + its_encode_cmd(cmd, GITS_CMD_VSGI); + its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id); + its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); + its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority); + its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group); + its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear); + its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable); + + its_fixup_cmd(cmd); + + return valid_vpe(its, desc->its_vsgi_cmd.vpe); +} + static u64 its_cmd_ptr_to_offset(struct its_node *its, struct its_cmd_block *ptr) { @@ -4061,6 +4115,26 @@ static struct irq_chip its_vpe_4_1_irq_chip = { .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, }; +static void its_configure_sgi(struct irq_data *d, bool clear) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + struct its_cmd_desc desc; + + desc.its_vsgi_cmd.vpe = vpe; + desc.its_vsgi_cmd.sgi = d->hwirq; + desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority; + desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled; + desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group; + desc.its_vsgi_cmd.clear = clear; + + /* + * GICv4.1 allows us to send VSGI commands to any ITS as long as the + * destination VPE is mapped there. Since we map them eagerly at + * activation time, we're pretty sure the first GICv4.1 ITS will do. + */ + its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc); +} + static int its_sgi_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) @@ -4111,13 +4185,29 @@ static void its_sgi_irq_domain_free(struct irq_domain *domain, static int its_sgi_irq_domain_activate(struct irq_domain *domain, struct irq_data *d, bool reserve) { + /* Write out the initial SGI configuration */ + its_configure_sgi(d, false); return 0; } static void its_sgi_irq_domain_deactivate(struct irq_domain *domain, struct irq_data *d) { - /* Nothing to do */ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + + /* + * The VSGI command is awkward: + * + * - To change the configuration, CLEAR must be set to false, + * leaving the pending bit unchanged. + * - To clear the pending bit, CLEAR must be set to true, leaving + * the configuration unchanged. + * + * You just can't do both at once, hence the two commands below. + */ + vpe->sgi_config[d->hwirq].enabled = false; + its_configure_sgi(d, false); + its_configure_sgi(d, true); } static const struct irq_domain_ops its_sgi_domain_ops = { diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 359d35562135..258a43242560 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -505,8 +505,9 @@ #define GITS_CMD_VMAPTI GITS_CMD_GICv4(GITS_CMD_MAPTI) #define GITS_CMD_VMOVI GITS_CMD_GICv4(GITS_CMD_MOVI) #define GITS_CMD_VSYNC GITS_CMD_GICv4(GITS_CMD_SYNC) -/* VMOVP and INVDB are the odd ones, as they dont have a physical counterpart */ +/* VMOVP, VSGI and INVDB are the odd ones, as they dont have a physical counterpart */ #define GITS_CMD_VMOVP GITS_CMD_GICv4(2) +#define GITS_CMD_VSGI GITS_CMD_GICv4(3) #define GITS_CMD_INVDB GITS_CMD_GICv4(0xe) /* -- Gitee From 3188e4c2a4c06826400a2d41dcf1c34a3259174f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:17 +0000 Subject: [PATCH 62/94] irqchip/gic-v4.1: Plumb mask/unmask SGI callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement mask/unmask for virtual SGIs by calling into the configuration helper. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-11-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e72480d05102..939dd0e1b7c3 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4135,6 +4135,22 @@ static void its_configure_sgi(struct irq_data *d, bool clear) its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc); } +static void its_sgi_mask_irq(struct irq_data *d) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + + vpe->sgi_config[d->hwirq].enabled = false; + its_configure_sgi(d, false); +} + +static void its_sgi_unmask_irq(struct irq_data *d) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + + vpe->sgi_config[d->hwirq].enabled = true; + its_configure_sgi(d, false); +} + static int its_sgi_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) @@ -4149,6 +4165,8 @@ static int its_sgi_set_affinity(struct irq_data *d, static struct irq_chip its_sgi_irq_chip = { .name = "GICv4.1-sgi", + .irq_mask = its_sgi_mask_irq, + .irq_unmask = its_sgi_unmask_irq, .irq_set_affinity = its_sgi_set_affinity, }; -- Gitee From cabb7dad80f3eaba01fb9c20a115fa745e762fd1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:18 +0000 Subject: [PATCH 63/94] irqchip/gic-v4.1: Plumb get/set_irqchip_state SGI callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To implement the get/set_irqchip_state callbacks (limited to the PENDING state), we have to use a particular set of hacks: - Reading the pending state is done by using a pair of new redistributor registers (GICR_VSGIR, GICR_VSGIPENDR), which allow the 16 interrupts state to be retrieved. - Setting the pending state is done by generating it as we'd otherwise do for a guest (writing to GITS_SGIR). - Clearing the pending state is done by emitting a VSGI command with the "clear" bit set. This requires some interesting locking though: - When talking to the redistributor, we must make sure that the VPE affinity doesn't change, hence taking the VPE lock. - At the same time, we must ensure that nobody accesses the same redistributor's GICR_VSGIR registers for a different VPE, which would corrupt the reading of the pending bits. We thus take the per-RD spinlock. Much fun. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-12-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 77 ++++++++++++++++++++++++++++++ include/linux/irqchip/arm-gic-v3.h | 14 ++++++ 2 files changed, 91 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 939dd0e1b7c3..97bcc79f5750 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4163,11 +4163,88 @@ static int its_sgi_set_affinity(struct irq_data *d, return IRQ_SET_MASK_OK; } +static int its_sgi_set_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool state) +{ + if (which != IRQCHIP_STATE_PENDING) + return -EINVAL; + + if (state) { + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + struct its_node *its = find_4_1_its(); + u64 val; + + val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id); + val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq); + writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); + } else { + its_configure_sgi(d, true); + } + + return 0; +} + +static int its_sgi_get_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, bool *val) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + void __iomem *base; + unsigned long flags; + u32 count = 1000000; /* 1s! */ + u32 status; + int cpu; + + if (which != IRQCHIP_STATE_PENDING) + return -EINVAL; + + /* + * Locking galore! We can race against two different events: + * + * - Concurent vPE affinity change: we must make sure it cannot + * happen, or we'll talk to the wrong redistributor. This is + * identical to what happens with vLPIs. + * + * - Concurrent VSGIPENDR access: As it involves accessing two + * MMIO registers, this must be made atomic one way or another. + */ + cpu = vpe_to_cpuid_lock(vpe, &flags); + raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); + base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K; + writel_relaxed(vpe->vpe_id, base + GICR_VSGIR); + do { + status = readl_relaxed(base + GICR_VSGIPENDR); + if (!(status & GICR_VSGIPENDR_BUSY)) + goto out; + + count--; + if (!count) { + pr_err_ratelimited("Unable to get SGI status\n"); + goto out; + } + cpu_relax(); + udelay(1); + } while (count); + +out: + raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); + vpe_to_cpuid_unlock(vpe, flags); + + if (!count) + return -ENXIO; + + *val = !!(status & (1 << d->hwirq)); + + return 0; +} + static struct irq_chip its_sgi_irq_chip = { .name = "GICv4.1-sgi", .irq_mask = its_sgi_mask_irq, .irq_unmask = its_sgi_unmask_irq, .irq_set_affinity = its_sgi_set_affinity, + .irq_set_irqchip_state = its_sgi_set_irqchip_state, + .irq_get_irqchip_state = its_sgi_get_irqchip_state, }; static int its_sgi_irq_domain_alloc(struct irq_domain *domain, diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 258a43242560..8fdfc7d3c6d2 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -348,6 +348,15 @@ #define GICR_VPENDBASER_4_1_VGRP1EN (1ULL << 58) #define GICR_VPENDBASER_4_1_VPEID GENMASK_ULL(15, 0) +#define GICR_VSGIR 0x0080 + +#define GICR_VSGIR_VPEID GENMASK(15, 0) + +#define GICR_VSGIPENDR 0x0088 + +#define GICR_VSGIPENDR_BUSY (1U << 31) +#define GICR_VSGIPENDR_PENDING GENMASK(15, 0) + /* * ITS registers, offsets from ITS_base */ @@ -371,6 +380,11 @@ #define GITS_TRANSLATER 0x10040 +#define GITS_SGIR 0x20020 + +#define GITS_SGIR_VPEID GENMASK_ULL(47, 32) +#define GITS_SGIR_VINTID GENMASK_ULL(3, 0) + #define GITS_CTLR_ENABLE (1U << 0) #define GITS_CTLR_ImDe (1U << 1) #define GITS_CTLR_ITS_NUMBER_SHIFT 4 -- Gitee From 3ca6d0a9a1b6cf2b4e451702e2d5329034c041c5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:19 +0000 Subject: [PATCH 64/94] irqchip/gic-v4.1: Plumb set_vcpu_affinity SGI callbacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just like for vLPIs, there is some configuration information that cannot be directly communicated through the normal irqchip API, and we have to use our good old friend set_vcpu_affinity as a side-band communication mechanism. This is used to configure group and priority for a given vSGI. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-13-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 18 ++++++++++++++++++ include/linux/irqchip/arm-gic-v4.h | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 97bcc79f5750..6111d1086bed 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4238,6 +4238,23 @@ static int its_sgi_get_irqchip_state(struct irq_data *d, return 0; } +static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) +{ + struct its_vpe *vpe = irq_data_get_irq_chip_data(d); + struct its_cmd_info *info = vcpu_info; + + switch (info->cmd_type) { + case PROP_UPDATE_VSGI: + vpe->sgi_config[d->hwirq].priority = info->priority; + vpe->sgi_config[d->hwirq].group = info->group; + its_configure_sgi(d, false); + return 0; + + default: + return -EINVAL; + } +} + static struct irq_chip its_sgi_irq_chip = { .name = "GICv4.1-sgi", .irq_mask = its_sgi_mask_irq, @@ -4245,6 +4262,7 @@ static struct irq_chip its_sgi_irq_chip = { .irq_set_affinity = its_sgi_set_affinity, .irq_set_irqchip_state = its_sgi_set_irqchip_state, .irq_get_irqchip_state = its_sgi_get_irqchip_state, + .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity, }; static int its_sgi_irq_domain_alloc(struct irq_domain *domain, diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 44e8c19e3d56..1b34100e3536 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -103,6 +103,7 @@ enum its_vcpu_info_cmd_type { SCHEDULE_VPE, DESCHEDULE_VPE, INVALL_VPE, + PROP_UPDATE_VSGI, }; struct its_cmd_info { @@ -115,6 +116,10 @@ struct its_cmd_info { bool g0en; bool g1en; }; + struct { + u8 priority; + bool group; + }; }; }; -- Gitee From dad6c7454275d12806eb69f8fdf4c3e7aa3f0d55 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:20 +0000 Subject: [PATCH 65/94] irqchip/gic-v4.1: Move doorbell management to the GICv4 abstraction layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to hide some of the differences between v4.0 and v4.1, move the doorbell management out of the KVM code, and into the GICv4-specific layer. This allows the calling code to ask for the doorbell when blocking, and otherwise to leave the doorbell permanently disabled. This matches the v4.1 code perfectly, and only results in a minor refactoring of the v4.0 code. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-14-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v4.c | 45 +++++++++++++++++++++++++++--- include/kvm/arm_vgic.h | 1 + include/linux/irqchip/arm-gic-v4.h | 3 +- virt/kvm/arm/vgic/vgic-v3.c | 4 ++- virt/kvm/arm/vgic/vgic-v4.c | 34 ++++++++++------------ 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index c01910d53f9e..117ba6db023d 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -87,6 +87,11 @@ static struct irq_domain *gic_domain; static const struct irq_domain_ops *vpe_domain_ops; static const struct irq_domain_ops *sgi_domain_ops; +static bool has_v4_1(void) +{ + return !!sgi_domain_ops; +} + int its_alloc_vcpu_irqs(struct its_vm *vm) { int vpe_base_irq, i; @@ -139,18 +144,50 @@ static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info) return irq_set_vcpu_affinity(vpe->irq, info); } -int its_schedule_vpe(struct its_vpe *vpe, bool on) +int its_make_vpe_non_resident(struct its_vpe *vpe, bool db) { - struct its_cmd_info info; + struct irq_desc *desc = irq_to_desc(vpe->irq); + struct its_cmd_info info = { }; int ret; WARN_ON(preemptible()); - info.cmd_type = on ? SCHEDULE_VPE : DESCHEDULE_VPE; + info.cmd_type = DESCHEDULE_VPE; + if (has_v4_1()) { + /* GICv4.1 can directly deal with doorbells */ + info.req_db = db; + } else { + /* Undo the nested disable_irq() calls... */ + while (db && irqd_irq_disabled(&desc->irq_data)) + enable_irq(vpe->irq); + } + + ret = its_send_vpe_cmd(vpe, &info); + if (!ret) + vpe->resident = false; + + return ret; +} + +int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en) +{ + struct its_cmd_info info = { }; + int ret; + + WARN_ON(preemptible()); + + info.cmd_type = SCHEDULE_VPE; + if (has_v4_1()) { + info.g0en = g0en; + info.g1en = g1en; + } else { + /* Disabled the doorbell, as we're about to enter the guest */ + disable_irq_nosync(vpe->irq); + } ret = its_send_vpe_cmd(vpe, &info); if (!ret) - vpe->resident = on; + vpe->resident = true; return ret; } diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 1ecd69c65ba6..465b5657a3e1 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -70,6 +70,7 @@ struct vgic_global { /* Hardware has GICv4? */ bool has_gicv4; + bool has_gicv4_1; /* GIC system register CPU interface */ struct static_key_false gicv3_cpuif; diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 1b34100e3536..34ed4b5754dd 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -125,7 +125,8 @@ struct its_cmd_info { int its_alloc_vcpu_irqs(struct its_vm *vm); void its_free_vcpu_irqs(struct its_vm *vm); -int its_schedule_vpe(struct its_vpe *vpe, bool on); +int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en); +int its_make_vpe_non_resident(struct its_vpe *vpe, bool db); int its_invall_vpe(struct its_vpe *vpe); int its_map_vlpi(int irq, struct its_vlpi_map *map); int its_get_vlpi(int irq, struct its_vlpi_map *map); diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index f45635a6f0ec..1bc09b523486 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -595,7 +595,9 @@ int vgic_v3_probe(const struct gic_kvm_info *info) /* GICv4 support? */ if (info->has_v4) { kvm_vgic_global_state.has_gicv4 = gicv4_enable; - kvm_info("GICv4 support %sabled\n", + kvm_vgic_global_state.has_gicv4_1 = info->has_v4_1 && gicv4_enable; + kvm_info("GICv4%s support %sabled\n", + kvm_vgic_global_state.has_gicv4_1 ? ".1" : "", gicv4_enable ? "en" : "dis"); } diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 46f875589c47..1eb0f8c76219 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -67,10 +67,10 @@ * it. And if we've migrated our vcpu from one CPU to another, we must * tell the ITS (so that the messages reach the right redistributor). * This is done in two steps: first issue a irq_set_affinity() on the - * irq corresponding to the vcpu, then call its_schedule_vpe(). You - * must be in a non-preemptible context. On exit, another call to - * its_schedule_vpe() tells the redistributor that we're done with the - * vcpu. + * irq corresponding to the vcpu, then call its_make_vpe_resident(). + * You must be in a non-preemptible context. On exit, a call to + * its_make_vpe_non_resident() tells the redistributor that we're done + * with the vcpu. * * Finally, the doorbell handling: Each vcpu is allocated an interrupt * which will fire each time a VLPI is made pending whilst the vcpu is @@ -86,7 +86,8 @@ static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info) struct kvm_vcpu *vcpu = info; /* We got the message, no need to fire again */ - if (!irqd_irq_disabled(&irq_to_desc(irq)->irq_data)) + if (!kvm_vgic_global_state.has_gicv4_1 && + !irqd_irq_disabled(&irq_to_desc(irq)->irq_data)) disable_irq_nosync(irq); vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true; @@ -199,19 +200,11 @@ void vgic_v4_teardown(struct kvm *kvm) int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db) { struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; - struct irq_desc *desc = irq_to_desc(vpe->irq); if (!vgic_supports_direct_msis(vcpu->kvm) || !vpe->resident) return 0; - /* - * If blocking, a doorbell is required. Undo the nested - * disable_irq() calls... - */ - while (need_db && irqd_irq_disabled(&desc->irq_data)) - enable_irq(vpe->irq); - - return its_schedule_vpe(vpe, false); + return its_make_vpe_non_resident(vpe, need_db); } int vgic_v4_load(struct kvm_vcpu *vcpu) @@ -232,18 +225,19 @@ int vgic_v4_load(struct kvm_vcpu *vcpu) if (err) return err; - /* Disabled the doorbell, as we're about to enter the guest */ - disable_irq_nosync(vpe->irq); - - err = its_schedule_vpe(vpe, true); + err = its_make_vpe_resident(vpe, false, vcpu->kvm->arch.vgic.enabled); if (err) return err; /* * Now that the VPE is resident, let's get rid of a potential - * doorbell interrupt that would still be pending. + * doorbell interrupt that would still be pending. This is a + * GICv4.0 only "feature"... */ - return irq_set_irqchip_state(vpe->irq, IRQCHIP_STATE_PENDING, false); + if (!kvm_vgic_global_state.has_gicv4_1) + err = irq_set_irqchip_state(vpe->irq, IRQCHIP_STATE_PENDING, false); + + return err; } static struct vgic_its *vgic_get_its(struct kvm *kvm, -- Gitee From 174c2fefe02ec40d8022805e5acdfe9a812767d1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 23 Jun 2020 10:44:08 +0100 Subject: [PATCH 66/94] KVM: arm64: vgic-v4: Plug race between non-residency and v4.1 doorbell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When making a vPE non-resident because it has hit a blocking WFI, the doorbell can fire at any time after the write to the RD. Crucially, it can fire right between the write to GICR_VPENDBASER and the write to the pending_last field in the its_vpe structure. This means that we would overwrite pending_last with stale data, and potentially not wakeup until some unrelated event (such as a timer interrupt) puts the vPE back on the CPU. GICv4 isn't affected by this as we actively mask the doorbell on entering the guest, while GICv4.1 automatically manages doorbell delivery without any hypervisor-driven masking. Use the vpe_lock to synchronize such update, which solves the problem altogether. Fixes: ae699ad348cdc ("irqchip/gic-v4.1: Move doorbell management to the GICv4 abstraction layer") Reported-by: Zenghui Yu Signed-off-by: Marc Zyngier Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 8 ++++++++ virt/kvm/arm/vgic/vgic-v4.c | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 6111d1086bed..36242b7b6ffb 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4040,16 +4040,24 @@ static void its_vpe_4_1_deschedule(struct its_vpe *vpe, u64 val; if (info->req_db) { + unsigned long flags; + /* * vPE is going to block: make the vPE non-resident with * PendingLast clear and DB set. The GIC guarantees that if * we read-back PendingLast clear, then a doorbell will be * delivered when an interrupt comes. + * + * Note the locking to deal with the concurrent update of + * pending_last from the doorbell interrupt handler that can + * run concurrently. */ + raw_spin_lock_irqsave(&vpe->vpe_lock, flags); val = its_clear_vpend_valid(vlpi_base, GICR_VPENDBASER_PendingLast, GICR_VPENDBASER_4_1_DB); vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); + raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); } else { /* * We're not blocking, so just make the vPE non-resident diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 1eb0f8c76219..7af4c8ce68fa 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -90,7 +90,15 @@ static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info) !irqd_irq_disabled(&irq_to_desc(irq)->irq_data)) disable_irq_nosync(irq); + /* + * The v4.1 doorbell can fire concurrently with the vPE being + * made non-resident. Ensure we only update pending_last + * *after* the non-residency sequence has completed. + */ + raw_spin_lock(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vpe_lock); vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true; + raw_spin_unlock(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vpe_lock); + kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); kvm_vcpu_kick(vcpu); -- Gitee From 30617c399317ff41725e48b37785f3c440d0174d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:21 +0000 Subject: [PATCH 67/94] irqchip/gic-v4.1: Add VSGI allocation/teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allocate per-VPE SGIs when initializing the GIC-specific part of the VPE data structure. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-15-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v4.c | 68 +++++++++++++++++++++++++++++- include/linux/irqchip/arm-gic-v4.h | 2 + 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 117ba6db023d..99b33f60ac63 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -92,6 +92,47 @@ static bool has_v4_1(void) return !!sgi_domain_ops; } +static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) +{ + char *name; + int sgi_base; + + if (!has_v4_1()) + return 0; + + name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current)); + if (!name) + goto err; + + vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx); + if (!vpe->fwnode) + goto err; + + kfree(name); + name = NULL; + + vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16, + sgi_domain_ops, vpe); + if (!vpe->sgi_domain) + goto err; + + sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16, + NUMA_NO_NODE, vpe, + false, NULL); + if (sgi_base <= 0) + goto err; + + return 0; + +err: + if (vpe->sgi_domain) + irq_domain_remove(vpe->sgi_domain); + if (vpe->fwnode) + irq_domain_free_fwnode(vpe->fwnode); + kfree(name); + return -ENOMEM; +} + int its_alloc_vcpu_irqs(struct its_vm *vm) { int vpe_base_irq, i; @@ -118,8 +159,13 @@ int its_alloc_vcpu_irqs(struct its_vm *vm) if (vpe_base_irq <= 0) goto err; - for (i = 0; i < vm->nr_vpes; i++) + for (i = 0; i < vm->nr_vpes; i++) { + int ret; vm->vpes[i]->irq = vpe_base_irq + i; + ret = its_alloc_vcpu_sgis(vm->vpes[i], i); + if (ret) + goto err; + } return 0; @@ -132,8 +178,28 @@ int its_alloc_vcpu_irqs(struct its_vm *vm) return -ENOMEM; } +static void its_free_sgi_irqs(struct its_vm *vm) +{ + int i; + + if (!has_v4_1()) + return; + + for (i = 0; i < vm->nr_vpes; i++) { + unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0); + + if (WARN_ON(!irq)) + continue; + + irq_domain_free_irqs(irq, 16); + irq_domain_remove(vm->vpes[i]->sgi_domain); + irq_domain_free_fwnode(vm->vpes[i]->fwnode); + } +} + void its_free_vcpu_irqs(struct its_vm *vm) { + its_free_sgi_irqs(vm); irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes); irq_domain_remove(vm->domain); irq_domain_free_fwnode(vm->fwnode); diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 34ed4b5754dd..b120a01952fe 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -49,6 +49,8 @@ struct its_vpe { }; /* GICv4.1 implementations */ struct { + struct fwnode_handle *fwnode; + struct irq_domain *sgi_domain; struct { u8 priority; bool enabled; -- Gitee From 24db44220510610b9afd710c1441da22da770a9b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:22 +0000 Subject: [PATCH 68/94] irqchip/gic-v4.1: Add VSGI property setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the SGI configuration entry point for KVM to use. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-16-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v4.c | 13 +++++++++++++ include/linux/irqchip/arm-gic-v4.h | 1 + 2 files changed, 14 insertions(+) diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 99b33f60ac63..0c18714ae13e 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -320,6 +320,19 @@ int its_prop_update_vlpi(int irq, u8 config, bool inv) return irq_set_vcpu_affinity(irq, &info); } +int its_prop_update_vsgi(int irq, u8 priority, bool group) +{ + struct its_cmd_info info = { + .cmd_type = PROP_UPDATE_VSGI, + { + .priority = priority, + .group = group, + }, + }; + + return irq_set_vcpu_affinity(irq, &info); +} + int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *vpe_ops, const struct irq_domain_ops *sgi_ops) diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index b120a01952fe..6976b8331b60 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -134,6 +134,7 @@ int its_map_vlpi(int irq, struct its_vlpi_map *map); int its_get_vlpi(int irq, struct its_vlpi_map *map); int its_unmap_vlpi(int irq); int its_prop_update_vlpi(int irq, u8 config, bool inv); +int its_prop_update_vsgi(int irq, u8 priority, bool group); struct irq_domain_ops; int its_init_v4(struct irq_domain *domain, -- Gitee From 313e7e44033a93cf2afe9ddfb704273e3c6c25f6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:23 +0000 Subject: [PATCH 69/94] irqchip/gic-v4.1: Eagerly vmap vPEs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we have HW-accelerated SGIs being delivered to VPEs, it becomes required to map the VPEs on all ITSs instead of relying on the lazy approach that we would use when using the ITS-list mechanism. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-17-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 52 ++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 36242b7b6ffb..ffa282d23b88 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -199,6 +199,15 @@ static DEFINE_IDA(its_vpeid_ida); #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) +/* + * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we + * always have vSGIs mapped. + */ +static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its) +{ + return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); +} + static u16 get_its_list(struct its_vm *vm) { struct its_node *its; @@ -208,7 +217,7 @@ static u16 get_its_list(struct its_vm *vm) if (!is_v4(its)) continue; - if (vm->vlpi_count[its->list_nr]) + if (require_its_list_vmovp(vm, its)) __set_bit(its->list_nr, &its_list); } @@ -1320,7 +1329,7 @@ static void its_send_vmovp(struct its_vpe *vpe) if (!is_v4(its)) continue; - if (!vpe->its_vm->vlpi_count[its->list_nr]) + if (!require_its_list_vmovp(vpe->its_vm, its)) continue; desc.its_vmovp_cmd.col = &its->collections[col_id]; @@ -1740,12 +1749,31 @@ static int its_irq_set_irqchip_state(struct irq_data *d, return 0; } +/* + * Two favourable cases: + * + * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times + * for vSGI delivery + * + * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough + * and we're better off mapping all VPEs always + * + * If neither (a) nor (b) is true, then we map vPEs on demand. + * + */ +static bool gic_requires_eager_mapping(void) +{ + if (!its_list_map || gic_rdists->has_rvpeid) + return true; + + return false; +} + static void its_map_vm(struct its_node *its, struct its_vm *vm) { unsigned long flags; - /* Not using the ITS list? Everything is always mapped. */ - if (!its_list_map) + if (gic_requires_eager_mapping()) return; raw_spin_lock_irqsave(&vmovp_lock, flags); @@ -1779,7 +1807,7 @@ static void its_unmap_vm(struct its_node *its, struct its_vm *vm) unsigned long flags; /* Not using the ITS list? Everything is always mapped. */ - if (!its_list_map) + if (gic_requires_eager_mapping()) return; raw_spin_lock_irqsave(&vmovp_lock, flags); @@ -4476,8 +4504,12 @@ static int its_vpe_irq_domain_activate(struct irq_domain *domain, struct its_vpe *vpe = irq_data_get_irq_chip_data(d); struct its_node *its; - /* If we use the list map, we issue VMAPP on demand... */ - if (its_list_map) + /* + * If we use the list map, we issue VMAPP on demand... Unless + * we're on a GICv4.1 and we eagerly map the VPE on all ITSs + * so that VSGIs can work. + */ + if (!gic_requires_eager_mapping()) return 0; /* Map the VPE to the first possible CPU */ @@ -4503,10 +4535,10 @@ static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, struct its_node *its; /* - * If we use the list map, we unmap the VPE once no VLPIs are - * associated with the VM. + * If we use the list map on GICv4.0, we unmap the VPE once no + * VLPIs are associated with the VM. */ - if (its_list_map) + if (!gic_requires_eager_mapping()) return; list_for_each_entry(its, &its_nodes, entry) { -- Gitee From def37b1a0ba3d177e1485a36a8330ada4d1aebd6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:24 +0000 Subject: [PATCH 70/94] KVM: arm64: GICv4.1: Let doorbells be auto-enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As GICv4.1 understands the life cycle of doorbells (instead of just randomly firing them at the most inconvenient time), just enable them at irq_request time, and be done with it. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-18-maz@kernel.org Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-v4.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 7af4c8ce68fa..9104a0523de1 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -149,6 +149,7 @@ int vgic_v4_init(struct kvm *kvm) kvm_for_each_vcpu(i, vcpu, kvm) { int irq = dist->its_vm.vpes[i]->irq; + unsigned long irq_flags = DB_IRQ_FLAGS; /* * Don't automatically enable the doorbell, as we're @@ -156,8 +157,14 @@ int vgic_v4_init(struct kvm *kvm) * blocked. Also disable the lazy disabling, as the * doorbell could kick us out of the guest too * early... + * + * On GICv4.1, the doorbell is managed in HW and must + * be left enabled. */ - irq_set_status_flags(irq, DB_IRQ_FLAGS); + if (kvm_vgic_global_state.has_gicv4_1) + irq_flags &= ~IRQ_NOAUTOEN; + irq_set_status_flags(irq, irq_flags); + ret = request_irq(irq, vgic_v4_doorbell_handler, 0, "vcpu", vcpu); if (ret) { -- Gitee From 5b5a520e3f33a548448e5c2dbd492b662dda870f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:25 +0000 Subject: [PATCH 71/94] KVM: arm64: GICv4.1: Add direct injection capability to SGI registers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most of the GICv3 emulation code that deals with SGIs now has to be aware of the v4.1 capabilities in order to benefit from it. Add such support, keyed on the interrupt having the hw flag set and being a SGI. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-19-maz@kernel.org Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-mmio-v3.c | 27 ++++++++-- virt/kvm/arm/vgic/vgic-mmio.c | 88 ++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 8 deletions(-) diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index b1e639ea22e9..4a72c1558916 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -274,8 +275,18 @@ static unsigned long vgic_v3_uaccess_read_pending(struct kvm_vcpu *vcpu, */ for (i = 0; i < len * 8; i++) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); + bool state = irq->pending_latch; - if (irq->pending_latch) + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + int err; + + err = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &state); + WARN_ON(err); + } + + if (state) value |= (1U << i); vgic_put_irq(vcpu->kvm, irq); @@ -957,8 +968,18 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1) * generate interrupts of either group. */ if (!irq->group || allow_group1) { - irq->pending_latch = true; - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + if (!irq->hw) { + irq->pending_latch = true; + vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + } else { + /* HW SGI? Ask the GIC to inject it */ + int err; + err = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + true); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + } } else { raw_spin_unlock_irqrestore(&irq->irq_lock, flags); } diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c index fb1dcd397b93..b384d46cbc7c 100644 --- a/virt/kvm/arm/vgic/vgic-mmio.c +++ b/virt/kvm/arm/vgic/vgic-mmio.c @@ -5,6 +5,8 @@ #include #include +#include +#include #include #include #include @@ -59,6 +61,11 @@ unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu, return value; } +static void vgic_update_vsgi(struct vgic_irq *irq) +{ + WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group)); +} + void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { @@ -71,7 +78,12 @@ void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, raw_spin_lock_irqsave(&irq->irq_lock, flags); irq->group = !!(val & BIT(i)); - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + vgic_update_vsgi(irq); + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + } else { + vgic_queue_irq_unlock(vcpu->kvm, irq, flags); + } vgic_put_irq(vcpu->kvm, irq); } @@ -113,7 +125,21 @@ void vgic_mmio_write_senable(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (vgic_irq_is_mapped_level(irq)) { + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (!irq->enabled) { + struct irq_data *data; + + irq->enabled = true; + data = &irq_to_desc(irq->host_irq)->irq_data; + while (irqd_irq_disabled(data)) + enable_irq(irq->host_irq); + } + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + continue; + } else if (vgic_irq_is_mapped_level(irq)) { bool was_high = irq->line_level; /* @@ -148,6 +174,8 @@ void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); + if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled) + disable_irq_nosync(irq->host_irq); irq->enabled = false; @@ -167,10 +195,22 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu, for (i = 0; i < len * 8; i++) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); unsigned long flags; + bool val; raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq_is_pending(irq)) - value |= (1U << i); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + int err; + + val = false; + err = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &val); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + } else { + val = irq_is_pending(irq); + } + + value |= ((u32)val << i); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); vgic_put_irq(vcpu->kvm, irq); @@ -236,6 +276,21 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, } raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + /* HW SGI? Ask the GIC to inject it */ + int err; + err = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + true); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + continue; + } + if (irq->hw) vgic_hw_irq_spending(vcpu, irq, is_uaccess); else @@ -290,6 +345,20 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + /* HW SGI? Ask the GIC to clear its pending bit */ + int err; + err = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + false); + WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + continue; + } + if (irq->hw) vgic_hw_irq_cpending(vcpu, irq, is_uaccess); else @@ -397,8 +466,15 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw) { + if (irq->hw && !vgic_irq_is_sgi(irq->intid)) { vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu); + } else if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + /* + * GICv4.1 VSGI feature doesn't track an active state, + * so let's not kid ourselves, there is nothing we can + * do here. + */ + irq->active = false; } else { u32 model = vcpu->kvm->arch.vgic.vgic_model; u8 active_source; @@ -542,6 +618,8 @@ void vgic_mmio_write_priority(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); /* Narrow the priority range to what we actually support */ irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); + if (irq->hw && vgic_irq_is_sgi(irq->intid)) + vgic_update_vsgi(irq); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); vgic_put_irq(vcpu->kvm, irq); -- Gitee From 83257a6cfe86ffb97ef33e1465bc01b094310f73 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:26 +0000 Subject: [PATCH 72/94] KVM: arm64: GICv4.1: Allow SGIs to switch between HW and SW interrupts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to let a guest buy in the new, active-less SGIs, we need to be able to switch between the two modes. Handle this by stopping all guest activity, transfer the state from one mode to the other, and resume the guest. Nothing calls this code so far, but a later patch will plug it into the MMIO emulation. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-20-maz@kernel.org Signed-off-by: 温家明 --- include/kvm/arm_vgic.h | 3 ++ virt/kvm/arm/vgic/vgic-v4.c | 98 +++++++++++++++++++++++++++++++++++++ virt/kvm/arm/vgic/vgic.h | 1 + 3 files changed, 102 insertions(+) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 465b5657a3e1..e474fe39869c 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -231,6 +231,9 @@ struct vgic_dist { /* distributor enabled */ bool enabled; + /* Wants SGIs without active state */ + bool nassgireq; + struct vgic_irq *spis; struct vgic_io_device dist_iodev; diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 9104a0523de1..b5fa73c9fd35 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -105,6 +105,104 @@ static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info) return IRQ_HANDLED; } +static void vgic_v4_sync_sgi_config(struct its_vpe *vpe, struct vgic_irq *irq) +{ + vpe->sgi_config[irq->intid].enabled = irq->enabled; + vpe->sgi_config[irq->intid].group = irq->group; + vpe->sgi_config[irq->intid].priority = irq->priority; +} + +static void vgic_v4_enable_vsgis(struct kvm_vcpu *vcpu) +{ + struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + int i; + + /* + * With GICv4.1, every virtual SGI can be directly injected. So + * let's pretend that they are HW interrupts, tied to a host + * IRQ. The SGI code will do its magic. + */ + for (i = 0; i < VGIC_NR_SGIS; i++) { + struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, i); + struct irq_desc *desc; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (irq->hw) + goto unlock; + + irq->hw = true; + irq->host_irq = irq_find_mapping(vpe->sgi_domain, i); + + /* Transfer the full irq state to the vPE */ + vgic_v4_sync_sgi_config(vpe, irq); + desc = irq_to_desc(irq->host_irq); + ret = irq_domain_activate_irq(irq_desc_get_irq_data(desc), + false); + if (!WARN_ON(ret)) { + /* Transfer pending state */ + ret = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + irq->pending_latch); + WARN_ON(ret); + irq->pending_latch = false; + } + unlock: + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + } +} + +static void vgic_v4_disable_vsgis(struct kvm_vcpu *vcpu) +{ + int i; + + for (i = 0; i < VGIC_NR_SGIS; i++) { + struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, i); + struct irq_desc *desc; + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (!irq->hw) + goto unlock; + + irq->hw = false; + ret = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &irq->pending_latch); + WARN_ON(ret); + + desc = irq_to_desc(irq->host_irq); + irq_domain_deactivate_irq(irq_desc_get_irq_data(desc)); + unlock: + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + } +} + +/* Must be called with the kvm lock held */ +void vgic_v4_configure_vsgis(struct kvm *kvm) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu; + int i; + + kvm_arm_halt_guest(kvm); + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (dist->nassgireq) + vgic_v4_enable_vsgis(vcpu); + else + vgic_v4_disable_vsgis(vcpu); + } + + kvm_arm_resume_guest(kvm); +} + /** * vgic_v4_init - Initialize the GICv4 data structures * @kvm: Pointer to the VM being initialized diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index c7fefd6b1c80..769e4802645e 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -316,5 +316,6 @@ void vgic_its_invalidate_cache(struct kvm *kvm); bool vgic_supports_direct_msis(struct kvm *kvm); int vgic_v4_init(struct kvm *kvm); void vgic_v4_teardown(struct kvm *kvm); +void vgic_v4_configure_vsgis(struct kvm *kvm); #endif -- Gitee From 32ec51e982fc602397cb5064d28027fbb29b6147 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:27 +0000 Subject: [PATCH 73/94] KVM: arm64: GICv4.1: Plumb SGI implementation selection in the distributor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GICv4.1 architecture gives the hypervisor the option to let the guest choose whether it wants the good old SGIs with an active state, or the new, HW-based ones that do not have one. For this, plumb the configuration of SGIs into the GICv3 MMIO handling, present the GICD_TYPER2.nASSGIcap to the guest, and handle the GICD_CTLR.nASSGIreq setting. In order to be able to deal with the restore of a guest, also apply the GICD_CTLR.nASSGIreq setting at first run so that we can move the restored SGIs to the HW if that's what the guest had selected in a previous life. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-21-maz@kernel.org Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-mmio-v3.c | 49 ++++++++++++++++++++++++++++++-- virt/kvm/arm/vgic/vgic-v3.c | 2 ++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index 4a72c1558916..de1c571d01c8 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -3,6 +3,7 @@ * VGICv3 MMIO handling functions */ +#include #include #include #include @@ -70,6 +71,8 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, if (vgic->enabled) value |= GICD_CTLR_ENABLE_SS_G1; value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS; + if (vgic->nassgireq) + value |= GICD_CTLR_nASSGIreq; break; case GICD_TYPER: value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS; @@ -81,6 +84,10 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19; } break; + case GICD_TYPER2: + if (kvm_vgic_global_state.has_gicv4_1) + value = GICD_TYPER2_nASSGIcap; + break; case GICD_IIDR: value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) | (vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) | @@ -98,17 +105,43 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, unsigned long val) { struct vgic_dist *dist = &vcpu->kvm->arch.vgic; - bool was_enabled = dist->enabled; switch (addr & 0x0c) { - case GICD_CTLR: + case GICD_CTLR: { + bool was_enabled, is_hwsgi; + + mutex_lock(&vcpu->kvm->lock); + + was_enabled = dist->enabled; + is_hwsgi = dist->nassgireq; + dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; + /* Not a GICv4.1? No HW SGIs */ + if (!kvm_vgic_global_state.has_gicv4_1) + val &= ~GICD_CTLR_nASSGIreq; + + /* Dist stays enabled? nASSGIreq is RO */ + if (was_enabled && dist->enabled) { + val &= ~GICD_CTLR_nASSGIreq; + val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi); + } + + /* Switching HW SGIs? */ + dist->nassgireq = val & GICD_CTLR_nASSGIreq; + if (is_hwsgi != dist->nassgireq) + vgic_v4_configure_vsgis(vcpu->kvm); + if (!was_enabled && dist->enabled) vgic_kick_vcpus(vcpu->kvm); + + mutex_unlock(&vcpu->kvm->lock); break; + } case GICD_TYPER: + case GICD_TYPER2: case GICD_IIDR: + /* This is at best for documentation purposes... */ return; } } @@ -117,10 +150,22 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { + struct vgic_dist *dist = &vcpu->kvm->arch.vgic; + switch (addr & 0x0c) { + case GICD_TYPER2: case GICD_IIDR: if (val != vgic_mmio_read_v3_misc(vcpu, addr, len)) return -EINVAL; + return 0; + case GICD_CTLR: + /* Not a GICv4.1? No HW SGIs */ + if (!kvm_vgic_global_state.has_gicv4_1) + val &= ~GICD_CTLR_nASSGIreq; + + dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; + dist->nassgireq = val & GICD_CTLR_nASSGIreq; + return 0; } vgic_mmio_write_v3_misc(vcpu, addr, len, val); diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index 1bc09b523486..2c9fc13e2c59 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -540,6 +540,8 @@ int vgic_v3_map_resources(struct kvm *kvm) goto out; } + if (kvm_vgic_global_state.has_gicv4_1) + vgic_v4_configure_vsgis(kvm); dist->ready = true; out: -- Gitee From 8b9204ee98c4327dd2884b7c6141e8c28dc03ea8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:28 +0000 Subject: [PATCH 74/94] KVM: arm64: GICv4.1: Reload VLPI configuration on distributor enable/disable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each time a Group-enable bit gets flipped, the state of these bits needs to be forwarded to the hardware. This is a pretty heavy handed operation, requiring all vcpus to reload their GICv4 configuration. It is thus implemented as a new request type. These enable bits are programmed into the HW by setting the VGrp{0,1}En fields of GICR_VPENDBASER when the vPEs are made resident again. Of course, we only support Group-1 for now... Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Link: https://lore.kernel.org/r/20200304203330.4967-22-maz@kernel.org Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-mmio-v3.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index de1c571d01c8..5e9afa3f2ae4 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -132,7 +132,10 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, if (is_hwsgi != dist->nassgireq) vgic_v4_configure_vsgis(vcpu->kvm); - if (!was_enabled && dist->enabled) + if (kvm_vgic_global_state.has_gicv4_1 && + was_enabled != dist->enabled) + kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4); + else if (!was_enabled && dist->enabled) vgic_kick_vcpus(vcpu->kvm); mutex_unlock(&vcpu->kvm->lock); -- Gitee From b1388f17f33b7f85befc5e86cbfe339c1fabddcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Fri, 6 Dec 2024 16:56:22 +0800 Subject: [PATCH 75/94] resolve patch Reload VLPI configuration conflict --- arch/arm/include/asm/kvm_host.h | 1 + arch/arm64/include/asm/kvm_host.h | 1 + virt/kvm/arm/arm.c | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index d8ac89879327..c3c3321676c9 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -39,6 +39,7 @@ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) +#define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index a639c7fee3a8..01886b83d120 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -44,6 +44,7 @@ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) +#define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index 3e3d95ea393c..6ddc7a6aa679 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -677,6 +677,14 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) * that a VCPU sees new virtual interrupts. */ kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); + + if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) { + /* The distributor enable bits were changed */ + preempt_disable(); + vgic_v4_put(vcpu, false); + vgic_v4_load(vcpu); + preempt_enable(); + } } } -- Gitee From 81dc7ebbd15fa0208e2a942ccbfcc97fde064fcd Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:29 +0000 Subject: [PATCH 76/94] KVM: arm64: GICv4.1: Allow non-trapping WFI when using HW SGIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just like for VLPIs, it is beneficial to avoid trapping on WFI when the vcpu is using the GICv4.1 SGIs. Add such a check to vcpu_clear_wfx_traps(). Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-23-maz@kernel.org Signed-off-by: 温家明 --- arch/arm64/include/asm/kvm_emulate.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 226339e1ed5b..1d4ceb096fa6 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -90,7 +90,8 @@ static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 &= ~HCR_TWE; - if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count)) + if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) || + vcpu->kvm->arch.vgic.nassgireq) vcpu->arch.hcr_el2 &= ~HCR_TWI; else vcpu->arch.hcr_el2 |= HCR_TWI; -- Gitee From 59f3bcca2fdb86bd616b77a53d9585890f890860 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 Mar 2020 20:33:30 +0000 Subject: [PATCH 77/94] KVM: arm64: GICv4.1: Expose HW-based SGIs in debugfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vgic-state debugfs file could do with showing the pending state of the HW-backed SGIs. Plug it into the low-level code. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20200304203330.4967-24-maz@kernel.org Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-debug.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/virt/kvm/arm/vgic/vgic-debug.c b/virt/kvm/arm/vgic/vgic-debug.c index cc12fe9b2df3..b13a9e3f99dd 100644 --- a/virt/kvm/arm/vgic/vgic-debug.c +++ b/virt/kvm/arm/vgic/vgic-debug.c @@ -178,6 +178,8 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, struct kvm_vcpu *vcpu) { char *type; + bool pending; + if (irq->intid < VGIC_NR_SGIS) type = "SGI"; else if (irq->intid < VGIC_NR_PRIVATE_IRQS) @@ -190,6 +192,16 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, if (irq->intid ==0 || irq->intid == VGIC_NR_PRIVATE_IRQS) print_header(s, irq, vcpu); + pending = irq->pending_latch; + if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + int err; + + err = irq_get_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + &pending); + WARN_ON_ONCE(err); + } + seq_printf(s, " %s %4d " " %2d " "%d%d%d%d%d%d%d " @@ -201,7 +213,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, "\n", type, irq->intid, (irq->target_vcpu) ? irq->target_vcpu->vcpu_id : -1, - irq->pending_latch, + pending, irq->line_level, irq->active, irq->enabled, -- Gitee From 53e5901aa01bee94c349c2257e373cf66e194ea4 Mon Sep 17 00:00:00 2001 From: luanshi Date: Thu, 12 Mar 2020 11:20:55 +0800 Subject: [PATCH 78/94] irqchip/gic-v3: Move irq_domain_update_bus_token to after checking for NULL domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit irq_domain_update_bus_token should be called after checking for NULL domain. Signed-off-by: Liguang Zhang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/1583983255-44115-1-git-send-email-zhangliguang@linux.alibaba.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index a8dd158bea40..1e4baac7bec8 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1568,7 +1568,6 @@ static int __init gic_init_bases(void __iomem *dist_base, gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, &gic_data); - irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); gic_data.rdists.has_rvpeid = true; gic_data.rdists.has_vlpis = true; @@ -1579,6 +1578,8 @@ static int __init gic_init_bases(void __iomem *dist_base, goto out_free; } + irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); + gic_data.has_rss = !!(typer & GICD_TYPER_RSS); pr_info("Distributor has %sRange Selector support\n", gic_data.has_rss ? "" : "no "); -- Gitee From ec581bf018e912639dacac5dfd15300aee11a584 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 10 Apr 2020 11:13:26 +0100 Subject: [PATCH 79/94] irqchip/gic-v4.1: Add support for VPENDBASER's Dirty+Valid signaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a vPE is made resident, the GIC starts parsing the virtual pending table to deliver pending interrupts. This takes place asynchronously, and can at times take a long while. Long enough that the vcpu enters the guest and hits WFI before any interrupt has been signaled yet. The vcpu then exits, blocks, and now gets a doorbell. Rince, repeat. In order to avoid the above, a (optional on GICv4, mandatory on v4.1) feature allows the GIC to feedback to the hypervisor whether it is done parsing the VPT by clearing the GICR_VPENDBASER.Dirty bit. The hypervisor can then wait until the GIC is ready before actually running the vPE. Plug the detection code as well as polling on vPE schedule. While at it, tidy-up the kernel message that displays the GICv4 optional features. Reviewed-by: Zenghui Yu Signed-off-by: Marc Zyngier Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 19 +++++++++++++++++++ drivers/irqchip/irq-gic-v3.c | 11 +++++++---- include/linux/irqchip/arm-gic-v3.h | 2 ++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index ffa282d23b88..05f3af02b161 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -3829,6 +3830,20 @@ static int its_vpe_set_affinity(struct irq_data *d, return IRQ_SET_MASK_OK_DONE; } +static void its_wait_vpt_parse_complete(void) +{ + void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); + u64 val; + + if (!gic_rdists->has_vpend_valid_dirty) + return; + + WARN_ON_ONCE(readq_relaxed_poll_timeout(vlpi_base + GICR_VPENDBASER, + val, + !(val & GICR_VPENDBASER_Dirty), + 10, 500)); +} + static void its_vpe_schedule(struct its_vpe *vpe) { void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); @@ -3859,6 +3874,8 @@ static void its_vpe_schedule(struct its_vpe *vpe) val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; val |= GICR_VPENDBASER_Valid; gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + + its_wait_vpt_parse_complete(); } static void its_vpe_deschedule(struct its_vpe *vpe) @@ -4059,6 +4076,8 @@ static void its_vpe_4_1_schedule(struct its_vpe *vpe, val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + + its_wait_vpt_parse_complete(); } static void its_vpe_4_1_deschedule(struct its_vpe *vpe, diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 1e4baac7bec8..6afaf5a44ffb 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -863,6 +863,7 @@ static int __gic_update_rdist_properties(struct redist_region *region, gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | gic_data.rdists.has_rvpeid); + gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); /* Detect non-sensical configurations */ if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { @@ -883,10 +884,11 @@ static void gic_update_rdist_properties(void) if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) gic_data.ppi_nr = 0; pr_info("%d PPIs implemented\n", gic_data.ppi_nr); - pr_info("%sVLPI support, %sdirect LPI support, %sRVPEID support\n", - !gic_data.rdists.has_vlpis ? "no " : "", - !gic_data.rdists.has_direct_lpi ? "no " : "", - !gic_data.rdists.has_rvpeid ? "no " : ""); + if (gic_data.rdists.has_vlpis) + pr_info("GICv4 features: %s%s%s\n", + gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", + gic_data.rdists.has_rvpeid ? "RVPEID " : "", + gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); } /* Check whether it's single security state view */ @@ -1572,6 +1574,7 @@ static int __init gic_init_bases(void __iomem *dist_base, gic_data.rdists.has_rvpeid = true; gic_data.rdists.has_vlpis = true; gic_data.rdists.has_direct_lpi = true; + gic_data.rdists.has_vpend_valid_dirty = true; if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { err = -ENOMEM; diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 8fdfc7d3c6d2..58050837b110 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -243,6 +243,7 @@ #define GICR_TYPER_PLPIS (1U << 0) #define GICR_TYPER_VLPIS (1U << 1) +#define GICR_TYPER_DIRTY (1U << 2) #define GICR_TYPER_DirectLPIS (1U << 3) #define GICR_TYPER_LAST (1U << 4) #define GICR_TYPER_RVPEID (1U << 7) @@ -684,6 +685,7 @@ struct rdists { bool has_vlpis; bool has_rvpeid; bool has_direct_lpi; + bool has_vpend_valid_dirty; }; struct irq_domain; -- Gitee From 7e84494aeaa671ac19cda84be279411ee0d551c4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 10 Apr 2020 12:11:39 +0100 Subject: [PATCH 80/94] irqchip/gic-v4.1: Update effective affinity of virtual SGIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the vSGIs are not directly visible to the host, they still get moved around by the CPU hotplug, for example. This results in the kernel moaning on the console, such as: genirq: irq_chip GICv4.1-sgi did not update eff. affinity mask of irq 38 Updating the effective affinity on set_affinity() fixes it. Reviewed-by: Zenghui Yu Signed-off-by: Marc Zyngier Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 05f3af02b161..d2844f399e87 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4215,6 +4215,7 @@ static int its_sgi_set_affinity(struct irq_data *d, * not on the host (since they can only be targetting a vPE). * Tell the kernel we've done whatever it asked for. */ + irq_data_update_effective_affinity(d, mask_val); return IRQ_SET_MASK_OK; } -- Gitee From 247bf44064320848d42251b7245d5f5426e38761 Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Fri, 5 Jun 2020 13:23:45 +0800 Subject: [PATCH 81/94] irqchip/gic-v4.1: Use readx_poll_timeout_atomic() to fix sleep in atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readx_poll_timeout() can sleep if @sleep_us is specified by the caller, and is therefore unsafe to be used inside the atomic context, which is this case when we use it to poll the GICR_VPENDBASER.Dirty bit in irq_set_vcpu_affinity() callback. Let's convert to its atomic version instead which helps to get the v4.1 board back to life! Fixes: 96806229ca03 ("irqchip/gic-v4.1: Add support for VPENDBASER's Dirty+Valid signaling") Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200605052345.1494-1-yuzenghui@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index d2844f399e87..98a592adcaac 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3838,10 +3838,10 @@ static void its_wait_vpt_parse_complete(void) if (!gic_rdists->has_vpend_valid_dirty) return; - WARN_ON_ONCE(readq_relaxed_poll_timeout(vlpi_base + GICR_VPENDBASER, - val, - !(val & GICR_VPENDBASER_Dirty), - 10, 500)); + WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER, + val, + !(val & GICR_VPENDBASER_Dirty), + 10, 500)); } static void its_vpe_schedule(struct its_vpe *vpe) -- Gitee From 0555219afd887c1aec03083c1c46705b86e13b8a Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Wed, 17 Mar 2021 10:07:19 +0000 Subject: [PATCH 82/94] irqchip/gic-v4.1: Disable vSGI upon (GIC CPUIF < v4.1) detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GIC CPU interfaces versions predating GIC v4.1 were not built to accommodate vINTID within the vSGI range; as reported in the GIC specifications (8.2 "Changes to the CPU interface"), it is CONSTRAINED UNPREDICTABLE to deliver a vSGI to a PE with ID_AA64PFR0_EL1.GIC < b0011. Check the GIC CPUIF version by reading the SYS_ID_AA64_PFR0_EL1. Disable vSGIs if a CPUIF version < 4.1 is detected to prevent using vSGIs on systems where they may misbehave. Signed-off-by: Lorenzo Pieralisi Cc: Marc Zyngier Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210317100719.3331-2-lorenzo.pieralisi@arm.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v4.c | 27 +++++++++++++++++++++++++-- include/linux/irqchip/arm-gic-v4.h | 2 ++ virt/kvm/arm/vgic/vgic-mmio-v3.c | 4 ++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 0c18714ae13e..0393dada43f0 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -87,17 +87,40 @@ static struct irq_domain *gic_domain; static const struct irq_domain_ops *vpe_domain_ops; static const struct irq_domain_ops *sgi_domain_ops; +#ifdef CONFIG_ARM64 +#include + +bool gic_cpuif_has_vsgi(void) +{ + unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + + fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT); + + return fld >= 0x3; +} +#else +bool gic_cpuif_has_vsgi(void) +{ + return false; +} +#endif + static bool has_v4_1(void) { return !!sgi_domain_ops; } +static bool has_v4_1_sgi(void) +{ + return has_v4_1() && gic_cpuif_has_vsgi(); +} + static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) { char *name; int sgi_base; - if (!has_v4_1()) + if (!has_v4_1_sgi()) return 0; name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current)); @@ -182,7 +205,7 @@ static void its_free_sgi_irqs(struct its_vm *vm) { int i; - if (!has_v4_1()) + if (!has_v4_1_sgi()) return; for (i = 0; i < vm->nr_vpes; i++) { diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 6976b8331b60..6d1352bf77c7 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -141,4 +141,6 @@ int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *vpe_ops, const struct irq_domain_ops *sgi_ops); +bool gic_cpuif_has_vsgi(void); + #endif diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index 5e9afa3f2ae4..d64a63682aef 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -85,7 +85,7 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, } break; case GICD_TYPER2: - if (kvm_vgic_global_state.has_gicv4_1) + if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi()) value = GICD_TYPER2_nASSGIcap; break; case GICD_IIDR: @@ -118,7 +118,7 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu, dist->enabled = val & GICD_CTLR_ENABLE_SS_G1; /* Not a GICv4.1? No HW SGIs */ - if (!kvm_vgic_global_state.has_gicv4_1) + if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi()) val &= ~GICD_CTLR_nASSGIreq; /* Dist stays enabled? nASSGIreq is RO */ -- Gitee From 1656c94fc3f476dfc3ddad258c7dbe9ac53027ed Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Mon, 22 Mar 2021 14:01:57 +0800 Subject: [PATCH 83/94] KVM: arm64: GICv4.1: Restore VLPI pending state to physical side MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When setting the forwarding path of a VLPI (switch to the HW mode), we can also transfer the pending state from irq->pending_latch to VPT (especially in migration, the pending states of VLPIs are restored into kvm’s vgic first). And we currently send "INT+VSYNC" to trigger a VLPI to pending. Signed-off-by: Zenghui Yu Signed-off-by: Shenming Lu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210322060158.1584-6-lushenming@huawei.com Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-v4.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index b5fa73c9fd35..77230ca6c50a 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -373,6 +373,7 @@ int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq, struct vgic_its *its; struct vgic_irq *irq; struct its_vlpi_map map; + unsigned long flags; int ret; if (!vgic_supports_direct_msis(kvm)) @@ -418,6 +419,24 @@ int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq, irq->host_irq = virq; atomic_inc(&map.vpe->vlpi_count); + /* Transfer pending state */ + raw_spin_lock_irqsave(&irq->irq_lock, flags); + if (irq->pending_latch) { + ret = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + irq->pending_latch); + WARN_RATELIMIT(ret, "IRQ %d", irq->host_irq); + + /* + * Clear pending_latch and communicate this state + * change via vgic_queue_irq_unlock. + */ + irq->pending_latch = false; + vgic_queue_irq_unlock(kvm, irq, flags); + } else { + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + } + out: mutex_unlock(&its->its_lock); return ret; -- Gitee From 5cb109fb87619286c1075834483d7b6ea115182c Mon Sep 17 00:00:00 2001 From: Shenming Lu Date: Mon, 22 Mar 2021 14:01:55 +0800 Subject: [PATCH 84/94] KVM: arm64: GICv4.1: Add function to get VLPI state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With GICv4.1 and the vPE unmapped, which indicates the invalidation of any VPT caches associated with the vPE, we can get the VLPI state by peeking at the VPT. So we add a function for this. Signed-off-by: Shenming Lu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210322060158.1584-4-lushenming@huawei.com Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-v4.c | 19 +++++++++++++++++++ virt/kvm/arm/vgic/vgic.h | 1 + 2 files changed, 20 insertions(+) diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 77230ca6c50a..08e929578700 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -203,6 +203,25 @@ void vgic_v4_configure_vsgis(struct kvm *kvm) kvm_arm_resume_guest(kvm); } +/* + * Must be called with GICv4.1 and the vPE unmapped, which + * indicates the invalidation of any VPT caches associated + * with the vPE, thus we can get the VLPI state by peeking + * at the VPT. + */ +void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val) +{ + struct its_vpe *vpe = &irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + int mask = BIT(irq->intid % BITS_PER_BYTE); + void *va; + u8 *ptr; + + va = page_address(vpe->vpt_page); + ptr = va + irq->intid / BITS_PER_BYTE; + + *val = !!(*ptr & mask); +} + /** * vgic_v4_init - Initialize the GICv4 data structures * @kvm: Pointer to the VM being initialized diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index 769e4802645e..aa6a223fe375 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -317,5 +317,6 @@ bool vgic_supports_direct_msis(struct kvm *kvm); int vgic_v4_init(struct kvm *kvm); void vgic_v4_teardown(struct kvm *kvm); void vgic_v4_configure_vsgis(struct kvm *kvm); +void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val); #endif -- Gitee From 73feecd49b379b057272f1afbf4b9089ee91be3b Mon Sep 17 00:00:00 2001 From: Shenming Lu Date: Mon, 22 Mar 2021 14:01:56 +0800 Subject: [PATCH 85/94] KVM: arm64: GICv4.1: Try to save VLPI state in save_pending_tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After pausing all vCPUs and devices capable of interrupting, in order to save the states of all interrupts, besides flushing the states in kvm’s vgic, we also try to flush the states of VLPIs in the virtual pending tables into guest RAM, but we need to have GICv4.1 and safely unmap the vPEs first. As for the saving of VSGIs, which needs the vPEs to be mapped and might conflict with the saving of VLPIs, but since we will map the vPEs back at the end of save_pending_tables and both savings require the kvm->lock to be held (thus only happen serially), it will work fine. Signed-off-by: Shenming Lu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20210322060158.1584-5-lushenming@huawei.com Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-v3.c | 66 +++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index 2c9fc13e2c59..7315d8ab0735 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include +#include #include #include #include @@ -356,6 +358,32 @@ int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq) return 0; } +/* + * The deactivation of the doorbell interrupt will trigger the + * unmapping of the associated vPE. + */ +static void unmap_all_vpes(struct vgic_dist *dist) +{ + struct irq_desc *desc; + int i; + + for (i = 0; i < dist->its_vm.nr_vpes; i++) { + desc = irq_to_desc(dist->its_vm.vpes[i]->irq); + irq_domain_deactivate_irq(irq_desc_get_irq_data(desc)); + } +} + +static void map_all_vpes(struct vgic_dist *dist) +{ + struct irq_desc *desc; + int i; + + for (i = 0; i < dist->its_vm.nr_vpes; i++) { + desc = irq_to_desc(dist->its_vm.vpes[i]->irq); + irq_domain_activate_irq(irq_desc_get_irq_data(desc), false); + } +} + /** * vgic_v3_save_pending_tables - Save the pending tables into guest RAM * kvm lock and all vcpu lock must be held @@ -365,13 +393,28 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) struct vgic_dist *dist = &kvm->arch.vgic; struct vgic_irq *irq; gpa_t last_ptr = ~(gpa_t)0; - int ret; + bool vlpi_avail = false; + int ret = 0; u8 val; + if (unlikely(!vgic_initialized(kvm))) + return -ENXIO; + + /* + * A preparation for getting any VLPI states. + * The above vgic initialized check also ensures that the allocation + * and enabling of the doorbells have already been done. + */ + if (kvm_vgic_global_state.has_gicv4_1) { + unmap_all_vpes(dist); + vlpi_avail = true; + } + list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { int byte_offset, bit_nr; struct kvm_vcpu *vcpu; gpa_t pendbase, ptr; + bool is_pending; bool stored; vcpu = irq->target_vcpu; @@ -387,24 +430,35 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) if (ptr != last_ptr) { ret = kvm_read_guest_lock(kvm, ptr, &val, 1); if (ret) - return ret; + goto out; last_ptr = ptr; } stored = val & (1U << bit_nr); - if (stored == irq->pending_latch) + + is_pending = irq->pending_latch; + + if (irq->hw && vlpi_avail) + vgic_v4_get_vlpi_state(irq, &is_pending); + + if (stored == is_pending) continue; - if (irq->pending_latch) + if (is_pending) val |= 1 << bit_nr; else val &= ~(1 << bit_nr); ret = kvm_write_guest_lock(kvm, ptr, &val, 1); if (ret) - return ret; + goto out; } - return 0; + +out: + if (vlpi_avail) + map_all_vpes(dist); + + return ret; } /** -- Gitee From 6496481ac58fdefe52188195373d3c373748eb23 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 19 Jan 2023 11:07:59 +0000 Subject: [PATCH 86/94] KVM: arm64: GICv4.1: Fix race with doorbell on VPE activation/deactivation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To save the vgic LPI pending state with GICv4.1, the VPEs must all be unmapped from the ITSs so that the sGIC caches can be flushed. The opposite is done once the state is saved. This is all done by using the activate/deactivate irqdomain callbacks directly from the vgic code. Crutially, this is done without holding the irqdesc lock for the interrupts that represent the VPE. And these callbacks are changing the state of the irqdesc. What could possibly go wrong? If a doorbell fires while we are messing with the irqdesc state, it will acquire the lock and change the interrupt state concurrently. Since we don't hole the lock, curruption occurs in on the interrupt state. Oh well. While acquiring the lock would fix this (and this was Shanker's initial approach), this is still a layering violation we could do without. A better approach is actually to free the VPE interrupt, do what we have to do, and re-request it. It is more work, but this usually happens only once in the lifetime of the VM and we don't really care about this sort of overhead. Fixes: f66b7b151e00 ("KVM: arm64: GICv4.1: Try to save VLPI state in save_pending_tables") Reported-by: Shanker Donthineni Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20230118022348.4137094-1-sdonthineni@nvidia.com Signed-off-by: 温家明 --- virt/kvm/arm/vgic/vgic-v3.c | 25 +++++++++++-------------- virt/kvm/arm/vgic/vgic-v4.c | 8 ++++++-- virt/kvm/arm/vgic/vgic.h | 1 + 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c index 7315d8ab0735..1ec64640b7df 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/virt/kvm/arm/vgic/vgic-v3.c @@ -362,26 +362,23 @@ int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq) * The deactivation of the doorbell interrupt will trigger the * unmapping of the associated vPE. */ -static void unmap_all_vpes(struct vgic_dist *dist) +static void unmap_all_vpes(struct kvm *kvm) { - struct irq_desc *desc; + struct vgic_dist *dist = &kvm->arch.vgic; int i; - for (i = 0; i < dist->its_vm.nr_vpes; i++) { - desc = irq_to_desc(dist->its_vm.vpes[i]->irq); - irq_domain_deactivate_irq(irq_desc_get_irq_data(desc)); - } + for (i = 0; i < dist->its_vm.nr_vpes; i++) + free_irq(dist->its_vm.vpes[i]->irq, kvm_get_vcpu(kvm, i)); } -static void map_all_vpes(struct vgic_dist *dist) +static void map_all_vpes(struct kvm *kvm) { - struct irq_desc *desc; + struct vgic_dist *dist = &kvm->arch.vgic; int i; - for (i = 0; i < dist->its_vm.nr_vpes; i++) { - desc = irq_to_desc(dist->its_vm.vpes[i]->irq); - irq_domain_activate_irq(irq_desc_get_irq_data(desc), false); - } + for (i = 0; i < dist->its_vm.nr_vpes; i++) + WARN_ON(vgic_v4_request_vpe_irq(kvm_get_vcpu(kvm, i), + dist->its_vm.vpes[i]->irq)); } /** @@ -406,7 +403,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) * and enabling of the doorbells have already been done. */ if (kvm_vgic_global_state.has_gicv4_1) { - unmap_all_vpes(dist); + unmap_all_vpes(kvm); vlpi_avail = true; } @@ -456,7 +453,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) out: if (vlpi_avail) - map_all_vpes(dist); + map_all_vpes(kvm); return ret; } diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 08e929578700..6e1222037ed0 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -222,6 +222,11 @@ void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val) *val = !!(*ptr & mask); } +int vgic_v4_request_vpe_irq(struct kvm_vcpu *vcpu, int irq) +{ + return request_irq(irq, vgic_v4_doorbell_handler, 0, "vcpu", vcpu); +} + /** * vgic_v4_init - Initialize the GICv4 data structures * @kvm: Pointer to the VM being initialized @@ -282,8 +287,7 @@ int vgic_v4_init(struct kvm *kvm) irq_flags &= ~IRQ_NOAUTOEN; irq_set_status_flags(irq, irq_flags); - ret = request_irq(irq, vgic_v4_doorbell_handler, - 0, "vcpu", vcpu); + ret = vgic_v4_request_vpe_irq(vcpu, irq); if (ret) { kvm_err("failed to allocate vcpu IRQ%d\n", irq); /* diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index aa6a223fe375..14a0d7310c13 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -318,5 +318,6 @@ int vgic_v4_init(struct kvm *kvm); void vgic_v4_teardown(struct kvm *kvm); void vgic_v4_configure_vsgis(struct kvm *kvm); void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val); +int vgic_v4_request_vpe_irq(struct kvm_vcpu *vcpu, int irq); #endif -- Gitee From d3b617ac0f437fe1c25cd6ac0c222b5c0a609317 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 16 Dec 2021 14:32:27 +0000 Subject: [PATCH 87/94] irqchip/gic-v4: Disable redistributors' view of the VPE table at boot time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jay Chen reported that using a kdump kernel on a GICv4.1 system results in a RAS error being delivered when the secondary kernel configures the ITS's view of the new VPE table. As it turns out, that's because each RD still has a pointer to the previous instance of the VPE table, and that particular implementation is very upset by seeing two bits of the HW that should point to the same table with different values. To solve this, let's invalidate any reference that any RD has to the VPE table when discovering the RDs. The ITS can then be programmed as expected. Reported-by: Jay Chen Signed-off-by: Marc Zyngier Cc: Lorenzo Pieralisi Link: https://lore.kernel.org/r/20211214064716.21407-1-jkchen@linux.alibaba.com Link: https://lore.kernel.org/r/20211216144804.1578566-1-maz@kernel.org Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 6afaf5a44ffb..f7dcb6332f68 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -857,6 +857,22 @@ static int __gic_update_rdist_properties(struct redist_region *region, { u64 typer = gic_read_typer(ptr + GICR_TYPER); + /* Boot-time cleanip */ + if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { + u64 val; + + /* Deactivate any present vPE */ + val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); + if (val & GICR_VPENDBASER_Valid) + gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, + ptr + SZ_128K + GICR_VPENDBASER); + + /* Mark the VPE table as invalid */ + val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); + val &= ~GICR_VPROPBASER_4_1_VALID; + gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); + } + gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); /* RVPEID implies some form of DirectLPI, no matter what the doc says... :-/ */ -- Gitee From 4973cbf66634dc5d38d48f7b4da9866b6e5e4f3d Mon Sep 17 00:00:00 2001 From: Shenming Lu Date: Sat, 28 Nov 2020 22:18:57 +0800 Subject: [PATCH 88/94] KVM: arm64: Delay the polling of the GICR_VPENDBASER.Dirty bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to reduce the impact of the VPT parsing happening on the GIC, we can split the vcpu reseidency in two phases: - programming GICR_VPENDBASER: this still happens in vcpu_load() - checking for the VPT parsing to be complete: this can happen on vcpu entry (in kvm_vgic_flush_hwstate()) This allows the GIC and the CPU to work in parallel, rewmoving some of the entry overhead. Suggested-by: Marc Zyngier Signed-off-by: Shenming Lu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20201128141857.983-3-lushenming@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 12 ++++++++---- drivers/irqchip/irq-gic-v4.c | 19 +++++++++++++++++++ include/kvm/arm_vgic.h | 1 + include/linux/irqchip/arm-gic-v4.h | 4 ++++ virt/kvm/arm/vgic/vgic-v4.c | 12 ++++++++++++ virt/kvm/arm/vgic/vgic.c | 3 +++ 6 files changed, 47 insertions(+), 4 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 98a592adcaac..dbc0223e2e60 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3874,8 +3874,6 @@ static void its_vpe_schedule(struct its_vpe *vpe) val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; val |= GICR_VPENDBASER_Valid; gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); - - its_wait_vpt_parse_complete(); } static void its_vpe_deschedule(struct its_vpe *vpe) @@ -3923,6 +3921,10 @@ static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) its_vpe_deschedule(vpe); return 0; + case COMMIT_VPE: + its_wait_vpt_parse_complete(); + return 0; + case INVALL_VPE: its_vpe_invall(vpe); return 0; @@ -4076,8 +4078,6 @@ static void its_vpe_4_1_schedule(struct its_vpe *vpe, val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); - - its_wait_vpt_parse_complete(); } static void its_vpe_4_1_deschedule(struct its_vpe *vpe, @@ -4152,6 +4152,10 @@ static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) its_vpe_4_1_deschedule(vpe, info); return 0; + case COMMIT_VPE: + its_wait_vpt_parse_complete(); + return 0; + case INVALL_VPE: its_vpe_4_1_invall(vpe); return 0; diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 0393dada43f0..4ea71b28f9f5 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -255,6 +255,8 @@ int its_make_vpe_non_resident(struct its_vpe *vpe, bool db) if (!ret) vpe->resident = false; + vpe->ready = false; + return ret; } @@ -281,6 +283,23 @@ int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en) return ret; } +int its_commit_vpe(struct its_vpe *vpe) +{ + struct its_cmd_info info = { + .cmd_type = COMMIT_VPE, + }; + int ret; + + WARN_ON(preemptible()); + + ret = its_send_vpe_cmd(vpe, &info); + if (!ret) + vpe->ready = true; + + return ret; +} + + int its_invall_vpe(struct its_vpe *vpe) { struct its_cmd_info info = { diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index e474fe39869c..39d3d849d926 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -401,6 +401,7 @@ int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int irq, struct kvm_kernel_irq_routing_entry *irq_entry); int vgic_v4_load(struct kvm_vcpu *vcpu); +void vgic_v4_commit(struct kvm_vcpu *vcpu); int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db); #endif /* __KVM_ARM_VGIC_H */ diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 6d1352bf77c7..2c63375bbd43 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -39,6 +39,8 @@ struct its_vpe { irq_hw_number_t vpe_db_lpi; /* VPE resident */ bool resident; + /* VPT parse complete */ + bool ready; union { /* GICv4.0 implementations */ struct { @@ -104,6 +106,7 @@ enum its_vcpu_info_cmd_type { PROP_UPDATE_AND_INV_VLPI, SCHEDULE_VPE, DESCHEDULE_VPE, + COMMIT_VPE, INVALL_VPE, PROP_UPDATE_VSGI, }; @@ -129,6 +132,7 @@ int its_alloc_vcpu_irqs(struct its_vm *vm); void its_free_vcpu_irqs(struct its_vm *vm); int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en); int its_make_vpe_non_resident(struct its_vpe *vpe, bool db); +int its_commit_vpe(struct its_vpe *vpe); int its_invall_vpe(struct its_vpe *vpe); int its_map_vlpi(int irq, struct its_vlpi_map *map); int its_get_vlpi(int irq, struct its_vlpi_map *map); diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/virt/kvm/arm/vgic/vgic-v4.c index 6e1222037ed0..f507e3fcffce 100644 --- a/virt/kvm/arm/vgic/vgic-v4.c +++ b/virt/kvm/arm/vgic/vgic-v4.c @@ -376,6 +376,18 @@ int vgic_v4_load(struct kvm_vcpu *vcpu) return err; } +void vgic_v4_commit(struct kvm_vcpu *vcpu) +{ + struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + + /* + * No need to wait for the vPE to be ready across a shallow guest + * exit, as only a vcpu_put will invalidate it. + */ + if (!vpe->ready) + its_commit_vpe(vpe); +} + static struct vgic_its *vgic_get_its(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *irq_entry) { diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c index 99b02ca730a8..2726523a1de9 100644 --- a/virt/kvm/arm/vgic/vgic.c +++ b/virt/kvm/arm/vgic/vgic.c @@ -906,6 +906,9 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) if (can_access_vgic_from_kernel()) vgic_restore_state(vcpu); + + if (vgic_supports_direct_msis(vcpu->kvm)) + vgic_v4_commit(vcpu); } void kvm_vgic_load(struct kvm_vcpu *vcpu) -- Gitee From 756e94e4304cc71b2a83fc6373fdd6723604ce27 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 17 Mar 2022 09:49:02 +0000 Subject: [PATCH 89/94] irqchip/gic-v4: Wait for GICR_VPENDBASER.Dirty to clear before descheduling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The way KVM drives GICv4.{0,1} is as follows: - vcpu_load() makes the VPE resident, instructing the RD to start scanning for interrupts - just before entering the guest, we check that the RD has finished scanning and that we can start running the vcpu - on preemption, we deschedule the VPE by making it invalid on the RD However, we are preemptible between the first two steps. If it so happens *and* that the RD was still scanning, we nonetheless write to the GICR_VPENDBASER register while Dirty is set, and bad things happen (we're in UNPRED land). This affects both the 4.0 and 4.1 implementations. Make sure Dirty is cleared before performing the deschedule, meaning that its_clear_vpend_valid() becomes a sort of full VPE residency barrier. Reported-by: Jingyi Wang Tested-by: Nianyao Tang Signed-off-by: Marc Zyngier Fixes: 57e3cebd022f ("KVM: arm64: Delay the polling of the GICR_VPENDBASER.Dirty bit") Link: https://lore.kernel.org/r/4aae10ba-b39a-5f84-754b-69c2eb0a2c03@huawei.com Signed-off-by: 温家明 --- drivers/irqchip/irq-gic-v3-its.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index dbc0223e2e60..5fe185984650 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3031,18 +3031,12 @@ static int __init allocate_lpi_tables(void) return 0; } -static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) +static u64 read_vpend_dirty_clear(void __iomem *vlpi_base) { u32 count = 1000000; /* 1s! */ bool clean; u64 val; - val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); - val &= ~GICR_VPENDBASER_Valid; - val &= ~clr; - val |= set; - gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); - do { val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); clean = !(val & GICR_VPENDBASER_Dirty); @@ -3053,10 +3047,26 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) } } while (!clean && count); - if (unlikely(val & GICR_VPENDBASER_Dirty)) { + if (unlikely(!clean)) pr_err_ratelimited("ITS virtual pending table not cleaning\n"); + + return val; +} + +static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) +{ + u64 val; + + /* Make sure we wait until the RD is done with the initial scan */ + val = read_vpend_dirty_clear(vlpi_base); + val &= ~GICR_VPENDBASER_Valid; + val &= ~clr; + val |= set; + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + + val = read_vpend_dirty_clear(vlpi_base); + if (unlikely(val & GICR_VPENDBASER_Dirty)) val |= GICR_VPENDBASER_PendingLast; - } return val; } -- Gitee From ce0f6cd34b227fd15d61125e9c460d328b95d476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Sat, 7 Dec 2024 16:08:38 +0800 Subject: [PATCH 90/94] cache: Workaround HiSilicon Taishan DC CVAU --- arch/arm64/include/asm/cpucaps.h | 1 + arch/arm64/include/asm/cputype.h | 3 +++ arch/arm64/kernel/cpu_errata.c | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index c400ea284f7f..1a8ecb5de491 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -61,6 +61,7 @@ #define ARM64_HAS_TLB_RANGE 51 #define ARM64_WORKAROUND_AMPERE_AC03_XXXXXX 52 #define ARM64_HAS_ECV 53 +#define ARM64_WORKAROUND_HISILICON_1980005 54 #define ARM64_NCAPS 80 diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 9b368906191d..5cb2b8f04bc2 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -107,6 +107,7 @@ #define FUJITSU_CPU_PART_A64FX 0x001 #define HISI_CPU_PART_TSV110 0xD01 +#define HISI_CPU_PART_TSV200 0xD02 #define AMPERE_CPU_PART_AMPERE1 0xAC3 @@ -145,6 +146,8 @@ #define MIDR_NVIDIA_CARMEL MIDR_CPU_MODEL(ARM_CPU_IMP_NVIDIA, NVIDIA_CPU_PART_CARMEL) #define MIDR_FUJITSU_A64FX MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) +#define MIDR_HISI_TSV200 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV200) + #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) /* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index ab8ebfa19ddc..2fbbeb001669 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -56,6 +56,31 @@ is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope) return model == entry->midr_range.model; } +#ifdef CONFIG_HISILICON_ERRATUM_1980005 +static bool +hisilicon_1980005_match(const struct arm64_cpu_capabilities *entry, + int scope) +{ + static const struct midr_range idc_support_list[] = { + MIDR_ALL_VERSIONS(MIDR_HISI_TSV110), + MIDR_REV(MIDR_HISI_TSV200, 1, 0), + MIDR_REV(MIDR_HISI_TSV200, 1, 2), + + { /* sentinel */ } + }; + + return is_midr_in_range_list(read_cpuid_id(), idc_support_list); +} + +static void +hisilicon_1980005_enable(const struct arm64_cpu_capabilities *__unused) +{ + cpus_set_cap(ARM64_HAS_CACHE_IDC); + arm64_ftr_reg_ctrel0.sys_val |= BIT(CTR_IDC_SHIFT); + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); +} +#endif + static bool has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry, int scope) @@ -831,6 +856,15 @@ static struct midr_range broken_aarch32_aes[] = { #endif const struct arm64_cpu_capabilities arm64_errata[] = { +#ifdef CONFIG_HISILICON_ERRATUM_1980005 + { + .desc = "Taishan IDC coherence workaround", + .capability = ARM64_WORKAROUND_HISILICON_1980005, + .matches = hisilicon_1980005_match, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .cpu_enable = hisilicon_1980005_enable, + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE { .desc = "ARM errata 826319, 827319, 824069, 819472", -- Gitee From be7cc0a7e8089c36de391dbd951dd3176f7a6aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Sat, 7 Dec 2024 16:29:20 +0800 Subject: [PATCH 91/94] irqchip/gicv3-its: Add workaround for hip09 ITS erratum 162100801 --- drivers/irqchip/irq-gic-v3-its.c | 36 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5fe185984650..e444266592e2 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -46,6 +46,7 @@ #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2) +#define ITS_FLAGS_WORKAROUND_HISILICON_162100801 (1ULL << 3) #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) @@ -1296,6 +1297,14 @@ static void its_send_vmapp(struct its_node *its, its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); } +static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) +{ + struct its_cmd_desc desc; + + desc.its_vinvall_cmd.vpe = vpe; + its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); +} + static void its_send_vmovp(struct its_vpe *vpe) { struct its_cmd_desc desc = {}; @@ -1335,19 +1344,14 @@ static void its_send_vmovp(struct its_vpe *vpe) desc.its_vmovp_cmd.col = &its->collections[col_id]; its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); + if (is_v4_1(its) && (its->flags & + ITS_FLAGS_WORKAROUND_HISILICON_162100801)) + its_send_vinvall(its, vpe); } raw_spin_unlock_irqrestore(&vmovp_lock, flags); } -static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) -{ - struct its_cmd_desc desc; - - desc.its_vinvall_cmd.vpe = vpe; - its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); -} - static void its_send_vinv(struct its_device *dev, u32 event_id) { struct its_cmd_desc desc; @@ -4708,6 +4712,14 @@ static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data) return true; } +static bool __maybe_unused its_enable_quirk_hip09_162100801(void *data) +{ + struct its_node *its = data; + + its->flags |= ITS_FLAGS_WORKAROUND_HISILICON_162100801; + return true; +} + static const struct gic_quirk its_quirks[] = { #ifdef CONFIG_CAVIUM_ERRATUM_22375 { @@ -4753,6 +4765,14 @@ static const struct gic_quirk its_quirks[] = { .mask = 0xffffffff, .init = its_enable_quirk_hip07_161600802, }, +#endif +#ifdef CONFIG_HISILICON_ERRATUM_162100801 + { + .desc = "ITS: Hip09 erratum 162100801", + .iidr = 0x00051736, + .mask = 0xffffffff, + .init = its_enable_quirk_hip09_162100801, + }, #endif { } -- Gitee From 1fb608fef7dddfc85b50b9a5239f03562f33db6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Sat, 7 Dec 2024 16:48:11 +0800 Subject: [PATCH 92/94] arm64: kernel: disable CNP on LINXICORE9100 --- arch/arm64/include/asm/cpucaps.h | 1 + arch/arm64/include/asm/cputype.h | 2 ++ arch/arm64/kernel/cpu_errata.c | 14 ++++++++++++++ arch/arm64/kernel/cpufeature.c | 3 +++ 4 files changed, 20 insertions(+) diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 1a8ecb5de491..b8aa8cd429a5 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -62,6 +62,7 @@ #define ARM64_WORKAROUND_AMPERE_AC03_XXXXXX 52 #define ARM64_HAS_ECV 53 #define ARM64_WORKAROUND_HISILICON_1980005 54 +#define ARM64_WORKAROUND_HISILICON_ERRATUM_162100125 55 #define ARM64_NCAPS 80 diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 5cb2b8f04bc2..3226929c8816 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -108,6 +108,7 @@ #define HISI_CPU_PART_TSV110 0xD01 #define HISI_CPU_PART_TSV200 0xD02 +#define HISI_CPU_PART_LINXICORE9100 0xD02 #define AMPERE_CPU_PART_AMPERE1 0xAC3 @@ -147,6 +148,7 @@ #define MIDR_FUJITSU_A64FX MIDR_CPU_MODEL(ARM_CPU_IMP_FUJITSU, FUJITSU_CPU_PART_A64FX) #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) #define MIDR_HISI_TSV200 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV200) +#define MIDR_HISI_LINXICORE9100 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_LINXICORE9100) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 2fbbeb001669..0bdd67401d86 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -777,6 +777,13 @@ static const struct midr_range cavium_erratum_30115_cpus[] = { }; #endif +#ifdef CONFIG_HISILICON_ERRATUM_162100125 +static const struct midr_range hisilicon_erratum_162100125_cpus[] = { + MIDR_REV(MIDR_HISI_LINXICORE9100, 0, 0), + {}, +}; +#endif + #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = { { @@ -938,6 +945,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .cpu_enable = cpu_enable_trap_ctr_access, }, +#ifdef CONFIG_HISILICON_ERRATUM_162100125 + { + .desc = "Hisilicon erratum 162100125", + .capability = ARM64_WORKAROUND_HISILICON_ERRATUM_162100125, + ERRATA_MIDR_RANGE_LIST(hisilicon_erratum_162100125_cpus), + }, +#endif #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 { .desc = "Qualcomm Technologies Falkor/Kryo erratum 1003", diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index f394047685df..6c0cdac84fb6 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -977,6 +977,9 @@ has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope) if (is_kdump_kernel()) return false; + if (cpus_have_const_cap(ARM64_WORKAROUND_HISILICON_ERRATUM_162100125)) + return false; + return has_cpuid_feature(entry, scope); } -- Gitee From 208178a9457340d3a2b0ee37a7116f9cfaaa979a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Sat, 7 Dec 2024 17:12:39 +0800 Subject: [PATCH 93/94] iommu/arm-smmu-v3: Add a SYNC command to avoid broken page table prefetch --- arch/arm64/Kconfig | 13 +++++++++++++ arch/arm64/include/asm/cpucaps.h | 1 + arch/arm64/kernel/cpu_errata.c | 14 ++++++++++++++ drivers/iommu/arm-smmu-v3.c | 20 ++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 8e1444c47769..2a267eb1e7a1 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -769,6 +769,19 @@ config HISILICON_ERRATUM_162102203 If unsure, say N. +config HISILICON_ERRATUM_162100602 + bool "Hisilicon erratum 162100602" + depends on ARM_SMMU_V3 + default y + help + On Hisilicon LINXICORE9100 cores, SMMU pagetable prefetch features may + prefetch and use a invalid PTE even the PTE is valid at that time. This + will cause the device trigger fake pagefaults. If the SMMU works in + terminate mode, transactions which occur fake pagefaults will be aborted, + and could result in unexpected errors. + + If unsure, say Y. + config QCOM_FALKOR_ERRATUM_1003 bool "Falkor E1003: Incorrect translation due to ASID change" default y diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index b8aa8cd429a5..163a38107d3d 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -63,6 +63,7 @@ #define ARM64_HAS_ECV 53 #define ARM64_WORKAROUND_HISILICON_1980005 54 #define ARM64_WORKAROUND_HISILICON_ERRATUM_162100125 55 +#define ARM64_WORKAROUND_HISILICON_ERRATUM_162100602 56 #define ARM64_NCAPS 80 diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 0bdd67401d86..6706d8ad8cd4 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -784,6 +784,13 @@ static const struct midr_range hisilicon_erratum_162100125_cpus[] = { }; #endif +#ifdef CONFIG_HISILICON_ERRATUM_162100602 +static const struct midr_range hisilicon_erratum_162100602_cpus[] = { + MIDR_REV(MIDR_HISI_LINXICORE9100, 0, 0), + {}, +}; +#endif + #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = { { @@ -952,6 +959,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(hisilicon_erratum_162100125_cpus), }, #endif +#ifdef CONFIG_HISILICON_ERRATUM_162100602 + { + .desc = "Hisilicon erratum 162100602", + .capability = ARM64_WORKAROUND_HISILICON_ERRATUM_162100602, + ERRATA_MIDR_RANGE_LIST(hisilicon_erratum_162100602_cpus), + }, +#endif #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003 { .desc = "Qualcomm Technologies Falkor/Kryo erratum 1003", diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 62b431831f2e..cfbf5393e255 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2549,6 +2549,23 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, gather->pgsize, true, smmu_domain); } +#ifdef CONFIG_HISILICON_ERRATUM_162100602 +static void arm_smmu_iotlb_sync_map(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + if (!cpus_have_const_cap(ARM64_WORKAROUND_HISILICON_ERRATUM_162100602)) + return; + + if(smmu_domain->smmu) + arm_smmu_tlb_inv_context(smmu_domain); + //granule_size = 1 << __ffs(smmu_domain->domain.pgsize_bitmap); + + /* Add a SYNC command to sync io-pgtale to avoid errors in pgtable prefetch*/ + //arm_smmu_tlb_inv_range_domain(iova, granule_size, granule_size, true, smmu_domain); +} +#endif + static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -2843,6 +2860,9 @@ static struct iommu_ops arm_smmu_ops = { .unmap = arm_smmu_unmap, .flush_iotlb_all = arm_smmu_flush_iotlb_all, .iotlb_sync = arm_smmu_iotlb_sync, +#ifdef CONFIG_HISILICON_ERRATUM_162100602 + .iotlb_sync_map = arm_smmu_iotlb_sync_map, +#endif .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, -- Gitee From 90f3faf12f0009a0416ffd41f2f2641652696bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A9=E5=AE=B6=E6=98=8E?= Date: Sat, 7 Dec 2024 17:50:13 +0800 Subject: [PATCH 94/94] resolve drivers/irqchip/irq-gic-phytium-2500-its.c error message --- drivers/irqchip/irq-gic-phytium-2500-its.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/irqchip/irq-gic-phytium-2500-its.c b/drivers/irqchip/irq-gic-phytium-2500-its.c index 7a77d9787f2e..1ff4895741bf 100644 --- a/drivers/irqchip/irq-gic-phytium-2500-its.c +++ b/drivers/irqchip/irq-gic-phytium-2500-its.c @@ -2162,12 +2162,12 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base) bool clean; u64 val; - val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); + val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); val &= ~GICR_VPENDBASER_Valid; - gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); do { - val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); + val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); clean = !(val & GICR_VPENDBASER_Dirty); if (!clean) { count--; @@ -2277,7 +2277,7 @@ static void its_cpu_init_lpis(void) val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", smp_processor_id(), val); - gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); /* * Also clear Valid bit of GICR_VPENDBASER, in case some @@ -2947,7 +2947,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; val |= GICR_VPROPBASER_RaWb; val |= GICR_VPROPBASER_InnerShareable; - gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); val = virt_to_phys(page_address(vpe->vpt_page)) & GENMASK_ULL(51, 16); @@ -2965,7 +2965,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) val |= GICR_VPENDBASER_PendingLast; val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; val |= GICR_VPENDBASER_Valid; - gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); + gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); } static void its_vpe_deschedule(struct its_vpe *vpe) @@ -4124,7 +4124,7 @@ int __init phytium_its_init(struct fwnode_handle *handle, struct rdists *rdists, if (has_v4 & rdists->has_vlpis) { if (its_init_vpe_domain() || - its_init_v4(parent_domain, &its_vpe_domain_ops)) { + its_init_v4(parent_domain, &its_vpe_domain_ops, NULL)) { rdists->has_vlpis = false; pr_err("ITS: Disabling GICv4 support\n"); } -- Gitee