[PATCH] KVM: arm64: Mitigate AmpereOne erratum AC03_CPU_36

[Oliver Upton]

The AmpereOne design suffers from an erratum where if an asynchronous
exception arrives while EL2 is modifying hypervisor exception controls
(i.e. HCR_EL2, SCTLR_EL2) the PE may take an invalid exception to
another EL.

KVM's guest entry/exit path is unaffected because DAIF is already masked
when modifying HCR_EL2. However, KVM only masks IRQs when switching to
the guest MMU context for a TLB invalidation, meaning it is possible for
an intervening exception to trigger the erratum. This becomes
immediately obvious when profiling a KVM VM with pseudo-NMIs enabled,
such as:

  perf record ./dirty_log_perf_test -m 2 -s anonymous_thp -v 4 -b 4G

Mitigate the erratum by masking DAIF completely before switching to the
guest MMU context.

Signed-off-by: Oliver Upton <oliver.upton@xxxxxxxxx>
---
 Documentation/arch/arm64/silicon-errata.rst |  2 ++
 arch/arm64/Kconfig                          | 16 ++++++++++++++++
 arch/arm64/kernel/cpu_errata.c              |  7 +++++++
 arch/arm64/kvm/hyp/vhe/tlb.c                | 20 ++++++++++++++++++--
 arch/arm64/tools/cpucaps                    |  1 +
 5 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index f47f63bcf67c..aea7a75ae434 100644
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -52,6 +52,8 @@ stable kernels.
 | Allwinner      | A64/R18         | UNKNOWN1        | SUN50I_ERRATUM_UNKNOWN1     |
 +----------------+-----------------+-----------------+-----------------------------+
 +----------------+-----------------+-----------------+-----------------------------+
+| Ampere         | AmpereOne       | AC03_CPU_36     | AMPERE_ERRATUM_AC03_CPU_36  |
++----------------+-----------------+-----------------+-----------------------------+
 | Ampere         | AmpereOne       | AC03_CPU_38     | AMPERE_ERRATUM_AC03_CPU_38  |
 +----------------+-----------------+-----------------+-----------------------------+
 +----------------+-----------------+-----------------+-----------------------------+
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 7b071a00425d..9ba04b90e945 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -434,6 +434,22 @@ config AMPERE_ERRATUM_AC03_CPU_38
 
 	  If unsure, say Y.
 
+config AMPERE_ERRATUM_AC03_CPU_36
+	bool "AmpereOne: AC03_CPU_36: CPU takes invalid asynchronous exception while changing exception controls"
+	default y
+	help
+	  This option adds an alternative code sequence to work around Ampere
+	  erratum AC03_CPU_36 on AmpereOne parts.
+
+	  The affected designs can take an invalid exception to an incorrect
+	  exception level if an asynchronous exception arrives while software
+	  is changing the EL2 exception controls (i.e. HCR_EL2, SCTLR_EL2).
+
+	  The workaround forces KVM to mask all asynchronous exception sources
+	  when switching to the guest MMU context for TLB invalidations.
+
+	  If unsure, say Y.
+
 config ARM64_WORKAROUND_CLEAN_CACHE
 	bool
 
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index e29e0fea63fb..7e2856360f38 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -727,6 +727,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.capability = ARM64_WORKAROUND_AMPERE_AC03_CPU_38,
 		ERRATA_MIDR_ALL_VERSIONS(MIDR_AMPERE1),
 	},
+#endif
+#ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_36
+	{
+		.desc = "AmpereOne erratum AC03_CPU_36",
+		.capability = ARM64_WORKAROUND_AMPERE_AC03_CPU_36,
+		ERRATA_MIDR_ALL_VERSIONS(MIDR_AMPERE1),
+	},
 #endif
 	{
 	}
diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c
index b636b4111dbf..cedfb0a32fa0 100644
--- a/arch/arm64/kvm/hyp/vhe/tlb.c
+++ b/arch/arm64/kvm/hyp/vhe/tlb.c
@@ -17,13 +17,29 @@ struct tlb_inv_context {
 	u64			sctlr;
 };
 
+#define __tlb_daif_save(flags)						\
+({									\
+	if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_36))	\
+		flags = local_daif_save();				\
+	else								\
+		local_irq_save(flags);					\
+})
+
+#define __tlb_daif_restore(flags)					\
+({									\
+	if (cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_36))	\
+		local_daif_restore(flags);				\
+	else								\
+		local_irq_restore(flags);				\
+})
+
 static void __tlb_switch_to_guest(struct kvm_s2_mmu *mmu,
 				  struct tlb_inv_context *cxt)
 {
 	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 	u64 val;
 
-	local_irq_save(cxt->flags);
+	__tlb_daif_save(cxt->flags);
 
 	if (vcpu && mmu != vcpu->arch.hw_mmu)
 		cxt->mmu = vcpu->arch.hw_mmu;
@@ -86,7 +102,7 @@ static void __tlb_switch_to_host(struct tlb_inv_context *cxt)
 		write_sysreg_el1(cxt->sctlr, SYS_SCTLR);
 	}
 
-	local_irq_restore(cxt->flags);
+	__tlb_daif_restore(cxt->flags);
 }
 
 void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index b98c38288a9d..cfaae3b4d2d0 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -85,6 +85,7 @@ WORKAROUND_2457168
 WORKAROUND_2645198
 WORKAROUND_2658417
 WORKAROUND_2966298
+WORKAROUND_AMPERE_AC03_CPU_36
 WORKAROUND_AMPERE_AC03_CPU_38
 WORKAROUND_TRBE_OVERWRITE_FILL_MODE
 WORKAROUND_TSB_FLUSH_FAILURE
-- 
2.43.0.472.g3155946c3a-goog

-- 
Thanks,
Oliver