irqchip/gicv3: Workaround for Cavium ThunderX erratum 23154

This patch implements Cavium ThunderX erratum 23154.

The gicv3 of ThunderX requires a modified version for reading the IAR
status to ensure data synchronization. Since this is in the fast-path
and called with each interrupt, runtime patching is used using jump
label patching for smallest overhead (no-op). This is the same
technique as used for tracepoints.

Signed-off-by: Robert Richter <rrichter@cavium.com>
Reviewed-by: Marc Zygnier <marc.zyngier@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Tirumalesh Chalamarla <tchalamarla@cavium.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Will Deacon <will.deacon@arm.com>
Link: http://lkml.kernel.org/r/1442869119-1814-3-git-send-email-rric@kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 07d1811aa03fcd1ecd5ee7c260688a59f7ab97e4..490df449377d822ccf6d3161be3944b851c1d443 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -348,6 +348,16 @@ config ARM64_ERRATUM_843419
 
          If unsure, say Y.
 
+config CAVIUM_ERRATUM_23154
+       bool "Cavium erratum 23154: Access to ICC_IAR1_EL1 is not sync'ed"
+       default y
+       help
+         The gicv3 of ThunderX requires a modified version for
+         reading the IAR status to ensure data synchronization
+         (access to icc_iar1_el1 is not sync'ed before and after).
+
+         If unsure, say Y.
+
 endmenu
 
 

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 171570702bb801431f141f46238eb0d9e1895fe5..dbc78d2b8cc6bbd43af2bf4781b6fffaa54a4fe1 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -27,8 +27,9 @@
 #define ARM64_HAS_SYSREG_GIC_CPUIF             3
 #define ARM64_HAS_PAN                          4
 #define ARM64_HAS_LSE_ATOMICS                  5
+#define ARM64_WORKAROUND_CAVIUM_23154          6
 
-#define ARM64_NCAPS                            6
+#define ARM64_NCAPS                            7
 
 #ifndef __ASSEMBLY__
 

diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index ee6403df9fe4c1f32e9a7b30172a7a141056ec40..100a3d1b17c854d6c1a2c465b56ad8f101ab5efa 100644 (file)
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -62,15 +62,18 @@
        (0xf                    << MIDR_ARCHITECTURE_SHIFT) | \
        ((partnum)              << MIDR_PARTNUM_SHIFT))
 
-#define ARM_CPU_IMP_ARM                0x41
-#define ARM_CPU_IMP_APM                0x50
+#define ARM_CPU_IMP_ARM                        0x41
+#define ARM_CPU_IMP_APM                        0x50
+#define ARM_CPU_IMP_CAVIUM             0x43
 
-#define ARM_CPU_PART_AEM_V8    0xD0F
-#define ARM_CPU_PART_FOUNDATION        0xD00
-#define ARM_CPU_PART_CORTEX_A57        0xD07
-#define ARM_CPU_PART_CORTEX_A53        0xD03
+#define ARM_CPU_PART_AEM_V8            0xD0F
+#define ARM_CPU_PART_FOUNDATION                0xD00
+#define ARM_CPU_PART_CORTEX_A57                0xD07
+#define ARM_CPU_PART_CORTEX_A53                0xD03
 
-#define APM_CPU_PART_POTENZA   0x000
+#define APM_CPU_PART_POTENZA           0x000
+
+#define CAVIUM_CPU_PART_THUNDERX       0x0A1
 
 #define ID_AA64MMFR0_BIGENDEL0_SHIFT   16
 #define ID_AA64MMFR0_BIGENDEL0_MASK    (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 6ffd914385609d0aab875813e4e9e8b690976421..574450c257a4d038e17d4eb8a47b954b20b2b9bb 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -23,6 +23,7 @@
 
 #define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
 #define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_THUNDERX  MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
 
 #define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
                        MIDR_ARCHITECTURE_MASK)
@@ -81,6 +82,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                .capability = ARM64_WORKAROUND_845719,
                MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
        },
+#endif
+#ifdef CONFIG_CAVIUM_ERRATUM_23154
+       {
+       /* Cavium ThunderX, pass 1.x */
+               .desc = "Cavium erratum 23154",
+               .capability = ARM64_WORKAROUND_CAVIUM_23154,
+               MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
+       },
 #endif
        {
        }

diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 36ecfc870e5a6cb86698cb218ac00cccc491e299..eecec71faa119c23f056908da4aa6bb72a20d957 100644 (file)
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -109,7 +109,7 @@ static void gic_redist_wait_for_rwp(void)
 }
 
 /* Low level accessors */
-static u64 __maybe_unused gic_read_iar(void)
+static u64 gic_read_iar_common(void)
 {
        u64 irqstat;
 
@@ -117,6 +117,38 @@ static u64 __maybe_unused gic_read_iar(void)
        return irqstat;
 }
 
+/*
+ * Cavium ThunderX erratum 23154
+ *
+ * The gicv3 of ThunderX requires a modified version for reading the
+ * IAR status to ensure data synchronization (access to icc_iar1_el1
+ * is not sync'ed before and after).
+ */
+static u64 gic_read_iar_cavium_thunderx(void)
+{
+       u64 irqstat;
+
+       asm volatile(
+               "nop;nop;nop;nop\n\t"
+               "nop;nop;nop;nop\n\t"
+               "mrs_s %0, " __stringify(ICC_IAR1_EL1) "\n\t"
+               "nop;nop;nop;nop"
+               : "=r" (irqstat));
+       mb();
+
+       return irqstat;
+}
+
+static struct static_key is_cavium_thunderx = STATIC_KEY_INIT_FALSE;
+
+static u64 __maybe_unused gic_read_iar(void)
+{
+       if (static_key_false(&is_cavium_thunderx))
+               return gic_read_iar_cavium_thunderx();
+       else
+               return gic_read_iar_common();
+}
+
 static void __maybe_unused gic_write_pmr(u64 val)
 {
        asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val));
@@ -836,6 +868,12 @@ static const struct irq_domain_ops gic_irq_domain_ops = {
        .free = gic_irq_domain_free,
 };
 
+static void gicv3_enable_quirks(void)
+{
+       if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154))
+               static_key_slow_inc(&is_cavium_thunderx);
+}
+
 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
 {
        void __iomem *dist_base;
@@ -901,6 +939,8 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
        gic_data.nr_redist_regions = nr_redist_regions;
        gic_data.redist_stride = redist_stride;
 
+       gicv3_enable_quirks();
+
        /*
         * Find out how many interrupts are supported.
         * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)