arm64: Fix minor issues with the dcache_by_line_op macro

The dcache_by_line_op macro suffers from a couple of small problems:

First, the GAS directives that are currently being used rely on
assembler behavior that is not documented, and probably not guaranteed
to produce the correct behavior going forward. As a result, we end up
with some undefined symbols in cache.o:

$ nm arch/arm64/mm/cache.o
         ...
         U civac
         ...
         U cvac
         U cvap
         U cvau

This is due to the fact that the comparisons used to select the
operation type in the dcache_by_line_op macro are comparing symbols
not strings, and even though it seems that GAS is doing the right
thing here (undefined symbols by the same name are equal to each
other), it seems unwise to rely on this.

Second, when patching in a DC CVAP instruction on CPUs that support it,
the fallback path consists of a DC CVAU instruction which may be
affected by CPU errata that require ARM64_WORKAROUND_CLEAN_CACHE.

Solve these issues by unrolling the various maintenance routines and
using the conditional directives that are documented as operating on
strings. To avoid the complexity of nested alternatives, we move the
DC CVAP patching to __clean_dcache_area_pop, falling back to a branch
to __clean_dcache_area_poc if DCPOP is not supported by the CPU.

Reported-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Suggested-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index d8b9d3a427d14b1de28d3ef22fc6dd77ec9b333c..d103c3ee7335e0f2ffa709a1ebbd76c748a14ee4 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -392,27 +392,33 @@ alternative_endif
  *     size:           size of the region
  *     Corrupts:       kaddr, size, tmp1, tmp2
  */
+       .macro __dcache_op_workaround_clean_cache, op, kaddr
+alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
+       dc      \op, \kaddr
+alternative_else
+       dc      civac, \kaddr
+alternative_endif
+       .endm
+
        .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
        dcache_line_size \tmp1, \tmp2
        add     \size, \kaddr, \size
        sub     \tmp2, \tmp1, #1
        bic     \kaddr, \kaddr, \tmp2
 9998:
-       .if     (\op == cvau || \op == cvac)
-alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
-       dc      \op, \kaddr
-alternative_else
-       dc      civac, \kaddr
-alternative_endif
-       .elseif (\op == cvap)
-alternative_if ARM64_HAS_DCPOP
-       sys 3, c7, c12, 1, \kaddr       // dc cvap
-alternative_else
-       dc      cvac, \kaddr
-alternative_endif
+       .ifc    \op, cvau
+       __dcache_op_workaround_clean_cache \op, \kaddr
+       .else
+       .ifc    \op, cvac
+       __dcache_op_workaround_clean_cache \op, \kaddr
+       .else
+       .ifc    \op, cvap
+       sys     3, c7, c12, 1, \kaddr   // dc cvap
        .else
        dc      \op, \kaddr
        .endif
+       .endif
+       .endif
        add     \kaddr, \kaddr, \tmp1
        cmp     \kaddr, \size
        b.lo    9998b

diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 0c22ede52f9062241d7db938d709a16087bd142c..a194fd0e837fb913abf10e35807cea6c672a2af5 100644 (file)
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -212,6 +212,9 @@ ENDPROC(__dma_clean_area)
  *     - size    - size in question
  */
 ENTRY(__clean_dcache_area_pop)
+       alternative_if_not ARM64_HAS_DCPOP
+       b       __clean_dcache_area_poc
+       alternative_else_nop_endif
        dcache_by_line_op cvap, sy, x0, x1, x2, x3
        ret
 ENDPIPROC(__clean_dcache_area_pop)