git.openwrt.org Git - openwrt/staging/neocturne.git/blob

   1 From 360be1a8f326ec5c0d20a134e228fb96a2eb351d Mon Sep 17 00:00:00 2001
   2 From: Ard Biesheuvel <ardb@kernel.org>
   3 Date: Fri, 8 Nov 2019 13:22:15 +0100
   4 Subject: [PATCH 009/124] crypto: arm/chacha - expose ARM ChaCha routine as
   5  library function
   6
   7 commit a44a3430d71bad4ee56788a59fff099b291ea54c upstream.
   8
   9 Expose the accelerated NEON ChaCha routine directly as a symbol
  10 export so that users of the ChaCha library API can use it directly.
  11
  12 Given that calls into the library API will always go through the
  13 routines in this module if it is enabled, switch to static keys
  14 to select the optimal implementation available (which may be none
  15 at all, in which case we defer to the generic implementation for
  16 all invocations).
  17
  18 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
  19 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  20 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
  21 ---
  22  arch/arm/crypto/Kconfig       |  1 +
  23  arch/arm/crypto/chacha-glue.c | 41 ++++++++++++++++++++++++++++++++++-
  24  2 files changed, 41 insertions(+), 1 deletion(-)
  25
  26 --- a/arch/arm/crypto/Kconfig
  27 +++ b/arch/arm/crypto/Kconfig
  28 @@ -129,6 +129,7 @@ config CRYPTO_CRC32_ARM_CE
  29  config CRYPTO_CHACHA20_NEON
  30         tristate "NEON and scalar accelerated ChaCha stream cipher algorithms"
  31         select CRYPTO_BLKCIPHER
  32 +       select CRYPTO_ARCH_HAVE_LIB_CHACHA
  33
  34  config CRYPTO_NHPOLY1305_NEON
  35         tristate "NEON accelerated NHPoly1305 hash function (for Adiantum)"
  36 --- a/arch/arm/crypto/chacha-glue.c
  37 +++ b/arch/arm/crypto/chacha-glue.c
  38 @@ -11,6 +11,7 @@
  39  #include <crypto/internal/chacha.h>
  40  #include <crypto/internal/simd.h>
  41  #include <crypto/internal/skcipher.h>
  42 +#include <linux/jump_label.h>
  43  #include <linux/kernel.h>
  44  #include <linux/module.h>
  45
  46 @@ -29,9 +30,11 @@ asmlinkage void hchacha_block_neon(const
  47  asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
  48                              const u32 *state, int nrounds);
  49
  50 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
  51 +
  52  static inline bool neon_usable(void)
  53  {
  54 -       return crypto_simd_usable();
  55 +       return static_branch_likely(&use_neon) && crypto_simd_usable();
  56  }
  57
  58  static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
  59 @@ -60,6 +63,40 @@ static void chacha_doneon(u32 *state, u8
  60         }
  61  }
  62
  63 +void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
  64 +{
  65 +       if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
  66 +               hchacha_block_arm(state, stream, nrounds);
  67 +       } else {
  68 +               kernel_neon_begin();
  69 +               hchacha_block_neon(state, stream, nrounds);
  70 +               kernel_neon_end();
  71 +       }
  72 +}
  73 +EXPORT_SYMBOL(hchacha_block_arch);
  74 +
  75 +void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
  76 +{
  77 +       chacha_init_generic(state, key, iv);
  78 +}
  79 +EXPORT_SYMBOL(chacha_init_arch);
  80 +
  81 +void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
  82 +                      int nrounds)
  83 +{
  84 +       if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
  85 +           bytes <= CHACHA_BLOCK_SIZE) {
  86 +               chacha_doarm(dst, src, bytes, state, nrounds);
  87 +               state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
  88 +               return;
  89 +       }
  90 +
  91 +       kernel_neon_begin();
  92 +       chacha_doneon(state, dst, src, bytes, nrounds);
  93 +       kernel_neon_end();
  94 +}
  95 +EXPORT_SYMBOL(chacha_crypt_arch);
  96 +
  97  static int chacha_stream_xor(struct skcipher_request *req,
  98                              const struct chacha_ctx *ctx, const u8 *iv,
  99                              bool neon)
 100 @@ -269,6 +306,8 @@ static int __init chacha_simd_mod_init(v
 101                         for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
 102                                 neon_algs[i].base.cra_priority = 0;
 103                         break;
 104 +               default:
 105 +                       static_branch_enable(&use_neon);
 106                 }
 107
 108                 err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));