1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2 From: Ard Biesheuvel <ardb@kernel.org>
3 Date: Fri, 8 Nov 2019 13:22:17 +0100
4 Subject: [PATCH] crypto: mips/chacha - wire up accelerated 32r2 code from Zinc
6 Content-Type: text/plain; charset=UTF-8
7 Content-Transfer-Encoding: 8bit
9 commit 3a2f58f3ba4f6f44e33d1a48240d5eadb882cb59 upstream.
11 This integrates the accelerated MIPS 32r2 implementation of ChaCha
12 into both the API and library interfaces of the kernel crypto stack.
14 The significance of this is that, in addition to becoming available
15 as an accelerated library implementation, it can also be used by
16 existing crypto API code such as Adiantum (for block encryption on
17 ultra low performance cores) or IPsec using chacha20poly1305. These
18 are use cases that have already opted into using the abstract crypto
19 API. In order to support Adiantum, the core assembler routine has
20 been adapted to take the round count as a function argument rather
21 than hardcoding it to 20.
23 Co-developed-by: René van Dorst <opensource@vdorst.com>
24 Signed-off-by: René van Dorst <opensource@vdorst.com>
25 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
26 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
27 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
29 arch/mips/Makefile | 2 +-
30 arch/mips/crypto/Makefile | 4 +
31 arch/mips/crypto/chacha-core.S | 159 ++++++++++++++++++++++++---------
32 arch/mips/crypto/chacha-glue.c | 150 +++++++++++++++++++++++++++++++
34 5 files changed, 277 insertions(+), 44 deletions(-)
35 create mode 100644 arch/mips/crypto/chacha-glue.c
37 --- a/arch/mips/Makefile
38 +++ b/arch/mips/Makefile
39 @@ -334,7 +334,7 @@ libs-$(CONFIG_MIPS_FP_SUPPORT) += arch/m
40 # See arch/mips/Kbuild for content of core part of the kernel
43 -drivers-$(CONFIG_MIPS_CRC_SUPPORT) += arch/mips/crypto/
44 +drivers-y += arch/mips/crypto/
45 drivers-$(CONFIG_OPROFILE) += arch/mips/oprofile/
47 # suspend and hibernation support
48 --- a/arch/mips/crypto/Makefile
49 +++ b/arch/mips/crypto/Makefile
53 obj-$(CONFIG_CRYPTO_CRC32_MIPS) += crc32-mips.o
55 +obj-$(CONFIG_CRYPTO_CHACHA_MIPS) += chacha-mips.o
56 +chacha-mips-y := chacha-core.o chacha-glue.o
57 +AFLAGS_chacha-core.o += -O2 # needed to fill branch delay slots
58 --- a/arch/mips/crypto/chacha-core.S
59 +++ b/arch/mips/crypto/chacha-core.S
61 #define CONCAT3(a,b,c) _CONCAT3(a,b,c)
63 #define STORE_UNALIGNED(x) \
64 -CONCAT3(.Lchacha20_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \
65 +CONCAT3(.Lchacha_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \
67 lw T0, (x*4)(STATE); \
69 @@ -142,7 +142,7 @@ CONCAT3(.Lchacha20_mips_xor_unaligned_,
70 swr X ## x, (x*4)+LSB ## (OUT);
72 #define STORE_ALIGNED(x) \
73 -CONCAT3(.Lchacha20_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \
74 +CONCAT3(.Lchacha_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \
76 lw T0, (x*4)(STATE); \
78 @@ -162,9 +162,9 @@ CONCAT3(.Lchacha20_mips_xor_aligned_, PL
79 * Every jumptable entry must be equal in size.
81 #define JMPTBL_ALIGNED(x) \
82 -.Lchacha20_mips_jmptbl_aligned_ ## x: ; \
83 +.Lchacha_mips_jmptbl_aligned_ ## x: ; \
85 - b .Lchacha20_mips_xor_aligned_ ## x ## _b; \
86 + b .Lchacha_mips_xor_aligned_ ## x ## _b; \
88 addu SAVED_X, X ## x, NONCE_0; \
90 @@ -173,9 +173,9 @@ CONCAT3(.Lchacha20_mips_xor_aligned_, PL
93 #define JMPTBL_UNALIGNED(x) \
94 -.Lchacha20_mips_jmptbl_unaligned_ ## x: ; \
95 +.Lchacha_mips_jmptbl_unaligned_ ## x: ; \
97 - b .Lchacha20_mips_xor_unaligned_ ## x ## _b; \
98 + b .Lchacha_mips_xor_unaligned_ ## x ## _b; \
100 addu SAVED_X, X ## x, NONCE_0; \
102 @@ -200,15 +200,18 @@ CONCAT3(.Lchacha20_mips_xor_aligned_, PL
106 -.globl chacha20_mips
109 +.globl chacha_crypt_arch
110 +.ent chacha_crypt_arch
112 .frame $sp, STACK_SIZE, $ra
114 + /* Load number of rounds */
117 addiu $sp, -STACK_SIZE
119 /* Return bytes = 0. */
120 - beqz BYTES, .Lchacha20_mips_end
121 + beqz BYTES, .Lchacha_mips_end
123 lw NONCE_0, 48(STATE)
125 @@ -228,18 +231,15 @@ chacha20_mips:
126 or IS_UNALIGNED, IN, OUT
127 andi IS_UNALIGNED, 0x3
129 - /* Set number of rounds */
132 - b .Lchacha20_rounds_start
133 + b .Lchacha_rounds_start
136 -.Loop_chacha20_rounds:
137 +.Loop_chacha_rounds:
138 addiu IN, CHACHA20_BLOCK_SIZE
139 addiu OUT, CHACHA20_BLOCK_SIZE
142 -.Lchacha20_rounds_start:
143 +.Lchacha_rounds_start:
147 @@ -259,7 +259,7 @@ chacha20_mips:
151 -.Loop_chacha20_xor_rounds:
152 +.Loop_chacha_xor_rounds:
154 AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 16);
155 AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 12);
156 @@ -269,31 +269,31 @@ chacha20_mips:
157 AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 12);
158 AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 8);
159 AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 7);
160 - bnez $at, .Loop_chacha20_xor_rounds
161 + bnez $at, .Loop_chacha_xor_rounds
163 addiu BYTES, -(CHACHA20_BLOCK_SIZE)
165 /* Is data src/dst unaligned? Jump */
166 - bnez IS_UNALIGNED, .Loop_chacha20_unaligned
167 + bnez IS_UNALIGNED, .Loop_chacha_unaligned
169 /* Set number rounds here to fill delayslot. */
171 + lw $at, (STACK_SIZE+16)($sp)
173 /* BYTES < 0, it has no full block. */
174 - bltz BYTES, .Lchacha20_mips_no_full_block_aligned
175 + bltz BYTES, .Lchacha_mips_no_full_block_aligned
177 FOR_EACH_WORD_REV(STORE_ALIGNED)
179 /* BYTES > 0? Loop again. */
180 - bgtz BYTES, .Loop_chacha20_rounds
181 + bgtz BYTES, .Loop_chacha_rounds
183 /* Place this here to fill delay slot */
186 /* BYTES < 0? Handle last bytes */
187 - bltz BYTES, .Lchacha20_mips_xor_bytes
188 + bltz BYTES, .Lchacha_mips_xor_bytes
190 -.Lchacha20_mips_xor_done:
191 +.Lchacha_mips_xor_done:
192 /* Restore used registers */
195 @@ -307,11 +307,11 @@ chacha20_mips:
196 /* Write NONCE_0 back to right location in state */
197 sw NONCE_0, 48(STATE)
199 -.Lchacha20_mips_end:
201 addiu $sp, STACK_SIZE
204 -.Lchacha20_mips_no_full_block_aligned:
205 +.Lchacha_mips_no_full_block_aligned:
206 /* Restore the offset on BYTES */
207 addiu BYTES, CHACHA20_BLOCK_SIZE
209 @@ -319,7 +319,7 @@ chacha20_mips:
210 andi $at, BYTES, MASK_U32
212 /* Load upper half of jump table addr */
213 - lui T0, %hi(.Lchacha20_mips_jmptbl_aligned_0)
214 + lui T0, %hi(.Lchacha_mips_jmptbl_aligned_0)
216 /* Calculate lower half jump table offset */
218 @@ -328,7 +328,7 @@ chacha20_mips:
221 /* Add lower half jump table addr */
222 - addiu T0, %lo(.Lchacha20_mips_jmptbl_aligned_0)
223 + addiu T0, %lo(.Lchacha_mips_jmptbl_aligned_0)
225 /* Read value from STATE */
227 @@ -342,31 +342,31 @@ chacha20_mips:
228 FOR_EACH_WORD(JMPTBL_ALIGNED)
231 -.Loop_chacha20_unaligned:
232 +.Loop_chacha_unaligned:
233 /* Set number rounds here to fill delayslot. */
235 + lw $at, (STACK_SIZE+16)($sp)
237 /* BYTES > 0, it has no full block. */
238 - bltz BYTES, .Lchacha20_mips_no_full_block_unaligned
239 + bltz BYTES, .Lchacha_mips_no_full_block_unaligned
241 FOR_EACH_WORD_REV(STORE_UNALIGNED)
243 /* BYTES > 0? Loop again. */
244 - bgtz BYTES, .Loop_chacha20_rounds
245 + bgtz BYTES, .Loop_chacha_rounds
247 /* Write NONCE_0 back to right location in state */
248 sw NONCE_0, 48(STATE)
251 /* Fall through to byte handling */
252 - bgez BYTES, .Lchacha20_mips_xor_done
253 -.Lchacha20_mips_xor_unaligned_0_b:
254 -.Lchacha20_mips_xor_aligned_0_b:
255 + bgez BYTES, .Lchacha_mips_xor_done
256 +.Lchacha_mips_xor_unaligned_0_b:
257 +.Lchacha_mips_xor_aligned_0_b:
258 /* Place this here to fill delay slot */
262 -.Lchacha20_mips_xor_bytes:
263 +.Lchacha_mips_xor_bytes:
267 @@ -376,22 +376,22 @@ chacha20_mips:
271 - beqz $at, .Lchacha20_mips_xor_done
272 + beqz $at, .Lchacha_mips_xor_done
279 - beqz $at, .Lchacha20_mips_xor_done
280 + beqz $at, .Lchacha_mips_xor_done
286 - b .Lchacha20_mips_xor_done
287 + b .Lchacha_mips_xor_done
289 -.Lchacha20_mips_no_full_block_unaligned:
290 +.Lchacha_mips_no_full_block_unaligned:
291 /* Restore the offset on BYTES */
292 addiu BYTES, CHACHA20_BLOCK_SIZE
294 @@ -399,7 +399,7 @@ chacha20_mips:
295 andi $at, BYTES, MASK_U32
297 /* Load upper half of jump table addr */
298 - lui T0, %hi(.Lchacha20_mips_jmptbl_unaligned_0)
299 + lui T0, %hi(.Lchacha_mips_jmptbl_unaligned_0)
301 /* Calculate lower half jump table offset */
303 @@ -408,7 +408,7 @@ chacha20_mips:
306 /* Add lower half jump table addr */
307 - addiu T0, %lo(.Lchacha20_mips_jmptbl_unaligned_0)
308 + addiu T0, %lo(.Lchacha_mips_jmptbl_unaligned_0)
310 /* Read value from STATE */
312 @@ -420,5 +420,78 @@ chacha20_mips:
315 FOR_EACH_WORD(JMPTBL_UNALIGNED)
317 +.end chacha_crypt_arch
337 +.globl hchacha_block_arch
338 +.ent hchacha_block_arch
340 + .frame $sp, STACK_SIZE, $ra
342 + addiu $sp, -STACK_SIZE
364 +.Loop_hchacha_xor_rounds:
366 + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 16);
367 + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 12);
368 + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 8);
369 + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 7);
370 + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 16);
371 + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 12);
372 + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 8);
373 + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 7);
374 + bnez $a2, .Loop_hchacha_xor_rounds
376 + /* Restore used register */
388 + addiu $sp, STACK_SIZE
390 +.end hchacha_block_arch
393 +++ b/arch/mips/crypto/chacha-glue.c
395 +// SPDX-License-Identifier: GPL-2.0
397 + * MIPS accelerated ChaCha and XChaCha stream ciphers,
398 + * including ChaCha20 (RFC7539)
400 + * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
403 +#include <asm/byteorder.h>
404 +#include <crypto/algapi.h>
405 +#include <crypto/internal/chacha.h>
406 +#include <crypto/internal/skcipher.h>
407 +#include <linux/kernel.h>
408 +#include <linux/module.h>
410 +asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src,
411 + unsigned int bytes, int nrounds);
412 +EXPORT_SYMBOL(chacha_crypt_arch);
414 +asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds);
415 +EXPORT_SYMBOL(hchacha_block_arch);
417 +void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
419 + chacha_init_generic(state, key, iv);
421 +EXPORT_SYMBOL(chacha_init_arch);
423 +static int chacha_mips_stream_xor(struct skcipher_request *req,
424 + const struct chacha_ctx *ctx, const u8 *iv)
426 + struct skcipher_walk walk;
430 + err = skcipher_walk_virt(&walk, req, false);
432 + chacha_init_generic(state, ctx->key, iv);
434 + while (walk.nbytes > 0) {
435 + unsigned int nbytes = walk.nbytes;
437 + if (nbytes < walk.total)
438 + nbytes = round_down(nbytes, walk.stride);
440 + chacha_crypt(state, walk.dst.virt.addr, walk.src.virt.addr,
441 + nbytes, ctx->nrounds);
442 + err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
448 +static int chacha_mips(struct skcipher_request *req)
450 + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
451 + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
453 + return chacha_mips_stream_xor(req, ctx, req->iv);
456 +static int xchacha_mips(struct skcipher_request *req)
458 + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
459 + struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
460 + struct chacha_ctx subctx;
464 + chacha_init_generic(state, ctx->key, req->iv);
466 + hchacha_block(state, subctx.key, ctx->nrounds);
467 + subctx.nrounds = ctx->nrounds;
469 + memcpy(&real_iv[0], req->iv + 24, 8);
470 + memcpy(&real_iv[8], req->iv + 16, 8);
471 + return chacha_mips_stream_xor(req, &subctx, real_iv);
474 +static struct skcipher_alg algs[] = {
476 + .base.cra_name = "chacha20",
477 + .base.cra_driver_name = "chacha20-mips",
478 + .base.cra_priority = 200,
479 + .base.cra_blocksize = 1,
480 + .base.cra_ctxsize = sizeof(struct chacha_ctx),
481 + .base.cra_module = THIS_MODULE,
483 + .min_keysize = CHACHA_KEY_SIZE,
484 + .max_keysize = CHACHA_KEY_SIZE,
485 + .ivsize = CHACHA_IV_SIZE,
486 + .chunksize = CHACHA_BLOCK_SIZE,
487 + .setkey = chacha20_setkey,
488 + .encrypt = chacha_mips,
489 + .decrypt = chacha_mips,
491 + .base.cra_name = "xchacha20",
492 + .base.cra_driver_name = "xchacha20-mips",
493 + .base.cra_priority = 200,
494 + .base.cra_blocksize = 1,
495 + .base.cra_ctxsize = sizeof(struct chacha_ctx),
496 + .base.cra_module = THIS_MODULE,
498 + .min_keysize = CHACHA_KEY_SIZE,
499 + .max_keysize = CHACHA_KEY_SIZE,
500 + .ivsize = XCHACHA_IV_SIZE,
501 + .chunksize = CHACHA_BLOCK_SIZE,
502 + .setkey = chacha20_setkey,
503 + .encrypt = xchacha_mips,
504 + .decrypt = xchacha_mips,
506 + .base.cra_name = "xchacha12",
507 + .base.cra_driver_name = "xchacha12-mips",
508 + .base.cra_priority = 200,
509 + .base.cra_blocksize = 1,
510 + .base.cra_ctxsize = sizeof(struct chacha_ctx),
511 + .base.cra_module = THIS_MODULE,
513 + .min_keysize = CHACHA_KEY_SIZE,
514 + .max_keysize = CHACHA_KEY_SIZE,
515 + .ivsize = XCHACHA_IV_SIZE,
516 + .chunksize = CHACHA_BLOCK_SIZE,
517 + .setkey = chacha12_setkey,
518 + .encrypt = xchacha_mips,
519 + .decrypt = xchacha_mips,
523 +static int __init chacha_simd_mod_init(void)
525 + return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
528 +static void __exit chacha_simd_mod_fini(void)
530 + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
533 +module_init(chacha_simd_mod_init);
534 +module_exit(chacha_simd_mod_fini);
536 +MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)");
537 +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
538 +MODULE_LICENSE("GPL v2");
539 +MODULE_ALIAS_CRYPTO("chacha20");
540 +MODULE_ALIAS_CRYPTO("chacha20-mips");
541 +MODULE_ALIAS_CRYPTO("xchacha20");
542 +MODULE_ALIAS_CRYPTO("xchacha20-mips");
543 +MODULE_ALIAS_CRYPTO("xchacha12");
544 +MODULE_ALIAS_CRYPTO("xchacha12-mips");
547 @@ -1423,6 +1423,12 @@ config CRYPTO_CHACHA20_X86_64
548 SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20,
549 XChaCha20, and XChaCha12 stream ciphers.
551 +config CRYPTO_CHACHA_MIPS
552 + tristate "ChaCha stream cipher algorithms (MIPS 32r2 optimized)"
553 + depends on CPU_MIPS32_R2
554 + select CRYPTO_BLKCIPHER
555 + select CRYPTO_ARCH_HAVE_LIB_CHACHA
558 tristate "SEED cipher algorithm"