projects / openwrt / openwrt.git / blob
? search:


60339381b59cbd9bf3938a39ca6661c914017cc4
[openwrt/openwrt.git] /
1 From 29c84baf5e125aa43265192a08cc4bd904db1d45 Mon Sep 17 00:00:00 2001
2 From: Ard Biesheuvel <ardb@kernel.org>
3 Date: Fri, 8 Nov 2019 13:22:09 +0100
4 Subject: [PATCH 003/124] crypto: x86/chacha - depend on generic chacha library
5 instead of crypto driver
6
7 commit 28e8d89b1ce8d2e7badfb5f69971dd635acb8863 upstream.
8
9 In preparation of extending the x86 ChaCha driver to also expose the ChaCha
10 library interface, drop the dependency on the chacha_generic crypto driver
11 as a non-SIMD fallback, and depend on the generic ChaCha library directly.
12 This way, we only pull in the code we actually need, without registering
13 a set of ChaCha skciphers that we will never use.
14
15 Since turning the FPU on and off is cheap these days, simplify the SIMD
16 routine by dropping the per-page yield, which makes for a cleaner switch
17 to the library API as well. This also allows use to invoke the skcipher
18 walk routines in non-atomic mode.
19
20 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
21 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
22 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
23 ---
24 arch/x86/crypto/chacha_glue.c | 90 ++++++++++++++---------------------
25 crypto/Kconfig | 2 +-
26 2 files changed, 36 insertions(+), 56 deletions(-)
27
28 --- a/arch/x86/crypto/chacha_glue.c
29 +++ b/arch/x86/crypto/chacha_glue.c
30 @@ -123,37 +123,38 @@ static void chacha_dosimd(u32 *state, u8
31 }
32 }
33
34 -static int chacha_simd_stream_xor(struct skcipher_walk *walk,
35 +static int chacha_simd_stream_xor(struct skcipher_request *req,
36 const struct chacha_ctx *ctx, const u8 *iv)
37 {
38 u32 *state, state_buf[16 + 2] __aligned(8);
39 - int next_yield = 4096; /* bytes until next FPU yield */
40 - int err = 0;
41 + struct skcipher_walk walk;
42 + int err;
43 +
44 + err = skcipher_walk_virt(&walk, req, false);
45
46 BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
47 state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);
48
49 - crypto_chacha_init(state, ctx, iv);
50 + chacha_init_generic(state, ctx->key, iv);
51
52 - while (walk->nbytes > 0) {
53 - unsigned int nbytes = walk->nbytes;
54 + while (walk.nbytes > 0) {
55 + unsigned int nbytes = walk.nbytes;
56
57 - if (nbytes < walk->total) {
58 - nbytes = round_down(nbytes, walk->stride);
59 - next_yield -= nbytes;
60 - }
61 -
62 - chacha_dosimd(state, walk->dst.virt.addr, walk->src.virt.addr,
63 - nbytes, ctx->nrounds);
64 + if (nbytes < walk.total)
65 + nbytes = round_down(nbytes, walk.stride);
66
67 - if (next_yield <= 0) {
68 - /* temporarily allow preemption */
69 - kernel_fpu_end();
70 + if (!crypto_simd_usable()) {
71 + chacha_crypt_generic(state, walk.dst.virt.addr,
72 + walk.src.virt.addr, nbytes,
73 + ctx->nrounds);
74 + } else {
75 kernel_fpu_begin();
76 - next_yield = 4096;
77 + chacha_dosimd(state, walk.dst.virt.addr,
78 + walk.src.virt.addr, nbytes,
79 + ctx->nrounds);
80 + kernel_fpu_end();
81 }
82 -
83 - err = skcipher_walk_done(walk, walk->nbytes - nbytes);
84 + err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
85 }
86
87 return err;
88 @@ -163,55 +164,34 @@ static int chacha_simd(struct skcipher_r
89 {
90 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
91 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
92 - struct skcipher_walk walk;
93 - int err;
94 -
95 - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
96 - return crypto_chacha_crypt(req);
97
98 - err = skcipher_walk_virt(&walk, req, true);
99 - if (err)
100 - return err;
101 -
102 - kernel_fpu_begin();
103 - err = chacha_simd_stream_xor(&walk, ctx, req->iv);
104 - kernel_fpu_end();
105 - return err;
106 + return chacha_simd_stream_xor(req, ctx, req->iv);
107 }
108
109 static int xchacha_simd(struct skcipher_request *req)
110 {
111 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
112 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
113 - struct skcipher_walk walk;
114 - struct chacha_ctx subctx;
115 u32 *state, state_buf[16 + 2] __aligned(8);
116 + struct chacha_ctx subctx;
117 u8 real_iv[16];
118 - int err;
119 -
120 - if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
121 - return crypto_xchacha_crypt(req);
122 -
123 - err = skcipher_walk_virt(&walk, req, true);
124 - if (err)
125 - return err;
126
127 BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
128 state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN);
129 - crypto_chacha_init(state, ctx, req->iv);
130 + chacha_init_generic(state, ctx->key, req->iv);
131
132 - kernel_fpu_begin();
133 -
134 - hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
135 + if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) {
136 + kernel_fpu_begin();
137 + hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
138 + kernel_fpu_end();
139 + } else {
140 + hchacha_block_generic(state, subctx.key, ctx->nrounds);
141 + }
142 subctx.nrounds = ctx->nrounds;
143
144 memcpy(&real_iv[0], req->iv + 24, 8);
145 memcpy(&real_iv[8], req->iv + 16, 8);
146 - err = chacha_simd_stream_xor(&walk, &subctx, real_iv);
147 -
148 - kernel_fpu_end();
149 -
150 - return err;
151 + return chacha_simd_stream_xor(req, &subctx, real_iv);
152 }
153
154 static struct skcipher_alg algs[] = {
155 @@ -227,7 +207,7 @@ static struct skcipher_alg algs[] = {
156 .max_keysize = CHACHA_KEY_SIZE,
157 .ivsize = CHACHA_IV_SIZE,
158 .chunksize = CHACHA_BLOCK_SIZE,
159 - .setkey = crypto_chacha20_setkey,
160 + .setkey = chacha20_setkey,
161 .encrypt = chacha_simd,
162 .decrypt = chacha_simd,
163 }, {
164 @@ -242,7 +222,7 @@ static struct skcipher_alg algs[] = {
165 .max_keysize = CHACHA_KEY_SIZE,
166 .ivsize = XCHACHA_IV_SIZE,
167 .chunksize = CHACHA_BLOCK_SIZE,
168 - .setkey = crypto_chacha20_setkey,
169 + .setkey = chacha20_setkey,
170 .encrypt = xchacha_simd,
171 .decrypt = xchacha_simd,
172 }, {
173 @@ -257,7 +237,7 @@ static struct skcipher_alg algs[] = {
174 .max_keysize = CHACHA_KEY_SIZE,
175 .ivsize = XCHACHA_IV_SIZE,
176 .chunksize = CHACHA_BLOCK_SIZE,
177 - .setkey = crypto_chacha12_setkey,
178 + .setkey = chacha12_setkey,
179 .encrypt = xchacha_simd,
180 .decrypt = xchacha_simd,
181 },
182 --- a/crypto/Kconfig
183 +++ b/crypto/Kconfig
184 @@ -1417,7 +1417,7 @@ config CRYPTO_CHACHA20_X86_64
185 tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)"
186 depends on X86 && 64BIT
187 select CRYPTO_BLKCIPHER
188 - select CRYPTO_CHACHA20
189 + select CRYPTO_LIB_CHACHA_GENERIC
190 help
191 SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20,
192 XChaCha20, and XChaCha12 stream ciphers.
OpenWrt Source Repository