Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
serpent_avx_glue.c
Go to the documentation of this file.
1 /*
2  * Glue Code for AVX assembler versions of Serpent Cipher
3  *
4  * Copyright (C) 2012 Johannes Goetzfried
6  *
7  * Glue code based on serpent_sse2_glue.c by:
8  * Copyright (C) 2011 Jussi Kivilinna <[email protected]>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23  * USA
24  *
25  */
26 
27 #include <linux/module.h>
28 #include <linux/hardirq.h>
29 #include <linux/types.h>
30 #include <linux/crypto.h>
31 #include <linux/err.h>
32 #include <crypto/algapi.h>
33 #include <crypto/serpent.h>
34 #include <crypto/cryptd.h>
35 #include <crypto/b128ops.h>
36 #include <crypto/ctr.h>
37 #include <crypto/lrw.h>
38 #include <crypto/xts.h>
39 #include <asm/xcr.h>
40 #include <asm/xsave.h>
41 #include <asm/crypto/serpent-avx.h>
42 #include <asm/crypto/ablk_helper.h>
43 #include <asm/crypto/glue_helper.h>
44 
45 static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
46 {
48  unsigned int j;
49 
50  for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
51  ivs[j] = src[j];
52 
53  serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
54 
55  for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
56  u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
57 }
58 
59 static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
60 {
61  be128 ctrblk;
62 
63  u128_to_be128(&ctrblk, iv);
64  u128_inc(iv);
65 
66  __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
67  u128_xor(dst, src, (u128 *)&ctrblk);
68 }
69 
70 static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
71  u128 *iv)
72 {
74  unsigned int i;
75 
76  for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
77  if (dst != src)
78  dst[i] = src[i];
79 
80  u128_to_be128(&ctrblks[i], iv);
81  u128_inc(iv);
82  }
83 
84  serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
85 }
86 
87 static const struct common_glue_ctx serpent_enc = {
88  .num_funcs = 2,
89  .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
90 
91  .funcs = { {
92  .num_blocks = SERPENT_PARALLEL_BLOCKS,
93  .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
94  }, {
95  .num_blocks = 1,
96  .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
97  } }
98 };
99 
100 static const struct common_glue_ctx serpent_ctr = {
101  .num_funcs = 2,
102  .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
103 
104  .funcs = { {
105  .num_blocks = SERPENT_PARALLEL_BLOCKS,
106  .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
107  }, {
108  .num_blocks = 1,
109  .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
110  } }
111 };
112 
113 static const struct common_glue_ctx serpent_dec = {
114  .num_funcs = 2,
115  .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
116 
117  .funcs = { {
118  .num_blocks = SERPENT_PARALLEL_BLOCKS,
119  .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
120  }, {
121  .num_blocks = 1,
122  .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
123  } }
124 };
125 
126 static const struct common_glue_ctx serpent_dec_cbc = {
127  .num_funcs = 2,
128  .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
129 
130  .funcs = { {
131  .num_blocks = SERPENT_PARALLEL_BLOCKS,
132  .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
133  }, {
134  .num_blocks = 1,
135  .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
136  } }
137 };
138 
139 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
140  struct scatterlist *src, unsigned int nbytes)
141 {
142  return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
143 }
144 
145 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
146  struct scatterlist *src, unsigned int nbytes)
147 {
148  return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
149 }
150 
151 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
152  struct scatterlist *src, unsigned int nbytes)
153 {
155  dst, src, nbytes);
156 }
157 
158 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
159  struct scatterlist *src, unsigned int nbytes)
160 {
161  return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
162  nbytes);
163 }
164 
165 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
166  struct scatterlist *src, unsigned int nbytes)
167 {
168  return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
169 }
170 
171 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
172 {
173  return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
174  NULL, fpu_enabled, nbytes);
175 }
176 
177 static inline void serpent_fpu_end(bool fpu_enabled)
178 {
179  glue_fpu_end(fpu_enabled);
180 }
181 
182 struct crypt_priv {
183  struct serpent_ctx *ctx;
184  bool fpu_enabled;
185 };
186 
187 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
188 {
189  const unsigned int bsize = SERPENT_BLOCK_SIZE;
190  struct crypt_priv *ctx = priv;
191  int i;
192 
193  ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
194 
195  if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
196  serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
197  return;
198  }
199 
200  for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
201  __serpent_encrypt(ctx->ctx, srcdst, srcdst);
202 }
203 
204 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
205 {
206  const unsigned int bsize = SERPENT_BLOCK_SIZE;
207  struct crypt_priv *ctx = priv;
208  int i;
209 
210  ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
211 
212  if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
213  serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
214  return;
215  }
216 
217  for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
218  __serpent_decrypt(ctx->ctx, srcdst, srcdst);
219 }
220 
224 };
225 
226 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
227  unsigned int keylen)
228 {
229  struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
230  int err;
231 
232  err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
234  if (err)
235  return err;
236 
237  return lrw_init_table(&ctx->lrw_table, key + keylen -
239 }
240 
241 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
242  struct scatterlist *src, unsigned int nbytes)
243 {
244  struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
246  struct crypt_priv crypt_ctx = {
247  .ctx = &ctx->serpent_ctx,
248  .fpu_enabled = false,
249  };
250  struct lrw_crypt_req req = {
251  .tbuf = buf,
252  .tbuflen = sizeof(buf),
253 
254  .table_ctx = &ctx->lrw_table,
255  .crypt_ctx = &crypt_ctx,
256  .crypt_fn = encrypt_callback,
257  };
258  int ret;
259 
261  ret = lrw_crypt(desc, dst, src, nbytes, &req);
262  serpent_fpu_end(crypt_ctx.fpu_enabled);
263 
264  return ret;
265 }
266 
267 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
268  struct scatterlist *src, unsigned int nbytes)
269 {
270  struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
272  struct crypt_priv crypt_ctx = {
273  .ctx = &ctx->serpent_ctx,
274  .fpu_enabled = false,
275  };
276  struct lrw_crypt_req req = {
277  .tbuf = buf,
278  .tbuflen = sizeof(buf),
279 
280  .table_ctx = &ctx->lrw_table,
281  .crypt_ctx = &crypt_ctx,
282  .crypt_fn = decrypt_callback,
283  };
284  int ret;
285 
287  ret = lrw_crypt(desc, dst, src, nbytes, &req);
288  serpent_fpu_end(crypt_ctx.fpu_enabled);
289 
290  return ret;
291 }
292 
293 static void lrw_exit_tfm(struct crypto_tfm *tfm)
294 {
295  struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
296 
297  lrw_free_table(&ctx->lrw_table);
298 }
299 
303 };
304 
305 static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
306  unsigned int keylen)
307 {
308  struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
309  u32 *flags = &tfm->crt_flags;
310  int err;
311 
312  /* key consists of keys of equal size concatenated, therefore
313  * the length must be even
314  */
315  if (keylen % 2) {
316  *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
317  return -EINVAL;
318  }
319 
320  /* first half of xts-key is for crypt */
321  err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
322  if (err)
323  return err;
324 
325  /* second half of xts-key is for tweak */
326  return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
327 }
328 
329 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330  struct scatterlist *src, unsigned int nbytes)
331 {
332  struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
334  struct crypt_priv crypt_ctx = {
335  .ctx = &ctx->crypt_ctx,
336  .fpu_enabled = false,
337  };
338  struct xts_crypt_req req = {
339  .tbuf = buf,
340  .tbuflen = sizeof(buf),
341 
342  .tweak_ctx = &ctx->tweak_ctx,
343  .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
344  .crypt_ctx = &crypt_ctx,
345  .crypt_fn = encrypt_callback,
346  };
347  int ret;
348 
350  ret = xts_crypt(desc, dst, src, nbytes, &req);
351  serpent_fpu_end(crypt_ctx.fpu_enabled);
352 
353  return ret;
354 }
355 
356 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
357  struct scatterlist *src, unsigned int nbytes)
358 {
359  struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
361  struct crypt_priv crypt_ctx = {
362  .ctx = &ctx->crypt_ctx,
363  .fpu_enabled = false,
364  };
365  struct xts_crypt_req req = {
366  .tbuf = buf,
367  .tbuflen = sizeof(buf),
368 
369  .tweak_ctx = &ctx->tweak_ctx,
370  .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
371  .crypt_ctx = &crypt_ctx,
372  .crypt_fn = decrypt_callback,
373  };
374  int ret;
375 
377  ret = xts_crypt(desc, dst, src, nbytes, &req);
378  serpent_fpu_end(crypt_ctx.fpu_enabled);
379 
380  return ret;
381 }
382 
383 static struct crypto_alg serpent_algs[10] = { {
384  .cra_name = "__ecb-serpent-avx",
385  .cra_driver_name = "__driver-ecb-serpent-avx",
386  .cra_priority = 0,
387  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
388  .cra_blocksize = SERPENT_BLOCK_SIZE,
389  .cra_ctxsize = sizeof(struct serpent_ctx),
390  .cra_alignmask = 0,
391  .cra_type = &crypto_blkcipher_type,
392  .cra_module = THIS_MODULE,
393  .cra_u = {
394  .blkcipher = {
395  .min_keysize = SERPENT_MIN_KEY_SIZE,
396  .max_keysize = SERPENT_MAX_KEY_SIZE,
397  .setkey = serpent_setkey,
398  .encrypt = ecb_encrypt,
399  .decrypt = ecb_decrypt,
400  },
401  },
402 }, {
403  .cra_name = "__cbc-serpent-avx",
404  .cra_driver_name = "__driver-cbc-serpent-avx",
405  .cra_priority = 0,
406  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
407  .cra_blocksize = SERPENT_BLOCK_SIZE,
408  .cra_ctxsize = sizeof(struct serpent_ctx),
409  .cra_alignmask = 0,
410  .cra_type = &crypto_blkcipher_type,
411  .cra_module = THIS_MODULE,
412  .cra_u = {
413  .blkcipher = {
414  .min_keysize = SERPENT_MIN_KEY_SIZE,
415  .max_keysize = SERPENT_MAX_KEY_SIZE,
416  .setkey = serpent_setkey,
417  .encrypt = cbc_encrypt,
418  .decrypt = cbc_decrypt,
419  },
420  },
421 }, {
422  .cra_name = "__ctr-serpent-avx",
423  .cra_driver_name = "__driver-ctr-serpent-avx",
424  .cra_priority = 0,
425  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
426  .cra_blocksize = 1,
427  .cra_ctxsize = sizeof(struct serpent_ctx),
428  .cra_alignmask = 0,
429  .cra_type = &crypto_blkcipher_type,
430  .cra_module = THIS_MODULE,
431  .cra_u = {
432  .blkcipher = {
433  .min_keysize = SERPENT_MIN_KEY_SIZE,
434  .max_keysize = SERPENT_MAX_KEY_SIZE,
435  .ivsize = SERPENT_BLOCK_SIZE,
436  .setkey = serpent_setkey,
437  .encrypt = ctr_crypt,
438  .decrypt = ctr_crypt,
439  },
440  },
441 }, {
442  .cra_name = "__lrw-serpent-avx",
443  .cra_driver_name = "__driver-lrw-serpent-avx",
444  .cra_priority = 0,
445  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
446  .cra_blocksize = SERPENT_BLOCK_SIZE,
447  .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
448  .cra_alignmask = 0,
449  .cra_type = &crypto_blkcipher_type,
450  .cra_module = THIS_MODULE,
451  .cra_exit = lrw_exit_tfm,
452  .cra_u = {
453  .blkcipher = {
454  .min_keysize = SERPENT_MIN_KEY_SIZE +
456  .max_keysize = SERPENT_MAX_KEY_SIZE +
458  .ivsize = SERPENT_BLOCK_SIZE,
459  .setkey = lrw_serpent_setkey,
460  .encrypt = lrw_encrypt,
461  .decrypt = lrw_decrypt,
462  },
463  },
464 }, {
465  .cra_name = "__xts-serpent-avx",
466  .cra_driver_name = "__driver-xts-serpent-avx",
467  .cra_priority = 0,
468  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
469  .cra_blocksize = SERPENT_BLOCK_SIZE,
470  .cra_ctxsize = sizeof(struct serpent_xts_ctx),
471  .cra_alignmask = 0,
472  .cra_type = &crypto_blkcipher_type,
473  .cra_module = THIS_MODULE,
474  .cra_u = {
475  .blkcipher = {
476  .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
477  .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
478  .ivsize = SERPENT_BLOCK_SIZE,
479  .setkey = xts_serpent_setkey,
480  .encrypt = xts_encrypt,
481  .decrypt = xts_decrypt,
482  },
483  },
484 }, {
485  .cra_name = "ecb(serpent)",
486  .cra_driver_name = "ecb-serpent-avx",
487  .cra_priority = 500,
489  .cra_blocksize = SERPENT_BLOCK_SIZE,
490  .cra_ctxsize = sizeof(struct async_helper_ctx),
491  .cra_alignmask = 0,
492  .cra_type = &crypto_ablkcipher_type,
493  .cra_module = THIS_MODULE,
494  .cra_init = ablk_init,
495  .cra_exit = ablk_exit,
496  .cra_u = {
497  .ablkcipher = {
498  .min_keysize = SERPENT_MIN_KEY_SIZE,
499  .max_keysize = SERPENT_MAX_KEY_SIZE,
500  .setkey = ablk_set_key,
501  .encrypt = ablk_encrypt,
502  .decrypt = ablk_decrypt,
503  },
504  },
505 }, {
506  .cra_name = "cbc(serpent)",
507  .cra_driver_name = "cbc-serpent-avx",
508  .cra_priority = 500,
510  .cra_blocksize = SERPENT_BLOCK_SIZE,
511  .cra_ctxsize = sizeof(struct async_helper_ctx),
512  .cra_alignmask = 0,
513  .cra_type = &crypto_ablkcipher_type,
514  .cra_module = THIS_MODULE,
515  .cra_init = ablk_init,
516  .cra_exit = ablk_exit,
517  .cra_u = {
518  .ablkcipher = {
519  .min_keysize = SERPENT_MIN_KEY_SIZE,
520  .max_keysize = SERPENT_MAX_KEY_SIZE,
521  .ivsize = SERPENT_BLOCK_SIZE,
522  .setkey = ablk_set_key,
523  .encrypt = __ablk_encrypt,
524  .decrypt = ablk_decrypt,
525  },
526  },
527 }, {
528  .cra_name = "ctr(serpent)",
529  .cra_driver_name = "ctr-serpent-avx",
530  .cra_priority = 500,
532  .cra_blocksize = 1,
533  .cra_ctxsize = sizeof(struct async_helper_ctx),
534  .cra_alignmask = 0,
535  .cra_type = &crypto_ablkcipher_type,
536  .cra_module = THIS_MODULE,
537  .cra_init = ablk_init,
538  .cra_exit = ablk_exit,
539  .cra_u = {
540  .ablkcipher = {
541  .min_keysize = SERPENT_MIN_KEY_SIZE,
542  .max_keysize = SERPENT_MAX_KEY_SIZE,
543  .ivsize = SERPENT_BLOCK_SIZE,
544  .setkey = ablk_set_key,
545  .encrypt = ablk_encrypt,
546  .decrypt = ablk_encrypt,
547  .geniv = "chainiv",
548  },
549  },
550 }, {
551  .cra_name = "lrw(serpent)",
552  .cra_driver_name = "lrw-serpent-avx",
553  .cra_priority = 500,
555  .cra_blocksize = SERPENT_BLOCK_SIZE,
556  .cra_ctxsize = sizeof(struct async_helper_ctx),
557  .cra_alignmask = 0,
558  .cra_type = &crypto_ablkcipher_type,
559  .cra_module = THIS_MODULE,
560  .cra_init = ablk_init,
561  .cra_exit = ablk_exit,
562  .cra_u = {
563  .ablkcipher = {
564  .min_keysize = SERPENT_MIN_KEY_SIZE +
566  .max_keysize = SERPENT_MAX_KEY_SIZE +
568  .ivsize = SERPENT_BLOCK_SIZE,
569  .setkey = ablk_set_key,
570  .encrypt = ablk_encrypt,
571  .decrypt = ablk_decrypt,
572  },
573  },
574 }, {
575  .cra_name = "xts(serpent)",
576  .cra_driver_name = "xts-serpent-avx",
577  .cra_priority = 500,
579  .cra_blocksize = SERPENT_BLOCK_SIZE,
580  .cra_ctxsize = sizeof(struct async_helper_ctx),
581  .cra_alignmask = 0,
582  .cra_type = &crypto_ablkcipher_type,
583  .cra_module = THIS_MODULE,
584  .cra_init = ablk_init,
585  .cra_exit = ablk_exit,
586  .cra_u = {
587  .ablkcipher = {
588  .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
589  .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
590  .ivsize = SERPENT_BLOCK_SIZE,
591  .setkey = ablk_set_key,
592  .encrypt = ablk_encrypt,
593  .decrypt = ablk_decrypt,
594  },
595  },
596 } };
597 
598 static int __init serpent_init(void)
599 {
600  u64 xcr0;
601 
602  if (!cpu_has_avx || !cpu_has_osxsave) {
603  printk(KERN_INFO "AVX instructions are not detected.\n");
604  return -ENODEV;
605  }
606 
607  xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
608  if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
609  printk(KERN_INFO "AVX detected but unusable.\n");
610  return -ENODEV;
611  }
612 
613  return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
614 }
615 
616 static void __exit serpent_exit(void)
617 {
618  crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
619 }
620 
621 module_init(serpent_init);
622 module_exit(serpent_exit);
623 
624 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
625 MODULE_LICENSE("GPL");
626 MODULE_ALIAS("serpent");