Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
padlock-aes.c
Go to the documentation of this file.
1 /*
2  * Cryptographic API.
3  *
4  * Support for VIA PadLock hardware crypto engine.
5  *
6  * Copyright (c) 2004 Michal Ludvig <[email protected]>
7  *
8  */
9 
10 #include <crypto/algapi.h>
11 #include <crypto/aes.h>
12 #include <crypto/padlock.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/types.h>
16 #include <linux/errno.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel.h>
19 #include <linux/percpu.h>
20 #include <linux/smp.h>
21 #include <linux/slab.h>
22 #include <asm/cpu_device_id.h>
23 #include <asm/byteorder.h>
24 #include <asm/processor.h>
25 #include <asm/i387.h>
26 
27 /*
28  * Number of data blocks actually fetched for each xcrypt insn.
29  * Processors with prefetch errata will fetch extra blocks.
30  */
31 static unsigned int ecb_fetch_blocks = 2;
32 #define MAX_ECB_FETCH_BLOCKS (8)
33 #define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
34 
35 static unsigned int cbc_fetch_blocks = 1;
36 #define MAX_CBC_FETCH_BLOCKS (4)
37 #define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
38 
39 /* Control word. */
40 struct cword {
41  unsigned int __attribute__ ((__packed__))
42  rounds:4,
43  algo:3,
44  keygen:1,
45  interm:1,
46  encdec:1,
47  ksize:2;
48 } __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
49 
50 /* Whenever making any changes to the following
51  * structure *make sure* you keep E, d_data
52  * and cword aligned on 16 Bytes boundaries and
53  * the Hardware can access 16 * 16 bytes of E and d_data
54  * (only the first 15 * 16 bytes matter but the HW reads
55  * more).
56  */
57 struct aes_ctx {
58  u32 E[AES_MAX_KEYLENGTH_U32]
59  __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
60  u32 d_data[AES_MAX_KEYLENGTH_U32]
61  __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
62  struct {
63  struct cword encrypt;
64  struct cword decrypt;
65  } cword;
66  u32 *D;
67 };
68 
69 static DEFINE_PER_CPU(struct cword *, paes_last_cword);
70 
71 /* Tells whether the ACE is capable to generate
72  the extended key for a given key_len. */
73 static inline int
74 aes_hw_extkey_available(uint8_t key_len)
75 {
76  /* TODO: We should check the actual CPU model/stepping
77  as it's possible that the capability will be
78  added in the next CPU revisions. */
79  if (key_len == 16)
80  return 1;
81  return 0;
82 }
83 
84 static inline struct aes_ctx *aes_ctx_common(void *ctx)
85 {
86  unsigned long addr = (unsigned long)ctx;
87  unsigned long align = PADLOCK_ALIGNMENT;
88 
89  if (align <= crypto_tfm_ctx_alignment())
90  align = 1;
91  return (struct aes_ctx *)ALIGN(addr, align);
92 }
93 
94 static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
95 {
96  return aes_ctx_common(crypto_tfm_ctx(tfm));
97 }
98 
99 static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
100 {
101  return aes_ctx_common(crypto_blkcipher_ctx(tfm));
102 }
103 
104 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
105  unsigned int key_len)
106 {
107  struct aes_ctx *ctx = aes_ctx(tfm);
108  const __le32 *key = (const __le32 *)in_key;
109  u32 *flags = &tfm->crt_flags;
110  struct crypto_aes_ctx gen_aes;
111  int cpu;
112 
113  if (key_len % 8) {
114  *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
115  return -EINVAL;
116  }
117 
118  /*
119  * If the hardware is capable of generating the extended key
120  * itself we must supply the plain key for both encryption
121  * and decryption.
122  */
123  ctx->D = ctx->E;
124 
125  ctx->E[0] = le32_to_cpu(key[0]);
126  ctx->E[1] = le32_to_cpu(key[1]);
127  ctx->E[2] = le32_to_cpu(key[2]);
128  ctx->E[3] = le32_to_cpu(key[3]);
129 
130  /* Prepare control words. */
131  memset(&ctx->cword, 0, sizeof(ctx->cword));
132 
133  ctx->cword.decrypt.encdec = 1;
134  ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
135  ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
136  ctx->cword.encrypt.ksize = (key_len - 16) / 8;
137  ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
138 
139  /* Don't generate extended keys if the hardware can do it. */
140  if (aes_hw_extkey_available(key_len))
141  goto ok;
142 
143  ctx->D = ctx->d_data;
144  ctx->cword.encrypt.keygen = 1;
145  ctx->cword.decrypt.keygen = 1;
146 
147  if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
148  *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
149  return -EINVAL;
150  }
151 
152  memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
153  memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
154 
155 ok:
156  for_each_online_cpu(cpu)
157  if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) ||
158  &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu))
159  per_cpu(paes_last_cword, cpu) = NULL;
160 
161  return 0;
162 }
163 
164 /* ====== Encryption/decryption routines ====== */
165 
166 /* These are the real call to PadLock. */
167 static inline void padlock_reset_key(struct cword *cword)
168 {
169  int cpu = raw_smp_processor_id();
170 
171  if (cword != per_cpu(paes_last_cword, cpu))
172 #ifndef CONFIG_X86_64
173  asm volatile ("pushfl; popfl");
174 #else
175  asm volatile ("pushfq; popfq");
176 #endif
177 }
178 
179 static inline void padlock_store_cword(struct cword *cword)
180 {
181  per_cpu(paes_last_cword, raw_smp_processor_id()) = cword;
182 }
183 
184 /*
185  * While the padlock instructions don't use FP/SSE registers, they
186  * generate a spurious DNA fault when cr0.ts is '1'. These instructions
187  * should be used only inside the irq_ts_save/restore() context
188  */
189 
190 static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
191  struct cword *control_word, int count)
192 {
193  asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
194  : "+S"(input), "+D"(output)
195  : "d"(control_word), "b"(key), "c"(count));
196 }
197 
198 static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key,
199  u8 *iv, struct cword *control_word, int count)
200 {
201  asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
202  : "+S" (input), "+D" (output), "+a" (iv)
203  : "d" (control_word), "b" (key), "c" (count));
204  return iv;
205 }
206 
207 static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
208  struct cword *cword, int count)
209 {
210  /*
211  * Padlock prefetches extra data so we must provide mapped input buffers.
212  * Assume there are at least 16 bytes of stack already in use.
213  */
214  u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
215  u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
216 
217  memcpy(tmp, in, count * AES_BLOCK_SIZE);
218  rep_xcrypt_ecb(tmp, out, key, cword, count);
219 }
220 
221 static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
222  u8 *iv, struct cword *cword, int count)
223 {
224  /*
225  * Padlock prefetches extra data so we must provide mapped input buffers.
226  * Assume there are at least 16 bytes of stack already in use.
227  */
228  u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
229  u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
230 
231  memcpy(tmp, in, count * AES_BLOCK_SIZE);
232  return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
233 }
234 
235 static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
236  struct cword *cword, int count)
237 {
238  /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
239  * We could avoid some copying here but it's probably not worth it.
240  */
241  if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) {
242  ecb_crypt_copy(in, out, key, cword, count);
243  return;
244  }
245 
246  rep_xcrypt_ecb(in, out, key, cword, count);
247 }
248 
249 static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
250  u8 *iv, struct cword *cword, int count)
251 {
252  /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
253  if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE))
254  return cbc_crypt_copy(in, out, key, iv, cword, count);
255 
256  return rep_xcrypt_cbc(in, out, key, iv, cword, count);
257 }
258 
259 static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
260  void *control_word, u32 count)
261 {
262  u32 initial = count & (ecb_fetch_blocks - 1);
263 
264  if (count < ecb_fetch_blocks) {
265  ecb_crypt(input, output, key, control_word, count);
266  return;
267  }
268 
269  if (initial)
270  asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
271  : "+S"(input), "+D"(output)
272  : "d"(control_word), "b"(key), "c"(initial));
273 
274  asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
275  : "+S"(input), "+D"(output)
276  : "d"(control_word), "b"(key), "c"(count - initial));
277 }
278 
279 static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
280  u8 *iv, void *control_word, u32 count)
281 {
282  u32 initial = count & (cbc_fetch_blocks - 1);
283 
284  if (count < cbc_fetch_blocks)
285  return cbc_crypt(input, output, key, iv, control_word, count);
286 
287  if (initial)
288  asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
289  : "+S" (input), "+D" (output), "+a" (iv)
290  : "d" (control_word), "b" (key), "c" (initial));
291 
292  asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
293  : "+S" (input), "+D" (output), "+a" (iv)
294  : "d" (control_word), "b" (key), "c" (count-initial));
295  return iv;
296 }
297 
298 static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
299 {
300  struct aes_ctx *ctx = aes_ctx(tfm);
301  int ts_state;
302 
303  padlock_reset_key(&ctx->cword.encrypt);
304  ts_state = irq_ts_save();
305  ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
306  irq_ts_restore(ts_state);
307  padlock_store_cword(&ctx->cword.encrypt);
308 }
309 
310 static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
311 {
312  struct aes_ctx *ctx = aes_ctx(tfm);
313  int ts_state;
314 
315  padlock_reset_key(&ctx->cword.encrypt);
316  ts_state = irq_ts_save();
317  ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
318  irq_ts_restore(ts_state);
319  padlock_store_cword(&ctx->cword.encrypt);
320 }
321 
322 static struct crypto_alg aes_alg = {
323  .cra_name = "aes",
324  .cra_driver_name = "aes-padlock",
325  .cra_priority = PADLOCK_CRA_PRIORITY,
326  .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
327  .cra_blocksize = AES_BLOCK_SIZE,
328  .cra_ctxsize = sizeof(struct aes_ctx),
329  .cra_alignmask = PADLOCK_ALIGNMENT - 1,
330  .cra_module = THIS_MODULE,
331  .cra_u = {
332  .cipher = {
333  .cia_min_keysize = AES_MIN_KEY_SIZE,
334  .cia_max_keysize = AES_MAX_KEY_SIZE,
335  .cia_setkey = aes_set_key,
336  .cia_encrypt = aes_encrypt,
337  .cia_decrypt = aes_decrypt,
338  }
339  }
340 };
341 
342 static int ecb_aes_encrypt(struct blkcipher_desc *desc,
343  struct scatterlist *dst, struct scatterlist *src,
344  unsigned int nbytes)
345 {
346  struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
347  struct blkcipher_walk walk;
348  int err;
349  int ts_state;
350 
351  padlock_reset_key(&ctx->cword.encrypt);
352 
353  blkcipher_walk_init(&walk, dst, src, nbytes);
354  err = blkcipher_walk_virt(desc, &walk);
355 
356  ts_state = irq_ts_save();
357  while ((nbytes = walk.nbytes)) {
358  padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
359  ctx->E, &ctx->cword.encrypt,
360  nbytes / AES_BLOCK_SIZE);
361  nbytes &= AES_BLOCK_SIZE - 1;
362  err = blkcipher_walk_done(desc, &walk, nbytes);
363  }
364  irq_ts_restore(ts_state);
365 
366  padlock_store_cword(&ctx->cword.encrypt);
367 
368  return err;
369 }
370 
371 static int ecb_aes_decrypt(struct blkcipher_desc *desc,
372  struct scatterlist *dst, struct scatterlist *src,
373  unsigned int nbytes)
374 {
375  struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
376  struct blkcipher_walk walk;
377  int err;
378  int ts_state;
379 
380  padlock_reset_key(&ctx->cword.decrypt);
381 
382  blkcipher_walk_init(&walk, dst, src, nbytes);
383  err = blkcipher_walk_virt(desc, &walk);
384 
385  ts_state = irq_ts_save();
386  while ((nbytes = walk.nbytes)) {
387  padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
388  ctx->D, &ctx->cword.decrypt,
389  nbytes / AES_BLOCK_SIZE);
390  nbytes &= AES_BLOCK_SIZE - 1;
391  err = blkcipher_walk_done(desc, &walk, nbytes);
392  }
393  irq_ts_restore(ts_state);
394 
395  padlock_store_cword(&ctx->cword.encrypt);
396 
397  return err;
398 }
399 
400 static struct crypto_alg ecb_aes_alg = {
401  .cra_name = "ecb(aes)",
402  .cra_driver_name = "ecb-aes-padlock",
403  .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
404  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
405  .cra_blocksize = AES_BLOCK_SIZE,
406  .cra_ctxsize = sizeof(struct aes_ctx),
407  .cra_alignmask = PADLOCK_ALIGNMENT - 1,
408  .cra_type = &crypto_blkcipher_type,
409  .cra_module = THIS_MODULE,
410  .cra_u = {
411  .blkcipher = {
412  .min_keysize = AES_MIN_KEY_SIZE,
413  .max_keysize = AES_MAX_KEY_SIZE,
414  .setkey = aes_set_key,
415  .encrypt = ecb_aes_encrypt,
416  .decrypt = ecb_aes_decrypt,
417  }
418  }
419 };
420 
421 static int cbc_aes_encrypt(struct blkcipher_desc *desc,
422  struct scatterlist *dst, struct scatterlist *src,
423  unsigned int nbytes)
424 {
425  struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
426  struct blkcipher_walk walk;
427  int err;
428  int ts_state;
429 
430  padlock_reset_key(&ctx->cword.encrypt);
431 
432  blkcipher_walk_init(&walk, dst, src, nbytes);
433  err = blkcipher_walk_virt(desc, &walk);
434 
435  ts_state = irq_ts_save();
436  while ((nbytes = walk.nbytes)) {
437  u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
438  walk.dst.virt.addr, ctx->E,
439  walk.iv, &ctx->cword.encrypt,
440  nbytes / AES_BLOCK_SIZE);
441  memcpy(walk.iv, iv, AES_BLOCK_SIZE);
442  nbytes &= AES_BLOCK_SIZE - 1;
443  err = blkcipher_walk_done(desc, &walk, nbytes);
444  }
445  irq_ts_restore(ts_state);
446 
447  padlock_store_cword(&ctx->cword.decrypt);
448 
449  return err;
450 }
451 
452 static int cbc_aes_decrypt(struct blkcipher_desc *desc,
453  struct scatterlist *dst, struct scatterlist *src,
454  unsigned int nbytes)
455 {
456  struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
457  struct blkcipher_walk walk;
458  int err;
459  int ts_state;
460 
461  padlock_reset_key(&ctx->cword.encrypt);
462 
463  blkcipher_walk_init(&walk, dst, src, nbytes);
464  err = blkcipher_walk_virt(desc, &walk);
465 
466  ts_state = irq_ts_save();
467  while ((nbytes = walk.nbytes)) {
468  padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
469  ctx->D, walk.iv, &ctx->cword.decrypt,
470  nbytes / AES_BLOCK_SIZE);
471  nbytes &= AES_BLOCK_SIZE - 1;
472  err = blkcipher_walk_done(desc, &walk, nbytes);
473  }
474 
475  irq_ts_restore(ts_state);
476 
477  padlock_store_cword(&ctx->cword.encrypt);
478 
479  return err;
480 }
481 
482 static struct crypto_alg cbc_aes_alg = {
483  .cra_name = "cbc(aes)",
484  .cra_driver_name = "cbc-aes-padlock",
485  .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
486  .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
487  .cra_blocksize = AES_BLOCK_SIZE,
488  .cra_ctxsize = sizeof(struct aes_ctx),
489  .cra_alignmask = PADLOCK_ALIGNMENT - 1,
490  .cra_type = &crypto_blkcipher_type,
491  .cra_module = THIS_MODULE,
492  .cra_u = {
493  .blkcipher = {
494  .min_keysize = AES_MIN_KEY_SIZE,
495  .max_keysize = AES_MAX_KEY_SIZE,
496  .ivsize = AES_BLOCK_SIZE,
497  .setkey = aes_set_key,
498  .encrypt = cbc_aes_encrypt,
499  .decrypt = cbc_aes_decrypt,
500  }
501  }
502 };
503 
504 static struct x86_cpu_id padlock_cpu_id[] = {
505  X86_FEATURE_MATCH(X86_FEATURE_XCRYPT),
506  {}
507 };
508 MODULE_DEVICE_TABLE(x86cpu, padlock_cpu_id);
509 
510 static int __init padlock_init(void)
511 {
512  int ret;
513  struct cpuinfo_x86 *c = &cpu_data(0);
514 
515  if (!x86_match_cpu(padlock_cpu_id))
516  return -ENODEV;
517 
518  if (!cpu_has_xcrypt_enabled) {
519  printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
520  return -ENODEV;
521  }
522 
523  if ((ret = crypto_register_alg(&aes_alg)))
524  goto aes_err;
525 
526  if ((ret = crypto_register_alg(&ecb_aes_alg)))
527  goto ecb_aes_err;
528 
529  if ((ret = crypto_register_alg(&cbc_aes_alg)))
530  goto cbc_aes_err;
531 
532  printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
533 
534  if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
535  ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
536  cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
537  printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
538  }
539 
540 out:
541  return ret;
542 
543 cbc_aes_err:
544  crypto_unregister_alg(&ecb_aes_alg);
545 ecb_aes_err:
546  crypto_unregister_alg(&aes_alg);
547 aes_err:
548  printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
549  goto out;
550 }
551 
552 static void __exit padlock_fini(void)
553 {
554  crypto_unregister_alg(&cbc_aes_alg);
555  crypto_unregister_alg(&ecb_aes_alg);
556  crypto_unregister_alg(&aes_alg);
557 }
558 
559 module_init(padlock_init);
560 module_exit(padlock_fini);
561 
562 MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
563 MODULE_LICENSE("GPL");
564 MODULE_AUTHOR("Michal Ludvig");
565 
566 MODULE_ALIAS("aes");
Generated on Thu Jan 10 2013 13:29:01 for Linux Kernel by   doxygen 1.8.2