28 #include <linux/types.h>
30 #include <linux/module.h>
32 #include <asm/byteorder.h>
40 #define UINT64_C(x) x##ULL
41 static const u64 p64 =
UINT64_C(0xfffffffffffffeff);
42 static const u64 m62 =
UINT64_C(0x3fffffffffffffff);
43 static const u64 m63 =
UINT64_C(0x7fffffffffffffff);
44 static const u64 m64 =
UINT64_C(0xffffffffffffffff);
45 static const u64 mpoly =
UINT64_C(0x1fffffff1fffffff);
47 #define pe64_to_cpup le64_to_cpup
49 #ifdef __LITTLE_ENDIAN
66 #define ADD128(rh, rl, ih, il) \
75 #define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2))
77 #define PMUL64(rh, rl, i1, i2) \
79 u64 _i1 = (i1), _i2 = (i2); \
80 u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \
81 rh = MUL32(_i1>>32, _i2>>32); \
82 rl = MUL32(_i1, _i2); \
83 ADD128(rh, rl, (m >> 32), (m << 32)); \
86 #define MUL64(rh, rl, i1, i2) \
88 u64 _i1 = (i1), _i2 = (i2); \
89 u64 m1 = MUL32(_i1, _i2>>32); \
90 u64 m2 = MUL32(_i1>>32, _i2); \
91 rh = MUL32(_i1>>32, _i2>>32); \
92 rl = MUL32(_i1, _i2); \
93 ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \
94 ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \
111 #define nh_16(mp, kp, nw, rh, rl) \
115 for (i = 0; i < nw; i += 2) { \
116 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
117 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
118 ADD128(rh, rl, th, tl); \
122 #define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \
125 rh1 = rl1 = rh = rl = 0; \
126 for (i = 0; i < nw; i += 2) { \
127 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
128 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
129 ADD128(rh, rl, th, tl); \
130 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
131 pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
132 ADD128(rh1, rl1, th, tl); \
136 #if (VMAC_NHBYTES >= 64)
137 #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
141 for (i = 0; i < nw; i += 8) { \
142 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
143 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
144 ADD128(rh, rl, th, tl); \
145 MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
146 pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
147 ADD128(rh, rl, th, tl); \
148 MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
149 pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
150 ADD128(rh, rl, th, tl); \
151 MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
152 pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
153 ADD128(rh, rl, th, tl); \
157 #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \
160 rh1 = rl1 = rh = rl = 0; \
161 for (i = 0; i < nw; i += 8) { \
162 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
163 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
164 ADD128(rh, rl, th, tl); \
165 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
166 pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
167 ADD128(rh1, rl1, th, tl); \
168 MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
169 pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
170 ADD128(rh, rl, th, tl); \
171 MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
172 pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \
173 ADD128(rh1, rl1, th, tl); \
174 MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
175 pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
176 ADD128(rh, rl, th, tl); \
177 MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
178 pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \
179 ADD128(rh1, rl1, th, tl); \
180 MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
181 pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
182 ADD128(rh, rl, th, tl); \
183 MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
184 pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \
185 ADD128(rh1, rl1, th, tl); \
190 #define poly_step(ah, al, kh, kl, mh, ml) \
192 u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \
194 PMUL64(t3h, t3l, al, kh); \
195 PMUL64(t2h, t2l, ah, kl); \
196 PMUL64(t1h, t1l, ah, 2*kh); \
197 PMUL64(ah, al, al, kl); \
199 ADD128(ah, al, t1h, t1l); \
201 ADD128(t2h, t2l, t3h, t3l); \
204 ADD128(t2h, ah, z, t2l); \
206 t2h = 2 * t2h + (ah >> 63); \
209 ADD128(ah, al, mh, ml); \
210 ADD128(ah, al, z, t2h); \
216 #define nh_16(mp, kp, nw, rh, rl) \
218 u64 t1, t2, m1, m2, t; \
221 for (i = 0; i < nw; i += 2) { \
222 t1 = pe64_to_cpup(mp+i) + kp[i]; \
223 t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \
224 m2 = MUL32(t1 >> 32, t2); \
225 m1 = MUL32(t1, t2 >> 32); \
226 ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \
228 rh += (u64)(u32)(m1 >> 32) \
230 t += (u64)(u32)m1 + (u32)m2; \
232 ADD128(rh, rl, (t >> 32), (t << 32)); \
236 static void poly_step_func(
u64 *
ahi,
u64 *alo,
238 const u64 *mh,
const u64 *ml)
240 #define a0 (*(((u32 *)alo)+INDEX_LOW))
241 #define a1 (*(((u32 *)alo)+INDEX_HIGH))
242 #define a2 (*(((u32 *)ahi)+INDEX_LOW))
243 #define a3 (*(((u32 *)ahi)+INDEX_HIGH))
244 #define k0 (*(((u32 *)kl)+INDEX_LOW))
245 #define k1 (*(((u32 *)kl)+INDEX_HIGH))
246 #define k2 (*(((u32 *)kh)+INDEX_LOW))
247 #define k3 (*(((u32 *)kh)+INDEX_HIGH))
264 t |= ((
u64)((
u32)p & 0x7fffffff)) << 32;
282 *(
u64 *)(alo) = (p << 32) | t2;
284 *(
u64 *)(ahi) = p +
t;
296 #define poly_step(ah, al, kh, kl, mh, ml) \
297 poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))
303 #define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \
305 nh_16(mp, kp, nw, rh, rl); \
306 nh_16(mp, ((kp)+2), nw, rh2, rl2); \
309 #ifndef nh_vmac_nhbytes
310 #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
311 nh_16(mp, kp, nw, rh, rl)
313 #ifndef nh_vmac_nhbytes_2
314 #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \
316 nh_vmac_nhbytes(mp, kp, nw, rh, rl); \
317 nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \
330 u64 rh, rl,
t, z = 0;
337 t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
344 t += (
u32)t > 0xfffffffeu;
350 p1 += (0 - (p1 <
k1)) & 257;
352 p2 += (0 - (p2 < k2)) & 257;
355 MUL64(rh, rl, p1, p2);
362 rl += (0 - (rl <
t)) & 257;
363 rl += (0 - (rl > p64-1)) & 257;
367 static void vhash_update(
const unsigned char *
m,
404 static u64 vhash(
unsigned char m[],
unsigned int mbytes,
427 }
else if (remaining) {
428 nh_16(mptr, kptr, 2*((remaining+15)/16), ch,
cl);
445 nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
453 return l3hash(ch,
cl, ctx->
l3key[0], ctx->
l3key[1], remaining);
456 static u64 vmac(
unsigned char m[],
unsigned int mbytes,
457 unsigned char n[16],
u64 *tagl,
468 if ((*(
u64 *)(n+8) != in_n[1]) || (*(
u64 *)(n) != in_n[0])) {
469 in_n[0] = *(
u64 *)(n);
470 in_n[1] = *(
u64 *)(n+8);
471 ((
unsigned char *)in_n)[15] &= 0xFE;
472 crypto_cipher_encrypt_one(ctx->
child,
473 (
unsigned char *)out_p, (
unsigned char *)in_n);
475 ((
unsigned char *)in_n)[15] |= (
unsigned char)(1-i);
482 static int vmac_set_key(
unsigned char user_key[],
struct vmac_ctx_t *ctx)
493 ((
unsigned char *)in)[0] = 0x80;
494 for (i = 0; i <
sizeof(ctx->
__vmac_ctx.nhkey)/8; i += 2) {
495 crypto_cipher_encrypt_one(ctx->
child,
496 (
unsigned char *)
out, (
unsigned char *)in);
499 ((
unsigned char *)in)[15] += 1;
503 ((
unsigned char *)in)[0] = 0xC0;
505 for (i = 0; i <
sizeof(ctx->
__vmac_ctx.polykey)/8; i += 2) {
506 crypto_cipher_encrypt_one(ctx->
child,
507 (
unsigned char *)
out, (
unsigned char *)in);
514 ((
unsigned char *)in)[15] += 1;
518 ((
unsigned char *)in)[0] = 0xE0;
520 for (i = 0; i <
sizeof(ctx->
__vmac_ctx.l3key)/8; i += 2) {
522 crypto_cipher_encrypt_one(ctx->
child,
523 (
unsigned char *)
out, (
unsigned char *)in);
526 ((
unsigned char *)in)[15] += 1;
540 const u8 *
key,
unsigned int keylen)
542 struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
549 return vmac_set_key((
u8 *)key, ctx);
561 struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
571 struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
575 mac = vmac(
NULL, 0, nonce,
NULL, ctx);
582 static int vmac_init_tfm(
struct crypto_tfm *tfm)
589 cipher = crypto_spawn_cipher(spawn);
591 return PTR_ERR(cipher);
597 static void vmac_exit_tfm(
struct crypto_tfm *tfm)
600 crypto_free_cipher(ctx->
child);
618 inst = shash_alloc_instance(
"vmac", alg);
624 shash_crypto_instance(inst),
635 inst->
alg.base.cra_init = vmac_init_tfm;
636 inst->
alg.base.cra_exit = vmac_exit_tfm;
638 inst->
alg.init = vmac_init;
639 inst->
alg.update = vmac_update;
640 inst->
alg.final = vmac_final;
641 inst->
alg.setkey = vmac_setkey;
656 .create = vmac_create,
661 static int __init vmac_module_init(
void)
666 static void __exit vmac_module_exit(
void)