8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
15 #include <linux/stddef.h>
17 #include <asm/uaccess.h>
21 #include <linux/list.h>
24 #include <asm/signal.h>
29 #include <hwregs/reg_map.h>
31 #include <hwregs/intr_vect_defs.h>
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
51 #define DESCR_ALLOC_PAD (31)
86 cryptocop_tfrm_id
tid;
91 struct cryptocop_tfrm_cfg *
tcfg;
131 struct cryptocop_transform_init
init;
166 struct cryptocop_operation *
oper;
182 static const char csum_zero_pad[1] = {0x00};
185 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
193 static int descr_pool_no_free;
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
215 static int cryptocop_open(
struct inode *,
struct file *);
217 static int cryptocop_release(
struct inode *,
struct file *);
219 static long cryptocop_ioctl(
struct file *
file,
220 unsigned int cmd,
unsigned long arg);
222 static void cryptocop_start_job(
void);
227 static int cryptocop_job_queue_init(
void);
228 static void cryptocop_job_queue_close(
void);
230 static int create_md5_pad(
int alloc_flag,
unsigned long long hashed_length,
char **
pad,
size_t *pad_length);
232 static int create_sha1_pad(
int alloc_flag,
unsigned long long hashed_length,
char **
pad,
size_t *pad_length);
234 static int transform_ok(
struct cryptocop_transform_init *tinit);
242 static void get_aes_decrypt_key(
unsigned char *
dec_key,
const unsigned char *
key,
unsigned int keylength);
244 static int init_stream_coprocessor(
void);
246 static void __exit exit_stream_coprocessor(
void);
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(
struct cryptocop_operation *cop);
255 static void print_lock_status(
void);
256 static void print_user_dma_lists(
struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
266 #define DES_BLOCK_LENGTH (8)
267 #define AES_BLOCK_LENGTH (16)
268 #define MD5_BLOCK_LENGTH (64)
269 #define SHA1_BLOCK_LENGTH (64)
270 #define CSUM_BLOCK_LENGTH (2)
271 #define MD5_STATE_LENGTH (16)
272 #define SHA1_STATE_LENGTH (20)
275 #define CRYPTOCOP_MAJOR (254)
276 #define CRYPTOCOP_MINOR (0)
282 .open = cryptocop_open,
283 .release = cryptocop_release,
284 .unlocked_ioctl = cryptocop_ioctl,
295 unsigned long int flags;
297 cdesc->
next = descr_pool_free_list;
298 descr_pool_free_list = cdesc;
299 ++descr_pool_no_free;
300 spin_unlock_irqrestore(&descr_pool_lock, flags);
309 int use_pool = (alloc_flag &
GFP_ATOMIC) ? 1 : 0;
313 unsigned long int flags;
315 if (!descr_pool_free_list) {
316 spin_unlock_irqrestore(&descr_pool_lock, flags);
320 cdesc = descr_pool_free_list;
321 descr_pool_free_list = descr_pool_free_list->
next;
322 --descr_pool_no_free;
323 spin_unlock_irqrestore(&descr_pool_lock, flags);
383 DEBUG(
printk(
"create_pad_descriptor: setting up padding.\n"));
384 cdesc = alloc_cdesc(alloc_flag);
391 error = create_md5_pad(alloc_flag, tc->
consumed, &pad, &plen);
402 error = create_sha1_pad(alloc_flag, tc->
consumed, &pad, &plen);
414 pad = (
char*)csum_zero_pad;
434 if (cdesc) free_cdesc(cdesc);
447 DEBUG_API(
printk(
"setup_key_dl_desc: failed descriptor allocation.\n"));
452 if ((tc->
tctx->init.alg == cryptocop_alg_aes) && (tc->
tcfg->flags & CRYPTOCOP_DECRYPT)) {
454 if (!tc->
tctx->dec_key_set){
455 get_aes_decrypt_key(tc->
tctx->dec_key, tc->
tctx->init.key, tc->
tctx->init.keylen);
456 tc->
tctx->dec_key_set = 1;
466 switch (tc->
tctx->init.keylen) {
505 DEBUG_API(
printk(
"setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
534 size_t iov_offset = 0;
544 if (((tc->
produced + tc->
tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->
produced && (operation->tfrm_op.outlen == 0))) {
545 DEBUG_API(
printk(
"create_input_descriptors: operation outdata too small\n"));
549 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->
tcfg->inject_ix)){
550 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
553 if (outiov_ix >= operation->tfrm_op.outcount){
554 DEBUG_API(
printk(
"create_input_descriptors: operation outdata too small\n"));
557 iov_offset = tc->
tcfg->inject_ix - out_ix;
561 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 outdesc->
next = alloc_cdesc(alloc_flag);
563 if (!outdesc->
next) {
568 outdesc = outdesc->
next;
569 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 dlength = (out_length < rem_length) ? out_length : rem_length;
573 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 "outcount=%d, outiov_ix=%d\n",
576 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
582 out_length -= dlength;
583 iov_offset += dlength;
584 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
590 DEBUG_API(
printk(
"create_input_descriptors: not enough room for output, %d remained\n", out_length));
603 outdesc = head.
next->next;
604 free_cdesc(head.
next);
611 static int create_output_descriptors(
struct cryptocop_operation *operation,
int *iniov_ix,
int *iniov_offset,
size_t desc_len,
struct cryptocop_dma_desc **current_out_cdesc,
struct strcop_meta_out *meta_out,
int alloc_flag)
613 while (desc_len != 0) {
615 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
618 cdesc = alloc_cdesc(alloc_flag);
623 (*current_out_cdesc)->next = cdesc;
624 (*current_out_cdesc) = cdesc;
631 assert(desc_len >= dlength);
633 *iniov_offset += dlength;
634 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
637 if (*iniov_ix > operation->tfrm_op.incount) {
638 DEBUG_API(
printk(
"create_output_descriptors: not enough indata in operation."));
645 (*current_out_cdesc)->dma_descr->wait = 1;
659 DEBUG(
printk(
"append_input_descriptors: append pad descriptors to DMA out list.\n"));
662 (*current_out_cdesc)->next = tc->
pad_descs;
664 (*current_out_cdesc) = (*current_out_cdesc)->next;
672 unsigned int start_ix = tc->
start_ix;
675 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 if (!(*current_in_cdesc)->next){
677 DEBUG_API(
printk(
"append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
680 (*current_in_cdesc) = (*current_in_cdesc)->next;
681 (*current_in_cdesc)->dma_descr->buf = (
char*)
virt_to_phys(mem2mem_discard_buf);
682 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
690 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
692 DEBUG_API(
printk(
"append_input_descriptors: output descriptor setup failed\n"));
695 DEBUG(
printk(
"append_input_descriptors: append output descriptors to DMA in list.\n"));
698 (*current_in_cdesc)->next = idescs;
699 idescs = idescs->
next;
700 (*current_in_cdesc) = (*current_in_cdesc)->next;
708 static int cryptocop_setup_dma_list(
struct cryptocop_operation *operation,
struct cryptocop_int_operation **int_op,
int alloc_flag)
717 .requires_padding = 1,
718 .strict_block_length = 0,
736 .requires_padding = 0,
737 .strict_block_length = 1,
756 .requires_padding = 1,
757 .strict_block_length = 0,
772 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
774 unsigned int indata_ix = 0;
778 int iniov_offset = 0;
781 struct cryptocop_desc *odsc;
797 DEBUG(
printk(
"cryptocop_setup_dma_list: start\n"));
798 DEBUG(print_cryptocop_operation(operation));
800 sess = get_session(operation->sid);
802 DEBUG_API(
printk(
"cryptocop_setup_dma_list: no session found for operation.\n"));
807 if (!iop_alloc_ptr) {
808 DEBUG_API(
printk(
"cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
814 (*int_op)->alloc_ptr = iop_alloc_ptr;
815 DEBUG(
printk(
"cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->
alloc_ptr));
817 (*int_op)->sid = operation->sid;
818 (*int_op)->cdesc_out =
NULL;
819 (*int_op)->cdesc_in =
NULL;
822 (*int_op)->ddesc_out =
NULL;
823 (*int_op)->ddesc_in =
NULL;
827 DEBUG_API(
printk(
"cryptocop_setup_dma_list: no configured transforms in operation.\n"));
832 tctx = get_transform_ctx(sess, tcfg->tid);
834 DEBUG_API(
printk(
"cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
838 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 DEBUG_API(
printk(
"cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
843 switch (tctx->
init.alg){
844 case cryptocop_alg_mem2mem:
846 DEBUG_API(
printk(
"cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
856 cipher_ctx.
tcfg = tcfg;
857 cipher_ctx.
tctx = tctx;
859 case cryptocop_alg_des:
860 case cryptocop_alg_3des:
861 case cryptocop_alg_aes:
864 DEBUG_API(
printk(
"cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
868 cipher_ctx.
tcfg = tcfg;
869 cipher_ctx.
tctx = tctx;
870 if (cipher_ctx.
tcfg->flags & CRYPTOCOP_DECRYPT){
873 switch (tctx->
init.cipher_mode) {
881 DEBUG_API(
printk(
"cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->
init.cipher_mode));
885 DEBUG(
printk(
"cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.
cbcmode));
886 switch (tctx->
init.alg){
887 case cryptocop_alg_des:
892 case cryptocop_alg_3des:
897 case cryptocop_alg_aes:
903 panic(
"cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->
init.alg);
905 (*int_op)->tdes_mode = tctx->
init.tdes_mode;
907 case cryptocop_alg_md5:
908 case cryptocop_alg_sha1:
911 DEBUG_API(
printk(
"cryptocop_setup_dma_list: multiple digests in operation.\n"));
915 digest_ctx.
tcfg = tcfg;
916 digest_ctx.
tctx = tctx;
918 switch (tctx->
init.alg){
919 case cryptocop_alg_md5:
924 case cryptocop_alg_sha1:
930 panic(
"cryptocop_setup_dma_list: impossible digest algorithm\n");
933 case cryptocop_alg_csum:
936 DEBUG_API(
printk(
"cryptocop_setup_dma_list: multiple checksums in operation.\n"));
940 (*int_op)->csum_mode = tctx->
init.csum_mode;
941 csum_ctx.
tcfg = tcfg;
942 csum_ctx.
tctx = tctx;
946 DEBUG_API(
printk(
"cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->
init.alg, tcfg->tid));
953 if (cipher_ctx.
tcfg && (cipher_ctx.
tctx->init.alg != cryptocop_alg_mem2mem)){
956 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
961 current_out_cdesc->
next = key_desc;
962 current_out_cdesc = key_desc;
969 DEBUG(
printk(
"cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
971 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
976 current_out_cdesc->
next = iv_desc;
977 current_out_cdesc = iv_desc;
983 odsc = operation->tfrm_op.desc;
985 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
987 size_t desc_len = odsc->length;
988 int active_count, eop_needed_count;
992 DEBUG(
printk(
"cryptocop_setup_dma_list: parsing an operation descriptor\n"));
997 DEBUG(
printk(
"cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
999 if (digest_ctx.
tcfg && (digest_ctx.
tcfg->tid == dcfg->tid)){
1001 }
else if (cipher_ctx.
tcfg && (cipher_ctx.
tcfg->tid == dcfg->tid)){
1003 }
else if (csum_ctx.
tcfg && (csum_ctx.
tcfg->tid == dcfg->tid)){
1007 DEBUG_API(
printk(
"cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1012 DEBUG_API(
printk(
"cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1025 case cryptocop_source_dma:
1028 case cryptocop_source_des:
1031 case cryptocop_source_3des:
1034 case cryptocop_source_aes:
1037 case cryptocop_source_md5:
1038 case cryptocop_source_sha1:
1039 case cryptocop_source_csum:
1040 case cryptocop_source_none:
1044 DEBUG_API(
printk(
"cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1076 DEBUG_API(
printk(
"cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->
prev_src ? tc->
prev_src->consumed :
INT_MIN, tc->
prev_src ? tc->
prev_src->produced :
INT_MIN, tc->
prev_src ? tc->
prev_src->blocklength :
INT_MIN, tc->
curr_src ? tc->
curr_src->consumed :
INT_MIN, tc->
curr_src ? tc->
curr_src->produced :
INT_MIN, tc->
curr_src ? tc->
curr_src->blocklength :
INT_MIN));
1088 DEBUG(
printk(
"cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1091 DEBUG_API(
printk(
"cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.
curr_src->unit_no));
1096 DEBUG_API(
printk(
"cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.
curr_src->unit_no));
1101 DEBUG_API(
printk(
"cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.
curr_src->unit_no));
1115 DEBUG(
printk(
"cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1119 if (csum_ctx.
done) {
1126 if (digest_ctx.
done) {
1140 if (cipher_ctx.
cbcmode && !(cipher_ctx.
tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.
produced){
1149 eop_needed_count = 0;
1162 if (cipher_ctx.
done) ++eop_needed_count;
1173 if (digest_ctx.
done) {
1175 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.
pad_descs, alloc_flag);
1177 DEBUG_API(
printk(
"cryptocop_setup_dma_list: failed digest pad creation.\n"));
1188 if (csum_ctx.
done) {
1190 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.
pad_descs, alloc_flag);
1192 DEBUG_API(
printk(
"cryptocop_setup_dma_list: failed csum pad creation.\n"));
1199 DEBUG(
printk(
"cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1201 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, ¤t_out_cdesc, &meta_out, alloc_flag);
1203 DEBUG_API(
printk(
"cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1210 assert(active_count >= eop_needed_count);
1211 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 if (eop_needed_count) {
1214 if (active_count > 1) {
1219 DEBUG_API(
printk(
"cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1243 current_out_cdesc->
next =
ed;
1244 current_out_cdesc =
ed;
1256 indata_ix += odsc->length;
1259 DEBUG(
printk(
"cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1261 DEBUG_API(
printk(
"cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1266 DEBUG_API(
printk(
"cryptocop_setup_dma_list: digest operation not terminated.\n"));
1271 DEBUG_API(
printk(
"cryptocop_setup_dma_list: csum operation not terminated.\n"));
1276 failed = append_input_descriptors(operation, ¤t_in_cdesc, ¤t_out_cdesc, &cipher_ctx, alloc_flag);
1278 DEBUG_API(
printk(
"cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1281 failed = append_input_descriptors(operation, ¤t_in_cdesc, ¤t_out_cdesc, &digest_ctx, alloc_flag);
1283 DEBUG_API(
printk(
"cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1286 failed = append_input_descriptors(operation, ¤t_in_cdesc, ¤t_out_cdesc, &csum_ctx, alloc_flag);
1288 DEBUG_API(
printk(
"cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1292 DEBUG(
printk(
"cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 (*int_op)->cdesc_out = out_cdesc_head.
next;
1294 (*int_op)->cdesc_in = in_cdesc_head.
next;
1295 DEBUG(
printk(
"cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1297 setup_descr_chain(out_cdesc_head.
next);
1298 setup_descr_chain(in_cdesc_head.
next);
1308 (*int_op)->ctx_out.next =
NULL;
1309 (*int_op)->ctx_out.eol = 1;
1310 (*int_op)->ctx_out.intr = 0;
1311 (*int_op)->ctx_out.store_mode = 0;
1312 (*int_op)->ctx_out.en = 0;
1313 (*int_op)->ctx_out.dis = 0;
1314 (*int_op)->ctx_out.md0 = 0;
1315 (*int_op)->ctx_out.md1 = 0;
1316 (*int_op)->ctx_out.md2 = 0;
1317 (*int_op)->ctx_out.md3 = 0;
1318 (*int_op)->ctx_out.md4 = 0;
1320 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf;
1322 (*int_op)->ctx_in.next =
NULL;
1323 (*int_op)->ctx_in.eol = 1;
1324 (*int_op)->ctx_in.intr = 0;
1325 (*int_op)->ctx_in.store_mode = 0;
1326 (*int_op)->ctx_in.en = 0;
1327 (*int_op)->ctx_in.dis = 0;
1328 (*int_op)->ctx_in.md0 = 0;
1329 (*int_op)->ctx_in.md1 = 0;
1330 (*int_op)->ctx_in.md2 = 0;
1331 (*int_op)->ctx_in.md3 = 0;
1332 (*int_op)->ctx_in.md4 = 0;
1335 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf;
1337 DEBUG(
printk(
"cryptocop_setup_dma_list: done\n"));
1356 if (*int_op !=
NULL) delete_internal_operation(*int_op);
1358 DEBUG_API(
printk(
"cryptocop_setup_dma_list: done with error %d\n", failed));
1369 DEBUG(
printk(
"delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1388 static int create_md5_pad(
int alloc_flag,
unsigned long long hashed_length,
char **pad,
size_t *pad_length)
1393 unsigned long long int bit_length = hashed_length << 3;
1397 p = kzalloc(padlen, alloc_flag);
1402 DEBUG(
printk(
"create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1405 while (bit_length != 0){
1406 p[i++] = bit_length % 0x100;
1411 *pad_length = padlen;
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1419 static int create_sha1_pad(
int alloc_flag,
unsigned long long hashed_length,
char **pad,
size_t *pad_length)
1424 unsigned long long int bit_length = hashed_length << 3;
1428 p = kzalloc(padlen, alloc_flag);
1433 DEBUG(
printk(
"create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1436 while (bit_length != 0){
1437 p[i--] = bit_length % 0x100;
1442 *pad_length = padlen;
1448 static int transform_ok(
struct cryptocop_transform_init *tinit)
1450 switch (tinit->alg){
1451 case cryptocop_alg_csum:
1452 switch (tinit->csum_mode){
1460 case cryptocop_alg_mem2mem:
1461 case cryptocop_alg_md5:
1462 case cryptocop_alg_sha1:
1463 if (tinit->keylen != 0) {
1464 DEBUG_API(
printk(
"transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1468 case cryptocop_alg_des:
1469 if (tinit->keylen != 64) {
1470 DEBUG_API(
printk(
"transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1474 case cryptocop_alg_3des:
1475 if (tinit->keylen != 192) {
1476 DEBUG_API(
printk(
"transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1480 case cryptocop_alg_aes:
1481 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482 DEBUG_API(
printk(
"transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1486 case cryptocop_no_alg:
1488 DEBUG_API(
printk(
"transform_ok: no such algorithm %d\n", tinit->alg));
1492 switch (tinit->alg){
1493 case cryptocop_alg_des:
1494 case cryptocop_alg_3des:
1495 case cryptocop_alg_aes:
1507 struct cryptocop_transform_init *tfrm_in = tinit;
1508 struct cryptocop_transform_init *tmp_in;
1511 unsigned long int flags;
1513 init_stream_coprocessor();
1518 if ((err = transform_ok(tfrm_in))) {
1522 tfrm_in = tfrm_in->next;
1524 if (0 == no_tfrms) {
1525 DEBUG_API(
printk(
"cryptocop_new_session, no transforms specified\n"));
1531 DEBUG_API(
printk(
"cryptocop_new_session, kmalloc cryptocop_session\n"));
1537 DEBUG_API(
printk(
"cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1543 for (i = 0; i < no_tfrms; i++){
1544 tmp_in = tfrm_in->next;
1546 if (tmp_in->tid == tfrm_in->tid) {
1547 DEBUG_API(
printk(
"cryptocop_new_session, duplicate transform ids\n"));
1552 tmp_in = tmp_in->next;
1554 memcpy(&sess->
tfrm_ctx[i].init, tfrm_in,
sizeof(
struct cryptocop_transform_init));
1558 tfrm_in = tfrm_in->next;
1563 sess->
sid = next_sid;
1567 if (next_sid == 0) next_sid = 1;
1570 sess->
next = cryptocop_sessions;
1571 cryptocop_sessions = sess;
1572 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1583 unsigned long int flags;
1589 DEBUG(
printk(
"cryptocop_free_session: sid=%lld\n", sid));
1592 sess = cryptocop_sessions;
1593 while (sess && sess->
sid != sid){
1601 cryptocop_sessions = sess->
next;
1604 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1606 if (!sess)
return -
EINVAL;
1612 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1615 if (pj->
oper->sid == sid) {
1616 list_move_tail(node, &remove_list);
1621 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1627 DEBUG(
printk(
"cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->
oper, pj->
iop));
1629 delete_internal_operation(pj->
iop);
1636 DEBUG(
printk(
"cryptocop_free_session: memset keys, tfrm id=%d\n", tc->
init.tid));
1650 unsigned long int flags;
1653 sess = cryptocop_sessions;
1654 while (sess && (sess->
sid != sid)){
1657 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1666 DEBUG(
printk(
"get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1668 while (tc && tc->
init.tid != tid){
1672 DEBUG(
printk(
"get_transform_ctx, returning tc=0x%p\n", tc));
1679 static const u8 aes_sbox[256] = {
1680 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1681 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1682 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1683 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1684 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1685 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1686 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1687 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1688 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1689 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1690 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1691 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1692 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1693 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1694 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1695 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1701 static u32 round_constant[11] = {
1702 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704 0x1B000000, 0x36000000, 0x6C000000
1708 static u32 aes_ks_subword(
const u32 w)
1712 *(
u32*)(&bytes[0]) =
w;
1713 bytes[0] = aes_sbox[bytes[0]];
1714 bytes[1] = aes_sbox[bytes[1]];
1715 bytes[2] = aes_sbox[bytes[2]];
1716 bytes[3] = aes_sbox[bytes[3]];
1717 return *(
u32*)(&bytes[0]);
1753 static void get_aes_decrypt_key(
unsigned char *
dec_key,
const unsigned char *
key,
unsigned int keylength)
1775 panic(
"stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1780 for (i = 0; i < nk; i+=1) {
1786 while (i < (4 * (nr + 2))) {
1787 temp = w_ring[w_last_ix];
1790 temp = (temp << 8) | (temp >> 24);
1791 temp = aes_ks_subword(temp);
1792 temp ^= round_constant[i/nk - 1];
1793 }
else if ((nk > 6) && ((i % nk) == 4)) {
1794 temp = aes_ks_subword(temp);
1796 w_last_ix = (w_last_ix + 1) % nk;
1797 temp ^= w_ring[w_last_ix];
1798 w_ring[w_last_ix] =
temp;
1805 if (i >= (4 * nr)) {
1837 unsigned long int flags;
1839 DEBUG(
printk(
"cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1841 if (!operation || !operation->cb){
1842 DEBUG_API(
printk(
"cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1846 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1854 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1857 cryptocop_start_job();
1861 static void cryptocop_do_tasklet(
unsigned long unused);
1864 static void cryptocop_do_tasklet(
unsigned long unused)
1868 unsigned long flags;
1870 DEBUG(
printk(
"cryptocop_do_tasklet: entering\n"));
1874 if (!list_empty(&cryptocop_completed_jobs)){
1875 node = cryptocop_completed_jobs.next;
1881 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1886 DEBUG(
printk(
"cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->
oper->cb, pj->
oper->cb_data));
1888 pj->
oper->operation_status = 0;
1890 delete_internal_operation(pj->
iop);
1893 }
while (pj !=
NULL);
1899 dma_done_interrupt(
int irq,
void *
dev_id)
1910 spin_lock(&running_job_lock);
1911 if (cryptocop_running_job ==
NULL){
1912 printk(
"stream co-processor got interrupt when not busy\n");
1913 spin_unlock(&running_job_lock);
1916 done_job = cryptocop_running_job;
1917 cryptocop_running_job =
NULL;
1918 spin_unlock(&running_job_lock);
1921 if (!spin_trylock(&cryptocop_process_lock)){
1922 DEBUG(
printk(
"cryptocop irq handler, not starting a job\n"));
1924 cryptocop_start_job();
1925 spin_unlock(&cryptocop_process_lock);
1928 done_job->
oper->operation_status = 0;
1929 if (done_job->
oper->fast_callback){
1931 done_job->
oper->cb(done_job->
oper, done_job->
oper->cb_data);
1932 delete_internal_operation(done_job->
iop);
1935 spin_lock(&cryptocop_completed_jobs_lock);
1937 spin_unlock(&cryptocop_completed_jobs_lock);
1938 tasklet_schedule(&cryptocop_tasklet);
1947 static int init_cryptocop(
void)
1949 unsigned long flags;
1963 "stream co-processor DMA",
NULL))
1964 panic(
"request_irq stream co-processor irq dma9");
1999 static void release_cryptocop(
void)
2001 unsigned long flags;
2028 static int cryptocop_job_queue_init(
void)
2032 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2036 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2042 static void cryptocop_job_queue_close(
void)
2046 unsigned long int process_flags,
flags;
2057 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2063 DEBUG(
printk(
"cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->
oper->cb, pj->
oper->cb_data));
2067 delete_internal_operation(pj->
iop);
2072 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2076 if (cryptocop_running_job){
2094 pj = cryptocop_running_job;
2095 cryptocop_running_job =
NULL;
2098 DEBUG(
printk(
"cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->
oper->cb, pj->
oper->cb_data));
2102 delete_internal_operation(pj->
iop);
2105 spin_unlock_irqrestore(&running_job_lock, flags);
2114 DEBUG(
printk(
"cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->
oper->cb, pj->
oper->cb_data));
2118 delete_internal_operation(pj->
iop);
2121 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2125 static void cryptocop_start_job(
void)
2129 unsigned long int flags;
2130 unsigned long int running_job_flags;
2136 if (cryptocop_running_job !=
NULL){
2138 DEBUG(
printk(
"cryptocop_start_job: already running, exit\n"));
2139 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2146 if (i == cryptocop_prio_no_prios) {
2147 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149 DEBUG(
printk(
"cryptocop_start_job: no jobs to run\n"));
2163 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164 cryptocop_running_job = pj;
2167 switch (pj->
iop->tdes_mode){
2209 DEBUG(
printk(
"cryptocop_setup_dma_list: bad 3DES mode\n"));
2211 switch (pj->
iop->csum_mode){
2219 DEBUG(
printk(
"cryptocop_setup_dma_list: bad checksum mode\n"));
2223 DEBUG(
printk(
"cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224 "ctx_in: 0x%p, phys: 0x%p\n"
2225 "ctx_out: 0x%p, phys: 0x%p\n",
2237 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2242 static int cryptocop_job_setup(
struct cryptocop_prio_job **pj,
struct cryptocop_operation *operation)
2246 void *iop_alloc_ptr =
NULL;
2249 if (!*pj)
return -
ENOMEM;
2251 DEBUG(
printk(
"cryptocop_job_setup: operation=0x%p\n", operation));
2254 DEBUG(
printk(
"cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2256 if (operation->use_dmalists) {
2257 DEBUG(print_user_dma_lists(&operation->list_op));
2258 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259 DEBUG_API(
printk(
"cryptocop_job_setup: bad indata (use_dmalists)\n"));
2264 if (!iop_alloc_ptr) {
2265 DEBUG_API(
printk(
"cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2271 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272 (*pj)->iop->sid = operation->sid;
2273 (*pj)->iop->cdesc_out =
NULL;
2274 (*pj)->iop->cdesc_in =
NULL;
2275 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2281 (*pj)->iop->ctx_out.next =
NULL;
2282 (*pj)->iop->ctx_out.eol = 1;
2283 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2286 (*pj)->iop->ctx_in.next =
NULL;
2287 (*pj)->iop->ctx_in.eol = 1;
2288 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2291 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292 DEBUG_API(
printk(
"cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2297 DEBUG(print_dma_descriptors((*pj)->iop));
2299 DEBUG(
printk(
"cryptocop_job_setup, DMA list setup successful\n"));
2304 static int cryptocop_open(
struct inode *
inode,
struct file *filp)
2306 int p = iminor(inode);
2315 static int cryptocop_release(
struct inode *inode,
struct file *filp)
2321 dev_next = dev->
next;
2333 static int cryptocop_ioctl_close_session(
struct inode *inode,
struct file *filp,
2334 unsigned int cmd,
unsigned long arg)
2349 while (dev && (dev->
sid != sop.ses_id)) {
2362 DEBUG_API(
printk(
"cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2369 static void ioctl_process_job_callback(
struct cryptocop_operation *
op,
void*
cb_data)
2373 DEBUG(
printk(
"ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2376 wake_up(&cryptocop_ioc_process_wq);
2380 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2381 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2382 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2392 DEBUG(
printk(
"first_cfg_change_ix: ix=%d\n", ch_ix));
2397 static size_t next_cfg_change_ix(
struct strcop_crypto_op *crp_op,
size_t ix)
2415 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2423 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2425 if (ch_ix ==
INT_MAX) ch_ix = ix;
2426 DEBUG(
printk(
"next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2433 static int map_pages_to_iovec(
struct iovec *iov,
int iovlen,
int *iovix,
struct page **
pages,
int nopages,
int *pageix,
int *pageoffset,
int map_length )
2443 DEBUG(
printk(
"map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2445 while (map_length > 0){
2446 DEBUG(
printk(
"map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447 if (*iovix >= iovlen){
2448 DEBUG_API(
printk(
"map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2451 if (*pageix >= nopages){
2452 DEBUG_API(
printk(
"map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2457 if (tmplen < map_length){
2461 tmplen = map_length;
2462 (*pageoffset) += map_length;
2464 DEBUG(
printk(
"mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2466 map_length -= tmplen;
2469 DEBUG(
printk(
"map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2475 static int cryptocop_ioctl_process(
struct inode *inode,
struct file *filp,
unsigned int cmd,
unsigned long arg)
2482 struct cryptocop_operation *cop =
NULL;
2491 struct cryptocop_desc descs[5];
2493 struct cryptocop_desc_cfg dcfgs[5*3];
2496 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2497 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2498 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2500 unsigned char *digest_result =
NULL;
2501 int digest_length = 0;
2514 int cipher_active, digest_active, csum_active;
2515 int end_digest, end_csum;
2516 int digest_done = 0;
2517 int cipher_done = 0;
2530 DEBUG(print_strcop_crypto_op(&oper));
2540 DEBUG_API(
printk(
"cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2567 cop->cb = ioctl_process_job_callback;
2568 cop->operation_status = 0;
2569 cop->use_dmalists = 0;
2570 cop->in_interrupt = 0;
2571 cop->fast_callback = 0;
2572 cop->tfrm_op.tfrm_cfg =
NULL;
2573 cop->tfrm_op.desc =
NULL;
2574 cop->tfrm_op.indata =
NULL;
2575 cop->tfrm_op.incount = 0;
2576 cop->tfrm_op.inlen = 0;
2577 cop->tfrm_op.outdata =
NULL;
2578 cop->tfrm_op.outcount = 0;
2579 cop->tfrm_op.outlen = 0;
2581 sess = get_session(oper.
ses_id);
2590 unsigned int cipher_outlen = 0;
2593 DEBUG_API(
printk(
"cryptocop_ioctl_process: no cipher transform in session.\n"));
2598 ciph_tcfg.inject_ix = 0;
2599 ciph_tcfg.flags = 0;
2610 DEBUG_API(
printk(
"cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2616 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2625 DEBUG_API(
printk(
"cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2637 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2639 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2641 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2647 DEBUG_API(
printk(
"cryptocop_ioctl_process: no digest transform in session.\n"));
2651 digest_length = tc->
init.alg == cryptocop_alg_md5 ? 16 : 20;
2653 if (!digest_result) {
2654 DEBUG_API(
printk(
"cryptocop_ioctl_process: kmalloc digest_result\n"));
2658 DEBUG(
memset(digest_result, 0xff, digest_length));
2661 digest_tcfg.inject_ix = 0;
2662 ciph_tcfg.inject_ix += digest_length;
2669 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2674 csum_tcfg.inject_ix = digest_length;
2675 ciph_tcfg.inject_ix += 2;
2684 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2688 prev_ix = first_cfg_change_ix(&oper);
2689 if (prev_ix > oper.
inlen) {
2691 nooutpages = noinpages = 0;
2699 DEBUG(
printk(
"cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2703 nooutpages = noinpages = 0;
2709 DEBUG(
printk(
"cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2713 nooutpages = noinpages = 0;
2724 (
unsigned long int)(oper.
indata + prev_ix),
2733 nooutpages = noinpages = 0;
2734 DEBUG_API(
printk(
"cryptocop_ioctl_process: get_user_pages indata\n"));
2750 DEBUG_API(
printk(
"cryptocop_ioctl_process: get_user_pages outdata\n"));
2762 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2768 cop->tfrm_op.inlen = oper.
inlen - prev_ix;
2769 cop->tfrm_op.outlen = 0;
2771 if (oper.
do_digest) cop->tfrm_op.outlen += digest_length;
2772 if (oper.
do_csum) cop->tfrm_op.outlen += 2;
2775 cop->tfrm_op.incount = noinpages;
2777 size_t tmplen = cop->tfrm_op.inlen;
2781 tmplen -= cop->tfrm_op.indata[0].iov_len;
2782 for (i = 1; i<noinpages; i++){
2784 cop->tfrm_op.indata[
i].iov_base = (
unsigned char*)
page_address(inpages[i]);
2788 cop->tfrm_op.indata[0].iov_len = oper.
inlen - prev_ix;
2792 iovlen = nooutpages + 6;
2795 next_ix = next_cfg_change_ix(&oper, prev_ix);
2796 if (prev_ix == next_ix){
2797 DEBUG_API(
printk(
"cryptocop_ioctl_process: length configuration broken.\n"));
2801 while (prev_ix != next_ix){
2802 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2803 descs[desc_ix].cfg =
NULL;
2804 descs[desc_ix].length = next_ix - prev_ix;
2808 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2813 dcfgs[dcfg_ix].last = 1;
2815 dcfgs[dcfg_ix].last = 0;
2817 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2818 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2824 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2828 dcfgs[dcfg_ix].last = 1;
2830 dcfgs[dcfg_ix].last = 0;
2832 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2833 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2839 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2842 dcfgs[dcfg_ix].last = 1;
2844 dcfgs[dcfg_ix].last = 0;
2846 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2847 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2850 if (!descs[desc_ix].
cfg){
2851 DEBUG_API(
printk(
"cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2855 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2858 next_ix = next_cfg_change_ix(&oper, prev_ix);
2861 descs[desc_ix-1].next =
NULL;
2863 descs[0].next =
NULL;
2866 DEBUG(
printk(
"cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2868 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2869 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2874 DEBUG(
printk(
"cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2876 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2877 cop->tfrm_op.outdata[iovix].iov_len = 2;
2881 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.
cipher_outlen)){
2882 DEBUG_API(
printk(
"cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2887 DEBUG(
printk(
"cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2888 cop->tfrm_op.outcount = iovix;
2889 assert(iovix <= (nooutpages + 6));
2892 cop->tfrm_op.desc = &descs[0];
2894 DEBUG(
printk(
"cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2902 DEBUG(
printk(
"cryptocop_ioctl_process: begin wait for result\n"));
2905 DEBUG(
printk(
"cryptocop_ioctl_process: end wait for result\n"));
2913 DEBUG(
printk(
"cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2914 if (cop->operation_status == 0){
2916 DEBUG(
printk(
"cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2919 DEBUG_API(
printk(
"cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2925 DEBUG(
printk(
"cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2928 DEBUG_API(
printk(
"cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2935 DEBUG(
printk(
"cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2936 err = cop->operation_status;
2941 for (i = 0; i < noinpages; i++){
2944 for (i = 0; i < nooutpages; i++){
2948 DEBUG(
if (spdl_err < 0)
printk(
"cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2950 for (i = 0; i < nooutpages; i++){
2954 kfree(digest_result);
2958 kfree(cop->tfrm_op.indata);
2959 kfree(cop->tfrm_op.outdata);
2964 DEBUG(print_lock_status());
2970 static int cryptocop_ioctl_create_session(
struct inode *inode,
struct file *filp,
unsigned int cmd,
unsigned long arg)
2977 struct cryptocop_transform_init *tis =
NULL;
2978 struct cryptocop_transform_init ti_cipher = {0};
2979 struct cryptocop_transform_init ti_digest = {0};
2980 struct cryptocop_transform_init ti_csum = {0};
2989 DEBUG(
printk(
"cryptocop_ioctl_create_session, sess_op:\n"));
2992 "\tcipher_mode:%d\n"
3002 switch (sop.cipher){
3004 ti_cipher.alg = cryptocop_alg_des;
3007 ti_cipher.alg = cryptocop_alg_3des;
3010 ti_cipher.alg = cryptocop_alg_aes;
3013 DEBUG_API(
printk(
"create session, bad cipher algorithm %d\n", sop.cipher));
3016 DEBUG(
printk(
"setting cipher transform %d\n", ti_cipher.alg));
3018 ti_cipher.keylen = sop.keylen;
3022 ti_cipher.cipher_mode = sop.cmode;
3028 DEBUG(
printk(
"cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3029 switch (sop.des3_mode){
3038 ti_cipher.tdes_mode = sop.des3_mode;
3041 DEBUG_API(
printk(
"create session, bad 3DES mode %d\n", sop.des3_mode));
3045 ti_cipher.next = tis;
3050 switch (sop.digest){
3052 ti_digest.alg = cryptocop_alg_md5;
3055 ti_digest.alg = cryptocop_alg_sha1;
3058 DEBUG_API(
printk(
"create session, bad digest algorithm %d\n", sop.digest));
3062 ti_digest.next = tis;
3070 ti_csum.csum_mode = sop.csum;
3073 DEBUG_API(
printk(
"create session, bad checksum algorithm %d\n", sop.csum));
3076 ti_csum.alg = cryptocop_alg_csum;
3088 DEBUG({
if (err)
printk(
"create session, cryptocop_new_session %d\n", err);});
3102 static long cryptocop_ioctl_unlocked(
struct inode *inode,
3103 struct file *filp,
unsigned int cmd,
unsigned long arg)
3123 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3125 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3127 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3136 cryptocop_ioctl(
struct file *filp,
unsigned int cmd,
unsigned long arg)
3138 struct inode *inode =
file->
f_path.dentry->d_inode;
3142 ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3156 printk(
"print_dma_descriptors start\n");
3168 "\tsaved_data: 0x%p\n"
3169 "\tsaved_data_buf: 0x%p\n",
3176 "\tsaved_data: 0x%p\n"
3177 "\tsaved_data_buf: 0x%p\n",
3207 cdesc_out = cdesc_out->
next;
3235 cdesc_in = cdesc_in->
next;
3239 printk(
"print_dma_descriptors end\n");
3245 printk(
"print_strcop_crypto_op, 0x%p\n", cop);
3252 "cipher_explicit=%d\n"
3282 static void print_cryptocop_operation(
struct cryptocop_operation *cop)
3284 struct cryptocop_desc *
d;
3285 struct cryptocop_tfrm_cfg *
tc;
3286 struct cryptocop_desc_cfg *
dc;
3289 printk(
"print_cryptocop_operation, cop=0x%p\n\n", cop);
3290 printk(
"sid: %lld\n", cop->sid);
3291 printk(
"operation_status=%d\n"
3294 "fast_callback=%d\n",
3295 cop->operation_status,
3298 cop->fast_callback);
3300 if (cop->use_dmalists){
3301 print_user_dma_lists(&cop->list_op);
3312 cop->tfrm_op.tfrm_cfg,
3314 cop->tfrm_op.indata,
3315 cop->tfrm_op.incount,
3317 cop->tfrm_op.outdata,
3318 cop->tfrm_op.outcount,
3319 cop->tfrm_op.outlen);
3321 tc = cop->tfrm_op.tfrm_cfg;
3323 printk(
"tfrm_cfg, 0x%p\n"
3335 d = cop->tfrm_op.desc;
3337 printk(
"\n======================desc, 0x%p\n"
3347 printk(
"=========desc_cfg, 0x%p\n"
3362 for (i = 0; i < cop->tfrm_op.incount; i++){
3367 cop->tfrm_op.indata[i].iov_base,
3368 cop->tfrm_op.indata[i].iov_len);
3370 printk(
"\n====outiov\n");
3371 for (i = 0; i < cop->tfrm_op.outcount; i++){
3376 cop->tfrm_op.outdata[i].iov_base,
3377 cop->tfrm_op.outdata[i].iov_len);
3380 printk(
"------------end print_cryptocop_operation\n");
3384 static void print_user_dma_lists(
struct cryptocop_dma_list_operation *dma_op)
3389 printk(
"print_user_dma_lists, dma_op=0x%p\n", dma_op);
3391 printk(
"out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf,
phys_to_virt((
unsigned long int)dma_op->out_data_buf));
3392 printk(
"in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf,
phys_to_virt((
unsigned long int)dma_op->in_data_buf));
3394 printk(
"##############outlist\n");
3397 while (dd !=
NULL) {
3398 printk(
"#%d phys_to_virt(desc) 0x%p\n", i, dd);
3425 printk(
"##############inlist\n");
3428 while (dd !=
NULL) {
3429 printk(
"#%d phys_to_virt(desc) 0x%p\n", i, dd);
3458 static void print_lock_status(
void)
3460 printk(
"**********************print_lock_status\n");
3461 printk(
"cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3462 printk(
"cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3463 printk(
"descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3464 printk(
"cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3465 printk(
"running_job_lock %d\n", spin_is_locked(running_job_lock));
3466 printk(
"cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3471 static const char cryptocop_name[] =
"ETRAX FS stream co-processor";
3473 static int init_stream_coprocessor(
void)
3484 printk(
"ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3486 err = register_chrdev(
CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3488 printk(
KERN_ERR "stream co-processor: could not get major number.\n");
3492 err = init_cryptocop();
3497 err = cryptocop_job_queue_init();
3499 release_cryptocop();
3505 descr_pool[
i].from_pool = 1;
3506 descr_pool[
i].next = &descr_pool[i + 1];
3508 descr_pool[
i].from_pool = 1;
3509 descr_pool[
i].next =
NULL;
3510 descr_pool_free_list = &descr_pool[0];
3520 cryptocop_sessions =
NULL;
3523 cryptocop_running_job =
NULL;
3525 printk(
"stream co-processor: init done.\n");
3529 static void __exit exit_stream_coprocessor(
void)
3531 release_cryptocop();
3532 cryptocop_job_queue_close();