Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
ib_srpt.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc. All rights reserved.
3  * Copyright (C) 2008 - 2011 Bart Van Assche <[email protected]>.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses. You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  * Redistribution and use in source and binary forms, with or
12  * without modification, are permitted provided that the following
13  * conditions are met:
14  *
15  * - Redistributions of source code must retain the above
16  * copyright notice, this list of conditions and the following
17  * disclaimer.
18  *
19  * - Redistributions in binary form must reproduce the above
20  * copyright notice, this list of conditions and the following
21  * disclaimer in the documentation and/or other materials
22  * provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  *
33  */
34 
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/err.h>
39 #include <linux/ctype.h>
40 #include <linux/kthread.h>
41 #include <linux/string.h>
42 #include <linux/delay.h>
43 #include <linux/atomic.h>
44 #include <scsi/scsi_tcq.h>
45 #include <target/configfs_macros.h>
50 #include "ib_srpt.h"
51 
52 /* Name of this kernel module. */
53 #define DRV_NAME "ib_srpt"
54 #define DRV_VERSION "2.0.0"
55 #define DRV_RELDATE "2011-02-14"
56 
57 #define SRPT_ID_STRING "Linux SRP target"
58 
59 #undef pr_fmt
60 #define pr_fmt(fmt) DRV_NAME " " fmt
61 
62 MODULE_AUTHOR("Vu Pham and Bart Van Assche");
63 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
64  "v" DRV_VERSION " (" DRV_RELDATE ")");
65 MODULE_LICENSE("Dual BSD/GPL");
66 
67 /*
68  * Global Variables
69  */
70 
71 static u64 srpt_service_guid;
72 static DEFINE_SPINLOCK(srpt_dev_lock); /* Protects srpt_dev_list. */
73 static LIST_HEAD(srpt_dev_list); /* List of srpt_device structures. */
74 
75 static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
76 module_param(srp_max_req_size, int, 0444);
77 MODULE_PARM_DESC(srp_max_req_size,
78  "Maximum size of SRP request messages in bytes.");
79 
80 static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
81 module_param(srpt_srq_size, int, 0444);
82 MODULE_PARM_DESC(srpt_srq_size,
83  "Shared receive queue (SRQ) size.");
84 
85 static int srpt_get_u64_x(char *buffer, struct kernel_param *kp)
86 {
87  return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
88 }
89 module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
90  0444);
91 MODULE_PARM_DESC(srpt_service_guid,
92  "Using this value for ioc_guid, id_ext, and cm_listen_id"
93  " instead of using the node_guid of the first HCA.");
94 
95 static struct ib_client srpt_client;
96 static struct target_fabric_configfs *srpt_target;
97 static void srpt_release_channel(struct srpt_rdma_ch *ch);
98 static int srpt_queue_status(struct se_cmd *cmd);
99 
103 static inline
104 enum dma_data_direction opposite_dma_dir(enum dma_data_direction dir)
105 {
106  switch (dir) {
107  case DMA_TO_DEVICE: return DMA_FROM_DEVICE;
108  case DMA_FROM_DEVICE: return DMA_TO_DEVICE;
109  default: return dir;
110  }
111 }
112 
118 static inline const char *srpt_sdev_name(struct srpt_device *sdev)
119 {
120  return sdev->device->name;
121 }
122 
123 static enum rdma_ch_state srpt_get_ch_state(struct srpt_rdma_ch *ch)
124 {
125  unsigned long flags;
126  enum rdma_ch_state state;
127 
128  spin_lock_irqsave(&ch->spinlock, flags);
129  state = ch->state;
130  spin_unlock_irqrestore(&ch->spinlock, flags);
131  return state;
132 }
133 
134 static enum rdma_ch_state
135 srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new_state)
136 {
137  unsigned long flags;
138  enum rdma_ch_state prev;
139 
140  spin_lock_irqsave(&ch->spinlock, flags);
141  prev = ch->state;
142  ch->state = new_state;
143  spin_unlock_irqrestore(&ch->spinlock, flags);
144  return prev;
145 }
146 
152 static bool
153 srpt_test_and_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state old,
154  enum rdma_ch_state new)
155 {
156  unsigned long flags;
157  enum rdma_ch_state prev;
158 
159  spin_lock_irqsave(&ch->spinlock, flags);
160  prev = ch->state;
161  if (prev == old)
162  ch->state = new;
163  spin_unlock_irqrestore(&ch->spinlock, flags);
164  return prev == old;
165 }
166 
175 static void srpt_event_handler(struct ib_event_handler *handler,
176  struct ib_event *event)
177 {
178  struct srpt_device *sdev;
179  struct srpt_port *sport;
180 
181  sdev = ib_get_client_data(event->device, &srpt_client);
182  if (!sdev || sdev->device != event->device)
183  return;
184 
185  pr_debug("ASYNC event= %d on device= %s\n", event->event,
186  srpt_sdev_name(sdev));
187 
188  switch (event->event) {
189  case IB_EVENT_PORT_ERR:
190  if (event->element.port_num <= sdev->device->phys_port_cnt) {
191  sport = &sdev->port[event->element.port_num - 1];
192  sport->lid = 0;
193  sport->sm_lid = 0;
194  }
195  break;
197  case IB_EVENT_LID_CHANGE:
199  case IB_EVENT_SM_CHANGE:
201  /* Refresh port data asynchronously. */
202  if (event->element.port_num <= sdev->device->phys_port_cnt) {
203  sport = &sdev->port[event->element.port_num - 1];
204  if (!sport->lid && !sport->sm_lid)
205  schedule_work(&sport->work);
206  }
207  break;
208  default:
209  printk(KERN_ERR "received unrecognized IB event %d\n",
210  event->event);
211  break;
212  }
213 }
214 
218 static void srpt_srq_event(struct ib_event *event, void *ctx)
219 {
220  printk(KERN_INFO "SRQ event %d\n", event->event);
221 }
222 
226 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
227 {
228  pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
229  event->event, ch->cm_id, ch->sess_name, srpt_get_ch_state(ch));
230 
231  switch (event->event) {
232  case IB_EVENT_COMM_EST:
233  ib_cm_notify(ch->cm_id, event->event);
234  break;
236  if (srpt_test_and_set_ch_state(ch, CH_DRAINING,
237  CH_RELEASING))
238  srpt_release_channel(ch);
239  else
240  pr_debug("%s: state %d - ignored LAST_WQE.\n",
241  ch->sess_name, srpt_get_ch_state(ch));
242  break;
243  default:
244  printk(KERN_ERR "received unrecognized IB QP event %d\n",
245  event->event);
246  break;
247  }
248 }
249 
259 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
260 {
261  u16 id;
262  u8 tmp;
263 
264  id = (slot - 1) / 2;
265  if (slot & 0x1) {
266  tmp = c_list[id] & 0xf;
267  c_list[id] = (value << 4) | tmp;
268  } else {
269  tmp = c_list[id] & 0xf0;
270  c_list[id] = (value & 0xf) | tmp;
271  }
272 }
273 
280 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
281 {
282  struct ib_class_port_info *cif;
283 
284  cif = (struct ib_class_port_info *)mad->data;
285  memset(cif, 0, sizeof *cif);
286  cif->base_version = 1;
287  cif->class_version = 1;
288  cif->resp_time_value = 20;
289 
290  mad->mad_hdr.status = 0;
291 }
292 
299 static void srpt_get_iou(struct ib_dm_mad *mad)
300 {
301  struct ib_dm_iou_info *ioui;
302  u8 slot;
303  int i;
304 
305  ioui = (struct ib_dm_iou_info *)mad->data;
307  ioui->max_controllers = 16;
308 
309  /* set present for slot 1 and empty for the rest */
310  srpt_set_ioc(ioui->controller_list, 1, 1);
311  for (i = 1, slot = 2; i < 16; i++, slot++)
312  srpt_set_ioc(ioui->controller_list, slot, 0);
313 
314  mad->mad_hdr.status = 0;
315 }
316 
324 static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
325  struct ib_dm_mad *mad)
326 {
327  struct srpt_device *sdev = sport->sdev;
328  struct ib_dm_ioc_profile *iocp;
329 
330  iocp = (struct ib_dm_ioc_profile *)mad->data;
331 
332  if (!slot || slot > 16) {
333  mad->mad_hdr.status
335  return;
336  }
337 
338  if (slot > 2) {
339  mad->mad_hdr.status
341  return;
342  }
343 
344  memset(iocp, 0, sizeof *iocp);
346  iocp->guid = cpu_to_be64(srpt_service_guid);
347  iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
348  iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id);
349  iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver);
350  iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id);
351  iocp->subsys_device_id = 0x0;
356  iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);
357  iocp->rdma_read_depth = 4;
358  iocp->send_size = cpu_to_be32(srp_max_req_size);
359  iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
360  1U << 24));
361  iocp->num_svc_entries = 1;
364 
365  mad->mad_hdr.status = 0;
366 }
367 
374 static void srpt_get_svc_entries(u64 ioc_guid,
375  u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
376 {
377  struct ib_dm_svc_entries *svc_entries;
378 
379  WARN_ON(!ioc_guid);
380 
381  if (!slot || slot > 16) {
382  mad->mad_hdr.status
384  return;
385  }
386 
387  if (slot > 2 || lo > hi || hi > 1) {
388  mad->mad_hdr.status
390  return;
391  }
392 
393  svc_entries = (struct ib_dm_svc_entries *)mad->data;
394  memset(svc_entries, 0, sizeof *svc_entries);
395  svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
396  snprintf(svc_entries->service_entries[0].name,
397  sizeof(svc_entries->service_entries[0].name),
398  "%s%016llx",
400  ioc_guid);
401 
402  mad->mad_hdr.status = 0;
403 }
404 
411 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
412  struct ib_dm_mad *rsp_mad)
413 {
414  u16 attr_id;
415  u32 slot;
416  u8 hi, lo;
417 
418  attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
419  switch (attr_id) {
421  srpt_get_class_port_info(rsp_mad);
422  break;
423  case DM_ATTR_IOU_INFO:
424  srpt_get_iou(rsp_mad);
425  break;
426  case DM_ATTR_IOC_PROFILE:
427  slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
428  srpt_get_ioc(sp, slot, rsp_mad);
429  break;
430  case DM_ATTR_SVC_ENTRIES:
431  slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
432  hi = (u8) ((slot >> 8) & 0xff);
433  lo = (u8) (slot & 0xff);
434  slot = (u16) ((slot >> 16) & 0xffff);
435  srpt_get_svc_entries(srpt_service_guid,
436  slot, hi, lo, rsp_mad);
437  break;
438  default:
439  rsp_mad->mad_hdr.status =
441  break;
442  }
443 }
444 
448 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
449  struct ib_mad_send_wc *mad_wc)
450 {
451  ib_destroy_ah(mad_wc->send_buf->ah);
452  ib_free_send_mad(mad_wc->send_buf);
453 }
454 
458 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
459  struct ib_mad_recv_wc *mad_wc)
460 {
461  struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
462  struct ib_ah *ah;
463  struct ib_mad_send_buf *rsp;
464  struct ib_dm_mad *dm_mad;
465 
466  if (!mad_wc || !mad_wc->recv_buf.mad)
467  return;
468 
469  ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
470  mad_wc->recv_buf.grh, mad_agent->port_num);
471  if (IS_ERR(ah))
472  goto err;
473 
475 
476  rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
477  mad_wc->wc->pkey_index, 0,
479  GFP_KERNEL);
480  if (IS_ERR(rsp))
481  goto err_rsp;
482 
483  rsp->ah = ah;
484 
485  dm_mad = rsp->mad;
486  memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad);
487  dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
488  dm_mad->mad_hdr.status = 0;
489 
490  switch (mad_wc->recv_buf.mad->mad_hdr.method) {
491  case IB_MGMT_METHOD_GET:
492  srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
493  break;
494  case IB_MGMT_METHOD_SET:
495  dm_mad->mad_hdr.status =
497  break;
498  default:
499  dm_mad->mad_hdr.status =
501  break;
502  }
503 
504  if (!ib_post_send_mad(rsp, NULL)) {
505  ib_free_recv_mad(mad_wc);
506  /* will destroy_ah & free_send_mad in send completion */
507  return;
508  }
509 
511 
512 err_rsp:
513  ib_destroy_ah(ah);
514 err:
515  ib_free_recv_mad(mad_wc);
516 }
517 
527 static int srpt_refresh_port(struct srpt_port *sport)
528 {
529  struct ib_mad_reg_req reg_req;
530  struct ib_port_modify port_modify;
531  struct ib_port_attr port_attr;
532  int ret;
533 
534  memset(&port_modify, 0, sizeof port_modify);
535  port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
536  port_modify.clr_port_cap_mask = 0;
537 
538  ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
539  if (ret)
540  goto err_mod_port;
541 
542  ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
543  if (ret)
544  goto err_query_port;
545 
546  sport->sm_lid = port_attr.sm_lid;
547  sport->lid = port_attr.lid;
548 
549  ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid);
550  if (ret)
551  goto err_query_port;
552 
553  if (!sport->mad_agent) {
554  memset(&reg_req, 0, sizeof reg_req);
555  reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
556  reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
557  set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
558  set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
559 
560  sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
561  sport->port,
562  IB_QPT_GSI,
563  &reg_req, 0,
564  srpt_mad_send_handler,
565  srpt_mad_recv_handler,
566  sport);
567  if (IS_ERR(sport->mad_agent)) {
568  ret = PTR_ERR(sport->mad_agent);
569  sport->mad_agent = NULL;
570  goto err_query_port;
571  }
572  }
573 
574  return 0;
575 
576 err_query_port:
577 
578  port_modify.set_port_cap_mask = 0;
579  port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
580  ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
581 
582 err_mod_port:
583 
584  return ret;
585 }
586 
592 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
593 {
594  struct ib_port_modify port_modify = {
596  };
597  struct srpt_port *sport;
598  int i;
599 
600  for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
601  sport = &sdev->port[i - 1];
602  WARN_ON(sport->port != i);
603  if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
604  printk(KERN_ERR "disabling MAD processing failed.\n");
605  if (sport->mad_agent) {
607  sport->mad_agent = NULL;
608  }
609  }
610 }
611 
615 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
616  int ioctx_size, int dma_size,
617  enum dma_data_direction dir)
618 {
619  struct srpt_ioctx *ioctx;
620 
621  ioctx = kmalloc(ioctx_size, GFP_KERNEL);
622  if (!ioctx)
623  goto err;
624 
625  ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
626  if (!ioctx->buf)
627  goto err_free_ioctx;
628 
629  ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
630  if (ib_dma_mapping_error(sdev->device, ioctx->dma))
631  goto err_free_buf;
632 
633  return ioctx;
634 
635 err_free_buf:
636  kfree(ioctx->buf);
637 err_free_ioctx:
638  kfree(ioctx);
639 err:
640  return NULL;
641 }
642 
646 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
647  int dma_size, enum dma_data_direction dir)
648 {
649  if (!ioctx)
650  return;
651 
652  ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
653  kfree(ioctx->buf);
654  kfree(ioctx);
655 }
656 
665 static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
666  int ring_size, int ioctx_size,
667  int dma_size, enum dma_data_direction dir)
668 {
669  struct srpt_ioctx **ring;
670  int i;
671 
672  WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
673  && ioctx_size != sizeof(struct srpt_send_ioctx));
674 
675  ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL);
676  if (!ring)
677  goto out;
678  for (i = 0; i < ring_size; ++i) {
679  ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
680  if (!ring[i])
681  goto err;
682  ring[i]->index = i;
683  }
684  goto out;
685 
686 err:
687  while (--i >= 0)
688  srpt_free_ioctx(sdev, ring[i], dma_size, dir);
689  kfree(ring);
690  ring = NULL;
691 out:
692  return ring;
693 }
694 
698 static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
699  struct srpt_device *sdev, int ring_size,
700  int dma_size, enum dma_data_direction dir)
701 {
702  int i;
703 
704  for (i = 0; i < ring_size; ++i)
705  srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
706  kfree(ioctx_ring);
707 }
708 
712 static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
713 {
715  unsigned long flags;
716 
717  BUG_ON(!ioctx);
718 
719  spin_lock_irqsave(&ioctx->spinlock, flags);
720  state = ioctx->state;
721  spin_unlock_irqrestore(&ioctx->spinlock, flags);
722  return state;
723 }
724 
731 static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
732  enum srpt_command_state new)
733 {
734  enum srpt_command_state previous;
735  unsigned long flags;
736 
737  BUG_ON(!ioctx);
738 
739  spin_lock_irqsave(&ioctx->spinlock, flags);
740  previous = ioctx->state;
741  if (previous != SRPT_STATE_DONE)
742  ioctx->state = new;
743  spin_unlock_irqrestore(&ioctx->spinlock, flags);
744 
745  return previous;
746 }
747 
753 static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
754  enum srpt_command_state old,
755  enum srpt_command_state new)
756 {
757  enum srpt_command_state previous;
758  unsigned long flags;
759 
760  WARN_ON(!ioctx);
761  WARN_ON(old == SRPT_STATE_DONE);
762  WARN_ON(new == SRPT_STATE_NEW);
763 
764  spin_lock_irqsave(&ioctx->spinlock, flags);
765  previous = ioctx->state;
766  if (previous == old)
767  ioctx->state = new;
768  spin_unlock_irqrestore(&ioctx->spinlock, flags);
769  return previous == old;
770 }
771 
775 static int srpt_post_recv(struct srpt_device *sdev,
776  struct srpt_recv_ioctx *ioctx)
777 {
778  struct ib_sge list;
779  struct ib_recv_wr wr, *bad_wr;
780 
781  BUG_ON(!sdev);
782  wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index);
783 
784  list.addr = ioctx->ioctx.dma;
785  list.length = srp_max_req_size;
786  list.lkey = sdev->mr->lkey;
787 
788  wr.next = NULL;
789  wr.sg_list = &list;
790  wr.num_sge = 1;
791 
792  return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
793 }
794 
800 static int srpt_post_send(struct srpt_rdma_ch *ch,
801  struct srpt_send_ioctx *ioctx, int len)
802 {
803  struct ib_sge list;
804  struct ib_send_wr wr, *bad_wr;
805  struct srpt_device *sdev = ch->sport->sdev;
806  int ret;
807 
808  atomic_inc(&ch->req_lim);
809 
810  ret = -ENOMEM;
811  if (unlikely(atomic_dec_return(&ch->sq_wr_avail) < 0)) {
812  printk(KERN_WARNING "IB send queue full (needed 1)\n");
813  goto out;
814  }
815 
816  ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, len,
817  DMA_TO_DEVICE);
818 
819  list.addr = ioctx->ioctx.dma;
820  list.length = len;
821  list.lkey = sdev->mr->lkey;
822 
823  wr.next = NULL;
824  wr.wr_id = encode_wr_id(SRPT_SEND, ioctx->ioctx.index);
825  wr.sg_list = &list;
826  wr.num_sge = 1;
827  wr.opcode = IB_WR_SEND;
828  wr.send_flags = IB_SEND_SIGNALED;
829 
830  ret = ib_post_send(ch->qp, &wr, &bad_wr);
831 
832 out:
833  if (ret < 0) {
834  atomic_inc(&ch->sq_wr_avail);
835  atomic_dec(&ch->req_lim);
836  }
837  return ret;
838 }
839 
854 static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
855  struct srp_cmd *srp_cmd,
856  enum dma_data_direction *dir, u64 *data_len)
857 {
858  struct srp_indirect_buf *idb;
859  struct srp_direct_buf *db;
860  unsigned add_cdb_offset;
861  int ret;
862 
863  /*
864  * The pointer computations below will only be compiled correctly
865  * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
866  * whether srp_cmd::add_data has been declared as a byte pointer.
867  */
868  BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0)
869  && !__same_type(srp_cmd->add_data[0], (u8)0));
870 
871  BUG_ON(!dir);
872  BUG_ON(!data_len);
873 
874  ret = 0;
875  *data_len = 0;
876 
877  /*
878  * The lower four bits of the buffer format field contain the DATA-IN
879  * buffer descriptor format, and the highest four bits contain the
880  * DATA-OUT buffer descriptor format.
881  */
882  *dir = DMA_NONE;
883  if (srp_cmd->buf_fmt & 0xf)
884  /* DATA-IN: transfer data from target to initiator (read). */
885  *dir = DMA_FROM_DEVICE;
886  else if (srp_cmd->buf_fmt >> 4)
887  /* DATA-OUT: transfer data from initiator to target (write). */
888  *dir = DMA_TO_DEVICE;
889 
890  /*
891  * According to the SRP spec, the lower two bits of the 'ADDITIONAL
892  * CDB LENGTH' field are reserved and the size in bytes of this field
893  * is four times the value specified in bits 3..7. Hence the "& ~3".
894  */
895  add_cdb_offset = srp_cmd->add_cdb_len & ~3;
896  if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
897  ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
898  ioctx->n_rbuf = 1;
899  ioctx->rbufs = &ioctx->single_rbuf;
900 
901  db = (struct srp_direct_buf *)(srp_cmd->add_data
902  + add_cdb_offset);
903  memcpy(ioctx->rbufs, db, sizeof *db);
904  *data_len = be32_to_cpu(db->len);
905  } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
906  ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
907  idb = (struct srp_indirect_buf *)(srp_cmd->add_data
908  + add_cdb_offset);
909 
910  ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;
911 
912  if (ioctx->n_rbuf >
913  (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
914  printk(KERN_ERR "received unsupported SRP_CMD request"
915  " type (%u out + %u in != %u / %zu)\n",
916  srp_cmd->data_out_desc_cnt,
917  srp_cmd->data_in_desc_cnt,
918  be32_to_cpu(idb->table_desc.len),
919  sizeof(*db));
920  ioctx->n_rbuf = 0;
921  ret = -EINVAL;
922  goto out;
923  }
924 
925  if (ioctx->n_rbuf == 1)
926  ioctx->rbufs = &ioctx->single_rbuf;
927  else {
928  ioctx->rbufs =
929  kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);
930  if (!ioctx->rbufs) {
931  ioctx->n_rbuf = 0;
932  ret = -ENOMEM;
933  goto out;
934  }
935  }
936 
937  db = idb->desc_list;
938  memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);
939  *data_len = be32_to_cpu(idb->len);
940  }
941 out:
942  return ret;
943 }
944 
951 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
952 {
953  struct ib_qp_attr *attr;
954  int ret;
955 
956  attr = kzalloc(sizeof *attr, GFP_KERNEL);
957  if (!attr)
958  return -ENOMEM;
959 
960  attr->qp_state = IB_QPS_INIT;
963  attr->port_num = ch->sport->port;
964  attr->pkey_index = 0;
965 
966  ret = ib_modify_qp(qp, attr,
969 
970  kfree(attr);
971  return ret;
972 }
973 
985 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
986 {
987  struct ib_qp_attr qp_attr;
988  int attr_mask;
989  int ret;
990 
991  qp_attr.qp_state = IB_QPS_RTR;
992  ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
993  if (ret)
994  goto out;
995 
996  qp_attr.max_dest_rd_atomic = 4;
997 
998  ret = ib_modify_qp(qp, &qp_attr, attr_mask);
999 
1000 out:
1001  return ret;
1002 }
1003 
1015 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1016 {
1017  struct ib_qp_attr qp_attr;
1018  int attr_mask;
1019  int ret;
1020 
1021  qp_attr.qp_state = IB_QPS_RTS;
1022  ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1023  if (ret)
1024  goto out;
1025 
1026  qp_attr.max_rd_atomic = 4;
1027 
1028  ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1029 
1030 out:
1031  return ret;
1032 }
1033 
1037 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1038 {
1039  struct ib_qp_attr qp_attr;
1040 
1041  qp_attr.qp_state = IB_QPS_ERR;
1042  return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1043 }
1044 
1048 static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch,
1049  struct srpt_send_ioctx *ioctx)
1050 {
1051  struct scatterlist *sg;
1052  enum dma_data_direction dir;
1053 
1054  BUG_ON(!ch);
1055  BUG_ON(!ioctx);
1056  BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius);
1057 
1058  while (ioctx->n_rdma)
1059  kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge);
1060 
1061  kfree(ioctx->rdma_ius);
1062  ioctx->rdma_ius = NULL;
1063 
1064  if (ioctx->mapped_sg_count) {
1065  sg = ioctx->sg;
1066  WARN_ON(!sg);
1067  dir = ioctx->cmd.data_direction;
1068  BUG_ON(dir == DMA_NONE);
1069  ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt,
1070  opposite_dma_dir(dir));
1071  ioctx->mapped_sg_count = 0;
1072  }
1073 }
1074 
1078 static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch,
1079  struct srpt_send_ioctx *ioctx)
1080 {
1081  struct se_cmd *cmd;
1082  struct scatterlist *sg, *sg_orig;
1083  int sg_cnt;
1084  enum dma_data_direction dir;
1085  struct rdma_iu *riu;
1086  struct srp_direct_buf *db;
1088  struct ib_sge *sge;
1089  u64 raddr;
1090  u32 rsize;
1091  u32 tsize;
1092  u32 dma_len;
1093  int count, nrdma;
1094  int i, j, k;
1095 
1096  BUG_ON(!ch);
1097  BUG_ON(!ioctx);
1098  cmd = &ioctx->cmd;
1099  dir = cmd->data_direction;
1100  BUG_ON(dir == DMA_NONE);
1101 
1102  ioctx->sg = sg = sg_orig = cmd->t_data_sg;
1103  ioctx->sg_cnt = sg_cnt = cmd->t_data_nents;
1104 
1105  count = ib_dma_map_sg(ch->sport->sdev->device, sg, sg_cnt,
1106  opposite_dma_dir(dir));
1107  if (unlikely(!count))
1108  return -EAGAIN;
1109 
1110  ioctx->mapped_sg_count = count;
1111 
1112  if (ioctx->rdma_ius && ioctx->n_rdma_ius)
1113  nrdma = ioctx->n_rdma_ius;
1114  else {
1115  nrdma = (count + SRPT_DEF_SG_PER_WQE - 1) / SRPT_DEF_SG_PER_WQE
1116  + ioctx->n_rbuf;
1117 
1118  ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu, GFP_KERNEL);
1119  if (!ioctx->rdma_ius)
1120  goto free_mem;
1121 
1122  ioctx->n_rdma_ius = nrdma;
1123  }
1124 
1125  db = ioctx->rbufs;
1126  tsize = cmd->data_length;
1127  dma_len = sg_dma_len(&sg[0]);
1128  riu = ioctx->rdma_ius;
1129 
1130  /*
1131  * For each remote desc - calculate the #ib_sge.
1132  * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then
1133  * each remote desc rdma_iu is required a rdma wr;
1134  * else
1135  * we need to allocate extra rdma_iu to carry extra #ib_sge in
1136  * another rdma wr
1137  */
1138  for (i = 0, j = 0;
1139  j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
1140  rsize = be32_to_cpu(db->len);
1141  raddr = be64_to_cpu(db->va);
1142  riu->raddr = raddr;
1143  riu->rkey = be32_to_cpu(db->key);
1144  riu->sge_cnt = 0;
1145 
1146  /* calculate how many sge required for this remote_buf */
1147  while (rsize > 0 && tsize > 0) {
1148 
1149  if (rsize >= dma_len) {
1150  tsize -= dma_len;
1151  rsize -= dma_len;
1152  raddr += dma_len;
1153 
1154  if (tsize > 0) {
1155  ++j;
1156  if (j < count) {
1157  sg = sg_next(sg);
1158  dma_len = sg_dma_len(sg);
1159  }
1160  }
1161  } else {
1162  tsize -= rsize;
1163  dma_len -= rsize;
1164  rsize = 0;
1165  }
1166 
1167  ++riu->sge_cnt;
1168 
1169  if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) {
1170  ++ioctx->n_rdma;
1171  riu->sge =
1172  kmalloc(riu->sge_cnt * sizeof *riu->sge,
1173  GFP_KERNEL);
1174  if (!riu->sge)
1175  goto free_mem;
1176 
1177  ++riu;
1178  riu->sge_cnt = 0;
1179  riu->raddr = raddr;
1180  riu->rkey = be32_to_cpu(db->key);
1181  }
1182  }
1183 
1184  ++ioctx->n_rdma;
1185  riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge,
1186  GFP_KERNEL);
1187  if (!riu->sge)
1188  goto free_mem;
1189  }
1190 
1191  db = ioctx->rbufs;
1192  tsize = cmd->data_length;
1193  riu = ioctx->rdma_ius;
1194  sg = sg_orig;
1195  dma_len = sg_dma_len(&sg[0]);
1196  dma_addr = sg_dma_address(&sg[0]);
1197 
1198  /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */
1199  for (i = 0, j = 0;
1200  j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {
1201  rsize = be32_to_cpu(db->len);
1202  sge = riu->sge;
1203  k = 0;
1204 
1205  while (rsize > 0 && tsize > 0) {
1206  sge->addr = dma_addr;
1207  sge->lkey = ch->sport->sdev->mr->lkey;
1208 
1209  if (rsize >= dma_len) {
1210  sge->length =
1211  (tsize < dma_len) ? tsize : dma_len;
1212  tsize -= dma_len;
1213  rsize -= dma_len;
1214 
1215  if (tsize > 0) {
1216  ++j;
1217  if (j < count) {
1218  sg = sg_next(sg);
1219  dma_len = sg_dma_len(sg);
1220  dma_addr = sg_dma_address(sg);
1221  }
1222  }
1223  } else {
1224  sge->length = (tsize < rsize) ? tsize : rsize;
1225  tsize -= rsize;
1226  dma_len -= rsize;
1227  dma_addr += rsize;
1228  rsize = 0;
1229  }
1230 
1231  ++k;
1232  if (k == riu->sge_cnt && rsize > 0 && tsize > 0) {
1233  ++riu;
1234  sge = riu->sge;
1235  k = 0;
1236  } else if (rsize > 0 && tsize > 0)
1237  ++sge;
1238  }
1239  }
1240 
1241  return 0;
1242 
1243 free_mem:
1244  srpt_unmap_sg_to_ib_sge(ch, ioctx);
1245 
1246  return -ENOMEM;
1247 }
1248 
1252 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1253 {
1254  struct srpt_send_ioctx *ioctx;
1255  unsigned long flags;
1256 
1257  BUG_ON(!ch);
1258 
1259  ioctx = NULL;
1260  spin_lock_irqsave(&ch->spinlock, flags);
1261  if (!list_empty(&ch->free_list)) {
1262  ioctx = list_first_entry(&ch->free_list,
1263  struct srpt_send_ioctx, free_list);
1264  list_del(&ioctx->free_list);
1265  }
1266  spin_unlock_irqrestore(&ch->spinlock, flags);
1267 
1268  if (!ioctx)
1269  return ioctx;
1270 
1271  BUG_ON(ioctx->ch != ch);
1272  kref_init(&ioctx->kref);
1273  spin_lock_init(&ioctx->spinlock);
1274  ioctx->state = SRPT_STATE_NEW;
1275  ioctx->n_rbuf = 0;
1276  ioctx->rbufs = NULL;
1277  ioctx->n_rdma = 0;
1278  ioctx->n_rdma_ius = 0;
1279  ioctx->rdma_ius = NULL;
1280  ioctx->mapped_sg_count = 0;
1281  init_completion(&ioctx->tx_done);
1282  ioctx->queue_status_only = false;
1283  /*
1284  * transport_init_se_cmd() does not initialize all fields, so do it
1285  * here.
1286  */
1287  memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1288  memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
1289 
1290  return ioctx;
1291 }
1292 
1296 static void srpt_put_send_ioctx(struct srpt_send_ioctx *ioctx)
1297 {
1298  struct srpt_rdma_ch *ch;
1299  unsigned long flags;
1300 
1301  BUG_ON(!ioctx);
1302  ch = ioctx->ch;
1303  BUG_ON(!ch);
1304 
1305  WARN_ON(srpt_get_cmd_state(ioctx) != SRPT_STATE_DONE);
1306 
1307  srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx);
1308  transport_generic_free_cmd(&ioctx->cmd, 0);
1309 
1310  if (ioctx->n_rbuf > 1) {
1311  kfree(ioctx->rbufs);
1312  ioctx->rbufs = NULL;
1313  ioctx->n_rbuf = 0;
1314  }
1315 
1316  spin_lock_irqsave(&ch->spinlock, flags);
1317  list_add(&ioctx->free_list, &ch->free_list);
1318  spin_unlock_irqrestore(&ch->spinlock, flags);
1319 }
1320 
1321 static void srpt_put_send_ioctx_kref(struct kref *kref)
1322 {
1323  srpt_put_send_ioctx(container_of(kref, struct srpt_send_ioctx, kref));
1324 }
1325 
1331 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1332 {
1334  unsigned long flags;
1335 
1336  BUG_ON(!ioctx);
1337 
1338  /*
1339  * If the command is in a state where the target core is waiting for
1340  * the ib_srpt driver, change the state to the next state. Changing
1341  * the state of the command from SRPT_STATE_NEED_DATA to
1342  * SRPT_STATE_DATA_IN ensures that srpt_xmit_response() will call this
1343  * function a second time.
1344  */
1345 
1346  spin_lock_irqsave(&ioctx->spinlock, flags);
1347  state = ioctx->state;
1348  switch (state) {
1349  case SRPT_STATE_NEED_DATA:
1350  ioctx->state = SRPT_STATE_DATA_IN;
1351  break;
1352  case SRPT_STATE_DATA_IN:
1355  ioctx->state = SRPT_STATE_DONE;
1356  break;
1357  default:
1358  break;
1359  }
1360  spin_unlock_irqrestore(&ioctx->spinlock, flags);
1361 
1362  if (state == SRPT_STATE_DONE)
1363  goto out;
1364 
1365  pr_debug("Aborting cmd with state %d and tag %lld\n", state,
1366  ioctx->tag);
1367 
1368  switch (state) {
1369  case SRPT_STATE_NEW:
1370  case SRPT_STATE_DATA_IN:
1371  case SRPT_STATE_MGMT:
1372  /*
1373  * Do nothing - defer abort processing until
1374  * srpt_queue_response() is invoked.
1375  */
1376  WARN_ON(!transport_check_aborted_status(&ioctx->cmd, false));
1377  break;
1378  case SRPT_STATE_NEED_DATA:
1379  /* DMA_TO_DEVICE (write) - RDMA read error. */
1380 
1381  /* XXX(hch): this is a horrible layering violation.. */
1382  spin_lock_irqsave(&ioctx->cmd.t_state_lock, flags);
1383  ioctx->cmd.transport_state |= CMD_T_LUN_STOP;
1384  ioctx->cmd.transport_state &= ~CMD_T_ACTIVE;
1385  spin_unlock_irqrestore(&ioctx->cmd.t_state_lock, flags);
1386 
1387  complete(&ioctx->cmd.transport_lun_stop_comp);
1388  break;
1390  /*
1391  * SRP_RSP sending failed or the SRP_RSP send completion has
1392  * not been received in time.
1393  */
1394  srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx);
1395  spin_lock_irqsave(&ioctx->cmd.t_state_lock, flags);
1396  ioctx->cmd.transport_state |= CMD_T_LUN_STOP;
1397  spin_unlock_irqrestore(&ioctx->cmd.t_state_lock, flags);
1398  kref_put(&ioctx->kref, srpt_put_send_ioctx_kref);
1399  break;
1401  srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1402  kref_put(&ioctx->kref, srpt_put_send_ioctx_kref);
1403  break;
1404  default:
1405  WARN_ON("ERROR: unexpected command state");
1406  break;
1407  }
1408 
1409 out:
1410  return state;
1411 }
1412 
1416 static void srpt_handle_send_err_comp(struct srpt_rdma_ch *ch, u64 wr_id)
1417 {
1418  struct srpt_send_ioctx *ioctx;
1420  struct se_cmd *cmd;
1421  u32 index;
1422 
1423  atomic_inc(&ch->sq_wr_avail);
1424 
1425  index = idx_from_wr_id(wr_id);
1426  ioctx = ch->ioctx_ring[index];
1427  state = srpt_get_cmd_state(ioctx);
1428  cmd = &ioctx->cmd;
1429 
1431  && state != SRPT_STATE_MGMT_RSP_SENT
1432  && state != SRPT_STATE_NEED_DATA
1433  && state != SRPT_STATE_DONE);
1434 
1435  /* If SRP_RSP sending failed, undo the ch->req_lim change. */
1436  if (state == SRPT_STATE_CMD_RSP_SENT
1437  || state == SRPT_STATE_MGMT_RSP_SENT)
1438  atomic_dec(&ch->req_lim);
1439 
1440  srpt_abort_cmd(ioctx);
1441 }
1442 
1446 static void srpt_handle_send_comp(struct srpt_rdma_ch *ch,
1447  struct srpt_send_ioctx *ioctx)
1448 {
1450 
1451  atomic_inc(&ch->sq_wr_avail);
1452 
1453  state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1454 
1455  if (WARN_ON(state != SRPT_STATE_CMD_RSP_SENT
1456  && state != SRPT_STATE_MGMT_RSP_SENT
1457  && state != SRPT_STATE_DONE))
1458  pr_debug("state = %d\n", state);
1459 
1460  if (state != SRPT_STATE_DONE)
1461  kref_put(&ioctx->kref, srpt_put_send_ioctx_kref);
1462  else
1463  printk(KERN_ERR "IB completion has been received too late for"
1464  " wr_id = %u.\n", ioctx->ioctx.index);
1465 }
1466 
1475 static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch,
1476  struct srpt_send_ioctx *ioctx,
1477  enum srpt_opcode opcode)
1478 {
1479  WARN_ON(ioctx->n_rdma <= 0);
1480  atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
1481 
1482  if (opcode == SRPT_RDMA_READ_LAST) {
1483  if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1485  target_execute_cmd(&ioctx->cmd);
1486  else
1487  printk(KERN_ERR "%s[%d]: wrong state = %d\n", __func__,
1488  __LINE__, srpt_get_cmd_state(ioctx));
1489  } else if (opcode == SRPT_RDMA_ABORT) {
1490  ioctx->rdma_aborted = true;
1491  } else {
1492  WARN(true, "unexpected opcode %d\n", opcode);
1493  }
1494 }
1495 
1499 static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch,
1500  struct srpt_send_ioctx *ioctx,
1501  enum srpt_opcode opcode)
1502 {
1503  struct se_cmd *cmd;
1505  unsigned long flags;
1506 
1507  cmd = &ioctx->cmd;
1508  state = srpt_get_cmd_state(ioctx);
1509  switch (opcode) {
1510  case SRPT_RDMA_READ_LAST:
1511  if (ioctx->n_rdma <= 0) {
1512  printk(KERN_ERR "Received invalid RDMA read"
1513  " error completion with idx %d\n",
1514  ioctx->ioctx.index);
1515  break;
1516  }
1517  atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
1518  if (state == SRPT_STATE_NEED_DATA)
1519  srpt_abort_cmd(ioctx);
1520  else
1521  printk(KERN_ERR "%s[%d]: wrong state = %d\n",
1522  __func__, __LINE__, state);
1523  break;
1524  case SRPT_RDMA_WRITE_LAST:
1525  spin_lock_irqsave(&ioctx->cmd.t_state_lock, flags);
1526  ioctx->cmd.transport_state |= CMD_T_LUN_STOP;
1527  spin_unlock_irqrestore(&ioctx->cmd.t_state_lock, flags);
1528  break;
1529  default:
1530  printk(KERN_ERR "%s[%d]: opcode = %u\n", __func__,
1531  __LINE__, opcode);
1532  break;
1533  }
1534 }
1535 
1551 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1552  struct srpt_send_ioctx *ioctx, u64 tag,
1553  int status)
1554 {
1555  struct srp_rsp *srp_rsp;
1556  const u8 *sense_data;
1557  int sense_data_len, max_sense_len;
1558 
1559  /*
1560  * The lowest bit of all SAM-3 status codes is zero (see also
1561  * paragraph 5.3 in SAM-3).
1562  */
1563  WARN_ON(status & 1);
1564 
1565  srp_rsp = ioctx->ioctx.buf;
1566  BUG_ON(!srp_rsp);
1567 
1568  sense_data = ioctx->sense_data;
1569  sense_data_len = ioctx->cmd.scsi_sense_length;
1570  WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1571 
1572  memset(srp_rsp, 0, sizeof *srp_rsp);
1573  srp_rsp->opcode = SRP_RSP;
1574  srp_rsp->req_lim_delta =
1576  srp_rsp->tag = tag;
1577  srp_rsp->status = status;
1578 
1579  if (sense_data_len) {
1580  BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1581  max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1582  if (sense_data_len > max_sense_len) {
1583  printk(KERN_WARNING "truncated sense data from %d to %d"
1584  " bytes\n", sense_data_len, max_sense_len);
1585  sense_data_len = max_sense_len;
1586  }
1587 
1588  srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1589  srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1590  memcpy(srp_rsp + 1, sense_data, sense_data_len);
1591  }
1592 
1593  return sizeof(*srp_rsp) + sense_data_len;
1594 }
1595 
1609 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1610  struct srpt_send_ioctx *ioctx,
1611  u8 rsp_code, u64 tag)
1612 {
1613  struct srp_rsp *srp_rsp;
1614  int resp_data_len;
1615  int resp_len;
1616 
1617  resp_data_len = (rsp_code == SRP_TSK_MGMT_SUCCESS) ? 0 : 4;
1618  resp_len = sizeof(*srp_rsp) + resp_data_len;
1619 
1620  srp_rsp = ioctx->ioctx.buf;
1621  BUG_ON(!srp_rsp);
1622  memset(srp_rsp, 0, sizeof *srp_rsp);
1623 
1624  srp_rsp->opcode = SRP_RSP;
1626  + atomic_xchg(&ch->req_lim_delta, 0));
1627  srp_rsp->tag = tag;
1628 
1629  if (rsp_code != SRP_TSK_MGMT_SUCCESS) {
1630  srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1631  srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1632  srp_rsp->data[3] = rsp_code;
1633  }
1634 
1635  return resp_len;
1636 }
1637 
1638 #define NO_SUCH_LUN ((uint64_t)-1LL)
1639 
1640 /*
1641  * SCSI LUN addressing method. See also SAM-2 and the section about
1642  * eight byte LUNs.
1643  */
1649 };
1650 
1651 /*
1652  * srpt_unpack_lun() - Convert from network LUN to linear LUN.
1653  *
1654  * Convert an 2-byte, 4-byte, 6-byte or 8-byte LUN structure in network byte
1655  * order (big endian) to a linear LUN. Supports three LUN addressing methods:
1656  * peripheral, flat and logical unit. See also SAM-2, section 4.9.4 (page 40).
1657  */
1658 static uint64_t srpt_unpack_lun(const uint8_t *lun, int len)
1659 {
1661  int addressing_method;
1662 
1663  if (unlikely(len < 2)) {
1664  printk(KERN_ERR "Illegal LUN length %d, expected 2 bytes or "
1665  "more", len);
1666  goto out;
1667  }
1668 
1669  switch (len) {
1670  case 8:
1671  if ((*((__be64 *)lun) &
1672  __constant_cpu_to_be64(0x0000FFFFFFFFFFFFLL)) != 0)
1673  goto out_err;
1674  break;
1675  case 4:
1676  if (*((__be16 *)&lun[2]) != 0)
1677  goto out_err;
1678  break;
1679  case 6:
1680  if (*((__be32 *)&lun[2]) != 0)
1681  goto out_err;
1682  break;
1683  case 2:
1684  break;
1685  default:
1686  goto out_err;
1687  }
1688 
1689  addressing_method = (*lun) >> 6; /* highest two bits of byte 0 */
1690  switch (addressing_method) {
1694  res = *(lun + 1) | (((*lun) & 0x3f) << 8);
1695  break;
1696 
1698  default:
1699  printk(KERN_ERR "Unimplemented LUN addressing method %u",
1700  addressing_method);
1701  break;
1702  }
1703 
1704 out:
1705  return res;
1706 
1707 out_err:
1708  printk(KERN_ERR "Support for multi-level LUNs has not yet been"
1709  " implemented");
1710  goto out;
1711 }
1712 
1713 static int srpt_check_stop_free(struct se_cmd *cmd)
1714 {
1715  struct srpt_send_ioctx *ioctx;
1716 
1717  ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
1718  return kref_put(&ioctx->kref, srpt_put_send_ioctx_kref);
1719 }
1720 
1724 static int srpt_handle_cmd(struct srpt_rdma_ch *ch,
1725  struct srpt_recv_ioctx *recv_ioctx,
1726  struct srpt_send_ioctx *send_ioctx)
1727 {
1728  struct se_cmd *cmd;
1729  struct srp_cmd *srp_cmd;
1730  uint64_t unpacked_lun;
1731  u64 data_len;
1732  enum dma_data_direction dir;
1733  int ret;
1734 
1735  BUG_ON(!send_ioctx);
1736 
1737  srp_cmd = recv_ioctx->ioctx.buf;
1738  kref_get(&send_ioctx->kref);
1739  cmd = &send_ioctx->cmd;
1740  send_ioctx->tag = srp_cmd->tag;
1741 
1742  switch (srp_cmd->task_attr) {
1743  case SRP_CMD_SIMPLE_Q:
1745  break;
1746  case SRP_CMD_ORDERED_Q:
1747  default:
1749  break;
1750  case SRP_CMD_HEAD_OF_Q:
1751  cmd->sam_task_attr = MSG_HEAD_TAG;
1752  break;
1753  case SRP_CMD_ACA:
1754  cmd->sam_task_attr = MSG_ACA_TAG;
1755  break;
1756  }
1757 
1758  ret = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &data_len);
1759  if (ret) {
1760  printk(KERN_ERR "0x%llx: parsing SRP descriptor table failed.\n",
1761  srp_cmd->tag);
1764  kref_put(&send_ioctx->kref, srpt_put_send_ioctx_kref);
1765  goto send_sense;
1766  }
1767 
1768  cmd->data_length = data_len;
1769  cmd->data_direction = dir;
1770  unpacked_lun = srpt_unpack_lun((uint8_t *)&srp_cmd->lun,
1771  sizeof(srp_cmd->lun));
1772  if (transport_lookup_cmd_lun(cmd, unpacked_lun) < 0) {
1773  kref_put(&send_ioctx->kref, srpt_put_send_ioctx_kref);
1774  goto send_sense;
1775  }
1776  ret = target_setup_cmd_from_cdb(cmd, srp_cmd->cdb);
1777  if (ret < 0) {
1778  kref_put(&send_ioctx->kref, srpt_put_send_ioctx_kref);
1780  srpt_queue_status(cmd);
1781  return 0;
1782  } else
1783  goto send_sense;
1784  }
1785 
1787  return 0;
1788 
1789 send_sense:
1791  0);
1792  return -1;
1793 }
1794 
1808 static int srpt_rx_mgmt_fn_tag(struct srpt_send_ioctx *ioctx, u64 tag)
1809 {
1810  struct srpt_device *sdev;
1811  struct srpt_rdma_ch *ch;
1812  struct srpt_send_ioctx *target;
1813  int ret, i;
1814 
1815  ret = -EINVAL;
1816  ch = ioctx->ch;
1817  BUG_ON(!ch);
1818  BUG_ON(!ch->sport);
1819  sdev = ch->sport->sdev;
1820  BUG_ON(!sdev);
1821  spin_lock_irq(&sdev->spinlock);
1822  for (i = 0; i < ch->rq_size; ++i) {
1823  target = ch->ioctx_ring[i];
1824  if (target->cmd.se_lun == ioctx->cmd.se_lun &&
1825  target->tag == tag &&
1826  srpt_get_cmd_state(target) != SRPT_STATE_DONE) {
1827  ret = 0;
1828  /* now let the target core abort &target->cmd; */
1829  break;
1830  }
1831  }
1832  spin_unlock_irq(&sdev->spinlock);
1833  return ret;
1834 }
1835 
1836 static int srp_tmr_to_tcm(int fn)
1837 {
1838  switch (fn) {
1839  case SRP_TSK_ABORT_TASK:
1840  return TMR_ABORT_TASK;
1842  return TMR_ABORT_TASK_SET;
1844  return TMR_CLEAR_TASK_SET;
1845  case SRP_TSK_LUN_RESET:
1846  return TMR_LUN_RESET;
1847  case SRP_TSK_CLEAR_ACA:
1848  return TMR_CLEAR_ACA;
1849  default:
1850  return -1;
1851  }
1852 }
1853 
1862 static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1863  struct srpt_recv_ioctx *recv_ioctx,
1864  struct srpt_send_ioctx *send_ioctx)
1865 {
1866  struct srp_tsk_mgmt *srp_tsk;
1867  struct se_cmd *cmd;
1868  uint64_t unpacked_lun;
1869  int tcm_tmr;
1870  int res;
1871 
1872  BUG_ON(!send_ioctx);
1873 
1874  srp_tsk = recv_ioctx->ioctx.buf;
1875  cmd = &send_ioctx->cmd;
1876 
1877  pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
1878  " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
1879  srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
1880 
1881  srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
1882  send_ioctx->tag = srp_tsk->tag;
1883  tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
1884  if (tcm_tmr < 0) {
1885  send_ioctx->cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1886  send_ioctx->cmd.se_tmr_req->response =
1888  goto process_tmr;
1889  }
1890  res = core_tmr_alloc_req(cmd, NULL, tcm_tmr, GFP_KERNEL);
1891  if (res < 0) {
1892  send_ioctx->cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1893  send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
1894  goto process_tmr;
1895  }
1896 
1897  unpacked_lun = srpt_unpack_lun((uint8_t *)&srp_tsk->lun,
1898  sizeof(srp_tsk->lun));
1899  res = transport_lookup_tmr_lun(&send_ioctx->cmd, unpacked_lun);
1900  if (res) {
1901  pr_debug("rejecting TMR for LUN %lld\n", unpacked_lun);
1902  send_ioctx->cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1903  send_ioctx->cmd.se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
1904  goto process_tmr;
1905  }
1906 
1907  if (srp_tsk->tsk_mgmt_func == SRP_TSK_ABORT_TASK)
1908  srpt_rx_mgmt_fn_tag(send_ioctx, srp_tsk->task_tag);
1909 
1910 process_tmr:
1911  kref_get(&send_ioctx->kref);
1912  if (!(send_ioctx->cmd.se_cmd_flags & SCF_SCSI_CDB_EXCEPTION))
1913  transport_generic_handle_tmr(&send_ioctx->cmd);
1914  else
1916  cmd->scsi_sense_reason, 0);
1917 
1918 }
1919 
1925 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1926  struct srpt_recv_ioctx *recv_ioctx,
1927  struct srpt_send_ioctx *send_ioctx)
1928 {
1929  struct srp_cmd *srp_cmd;
1930  enum rdma_ch_state ch_state;
1931 
1932  BUG_ON(!ch);
1933  BUG_ON(!recv_ioctx);
1934 
1935  ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1936  recv_ioctx->ioctx.dma, srp_max_req_size,
1937  DMA_FROM_DEVICE);
1938 
1939  ch_state = srpt_get_ch_state(ch);
1940  if (unlikely(ch_state == CH_CONNECTING)) {
1941  list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
1942  goto out;
1943  }
1944 
1945  if (unlikely(ch_state != CH_LIVE))
1946  goto out;
1947 
1948  srp_cmd = recv_ioctx->ioctx.buf;
1949  if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
1950  if (!send_ioctx)
1951  send_ioctx = srpt_get_send_ioctx(ch);
1952  if (unlikely(!send_ioctx)) {
1953  list_add_tail(&recv_ioctx->wait_list,
1954  &ch->cmd_wait_list);
1955  goto out;
1956  }
1957  }
1958 
1959  transport_init_se_cmd(&send_ioctx->cmd, &srpt_target->tf_ops, ch->sess,
1961  send_ioctx->sense_data);
1962 
1963  switch (srp_cmd->opcode) {
1964  case SRP_CMD:
1965  srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1966  break;
1967  case SRP_TSK_MGMT:
1968  srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1969  break;
1970  case SRP_I_LOGOUT:
1971  printk(KERN_ERR "Not yet implemented: SRP_I_LOGOUT\n");
1972  break;
1973  case SRP_CRED_RSP:
1974  pr_debug("received SRP_CRED_RSP\n");
1975  break;
1976  case SRP_AER_RSP:
1977  pr_debug("received SRP_AER_RSP\n");
1978  break;
1979  case SRP_RSP:
1980  printk(KERN_ERR "Received SRP_RSP\n");
1981  break;
1982  default:
1983  printk(KERN_ERR "received IU with unknown opcode 0x%x\n",
1984  srp_cmd->opcode);
1985  break;
1986  }
1987 
1988  srpt_post_recv(ch->sport->sdev, recv_ioctx);
1989 out:
1990  return;
1991 }
1992 
1993 static void srpt_process_rcv_completion(struct ib_cq *cq,
1994  struct srpt_rdma_ch *ch,
1995  struct ib_wc *wc)
1996 {
1997  struct srpt_device *sdev = ch->sport->sdev;
1998  struct srpt_recv_ioctx *ioctx;
1999  u32 index;
2000 
2001  index = idx_from_wr_id(wc->wr_id);
2002  if (wc->status == IB_WC_SUCCESS) {
2003  int req_lim;
2004 
2005  req_lim = atomic_dec_return(&ch->req_lim);
2006  if (unlikely(req_lim < 0))
2007  printk(KERN_ERR "req_lim = %d < 0\n", req_lim);
2008  ioctx = sdev->ioctx_ring[index];
2009  srpt_handle_new_iu(ch, ioctx, NULL);
2010  } else {
2011  printk(KERN_INFO "receiving failed for idx %u with status %d\n",
2012  index, wc->status);
2013  }
2014 }
2015 
2031 static void srpt_process_send_completion(struct ib_cq *cq,
2032  struct srpt_rdma_ch *ch,
2033  struct ib_wc *wc)
2034 {
2035  struct srpt_send_ioctx *send_ioctx;
2036  uint32_t index;
2037  enum srpt_opcode opcode;
2038 
2039  index = idx_from_wr_id(wc->wr_id);
2040  opcode = opcode_from_wr_id(wc->wr_id);
2041  send_ioctx = ch->ioctx_ring[index];
2042  if (wc->status == IB_WC_SUCCESS) {
2043  if (opcode == SRPT_SEND)
2044  srpt_handle_send_comp(ch, send_ioctx);
2045  else {
2046  WARN_ON(opcode != SRPT_RDMA_ABORT &&
2047  wc->opcode != IB_WC_RDMA_READ);
2048  srpt_handle_rdma_comp(ch, send_ioctx, opcode);
2049  }
2050  } else {
2051  if (opcode == SRPT_SEND) {
2052  printk(KERN_INFO "sending response for idx %u failed"
2053  " with status %d\n", index, wc->status);
2054  srpt_handle_send_err_comp(ch, wc->wr_id);
2055  } else if (opcode != SRPT_RDMA_MID) {
2056  printk(KERN_INFO "RDMA t %d for idx %u failed with"
2057  " status %d", opcode, index, wc->status);
2058  srpt_handle_rdma_err_comp(ch, send_ioctx, opcode);
2059  }
2060  }
2061 
2062  while (unlikely(opcode == SRPT_SEND
2063  && !list_empty(&ch->cmd_wait_list)
2064  && srpt_get_ch_state(ch) == CH_LIVE
2065  && (send_ioctx = srpt_get_send_ioctx(ch)) != NULL)) {
2066  struct srpt_recv_ioctx *recv_ioctx;
2067 
2068  recv_ioctx = list_first_entry(&ch->cmd_wait_list,
2069  struct srpt_recv_ioctx,
2070  wait_list);
2071  list_del(&recv_ioctx->wait_list);
2072  srpt_handle_new_iu(ch, recv_ioctx, send_ioctx);
2073  }
2074 }
2075 
2076 static void srpt_process_completion(struct ib_cq *cq, struct srpt_rdma_ch *ch)
2077 {
2078  struct ib_wc *const wc = ch->wc;
2079  int i, n;
2080 
2081  WARN_ON(cq != ch->cq);
2082 
2083  ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
2084  while ((n = ib_poll_cq(cq, ARRAY_SIZE(ch->wc), wc)) > 0) {
2085  for (i = 0; i < n; i++) {
2086  if (opcode_from_wr_id(wc[i].wr_id) == SRPT_RECV)
2087  srpt_process_rcv_completion(cq, ch, &wc[i]);
2088  else
2089  srpt_process_send_completion(cq, ch, &wc[i]);
2090  }
2091  }
2092 }
2093 
2105 static void srpt_completion(struct ib_cq *cq, void *ctx)
2106 {
2107  struct srpt_rdma_ch *ch = ctx;
2108 
2110 }
2111 
2112 static int srpt_compl_thread(void *arg)
2113 {
2114  struct srpt_rdma_ch *ch;
2115 
2116  /* Hibernation / freezing of the SRPT kernel thread is not supported. */
2117  current->flags |= PF_NOFREEZE;
2118 
2119  ch = arg;
2120  BUG_ON(!ch);
2121  printk(KERN_INFO "Session %s: kernel thread %s (PID %d) started\n",
2122  ch->sess_name, ch->thread->comm, current->pid);
2123  while (!kthread_should_stop()) {
2125  (srpt_process_completion(ch->cq, ch),
2126  kthread_should_stop()));
2127  }
2128  printk(KERN_INFO "Session %s: kernel thread %s (PID %d) stopped\n",
2129  ch->sess_name, ch->thread->comm, current->pid);
2130  return 0;
2131 }
2132 
2136 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
2137 {
2138  struct ib_qp_init_attr *qp_init;
2139  struct srpt_port *sport = ch->sport;
2140  struct srpt_device *sdev = sport->sdev;
2141  u32 srp_sq_size = sport->port_attrib.srp_sq_size;
2142  int ret;
2143 
2144  WARN_ON(ch->rq_size < 1);
2145 
2146  ret = -ENOMEM;
2147  qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL);
2148  if (!qp_init)
2149  goto out;
2150 
2151  ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch,
2152  ch->rq_size + srp_sq_size, 0);
2153  if (IS_ERR(ch->cq)) {
2154  ret = PTR_ERR(ch->cq);
2155  printk(KERN_ERR "failed to create CQ cqe= %d ret= %d\n",
2156  ch->rq_size + srp_sq_size, ret);
2157  goto out;
2158  }
2159 
2160  qp_init->qp_context = (void *)ch;
2161  qp_init->event_handler
2162  = (void(*)(struct ib_event *, void*))srpt_qp_event;
2163  qp_init->send_cq = ch->cq;
2164  qp_init->recv_cq = ch->cq;
2165  qp_init->srq = sdev->srq;
2166  qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
2167  qp_init->qp_type = IB_QPT_RC;
2168  qp_init->cap.max_send_wr = srp_sq_size;
2169  qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
2170 
2171  ch->qp = ib_create_qp(sdev->pd, qp_init);
2172  if (IS_ERR(ch->qp)) {
2173  ret = PTR_ERR(ch->qp);
2174  printk(KERN_ERR "failed to create_qp ret= %d\n", ret);
2175  goto err_destroy_cq;
2176  }
2177 
2178  atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
2179 
2180  pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
2181  __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
2182  qp_init->cap.max_send_wr, ch->cm_id);
2183 
2184  ret = srpt_init_ch_qp(ch, ch->qp);
2185  if (ret)
2186  goto err_destroy_qp;
2187 
2189 
2190  pr_debug("creating thread for session %s\n", ch->sess_name);
2191 
2192  ch->thread = kthread_run(srpt_compl_thread, ch, "ib_srpt_compl");
2193  if (IS_ERR(ch->thread)) {
2194  printk(KERN_ERR "failed to create kernel thread %ld\n",
2195  PTR_ERR(ch->thread));
2196  ch->thread = NULL;
2197  goto err_destroy_qp;
2198  }
2199 
2200 out:
2201  kfree(qp_init);
2202  return ret;
2203 
2204 err_destroy_qp:
2205  ib_destroy_qp(ch->qp);
2206 err_destroy_cq:
2207  ib_destroy_cq(ch->cq);
2208  goto out;
2209 }
2210 
2211 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
2212 {
2213  if (ch->thread)
2214  kthread_stop(ch->thread);
2215 
2216  ib_destroy_qp(ch->qp);
2217  ib_destroy_cq(ch->cq);
2218 }
2219 
2228 static void __srpt_close_ch(struct srpt_rdma_ch *ch)
2229 {
2230  struct srpt_device *sdev;
2232  unsigned long flags;
2233 
2234  sdev = ch->sport->sdev;
2235 
2236  spin_lock_irqsave(&ch->spinlock, flags);
2237  prev_state = ch->state;
2238  switch (prev_state) {
2239  case CH_CONNECTING:
2240  case CH_LIVE:
2241  ch->state = CH_DISCONNECTING;
2242  break;
2243  default:
2244  break;
2245  }
2246  spin_unlock_irqrestore(&ch->spinlock, flags);
2247 
2248  switch (prev_state) {
2249  case CH_CONNECTING:
2251  NULL, 0);
2252  /* fall through */
2253  case CH_LIVE:
2254  if (ib_send_cm_dreq(ch->cm_id, NULL, 0) < 0)
2255  printk(KERN_ERR "sending CM DREQ failed.\n");
2256  break;
2257  case CH_DISCONNECTING:
2258  break;
2259  case CH_DRAINING:
2260  case CH_RELEASING:
2261  break;
2262  }
2263 }
2264 
2268 static void srpt_close_ch(struct srpt_rdma_ch *ch)
2269 {
2270  struct srpt_device *sdev;
2271 
2272  sdev = ch->sport->sdev;
2273  spin_lock_irq(&sdev->spinlock);
2274  __srpt_close_ch(ch);
2275  spin_unlock_irq(&sdev->spinlock);
2276 }
2277 
2290 static void srpt_drain_channel(struct ib_cm_id *cm_id)
2291 {
2292  struct srpt_device *sdev;
2293  struct srpt_rdma_ch *ch;
2294  int ret;
2295  bool do_reset = false;
2296 
2298 
2299  sdev = cm_id->context;
2300  BUG_ON(!sdev);
2301  spin_lock_irq(&sdev->spinlock);
2302  list_for_each_entry(ch, &sdev->rch_list, list) {
2303  if (ch->cm_id == cm_id) {
2304  do_reset = srpt_test_and_set_ch_state(ch,
2306  srpt_test_and_set_ch_state(ch,
2307  CH_LIVE, CH_DRAINING) ||
2308  srpt_test_and_set_ch_state(ch,
2310  break;
2311  }
2312  }
2313  spin_unlock_irq(&sdev->spinlock);
2314 
2315  if (do_reset) {
2316  ret = srpt_ch_qp_err(ch);
2317  if (ret < 0)
2318  printk(KERN_ERR "Setting queue pair in error state"
2319  " failed: %d\n", ret);
2320  }
2321 }
2322 
2329 static struct srpt_rdma_ch *srpt_find_channel(struct srpt_device *sdev,
2330  struct ib_cm_id *cm_id)
2331 {
2332  struct srpt_rdma_ch *ch;
2333  bool found;
2334 
2336  BUG_ON(!sdev);
2337 
2338  found = false;
2339  spin_lock_irq(&sdev->spinlock);
2340  list_for_each_entry(ch, &sdev->rch_list, list) {
2341  if (ch->cm_id == cm_id) {
2342  found = true;
2343  break;
2344  }
2345  }
2346  spin_unlock_irq(&sdev->spinlock);
2347 
2348  return found ? ch : NULL;
2349 }
2350 
2359 static void srpt_release_channel(struct srpt_rdma_ch *ch)
2360 {
2362 }
2363 
2364 static void srpt_release_channel_work(struct work_struct *w)
2365 {
2366  struct srpt_rdma_ch *ch;
2367  struct srpt_device *sdev;
2368 
2369  ch = container_of(w, struct srpt_rdma_ch, release_work);
2370  pr_debug("ch = %p; ch->sess = %p; release_done = %p\n", ch, ch->sess,
2371  ch->release_done);
2372 
2373  sdev = ch->sport->sdev;
2374  BUG_ON(!sdev);
2375 
2378  ch->sess = NULL;
2379 
2380  srpt_destroy_ch_ib(ch);
2381 
2382  srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2383  ch->sport->sdev, ch->rq_size,
2384  ch->rsp_size, DMA_TO_DEVICE);
2385 
2386  spin_lock_irq(&sdev->spinlock);
2387  list_del(&ch->list);
2388  spin_unlock_irq(&sdev->spinlock);
2389 
2390  ib_destroy_cm_id(ch->cm_id);
2391 
2392  if (ch->release_done)
2393  complete(ch->release_done);
2394 
2395  wake_up(&sdev->ch_releaseQ);
2396 
2397  kfree(ch);
2398 }
2399 
2400 static struct srpt_node_acl *__srpt_lookup_acl(struct srpt_port *sport,
2401  u8 i_port_id[16])
2402 {
2403  struct srpt_node_acl *nacl;
2404 
2405  list_for_each_entry(nacl, &sport->port_acl_list, list)
2406  if (memcmp(nacl->i_port_id, i_port_id,
2407  sizeof(nacl->i_port_id)) == 0)
2408  return nacl;
2409 
2410  return NULL;
2411 }
2412 
2413 static struct srpt_node_acl *srpt_lookup_acl(struct srpt_port *sport,
2414  u8 i_port_id[16])
2415 {
2416  struct srpt_node_acl *nacl;
2417 
2418  spin_lock_irq(&sport->port_acl_lock);
2419  nacl = __srpt_lookup_acl(sport, i_port_id);
2420  spin_unlock_irq(&sport->port_acl_lock);
2421 
2422  return nacl;
2423 }
2424 
2431 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
2432  struct ib_cm_req_event_param *param,
2433  void *private_data)
2434 {
2435  struct srpt_device *sdev = cm_id->context;
2436  struct srpt_port *sport = &sdev->port[param->port - 1];
2437  struct srp_login_req *req;
2438  struct srp_login_rsp *rsp;
2439  struct srp_login_rej *rej;
2440  struct ib_cm_rep_param *rep_param;
2441  struct srpt_rdma_ch *ch, *tmp_ch;
2442  struct srpt_node_acl *nacl;
2443  u32 it_iu_len;
2444  int i;
2445  int ret = 0;
2446 
2448 
2449  if (WARN_ON(!sdev || !private_data))
2450  return -EINVAL;
2451 
2452  req = (struct srp_login_req *)private_data;
2453 
2454  it_iu_len = be32_to_cpu(req->req_it_iu_len);
2455 
2456  printk(KERN_INFO "Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
2457  " t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
2458  " (guid=0x%llx:0x%llx)\n",
2459  be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
2460  be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
2461  be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
2462  be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
2463  it_iu_len,
2464  param->port,
2465  be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
2466  be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
2467 
2468  rsp = kzalloc(sizeof *rsp, GFP_KERNEL);
2469  rej = kzalloc(sizeof *rej, GFP_KERNEL);
2470  rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL);
2471 
2472  if (!rsp || !rej || !rep_param) {
2473  ret = -ENOMEM;
2474  goto out;
2475  }
2476 
2477  if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
2480  ret = -EINVAL;
2481  printk(KERN_ERR "rejected SRP_LOGIN_REQ because its"
2482  " length (%d bytes) is out of range (%d .. %d)\n",
2483  it_iu_len, 64, srp_max_req_size);
2484  goto reject;
2485  }
2486 
2487  if (!sport->enabled) {
2490  ret = -EINVAL;
2491  printk(KERN_ERR "rejected SRP_LOGIN_REQ because the target port"
2492  " has not yet been enabled\n");
2493  goto reject;
2494  }
2495 
2498 
2499  spin_lock_irq(&sdev->spinlock);
2500 
2501  list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
2502  if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
2503  && !memcmp(ch->t_port_id, req->target_port_id, 16)
2504  && param->port == ch->sport->port
2505  && param->listen_id == ch->sport->sdev->cm_id
2506  && ch->cm_id) {
2507  enum rdma_ch_state ch_state;
2508 
2509  ch_state = srpt_get_ch_state(ch);
2510  if (ch_state != CH_CONNECTING
2511  && ch_state != CH_LIVE)
2512  continue;
2513 
2514  /* found an existing channel */
2515  pr_debug("Found existing channel %s"
2516  " cm_id= %p state= %d\n",
2517  ch->sess_name, ch->cm_id, ch_state);
2518 
2519  __srpt_close_ch(ch);
2520 
2521  rsp->rsp_flags =
2523  }
2524  }
2525 
2526  spin_unlock_irq(&sdev->spinlock);
2527 
2528  } else
2530 
2531  if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
2532  || *(__be64 *)(req->target_port_id + 8) !=
2533  cpu_to_be64(srpt_service_guid)) {
2536  ret = -ENOMEM;
2537  printk(KERN_ERR "rejected SRP_LOGIN_REQ because it"
2538  " has an invalid target port identifier.\n");
2539  goto reject;
2540  }
2541 
2542  ch = kzalloc(sizeof *ch, GFP_KERNEL);
2543  if (!ch) {
2546  printk(KERN_ERR "rejected SRP_LOGIN_REQ because no memory.\n");
2547  ret = -ENOMEM;
2548  goto reject;
2549  }
2550 
2551  INIT_WORK(&ch->release_work, srpt_release_channel_work);
2552  memcpy(ch->i_port_id, req->initiator_port_id, 16);
2553  memcpy(ch->t_port_id, req->target_port_id, 16);
2554  ch->sport = &sdev->port[param->port - 1];
2555  ch->cm_id = cm_id;
2556  /*
2557  * Avoid QUEUE_FULL conditions by limiting the number of buffers used
2558  * for the SRP protocol to the command queue size.
2559  */
2560  ch->rq_size = SRPT_RQ_SIZE;
2561  spin_lock_init(&ch->spinlock);
2562  ch->state = CH_CONNECTING;
2563  INIT_LIST_HEAD(&ch->cmd_wait_list);
2564  ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
2565 
2566  ch->ioctx_ring = (struct srpt_send_ioctx **)
2567  srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
2568  sizeof(*ch->ioctx_ring[0]),
2569  ch->rsp_size, DMA_TO_DEVICE);
2570  if (!ch->ioctx_ring)
2571  goto free_ch;
2572 
2573  INIT_LIST_HEAD(&ch->free_list);
2574  for (i = 0; i < ch->rq_size; i++) {
2575  ch->ioctx_ring[i]->ch = ch;
2576  list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
2577  }
2578 
2579  ret = srpt_create_ch_ib(ch);
2580  if (ret) {
2583  printk(KERN_ERR "rejected SRP_LOGIN_REQ because creating"
2584  " a new RDMA channel failed.\n");
2585  goto free_ring;
2586  }
2587 
2588  ret = srpt_ch_qp_rtr(ch, ch->qp);
2589  if (ret) {
2592  printk(KERN_ERR "rejected SRP_LOGIN_REQ because enabling"
2593  " RTR failed (error code = %d)\n", ret);
2594  goto destroy_ib;
2595  }
2596  /*
2597  * Use the initator port identifier as the session name.
2598  */
2599  snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
2600  be64_to_cpu(*(__be64 *)ch->i_port_id),
2601  be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
2602 
2603  pr_debug("registering session %s\n", ch->sess_name);
2604 
2605  nacl = srpt_lookup_acl(sport, ch->i_port_id);
2606  if (!nacl) {
2607  printk(KERN_INFO "Rejected login because no ACL has been"
2608  " configured yet for initiator %s.\n", ch->sess_name);
2611  goto destroy_ib;
2612  }
2613 
2614  ch->sess = transport_init_session();
2615  if (IS_ERR(ch->sess)) {
2618  pr_debug("Failed to create session\n");
2619  goto deregister_session;
2620  }
2621  ch->sess->se_node_acl = &nacl->nacl;
2622  transport_register_session(&sport->port_tpg_1, &nacl->nacl, ch->sess, ch);
2623 
2624  pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
2625  ch->sess_name, ch->cm_id);
2626 
2627  /* create srp_login_response */
2628  rsp->opcode = SRP_LOGIN_RSP;
2629  rsp->tag = req->tag;
2630  rsp->max_it_iu_len = req->req_it_iu_len;
2631  rsp->max_ti_iu_len = req->req_it_iu_len;
2632  ch->max_ti_iu_len = it_iu_len;
2635  rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2636  atomic_set(&ch->req_lim, ch->rq_size);
2637  atomic_set(&ch->req_lim_delta, 0);
2638 
2639  /* create cm reply */
2640  rep_param->qp_num = ch->qp->qp_num;
2641  rep_param->private_data = (void *)rsp;
2642  rep_param->private_data_len = sizeof *rsp;
2643  rep_param->rnr_retry_count = 7;
2644  rep_param->flow_control = 1;
2645  rep_param->failover_accepted = 0;
2646  rep_param->srq = 1;
2647  rep_param->responder_resources = 4;
2648  rep_param->initiator_depth = 4;
2649 
2650  ret = ib_send_cm_rep(cm_id, rep_param);
2651  if (ret) {
2652  printk(KERN_ERR "sending SRP_LOGIN_REQ response failed"
2653  " (error code = %d)\n", ret);
2654  goto release_channel;
2655  }
2656 
2657  spin_lock_irq(&sdev->spinlock);
2658  list_add_tail(&ch->list, &sdev->rch_list);
2659  spin_unlock_irq(&sdev->spinlock);
2660 
2661  goto out;
2662 
2663 release_channel:
2664  srpt_set_ch_state(ch, CH_RELEASING);
2666 
2667 deregister_session:
2669  ch->sess = NULL;
2670 
2671 destroy_ib:
2672  srpt_destroy_ch_ib(ch);
2673 
2674 free_ring:
2675  srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2676  ch->sport->sdev, ch->rq_size,
2677  ch->rsp_size, DMA_TO_DEVICE);
2678 free_ch:
2679  kfree(ch);
2680 
2681 reject:
2682  rej->opcode = SRP_LOGIN_REJ;
2683  rej->tag = req->tag;
2686 
2688  (void *)rej, sizeof *rej);
2689 
2690 out:
2691  kfree(rep_param);
2692  kfree(rsp);
2693  kfree(rej);
2694 
2695  return ret;
2696 }
2697 
2698 static void srpt_cm_rej_recv(struct ib_cm_id *cm_id)
2699 {
2700  printk(KERN_INFO "Received IB REJ for cm_id %p.\n", cm_id);
2701  srpt_drain_channel(cm_id);
2702 }
2703 
2710 static void srpt_cm_rtu_recv(struct ib_cm_id *cm_id)
2711 {
2712  struct srpt_rdma_ch *ch;
2713  int ret;
2714 
2715  ch = srpt_find_channel(cm_id->context, cm_id);
2716  BUG_ON(!ch);
2717 
2718  if (srpt_test_and_set_ch_state(ch, CH_CONNECTING, CH_LIVE)) {
2719  struct srpt_recv_ioctx *ioctx, *ioctx_tmp;
2720 
2721  ret = srpt_ch_qp_rts(ch, ch->qp);
2722 
2723  list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,
2724  wait_list) {
2725  list_del(&ioctx->wait_list);
2726  srpt_handle_new_iu(ch, ioctx, NULL);
2727  }
2728  if (ret)
2729  srpt_close_ch(ch);
2730  }
2731 }
2732 
2733 static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id)
2734 {
2735  printk(KERN_INFO "Received IB TimeWait exit for cm_id %p.\n", cm_id);
2736  srpt_drain_channel(cm_id);
2737 }
2738 
2739 static void srpt_cm_rep_error(struct ib_cm_id *cm_id)
2740 {
2741  printk(KERN_INFO "Received IB REP error for cm_id %p.\n", cm_id);
2742  srpt_drain_channel(cm_id);
2743 }
2744 
2748 static void srpt_cm_dreq_recv(struct ib_cm_id *cm_id)
2749 {
2750  struct srpt_rdma_ch *ch;
2751  unsigned long flags;
2752  bool send_drep = false;
2753 
2754  ch = srpt_find_channel(cm_id->context, cm_id);
2755  BUG_ON(!ch);
2756 
2757  pr_debug("cm_id= %p ch->state= %d\n", cm_id, srpt_get_ch_state(ch));
2758 
2759  spin_lock_irqsave(&ch->spinlock, flags);
2760  switch (ch->state) {
2761  case CH_CONNECTING:
2762  case CH_LIVE:
2763  send_drep = true;
2764  ch->state = CH_DISCONNECTING;
2765  break;
2766  case CH_DISCONNECTING:
2767  case CH_DRAINING:
2768  case CH_RELEASING:
2769  WARN(true, "unexpected channel state %d\n", ch->state);
2770  break;
2771  }
2772  spin_unlock_irqrestore(&ch->spinlock, flags);
2773 
2774  if (send_drep) {
2775  if (ib_send_cm_drep(ch->cm_id, NULL, 0) < 0)
2776  printk(KERN_ERR "Sending IB DREP failed.\n");
2777  printk(KERN_INFO "Received DREQ and sent DREP for session %s.\n",
2778  ch->sess_name);
2779  }
2780 }
2781 
2785 static void srpt_cm_drep_recv(struct ib_cm_id *cm_id)
2786 {
2787  printk(KERN_INFO "Received InfiniBand DREP message for cm_id %p.\n",
2788  cm_id);
2789  srpt_drain_channel(cm_id);
2790 }
2791 
2802 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2803 {
2804  int ret;
2805 
2806  ret = 0;
2807  switch (event->event) {
2808  case IB_CM_REQ_RECEIVED:
2809  ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2810  event->private_data);
2811  break;
2812  case IB_CM_REJ_RECEIVED:
2813  srpt_cm_rej_recv(cm_id);
2814  break;
2815  case IB_CM_RTU_RECEIVED:
2817  srpt_cm_rtu_recv(cm_id);
2818  break;
2819  case IB_CM_DREQ_RECEIVED:
2820  srpt_cm_dreq_recv(cm_id);
2821  break;
2822  case IB_CM_DREP_RECEIVED:
2823  srpt_cm_drep_recv(cm_id);
2824  break;
2825  case IB_CM_TIMEWAIT_EXIT:
2826  srpt_cm_timewait_exit(cm_id);
2827  break;
2828  case IB_CM_REP_ERROR:
2829  srpt_cm_rep_error(cm_id);
2830  break;
2831  case IB_CM_DREQ_ERROR:
2832  printk(KERN_INFO "Received IB DREQ ERROR event.\n");
2833  break;
2834  case IB_CM_MRA_RECEIVED:
2835  printk(KERN_INFO "Received IB MRA event\n");
2836  break;
2837  default:
2838  printk(KERN_ERR "received unrecognized IB CM event %d\n",
2839  event->event);
2840  break;
2841  }
2842 
2843  return ret;
2844 }
2845 
2851 static int srpt_perform_rdmas(struct srpt_rdma_ch *ch,
2852  struct srpt_send_ioctx *ioctx)
2853 {
2854  struct ib_send_wr wr;
2855  struct ib_send_wr *bad_wr;
2856  struct rdma_iu *riu;
2857  int i;
2858  int ret;
2859  int sq_wr_avail;
2860  enum dma_data_direction dir;
2861  const int n_rdma = ioctx->n_rdma;
2862 
2863  dir = ioctx->cmd.data_direction;
2864  if (dir == DMA_TO_DEVICE) {
2865  /* write */
2866  ret = -ENOMEM;
2867  sq_wr_avail = atomic_sub_return(n_rdma, &ch->sq_wr_avail);
2868  if (sq_wr_avail < 0) {
2869  printk(KERN_WARNING "IB send queue full (needed %d)\n",
2870  n_rdma);
2871  goto out;
2872  }
2873  }
2874 
2875  ioctx->rdma_aborted = false;
2876  ret = 0;
2877  riu = ioctx->rdma_ius;
2878  memset(&wr, 0, sizeof wr);
2879 
2880  for (i = 0; i < n_rdma; ++i, ++riu) {
2881  if (dir == DMA_FROM_DEVICE) {
2882  wr.opcode = IB_WR_RDMA_WRITE;
2883  wr.wr_id = encode_wr_id(i == n_rdma - 1 ?
2885  SRPT_RDMA_MID,
2886  ioctx->ioctx.index);
2887  } else {
2888  wr.opcode = IB_WR_RDMA_READ;
2889  wr.wr_id = encode_wr_id(i == n_rdma - 1 ?
2891  SRPT_RDMA_MID,
2892  ioctx->ioctx.index);
2893  }
2894  wr.next = NULL;
2895  wr.wr.rdma.remote_addr = riu->raddr;
2896  wr.wr.rdma.rkey = riu->rkey;
2897  wr.num_sge = riu->sge_cnt;
2898  wr.sg_list = riu->sge;
2899 
2900  /* only get completion event for the last rdma write */
2901  if (i == (n_rdma - 1) && dir == DMA_TO_DEVICE)
2902  wr.send_flags = IB_SEND_SIGNALED;
2903 
2904  ret = ib_post_send(ch->qp, &wr, &bad_wr);
2905  if (ret)
2906  break;
2907  }
2908 
2909  if (ret)
2910  printk(KERN_ERR "%s[%d]: ib_post_send() returned %d for %d/%d",
2911  __func__, __LINE__, ret, i, n_rdma);
2912  if (ret && i > 0) {
2913  wr.num_sge = 0;
2914  wr.wr_id = encode_wr_id(SRPT_RDMA_ABORT, ioctx->ioctx.index);
2915  wr.send_flags = IB_SEND_SIGNALED;
2916  while (ch->state == CH_LIVE &&
2917  ib_post_send(ch->qp, &wr, &bad_wr) != 0) {
2918  printk(KERN_INFO "Trying to abort failed RDMA transfer [%d]",
2919  ioctx->ioctx.index);
2920  msleep(1000);
2921  }
2922  while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) {
2923  printk(KERN_INFO "Waiting until RDMA abort finished [%d]",
2924  ioctx->ioctx.index);
2925  msleep(1000);
2926  }
2927  }
2928 out:
2929  if (unlikely(dir == DMA_TO_DEVICE && ret < 0))
2930  atomic_add(n_rdma, &ch->sq_wr_avail);
2931  return ret;
2932 }
2933 
2937 static int srpt_xfer_data(struct srpt_rdma_ch *ch,
2938  struct srpt_send_ioctx *ioctx)
2939 {
2940  int ret;
2941 
2942  ret = srpt_map_sg_to_ib_sge(ch, ioctx);
2943  if (ret) {
2944  printk(KERN_ERR "%s[%d] ret=%d\n", __func__, __LINE__, ret);
2945  goto out;
2946  }
2947 
2948  ret = srpt_perform_rdmas(ch, ioctx);
2949  if (ret) {
2950  if (ret == -EAGAIN || ret == -ENOMEM)
2951  printk(KERN_INFO "%s[%d] queue full -- ret=%d\n",
2952  __func__, __LINE__, ret);
2953  else
2954  printk(KERN_ERR "%s[%d] fatal error -- ret=%d\n",
2955  __func__, __LINE__, ret);
2956  goto out_unmap;
2957  }
2958 
2959 out:
2960  return ret;
2961 out_unmap:
2962  srpt_unmap_sg_to_ib_sge(ch, ioctx);
2963  goto out;
2964 }
2965 
2966 static int srpt_write_pending_status(struct se_cmd *se_cmd)
2967 {
2968  struct srpt_send_ioctx *ioctx;
2969 
2970  ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2971  return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
2972 }
2973 
2974 /*
2975  * srpt_write_pending() - Start data transfer from initiator to target (write).
2976  */
2977 static int srpt_write_pending(struct se_cmd *se_cmd)
2978 {
2979  struct srpt_rdma_ch *ch;
2980  struct srpt_send_ioctx *ioctx;
2982  enum rdma_ch_state ch_state;
2983  int ret;
2984 
2985  ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2986 
2987  new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2988  WARN_ON(new_state == SRPT_STATE_DONE);
2989 
2990  ch = ioctx->ch;
2991  BUG_ON(!ch);
2992 
2993  ch_state = srpt_get_ch_state(ch);
2994  switch (ch_state) {
2995  case CH_CONNECTING:
2996  WARN(true, "unexpected channel state %d\n", ch_state);
2997  ret = -EINVAL;
2998  goto out;
2999  case CH_LIVE:
3000  break;
3001  case CH_DISCONNECTING:
3002  case CH_DRAINING:
3003  case CH_RELEASING:
3004  pr_debug("cmd with tag %lld: channel disconnecting\n",
3005  ioctx->tag);
3006  srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN);
3007  ret = -EINVAL;
3008  goto out;
3009  }
3010  ret = srpt_xfer_data(ch, ioctx);
3011 
3012 out:
3013  return ret;
3014 }
3015 
3016 static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
3017 {
3018  switch (tcm_mgmt_status) {
3019  case TMR_FUNCTION_COMPLETE:
3020  return SRP_TSK_MGMT_SUCCESS;
3021  case TMR_FUNCTION_REJECTED:
3023  }
3024  return SRP_TSK_MGMT_FAILED;
3025 }
3026 
3033 static int srpt_queue_response(struct se_cmd *cmd)
3034 {
3035  struct srpt_rdma_ch *ch;
3036  struct srpt_send_ioctx *ioctx;
3038  unsigned long flags;
3039  int ret;
3040  enum dma_data_direction dir;
3041  int resp_len;
3042  u8 srp_tm_status;
3043 
3044  ret = 0;
3045 
3046  ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
3047  ch = ioctx->ch;
3048  BUG_ON(!ch);
3049 
3050  spin_lock_irqsave(&ioctx->spinlock, flags);
3051  state = ioctx->state;
3052  switch (state) {
3053  case SRPT_STATE_NEW:
3054  case SRPT_STATE_DATA_IN:
3055  ioctx->state = SRPT_STATE_CMD_RSP_SENT;
3056  break;
3057  case SRPT_STATE_MGMT:
3059  break;
3060  default:
3061  WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
3062  ch, ioctx->ioctx.index, ioctx->state);
3063  break;
3064  }
3065  spin_unlock_irqrestore(&ioctx->spinlock, flags);
3066 
3067  if (unlikely(transport_check_aborted_status(&ioctx->cmd, false)
3068  || WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))) {
3069  atomic_inc(&ch->req_lim_delta);
3070  srpt_abort_cmd(ioctx);
3071  goto out;
3072  }
3073 
3074  dir = ioctx->cmd.data_direction;
3075 
3076  /* For read commands, transfer the data to the initiator. */
3077  if (dir == DMA_FROM_DEVICE && ioctx->cmd.data_length &&
3078  !ioctx->queue_status_only) {
3079  ret = srpt_xfer_data(ch, ioctx);
3080  if (ret) {
3081  printk(KERN_ERR "xfer_data failed for tag %llu\n",
3082  ioctx->tag);
3083  goto out;
3084  }
3085  }
3086 
3087  if (state != SRPT_STATE_MGMT)
3088  resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->tag,
3089  cmd->scsi_status);
3090  else {
3091  srp_tm_status
3092  = tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
3093  resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
3094  ioctx->tag);
3095  }
3096  ret = srpt_post_send(ch, ioctx, resp_len);
3097  if (ret) {
3098  printk(KERN_ERR "sending cmd response failed for tag %llu\n",
3099  ioctx->tag);
3100  srpt_unmap_sg_to_ib_sge(ch, ioctx);
3101  srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
3102  kref_put(&ioctx->kref, srpt_put_send_ioctx_kref);
3103  }
3104 
3105 out:
3106  return ret;
3107 }
3108 
3109 static int srpt_queue_status(struct se_cmd *cmd)
3110 {
3111  struct srpt_send_ioctx *ioctx;
3112 
3113  ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
3114  BUG_ON(ioctx->sense_data != cmd->sense_buffer);
3115  if (cmd->se_cmd_flags &
3118  ioctx->queue_status_only = true;
3119  return srpt_queue_response(cmd);
3120 }
3121 
3122 static void srpt_refresh_port_work(struct work_struct *work)
3123 {
3124  struct srpt_port *sport = container_of(work, struct srpt_port, work);
3125 
3126  srpt_refresh_port(sport);
3127 }
3128 
3129 static int srpt_ch_list_empty(struct srpt_device *sdev)
3130 {
3131  int res;
3132 
3133  spin_lock_irq(&sdev->spinlock);
3134  res = list_empty(&sdev->rch_list);
3135  spin_unlock_irq(&sdev->spinlock);
3136 
3137  return res;
3138 }
3139 
3143 static int srpt_release_sdev(struct srpt_device *sdev)
3144 {
3145  struct srpt_rdma_ch *ch, *tmp_ch;
3146  int res;
3147 
3149 
3150  BUG_ON(!sdev);
3151 
3152  spin_lock_irq(&sdev->spinlock);
3153  list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list)
3154  __srpt_close_ch(ch);
3155  spin_unlock_irq(&sdev->spinlock);
3156 
3157  res = wait_event_interruptible(sdev->ch_releaseQ,
3158  srpt_ch_list_empty(sdev));
3159  if (res)
3160  printk(KERN_ERR "%s: interrupted.\n", __func__);
3161 
3162  return 0;
3163 }
3164 
3165 static struct srpt_port *__srpt_lookup_port(const char *name)
3166 {
3167  struct ib_device *dev;
3168  struct srpt_device *sdev;
3169  struct srpt_port *sport;
3170  int i;
3171 
3172  list_for_each_entry(sdev, &srpt_dev_list, list) {
3173  dev = sdev->device;
3174  if (!dev)
3175  continue;
3176 
3177  for (i = 0; i < dev->phys_port_cnt; i++) {
3178  sport = &sdev->port[i];
3179 
3180  if (!strcmp(sport->port_guid, name))
3181  return sport;
3182  }
3183  }
3184 
3185  return NULL;
3186 }
3187 
3188 static struct srpt_port *srpt_lookup_port(const char *name)
3189 {
3190  struct srpt_port *sport;
3191 
3192  spin_lock(&srpt_dev_lock);
3193  sport = __srpt_lookup_port(name);
3194  spin_unlock(&srpt_dev_lock);
3195 
3196  return sport;
3197 }
3198 
3202 static void srpt_add_one(struct ib_device *device)
3203 {
3204  struct srpt_device *sdev;
3205  struct srpt_port *sport;
3206  struct ib_srq_init_attr srq_attr;
3207  int i;
3208 
3209  pr_debug("device = %p, device->dma_ops = %p\n", device,
3210  device->dma_ops);
3211 
3212  sdev = kzalloc(sizeof *sdev, GFP_KERNEL);
3213  if (!sdev)
3214  goto err;
3215 
3216  sdev->device = device;
3217  INIT_LIST_HEAD(&sdev->rch_list);
3219  spin_lock_init(&sdev->spinlock);
3220 
3221  if (ib_query_device(device, &sdev->dev_attr))
3222  goto free_dev;
3223 
3224  sdev->pd = ib_alloc_pd(device);
3225  if (IS_ERR(sdev->pd))
3226  goto free_dev;
3227 
3228  sdev->mr = ib_get_dma_mr(sdev->pd, IB_ACCESS_LOCAL_WRITE);
3229  if (IS_ERR(sdev->mr))
3230  goto err_pd;
3231 
3232  sdev->srq_size = min(srpt_srq_size, sdev->dev_attr.max_srq_wr);
3233 
3234  srq_attr.event_handler = srpt_srq_event;
3235  srq_attr.srq_context = (void *)sdev;
3236  srq_attr.attr.max_wr = sdev->srq_size;
3237  srq_attr.attr.max_sge = 1;
3238  srq_attr.attr.srq_limit = 0;
3239  srq_attr.srq_type = IB_SRQT_BASIC;
3240 
3241  sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
3242  if (IS_ERR(sdev->srq))
3243  goto err_mr;
3244 
3245  pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
3246  __func__, sdev->srq_size, sdev->dev_attr.max_srq_wr,
3247  device->name);
3248 
3249  if (!srpt_service_guid)
3250  srpt_service_guid = be64_to_cpu(device->node_guid);
3251 
3252  sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
3253  if (IS_ERR(sdev->cm_id))
3254  goto err_srq;
3255 
3256  /* print out target login information */
3257  pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
3258  "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
3259  srpt_service_guid, srpt_service_guid);
3260 
3261  /*
3262  * We do not have a consistent service_id (ie. also id_ext of target_id)
3263  * to identify this target. We currently use the guid of the first HCA
3264  * in the system as service_id; therefore, the target_id will change
3265  * if this HCA is gone bad and replaced by different HCA
3266  */
3267  if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0, NULL))
3268  goto err_cm;
3269 
3271  srpt_event_handler);
3273  goto err_cm;
3274 
3275  sdev->ioctx_ring = (struct srpt_recv_ioctx **)
3276  srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
3277  sizeof(*sdev->ioctx_ring[0]),
3278  srp_max_req_size, DMA_FROM_DEVICE);
3279  if (!sdev->ioctx_ring)
3280  goto err_event;
3281 
3282  for (i = 0; i < sdev->srq_size; ++i)
3283  srpt_post_recv(sdev, sdev->ioctx_ring[i]);
3284 
3285  WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));
3286 
3287  for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
3288  sport = &sdev->port[i - 1];
3289  sport->sdev = sdev;
3290  sport->port = i;
3291  sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
3292  sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
3293  sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
3294  INIT_WORK(&sport->work, srpt_refresh_port_work);
3295  INIT_LIST_HEAD(&sport->port_acl_list);
3296  spin_lock_init(&sport->port_acl_lock);
3297 
3298  if (srpt_refresh_port(sport)) {
3299  printk(KERN_ERR "MAD registration failed for %s-%d.\n",
3300  srpt_sdev_name(sdev), i);
3301  goto err_ring;
3302  }
3303  snprintf(sport->port_guid, sizeof(sport->port_guid),
3304  "0x%016llx%016llx",
3305  be64_to_cpu(sport->gid.global.subnet_prefix),
3306  be64_to_cpu(sport->gid.global.interface_id));
3307  }
3308 
3309  spin_lock(&srpt_dev_lock);
3310  list_add_tail(&sdev->list, &srpt_dev_list);
3311  spin_unlock(&srpt_dev_lock);
3312 
3313 out:
3314  ib_set_client_data(device, &srpt_client, sdev);
3315  pr_debug("added %s.\n", device->name);
3316  return;
3317 
3318 err_ring:
3319  srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
3320  sdev->srq_size, srp_max_req_size,
3321  DMA_FROM_DEVICE);
3322 err_event:
3324 err_cm:
3325  ib_destroy_cm_id(sdev->cm_id);
3326 err_srq:
3327  ib_destroy_srq(sdev->srq);
3328 err_mr:
3329  ib_dereg_mr(sdev->mr);
3330 err_pd:
3331  ib_dealloc_pd(sdev->pd);
3332 free_dev:
3333  kfree(sdev);
3334 err:
3335  sdev = NULL;
3336  printk(KERN_INFO "%s(%s) failed.\n", __func__, device->name);
3337  goto out;
3338 }
3339 
3343 static void srpt_remove_one(struct ib_device *device)
3344 {
3345  struct srpt_device *sdev;
3346  int i;
3347 
3348  sdev = ib_get_client_data(device, &srpt_client);
3349  if (!sdev) {
3350  printk(KERN_INFO "%s(%s): nothing to do.\n", __func__,
3351  device->name);
3352  return;
3353  }
3354 
3355  srpt_unregister_mad_agent(sdev);
3356 
3358 
3359  /* Cancel any work queued by the just unregistered IB event handler. */
3360  for (i = 0; i < sdev->device->phys_port_cnt; i++)
3361  cancel_work_sync(&sdev->port[i].work);
3362 
3363  ib_destroy_cm_id(sdev->cm_id);
3364 
3365  /*
3366  * Unregistering a target must happen after destroying sdev->cm_id
3367  * such that no new SRP_LOGIN_REQ information units can arrive while
3368  * destroying the target.
3369  */
3370  spin_lock(&srpt_dev_lock);
3371  list_del(&sdev->list);
3372  spin_unlock(&srpt_dev_lock);
3373  srpt_release_sdev(sdev);
3374 
3375  ib_destroy_srq(sdev->srq);
3376  ib_dereg_mr(sdev->mr);
3377  ib_dealloc_pd(sdev->pd);
3378 
3379  srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
3380  sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
3381  sdev->ioctx_ring = NULL;
3382  kfree(sdev);
3383 }
3384 
3385 static struct ib_client srpt_client = {
3386  .name = DRV_NAME,
3387  .add = srpt_add_one,
3388  .remove = srpt_remove_one
3389 };
3390 
3391 static int srpt_check_true(struct se_portal_group *se_tpg)
3392 {
3393  return 1;
3394 }
3395 
3396 static int srpt_check_false(struct se_portal_group *se_tpg)
3397 {
3398  return 0;
3399 }
3400 
3401 static char *srpt_get_fabric_name(void)
3402 {
3403  return "srpt";
3404 }
3405 
3406 static u8 srpt_get_fabric_proto_ident(struct se_portal_group *se_tpg)
3407 {
3409 }
3410 
3411 static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
3412 {
3413  struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
3414 
3415  return sport->port_guid;
3416 }
3417 
3418 static u16 srpt_get_tag(struct se_portal_group *tpg)
3419 {
3420  return 1;
3421 }
3422 
3423 static u32 srpt_get_default_depth(struct se_portal_group *se_tpg)
3424 {
3425  return 1;
3426 }
3427 
3428 static u32 srpt_get_pr_transport_id(struct se_portal_group *se_tpg,
3429  struct se_node_acl *se_nacl,
3430  struct t10_pr_registration *pr_reg,
3431  int *format_code, unsigned char *buf)
3432 {
3433  struct srpt_node_acl *nacl;
3434  struct spc_rdma_transport_id *tr_id;
3435 
3436  nacl = container_of(se_nacl, struct srpt_node_acl, nacl);
3437  tr_id = (void *)buf;
3439  memcpy(tr_id->i_port_id, nacl->i_port_id, sizeof(tr_id->i_port_id));
3440  return sizeof(*tr_id);
3441 }
3442 
3443 static u32 srpt_get_pr_transport_id_len(struct se_portal_group *se_tpg,
3444  struct se_node_acl *se_nacl,
3445  struct t10_pr_registration *pr_reg,
3446  int *format_code)
3447 {
3448  *format_code = 0;
3449  return sizeof(struct spc_rdma_transport_id);
3450 }
3451 
3452 static char *srpt_parse_pr_out_transport_id(struct se_portal_group *se_tpg,
3453  const char *buf, u32 *out_tid_len,
3454  char **port_nexus_ptr)
3455 {
3456  struct spc_rdma_transport_id *tr_id;
3457 
3458  *port_nexus_ptr = NULL;
3459  *out_tid_len = sizeof(struct spc_rdma_transport_id);
3460  tr_id = (void *)buf;
3461  return (char *)tr_id->i_port_id;
3462 }
3463 
3464 static struct se_node_acl *srpt_alloc_fabric_acl(struct se_portal_group *se_tpg)
3465 {
3466  struct srpt_node_acl *nacl;
3467 
3468  nacl = kzalloc(sizeof(struct srpt_node_acl), GFP_KERNEL);
3469  if (!nacl) {
3470  printk(KERN_ERR "Unable to allocate struct srpt_node_acl\n");
3471  return NULL;
3472  }
3473 
3474  return &nacl->nacl;
3475 }
3476 
3477 static void srpt_release_fabric_acl(struct se_portal_group *se_tpg,
3478  struct se_node_acl *se_nacl)
3479 {
3480  struct srpt_node_acl *nacl;
3481 
3482  nacl = container_of(se_nacl, struct srpt_node_acl, nacl);
3483  kfree(nacl);
3484 }
3485 
3486 static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
3487 {
3488  return 1;
3489 }
3490 
3491 static void srpt_release_cmd(struct se_cmd *se_cmd)
3492 {
3493 }
3494 
3498 static int srpt_shutdown_session(struct se_session *se_sess)
3499 {
3500  return true;
3501 }
3502 
3510 static void srpt_close_session(struct se_session *se_sess)
3511 {
3512  DECLARE_COMPLETION_ONSTACK(release_done);
3513  struct srpt_rdma_ch *ch;
3514  struct srpt_device *sdev;
3515  int res;
3516 
3517  ch = se_sess->fabric_sess_ptr;
3518  WARN_ON(ch->sess != se_sess);
3519 
3520  pr_debug("ch %p state %d\n", ch, srpt_get_ch_state(ch));
3521 
3522  sdev = ch->sport->sdev;
3523  spin_lock_irq(&sdev->spinlock);
3524  BUG_ON(ch->release_done);
3525  ch->release_done = &release_done;
3526  __srpt_close_ch(ch);
3527  spin_unlock_irq(&sdev->spinlock);
3528 
3529  res = wait_for_completion_timeout(&release_done, 60 * HZ);
3530  WARN_ON(res <= 0);
3531 }
3532 
3541 static u32 srpt_sess_get_index(struct se_session *se_sess)
3542 {
3543  return 0;
3544 }
3545 
3546 static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
3547 {
3548 }
3549 
3550 static u32 srpt_get_task_tag(struct se_cmd *se_cmd)
3551 {
3552  struct srpt_send_ioctx *ioctx;
3553 
3554  ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
3555  return ioctx->tag;
3556 }
3557 
3558 /* Note: only used from inside debug printk's by the TCM core. */
3559 static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
3560 {
3561  struct srpt_send_ioctx *ioctx;
3562 
3563  ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
3564  return srpt_get_cmd_state(ioctx);
3565 }
3566 
3572 static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
3573 {
3574  const char *p;
3575  unsigned len, count, leading_zero_bytes;
3576  int ret, rc;
3577 
3578  p = name;
3579  if (strnicmp(p, "0x", 2) == 0)
3580  p += 2;
3581  ret = -EINVAL;
3582  len = strlen(p);
3583  if (len % 2)
3584  goto out;
3585  count = min(len / 2, 16U);
3586  leading_zero_bytes = 16 - count;
3587  memset(i_port_id, 0, leading_zero_bytes);
3588  rc = hex2bin(i_port_id + leading_zero_bytes, p, count);
3589  if (rc < 0)
3590  pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", rc);
3591  ret = 0;
3592 out:
3593  return ret;
3594 }
3595 
3596 /*
3597  * configfs callback function invoked for
3598  * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
3599  */
3600 static struct se_node_acl *srpt_make_nodeacl(struct se_portal_group *tpg,
3601  struct config_group *group,
3602  const char *name)
3603 {
3604  struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
3605  struct se_node_acl *se_nacl, *se_nacl_new;
3606  struct srpt_node_acl *nacl;
3607  int ret = 0;
3608  u32 nexus_depth = 1;
3609  u8 i_port_id[16];
3610 
3611  if (srpt_parse_i_port_id(i_port_id, name) < 0) {
3612  printk(KERN_ERR "invalid initiator port ID %s\n", name);
3613  ret = -EINVAL;
3614  goto err;
3615  }
3616 
3617  se_nacl_new = srpt_alloc_fabric_acl(tpg);
3618  if (!se_nacl_new) {
3619  ret = -ENOMEM;
3620  goto err;
3621  }
3622  /*
3623  * nacl_new may be released by core_tpg_add_initiator_node_acl()
3624  * when converting a node ACL from demo mode to explict
3625  */
3626  se_nacl = core_tpg_add_initiator_node_acl(tpg, se_nacl_new, name,
3627  nexus_depth);
3628  if (IS_ERR(se_nacl)) {
3629  ret = PTR_ERR(se_nacl);
3630  goto err;
3631  }
3632  /* Locate our struct srpt_node_acl and set sdev and i_port_id. */
3633  nacl = container_of(se_nacl, struct srpt_node_acl, nacl);
3634  memcpy(&nacl->i_port_id[0], &i_port_id[0], 16);
3635  nacl->sport = sport;
3636 
3637  spin_lock_irq(&sport->port_acl_lock);
3638  list_add_tail(&nacl->list, &sport->port_acl_list);
3639  spin_unlock_irq(&sport->port_acl_lock);
3640 
3641  return se_nacl;
3642 err:
3643  return ERR_PTR(ret);
3644 }
3645 
3646 /*
3647  * configfs callback function invoked for
3648  * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
3649  */
3650 static void srpt_drop_nodeacl(struct se_node_acl *se_nacl)
3651 {
3652  struct srpt_node_acl *nacl;
3653  struct srpt_device *sdev;
3654  struct srpt_port *sport;
3655 
3656  nacl = container_of(se_nacl, struct srpt_node_acl, nacl);
3657  sport = nacl->sport;
3658  sdev = sport->sdev;
3659  spin_lock_irq(&sport->port_acl_lock);
3660  list_del(&nacl->list);
3661  spin_unlock_irq(&sport->port_acl_lock);
3662  core_tpg_del_initiator_node_acl(&sport->port_tpg_1, se_nacl, 1);
3663  srpt_release_fabric_acl(NULL, se_nacl);
3664 }
3665 
3666 static ssize_t srpt_tpg_attrib_show_srp_max_rdma_size(
3667  struct se_portal_group *se_tpg,
3668  char *page)
3669 {
3670  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3671 
3672  return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
3673 }
3674 
3675 static ssize_t srpt_tpg_attrib_store_srp_max_rdma_size(
3676  struct se_portal_group *se_tpg,
3677  const char *page,
3678  size_t count)
3679 {
3680  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3681  unsigned long val;
3682  int ret;
3683 
3684  ret = strict_strtoul(page, 0, &val);
3685  if (ret < 0) {
3686  pr_err("strict_strtoul() failed with ret: %d\n", ret);
3687  return -EINVAL;
3688  }
3689  if (val > MAX_SRPT_RDMA_SIZE) {
3690  pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
3692  return -EINVAL;
3693  }
3694  if (val < DEFAULT_MAX_RDMA_SIZE) {
3695  pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
3696  val, DEFAULT_MAX_RDMA_SIZE);
3697  return -EINVAL;
3698  }
3699  sport->port_attrib.srp_max_rdma_size = val;
3700 
3701  return count;
3702 }
3703 
3704 TF_TPG_ATTRIB_ATTR(srpt, srp_max_rdma_size, S_IRUGO | S_IWUSR);
3705 
3706 static ssize_t srpt_tpg_attrib_show_srp_max_rsp_size(
3707  struct se_portal_group *se_tpg,
3708  char *page)
3709 {
3710  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3711 
3712  return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
3713 }
3714 
3715 static ssize_t srpt_tpg_attrib_store_srp_max_rsp_size(
3716  struct se_portal_group *se_tpg,
3717  const char *page,
3718  size_t count)
3719 {
3720  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3721  unsigned long val;
3722  int ret;
3723 
3724  ret = strict_strtoul(page, 0, &val);
3725  if (ret < 0) {
3726  pr_err("strict_strtoul() failed with ret: %d\n", ret);
3727  return -EINVAL;
3728  }
3729  if (val > MAX_SRPT_RSP_SIZE) {
3730  pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
3732  return -EINVAL;
3733  }
3734  if (val < MIN_MAX_RSP_SIZE) {
3735  pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
3737  return -EINVAL;
3738  }
3739  sport->port_attrib.srp_max_rsp_size = val;
3740 
3741  return count;
3742 }
3743 
3744 TF_TPG_ATTRIB_ATTR(srpt, srp_max_rsp_size, S_IRUGO | S_IWUSR);
3745 
3746 static ssize_t srpt_tpg_attrib_show_srp_sq_size(
3747  struct se_portal_group *se_tpg,
3748  char *page)
3749 {
3750  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3751 
3752  return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
3753 }
3754 
3755 static ssize_t srpt_tpg_attrib_store_srp_sq_size(
3756  struct se_portal_group *se_tpg,
3757  const char *page,
3758  size_t count)
3759 {
3760  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3761  unsigned long val;
3762  int ret;
3763 
3764  ret = strict_strtoul(page, 0, &val);
3765  if (ret < 0) {
3766  pr_err("strict_strtoul() failed with ret: %d\n", ret);
3767  return -EINVAL;
3768  }
3769  if (val > MAX_SRPT_SRQ_SIZE) {
3770  pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
3772  return -EINVAL;
3773  }
3774  if (val < MIN_SRPT_SRQ_SIZE) {
3775  pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
3777  return -EINVAL;
3778  }
3779  sport->port_attrib.srp_sq_size = val;
3780 
3781  return count;
3782 }
3783 
3784 TF_TPG_ATTRIB_ATTR(srpt, srp_sq_size, S_IRUGO | S_IWUSR);
3785 
3786 static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
3787  &srpt_tpg_attrib_srp_max_rdma_size.attr,
3788  &srpt_tpg_attrib_srp_max_rsp_size.attr,
3789  &srpt_tpg_attrib_srp_sq_size.attr,
3790  NULL,
3791 };
3792 
3793 static ssize_t srpt_tpg_show_enable(
3794  struct se_portal_group *se_tpg,
3795  char *page)
3796 {
3797  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3798 
3799  return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
3800 }
3801 
3802 static ssize_t srpt_tpg_store_enable(
3803  struct se_portal_group *se_tpg,
3804  const char *page,
3805  size_t count)
3806 {
3807  struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
3808  unsigned long tmp;
3809  int ret;
3810 
3811  ret = strict_strtoul(page, 0, &tmp);
3812  if (ret < 0) {
3813  printk(KERN_ERR "Unable to extract srpt_tpg_store_enable\n");
3814  return -EINVAL;
3815  }
3816 
3817  if ((tmp != 0) && (tmp != 1)) {
3818  printk(KERN_ERR "Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
3819  return -EINVAL;
3820  }
3821  if (tmp == 1)
3822  sport->enabled = true;
3823  else
3824  sport->enabled = false;
3825 
3826  return count;
3827 }
3828 
3830 
3831 static struct configfs_attribute *srpt_tpg_attrs[] = {
3832  &srpt_tpg_enable.attr,
3833  NULL,
3834 };
3835 
3840 static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
3841  struct config_group *group,
3842  const char *name)
3843 {
3844  struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3845  int res;
3846 
3847  /* Initialize sport->port_wwn and sport->port_tpg_1 */
3848  res = core_tpg_register(&srpt_target->tf_ops, &sport->port_wwn,
3849  &sport->port_tpg_1, sport, TRANSPORT_TPG_TYPE_NORMAL);
3850  if (res)
3851  return ERR_PTR(res);
3852 
3853  return &sport->port_tpg_1;
3854 }
3855 
3860 static void srpt_drop_tpg(struct se_portal_group *tpg)
3861 {
3862  struct srpt_port *sport = container_of(tpg,
3863  struct srpt_port, port_tpg_1);
3864 
3865  sport->enabled = false;
3867 }
3868 
3873 static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3874  struct config_group *group,
3875  const char *name)
3876 {
3877  struct srpt_port *sport;
3878  int ret;
3879 
3880  sport = srpt_lookup_port(name);
3881  pr_debug("make_tport(%s)\n", name);
3882  ret = -EINVAL;
3883  if (!sport)
3884  goto err;
3885 
3886  return &sport->port_wwn;
3887 
3888 err:
3889  return ERR_PTR(ret);
3890 }
3891 
3896 static void srpt_drop_tport(struct se_wwn *wwn)
3897 {
3898  struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3899 
3900  pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item));
3901 }
3902 
3903 static ssize_t srpt_wwn_show_attr_version(struct target_fabric_configfs *tf,
3904  char *buf)
3905 {
3906  return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
3907 }
3908 
3909 TF_WWN_ATTR_RO(srpt, version);
3910 
3911 static struct configfs_attribute *srpt_wwn_attrs[] = {
3912  &srpt_wwn_version.attr,
3913  NULL,
3914 };
3915 
3916 static struct target_core_fabric_ops srpt_template = {
3917  .get_fabric_name = srpt_get_fabric_name,
3918  .get_fabric_proto_ident = srpt_get_fabric_proto_ident,
3919  .tpg_get_wwn = srpt_get_fabric_wwn,
3920  .tpg_get_tag = srpt_get_tag,
3921  .tpg_get_default_depth = srpt_get_default_depth,
3922  .tpg_get_pr_transport_id = srpt_get_pr_transport_id,
3923  .tpg_get_pr_transport_id_len = srpt_get_pr_transport_id_len,
3924  .tpg_parse_pr_out_transport_id = srpt_parse_pr_out_transport_id,
3925  .tpg_check_demo_mode = srpt_check_false,
3926  .tpg_check_demo_mode_cache = srpt_check_true,
3927  .tpg_check_demo_mode_write_protect = srpt_check_true,
3928  .tpg_check_prod_mode_write_protect = srpt_check_false,
3929  .tpg_alloc_fabric_acl = srpt_alloc_fabric_acl,
3930  .tpg_release_fabric_acl = srpt_release_fabric_acl,
3931  .tpg_get_inst_index = srpt_tpg_get_inst_index,
3932  .release_cmd = srpt_release_cmd,
3933  .check_stop_free = srpt_check_stop_free,
3934  .shutdown_session = srpt_shutdown_session,
3935  .close_session = srpt_close_session,
3936  .sess_get_index = srpt_sess_get_index,
3937  .sess_get_initiator_sid = NULL,
3938  .write_pending = srpt_write_pending,
3939  .write_pending_status = srpt_write_pending_status,
3940  .set_default_node_attributes = srpt_set_default_node_attrs,
3941  .get_task_tag = srpt_get_task_tag,
3942  .get_cmd_state = srpt_get_tcm_cmd_state,
3943  .queue_data_in = srpt_queue_response,
3944  .queue_status = srpt_queue_status,
3945  .queue_tm_rsp = srpt_queue_response,
3946  /*
3947  * Setup function pointers for generic logic in
3948  * target_core_fabric_configfs.c
3949  */
3950  .fabric_make_wwn = srpt_make_tport,
3951  .fabric_drop_wwn = srpt_drop_tport,
3952  .fabric_make_tpg = srpt_make_tpg,
3953  .fabric_drop_tpg = srpt_drop_tpg,
3954  .fabric_post_link = NULL,
3955  .fabric_pre_unlink = NULL,
3956  .fabric_make_np = NULL,
3957  .fabric_drop_np = NULL,
3958  .fabric_make_nodeacl = srpt_make_nodeacl,
3959  .fabric_drop_nodeacl = srpt_drop_nodeacl,
3960 };
3961 
3970 static int __init srpt_init_module(void)
3971 {
3972  int ret;
3973 
3974  ret = -EINVAL;
3975  if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
3976  printk(KERN_ERR "invalid value %d for kernel module parameter"
3977  " srp_max_req_size -- must be at least %d.\n",
3978  srp_max_req_size, MIN_MAX_REQ_SIZE);
3979  goto out;
3980  }
3981 
3982  if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3983  || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
3984  printk(KERN_ERR "invalid value %d for kernel module parameter"
3985  " srpt_srq_size -- must be in the range [%d..%d].\n",
3986  srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3987  goto out;
3988  }
3989 
3990  srpt_target = target_fabric_configfs_init(THIS_MODULE, "srpt");
3991  if (IS_ERR(srpt_target)) {
3992  printk(KERN_ERR "couldn't register\n");
3993  ret = PTR_ERR(srpt_target);
3994  goto out;
3995  }
3996 
3997  srpt_target->tf_ops = srpt_template;
3998 
3999  /*
4000  * Set up default attribute lists.
4001  */
4002  srpt_target->tf_cit_tmpl.tfc_wwn_cit.ct_attrs = srpt_wwn_attrs;
4003  srpt_target->tf_cit_tmpl.tfc_tpg_base_cit.ct_attrs = srpt_tpg_attrs;
4004  srpt_target->tf_cit_tmpl.tfc_tpg_attrib_cit.ct_attrs = srpt_tpg_attrib_attrs;
4005  srpt_target->tf_cit_tmpl.tfc_tpg_param_cit.ct_attrs = NULL;
4006  srpt_target->tf_cit_tmpl.tfc_tpg_np_base_cit.ct_attrs = NULL;
4007  srpt_target->tf_cit_tmpl.tfc_tpg_nacl_base_cit.ct_attrs = NULL;
4008  srpt_target->tf_cit_tmpl.tfc_tpg_nacl_attrib_cit.ct_attrs = NULL;
4009  srpt_target->tf_cit_tmpl.tfc_tpg_nacl_auth_cit.ct_attrs = NULL;
4010  srpt_target->tf_cit_tmpl.tfc_tpg_nacl_param_cit.ct_attrs = NULL;
4011 
4012  ret = target_fabric_configfs_register(srpt_target);
4013  if (ret < 0) {
4014  printk(KERN_ERR "couldn't register\n");
4015  goto out_free_target;
4016  }
4017 
4018  ret = ib_register_client(&srpt_client);
4019  if (ret) {
4020  printk(KERN_ERR "couldn't register IB client\n");
4021  goto out_unregister_target;
4022  }
4023 
4024  return 0;
4025 
4026 out_unregister_target:
4027  target_fabric_configfs_deregister(srpt_target);
4028  srpt_target = NULL;
4029 out_free_target:
4030  if (srpt_target)
4031  target_fabric_configfs_free(srpt_target);
4032 out:
4033  return ret;
4034 }
4035 
4036 static void __exit srpt_cleanup_module(void)
4037 {
4038  ib_unregister_client(&srpt_client);
4039  target_fabric_configfs_deregister(srpt_target);
4040  srpt_target = NULL;
4041 }
4042 
4043 module_init(srpt_init_module);
4044 module_exit(srpt_cleanup_module);