36 #include <linux/slab.h>
39 #include <linux/if_vlan.h>
41 #include <linux/tcp.h>
75 en_dbg(DRV, priv,
"Allocated tx_info ring at addr:%p size:%d\n",
88 en_err(priv,
"Failed allocating hwq resources\n");
94 en_err(priv,
"Failed to map TX buffer\n");
98 ring->
buf = ring->
wqres.buf.direct.buf;
100 en_dbg(DRV, priv,
"Allocated TX ring (addr:%p) - buf:%p size:%d "
101 "buf_size:%d dma:%llx\n", ring, ring->
buf, ring->
size,
102 ring->
buf_size, (
unsigned long long) ring->
wqres.buf.direct.map);
107 en_err(priv,
"Failed allocating qp %d\n", ring->
qpn);
114 en_dbg(DRV, priv,
"working without blueflame (%d)", err);
140 en_dbg(DRV, priv,
"Destroying tx ring, qpn: %d\n", ring->
qpn);
156 int cq,
int user_prio)
163 ring->
cons = 0xffffffff;
203 int frags = skb_shinfo(
skb)->nr_frags;
219 for (i = 0; i < frags; i++) {
234 if ((
void *) data >=
end) {
235 data = ring->
buf + ((
void *)data -
end);
246 for (i = 0; i < frags; i++) {
248 if ((
void *) data >=
end)
261 if ((
void *) ptr >=
end) {
280 en_dbg(DRV, priv,
"Freeing Tx buf - cons:0x%x prod:0x%x\n",
285 en_warn(priv,
"Tx consumer passed producer!\n");
298 en_dbg(DRV, priv,
"Freed %d uncompleted tx descriptors\n", cnt);
310 u16 new_index, ring_index;
311 u32 txbbs_skipped = 0;
322 index = cons_index & size_mask;
324 ring_index = ring->
cons & size_mask;
328 cons_index & size)) {
340 ring_index = (ring_index + ring->
last_nr_txbb) & size_mask;
343 priv, ring, ring_index,
344 !!((ring->
cons + txbbs_skipped) &
347 bytes += ring->
tx_info[ring_index].nr_bytes;
348 }
while (ring_index != new_index);
351 index = cons_index & size_mask;
363 ring->
cons += txbbs_skipped;
364 netdev_tx_completed_queue(ring->
tx_queue, packets, bytes);
370 if (netif_tx_queue_stopped(ring->
tx_queue) && txbbs_skipped > 0) {
371 netif_tx_wake_queue(ring->
tx_queue);
381 mlx4_en_process_tx_cq(cq->
dev, cq);
389 unsigned int desc_size)
394 for (i = desc_size - copy - 4; i >= 0; i -= 4) {
395 if ((i & (TXBB_SIZE - 1)) == 0)
398 *((
u32 *) (ring->
buf + i)) =
402 for (i = copy - 4; i >= 4 ; i -= 4) {
403 if ((i & (TXBB_SIZE - 1)) == 0)
406 *((
u32 *) (ring->
buf + index * TXBB_SIZE + i)) =
414 static int is_inline(
struct sk_buff *
skb,
void **pfrag)
418 if (inline_thold && !skb_is_gso(skb) && skb->
len <= inline_thold) {
419 if (skb_shinfo(skb)->nr_frags == 1) {
420 ptr = skb_frag_address_safe(&skb_shinfo(skb)->frags[0]);
428 }
else if (
unlikely(skb_shinfo(skb)->nr_frags))
437 static int inline_size(
struct sk_buff *skb)
449 int *lso_header_size)
454 if (skb_is_gso(skb)) {
455 *lso_header_size = skb_transport_offset(skb) + tcp_hdrlen(skb);
458 if (
unlikely(*lso_header_size != skb_headlen(skb))) {
461 if (*lso_header_size < skb_headlen(skb))
465 en_warn(priv,
"Non-linear headers\n");
470 *lso_header_size = 0;
471 if (!is_inline(skb,
NULL))
474 real_size = inline_size(skb);
481 int real_size,
u16 *vlan_tag,
int tx_ind,
void *fragptr)
486 if (skb->
len <= spc) {
488 skb_copy_from_linear_data(skb, inl + 1, skb_headlen(skb));
489 if (skb_shinfo(skb)->nr_frags)
490 memcpy(((
void *)(inl + 1)) + skb_headlen(skb), fragptr,
491 skb_frag_size(&skb_shinfo(skb)->frags[0]));
495 if (skb_headlen(skb) <= spc) {
496 skb_copy_from_linear_data(skb, inl + 1, skb_headlen(skb));
497 if (skb_headlen(skb) < spc) {
498 memcpy(((
void *)(inl + 1)) + skb_headlen(skb),
499 fragptr, spc - skb_headlen(skb));
500 fragptr += spc - skb_headlen(skb);
502 inl = (
void *) (inl + 1) +
spc;
503 memcpy(((
void *)(inl + 1)), fragptr, skb->
len - spc);
505 skb_copy_from_linear_data(skb, inl + 1, spc);
506 inl = (
void *) (inl + 1) +
spc;
507 skb_copy_from_linear_data_offset(skb, spc, inl + 1,
508 skb_headlen(skb) - spc);
509 if (skb_shinfo(skb)->nr_frags)
510 memcpy(((
void *)(inl + 1)) + skb_headlen(skb) - spc,
511 fragptr, skb_frag_size(&skb_shinfo(skb)->frags[0]));
520 tx_desc->
ctrl.fence_size = (real_size / 16) & 0x3f;
526 u16 rings_p_up = priv->
mdev->profile.num_tx_rings_p_up;
530 return skb_tx_hash(dev, skb);
535 return __skb_tx_hash(dev, skb, rings_p_up) + up * rings_p_up;
569 real_size = get_real_size(skb, dev, &lso_header_size);
574 desc_size =
ALIGN(real_size, TXBB_SIZE);
578 en_warn(priv,
"Oversized header or SG list\n");
591 netif_tx_stop_queue(ring->
tx_queue);
603 bf_index = ring->
prod;
607 if (
likely(index + nr_txbb <= ring->size))
624 tx_desc->
ctrl.fence_size = (real_size / 16) & 0x3f;
637 if (lso_header_size) {
645 skb_shinfo(skb)->gso_size << 16 | lso_header_size);
650 data = ((
void *) &tx_desc->
lso +
654 i = ((skb->
len - lso_header_size) / skb_shinfo(skb)->gso_size) +
655 !!((skb->
len - lso_header_size) % skb_shinfo(skb)->gso_size);
656 tx_info->
nr_bytes = skb->
len + (i - 1) * lso_header_size;
663 data = &tx_desc->
data;
674 tx_info->
data_offset = (
void *) data - (
void *) tx_desc;
676 tx_info->
linear = (lso_header_size < skb_headlen(skb) && !is_inline(skb,
NULL)) ? 1 : 0;
677 data += skb_shinfo(skb)->nr_frags + tx_info->
linear - 1;
679 if (!is_inline(skb, &fragptr)) {
681 for (i = skb_shinfo(skb)->nr_frags - 1; i >= 0; i--) {
682 frag = &skb_shinfo(skb)->frags[
i];
683 dma = skb_frag_dma_map(priv->
ddev, frag,
684 0, skb_frag_size(frag),
704 build_inline_wqe(tx_desc, skb, real_size, &vlan_tag, tx_ind, fragptr);
708 ring->
prod += nr_txbb;
712 tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size);
716 op_own |=
htonl((bf_index & 0xffff) << 8);
720 tx_desc->
ctrl.owner_opcode = op_own;
724 mlx4_bf_copy(ring->
bf.reg + ring->
bf.offset, (
unsigned long *) &tx_desc->
ctrl,
729 ring->
bf.offset ^= ring->
bf.buf_size;
734 tx_desc->
ctrl.owner_opcode = op_own;
743 priv->
stats.tx_dropped++;