Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
tilepro.c
Go to the documentation of this file.
1 /*
2  * Copyright 2011 Tilera Corporation. All Rights Reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation, version 2.
7  *
8  * This program is distributed in the hope that it will be useful, but
9  * WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11  * NON INFRINGEMENT. See the GNU General Public License for
12  * more details.
13  */
14 
15 #include <linux/module.h>
16 #include <linux/init.h>
17 #include <linux/moduleparam.h>
18 #include <linux/sched.h>
19 #include <linux/kernel.h> /* printk() */
20 #include <linux/slab.h> /* kmalloc() */
21 #include <linux/errno.h> /* error codes */
22 #include <linux/types.h> /* size_t */
23 #include <linux/interrupt.h>
24 #include <linux/in.h>
25 #include <linux/netdevice.h> /* struct device, and other headers */
26 #include <linux/etherdevice.h> /* eth_type_trans */
27 #include <linux/skbuff.h>
28 #include <linux/ioctl.h>
29 #include <linux/cdev.h>
30 #include <linux/hugetlb.h>
31 #include <linux/in6.h>
32 #include <linux/timer.h>
33 #include <linux/io.h>
34 #include <asm/checksum.h>
35 #include <asm/homecache.h>
36 
37 #include <hv/drv_xgbe_intf.h>
38 #include <hv/drv_xgbe_impl.h>
39 #include <hv/hypervisor.h>
40 #include <hv/netio_intf.h>
41 
42 /* For TSO */
43 #include <linux/ip.h>
44 #include <linux/tcp.h>
45 
46 
47 /*
48  * First, "tile_net_init_module()" initializes all four "devices" which
49  * can be used by linux.
50  *
51  * Then, "ifconfig DEVICE up" calls "tile_net_open()", which analyzes
52  * the network cpus, then uses "tile_net_open_aux()" to initialize
53  * LIPP/LEPP, and then uses "tile_net_open_inner()" to register all
54  * the tiles, provide buffers to LIPP, allow ingress to start, and
55  * turn on hypervisor interrupt handling (and NAPI) on all tiles.
56  *
57  * If registration fails due to the link being down, then "retry_work"
58  * is used to keep calling "tile_net_open_inner()" until it succeeds.
59  *
60  * If "ifconfig DEVICE down" is called, it uses "tile_net_stop()" to
61  * stop egress, drain the LIPP buffers, unregister all the tiles, stop
62  * LIPP/LEPP, and wipe the LEPP queue.
63  *
64  * We start out with the ingress interrupt enabled on each CPU. When
65  * this interrupt fires, we disable it, and call "napi_schedule()".
66  * This will cause "tile_net_poll()" to be called, which will pull
67  * packets from the netio queue, filtering them out, or passing them
68  * to "netif_receive_skb()". If our budget is exhausted, we will
69  * return, knowing we will be called again later. Otherwise, we
70  * reenable the ingress interrupt, and call "napi_complete()".
71  *
72  * HACK: Since disabling the ingress interrupt is not reliable, we
73  * ignore the interrupt if the global "active" flag is false.
74  *
75  *
76  * NOTE: The use of "native_driver" ensures that EPP exists, and that
77  * we are using "LIPP" and "LEPP".
78  *
79  * NOTE: Failing to free completions for an arbitrarily long time
80  * (which is defined to be illegal) does in fact cause bizarre
81  * problems. The "egress_timer" helps prevent this from happening.
82  */
83 
84 
85 /* HACK: Allow use of "jumbo" packets. */
86 /* This should be 1500 if "jumbo" is not set in LIPP. */
87 /* This should be at most 10226 (10240 - 14) if "jumbo" is set in LIPP. */
88 /* ISSUE: This has not been thoroughly tested (except at 1500). */
89 #define TILE_NET_MTU 1500
90 
91 /* HACK: Define to support GSO. */
92 /* ISSUE: This may actually hurt performance of the TCP blaster. */
93 /* #define TILE_NET_GSO */
94 
95 /* Define this to collapse "duplicate" acks. */
96 /* #define IGNORE_DUP_ACKS */
97 
98 /* HACK: Define this to verify incoming packets. */
99 /* #define TILE_NET_VERIFY_INGRESS */
100 
101 /* Use 3000 to enable the Linux Traffic Control (QoS) layer, else 0. */
102 #define TILE_NET_TX_QUEUE_LEN 0
103 
104 /* Define to dump packets (prints out the whole packet on tx and rx). */
105 /* #define TILE_NET_DUMP_PACKETS */
106 
107 /* Define to enable debug spew (all PDEBUG's are enabled). */
108 /* #define TILE_NET_DEBUG */
109 
110 
111 /* Define to activate paranoia checks. */
112 /* #define TILE_NET_PARANOIA */
113 
114 /* Default transmit lockup timeout period, in jiffies. */
115 #define TILE_NET_TIMEOUT (5 * HZ)
116 
117 /* Default retry interval for bringing up the NetIO interface, in jiffies. */
118 #define TILE_NET_RETRY_INTERVAL (5 * HZ)
119 
120 /* Number of ports (xgbe0, xgbe1, gbe0, gbe1). */
121 #define TILE_NET_DEVS 4
122 
123 
124 
125 /* Paranoia. */
126 #if NET_IP_ALIGN != LIPP_PACKET_PADDING
127 #error "NET_IP_ALIGN must match LIPP_PACKET_PADDING."
128 #endif
129 
130 
131 /* Debug print. */
132 #ifdef TILE_NET_DEBUG
133 #define PDEBUG(fmt, args...) net_printk(fmt, ## args)
134 #else
135 #define PDEBUG(fmt, args...)
136 #endif
137 
138 
139 MODULE_AUTHOR("Tilera");
140 MODULE_LICENSE("GPL");
141 
142 
143 /*
144  * Queue of incoming packets for a specific cpu and device.
145  *
146  * Includes a pointer to the "system" data, and the actual "user" data.
147  */
151 
152 };
153 
154 
155 /*
156  * Statistics counters for a specific cpu and device.
157  */
163 };
164 
165 
166 /*
167  * Info for a specific cpu and device.
168  *
169  * ISSUE: There is a "dev" pointer in "napi" as well.
170  */
171 struct tile_net_cpu {
172  /* The NAPI struct. */
174  /* Packet queue. */
176  /* Statistics. */
178  /* True iff NAPI is enabled. */
180  /* True if this tile has successfully registered with the IPP. */
182  /* True if the link was down last time we tried to register. */
183  bool link_down;
184  /* True if "egress_timer" is scheduled. */
186  /* Number of small sk_buffs which must still be provided. */
188  /* Number of large sk_buffs which must still be provided. */
190  /* A timer for handling egress completions. */
192 };
193 
194 
195 /*
196  * Info for a specific device.
197  */
198 struct tile_net_priv {
199  /* Our network device. */
200  struct net_device *dev;
201  /* Pages making up the egress queue. */
202  struct page *eq_pages;
203  /* Address of the actual egress queue. */
205  /* Protects "eq". */
207  /* The hypervisor handle for this interface. */
209  /* The intr bit mask that IDs this device. */
211  /* True iff "tile_net_open_aux()" has succeeded. */
213  /* True iff the device is "active". */
214  bool active;
215  /* Effective network cpus. */
217  /* Number of network cpus. */
219  /* Credits per network cpu. */
221  /* Network stats. */
222  struct net_device_stats stats;
223  /* For NetIO bringup retries. */
225  /* Quick access to per cpu data. */
227 };
228 
229 /* Log2 of the number of small pages needed for the egress queue. */
230 #define EQ_ORDER get_order(sizeof(lepp_queue_t))
231 /* Size of the egress queue's pages. */
232 #define EQ_SIZE (1 << (PAGE_SHIFT + EQ_ORDER))
233 
234 /*
235  * The actual devices (xgbe0, xgbe1, gbe0, gbe1).
236  */
237 static struct net_device *tile_net_devs[TILE_NET_DEVS];
238 
239 /*
240  * The "tile_net_cpu" structures for each device.
241  */
242 static DEFINE_PER_CPU(struct tile_net_cpu, hv_xgbe0);
243 static DEFINE_PER_CPU(struct tile_net_cpu, hv_xgbe1);
244 static DEFINE_PER_CPU(struct tile_net_cpu, hv_gbe0);
245 static DEFINE_PER_CPU(struct tile_net_cpu, hv_gbe1);
246 
247 
248 /*
249  * True if "network_cpus" was specified.
250  */
251 static bool network_cpus_used;
252 
253 /*
254  * The actual cpus in "network_cpus".
255  */
256 static struct cpumask network_cpus_map;
257 
258 
259 
260 #ifdef TILE_NET_DEBUG
261 /*
262  * printk with extra stuff.
263  *
264  * We print the CPU we're running in brackets.
265  */
266 static void net_printk(char *fmt, ...)
267 {
268  int i;
269  int len;
270  va_list args;
271  static char buf[256];
272 
273  len = sprintf(buf, "tile_net[%2.2d]: ", smp_processor_id());
274  va_start(args, fmt);
275  i = vscnprintf(buf + len, sizeof(buf) - len - 1, fmt, args);
276  va_end(args);
277  buf[255] = '\0';
278  pr_notice(buf);
279 }
280 #endif
281 
282 
283 #ifdef TILE_NET_DUMP_PACKETS
284 /*
285  * Dump a packet.
286  */
287 static void dump_packet(unsigned char *data, unsigned long length, char *s)
288 {
289  int my_cpu = smp_processor_id();
290 
291  unsigned long i;
292  char buf[128];
293 
294  static unsigned int count;
295 
296  pr_info("dump_packet(data %p, length 0x%lx s %s count 0x%x)\n",
297  data, length, s, count++);
298 
299  pr_info("\n");
300 
301  for (i = 0; i < length; i++) {
302  if ((i & 0xf) == 0)
303  sprintf(buf, "[%02d] %8.8lx:", my_cpu, i);
304  sprintf(buf + strlen(buf), " %2.2x", data[i]);
305  if ((i & 0xf) == 0xf || i == length - 1) {
306  strcat(buf, "\n");
307  pr_info("%s", buf);
308  }
309  }
310 }
311 #endif
312 
313 
314 /*
315  * Provide support for the __netio_fastio1() swint
316  * (see <hv/drv_xgbe_intf.h> for how it is used).
317  *
318  * The fastio swint2 call may clobber all the caller-saved registers.
319  * It rarely clobbers memory, but we allow for the possibility in
320  * the signature just to be on the safe side.
321  *
322  * Also, gcc doesn't seem to allow an input operand to be
323  * clobbered, so we fake it with dummy outputs.
324  *
325  * This function can't be static because of the way it is declared
326  * in the netio header.
327  */
328 inline int __netio_fastio1(u32 fastio_index, u32 arg0)
329 {
330  long result, clobber_r1, clobber_r10;
331  asm volatile("swint2"
332  : "=R00" (result),
333  "=R01" (clobber_r1), "=R10" (clobber_r10)
334  : "R10" (fastio_index), "R01" (arg0)
335  : "memory", "r2", "r3", "r4",
336  "r5", "r6", "r7", "r8", "r9",
337  "r11", "r12", "r13", "r14",
338  "r15", "r16", "r17", "r18", "r19",
339  "r20", "r21", "r22", "r23", "r24",
340  "r25", "r26", "r27", "r28", "r29");
341  return result;
342 }
343 
344 
345 static void tile_net_return_credit(struct tile_net_cpu *info)
346 {
347  struct tile_netio_queue *queue = &info->queue;
348  netio_queue_user_impl_t *qup = &queue->__user_part;
349 
350  /* Return four credits after every fourth packet. */
351  if (--qup->__receive_credit_remaining == 0) {
355  }
356 }
357 
358 
359 
360 /*
361  * Provide a linux buffer to LIPP.
362  */
363 static void tile_net_provide_linux_buffer(struct tile_net_cpu *info,
364  void *va, bool small)
365 {
366  struct tile_netio_queue *queue = &info->queue;
367 
368  /* Convert "va" and "small" to "linux_buffer_t". */
369  unsigned int buffer = ((unsigned int)(__pa(va) >> 7) << 1) + small;
370 
372 }
373 
374 
375 /*
376  * Provide a linux buffer for LIPP.
377  *
378  * Note that the ACTUAL allocation for each buffer is a "struct sk_buff",
379  * plus a chunk of memory that includes not only the requested bytes, but
380  * also NET_SKB_PAD bytes of initial padding, and a "struct skb_shared_info".
381  *
382  * Note that "struct skb_shared_info" is 88 bytes with 64K pages and
383  * 268 bytes with 4K pages (since the frags[] array needs 18 entries).
384  *
385  * Without jumbo packets, the maximum packet size will be 1536 bytes,
386  * and we use 2 bytes (NET_IP_ALIGN) of padding. ISSUE: If we told
387  * the hardware to clip at 1518 bytes instead of 1536 bytes, then we
388  * could save an entire cache line, but in practice, we don't need it.
389  *
390  * Since CPAs are 38 bits, and we can only encode the high 31 bits in
391  * a "linux_buffer_t", the low 7 bits must be zero, and thus, we must
392  * align the actual "va" mod 128.
393  *
394  * We assume that the underlying "head" will be aligned mod 64. Note
395  * that in practice, we have seen "head" NOT aligned mod 128 even when
396  * using 2048 byte allocations, which is surprising.
397  *
398  * If "head" WAS always aligned mod 128, we could change LIPP to
399  * assume that the low SIX bits are zero, and the 7th bit is one, that
400  * is, align the actual "va" mod 128 plus 64, which would be "free".
401  *
402  * For now, the actual "head" pointer points at NET_SKB_PAD bytes of
403  * padding, plus 28 or 92 bytes of extra padding, plus the sk_buff
404  * pointer, plus the NET_IP_ALIGN padding, plus 126 or 1536 bytes for
405  * the actual packet, plus 62 bytes of empty padding, plus some
406  * padding and the "struct skb_shared_info".
407  *
408  * With 64K pages, a large buffer thus needs 32+92+4+2+1536+62+88
409  * bytes, or 1816 bytes, which fits comfortably into 2048 bytes.
410  *
411  * With 64K pages, a small buffer thus needs 32+92+4+2+126+88
412  * bytes, or 344 bytes, which means we are wasting 64+ bytes, and
413  * could presumably increase the size of small buffers.
414  *
415  * With 4K pages, a large buffer thus needs 32+92+4+2+1536+62+268
416  * bytes, or 1996 bytes, which fits comfortably into 2048 bytes.
417  *
418  * With 4K pages, a small buffer thus needs 32+92+4+2+126+268
419  * bytes, or 524 bytes, which is annoyingly wasteful.
420  *
421  * Maybe we should increase LIPP_SMALL_PACKET_SIZE to 192?
422  *
423  * ISSUE: Maybe we should increase "NET_SKB_PAD" to 64?
424  */
425 static bool tile_net_provide_needed_buffer(struct tile_net_cpu *info,
426  bool small)
427 {
428 #if TILE_NET_MTU <= 1536
429  /* Without "jumbo", 2 + 1536 should be sufficient. */
430  unsigned int large_size = NET_IP_ALIGN + 1536;
431 #else
432  /* ISSUE: This has not been tested. */
433  unsigned int large_size = NET_IP_ALIGN + TILE_NET_MTU + 100;
434 #endif
435 
436  /* Avoid "false sharing" with last cache line. */
437  /* ISSUE: This is already done by "netdev_alloc_skb()". */
438  unsigned int len =
439  (((small ? LIPP_SMALL_PACKET_SIZE : large_size) +
441 
442  unsigned int padding = 128 - NET_SKB_PAD;
443  unsigned int align;
444 
445  struct sk_buff *skb;
446  void *va;
447 
448  struct sk_buff **skb_ptr;
449 
450  /* Request 96 extra bytes for alignment purposes. */
451  skb = netdev_alloc_skb(info->napi.dev, len + padding);
452  if (skb == NULL)
453  return false;
454 
455  /* Skip 32 or 96 bytes to align "data" mod 128. */
456  align = -(long)skb->data & (128 - 1);
457  BUG_ON(align > padding);
458  skb_reserve(skb, align);
459 
460  /* This address is given to IPP. */
461  va = skb->data;
462 
463  /* Buffers must not span a huge page. */
464  BUG_ON(((((long)va & ~HPAGE_MASK) + len) & HPAGE_MASK) != 0);
465 
466 #ifdef TILE_NET_PARANOIA
467 #if CHIP_HAS_CBOX_HOME_MAP()
468  if (hash_default) {
469  HV_PTE pte = *virt_to_pte(current->mm, (unsigned long)va);
470  if (hv_pte_get_mode(pte) != HV_PTE_MODE_CACHE_HASH_L3)
471  panic("Non-HFH ingress buffer! VA=%p Mode=%d PTE=%llx",
472  va, hv_pte_get_mode(pte), hv_pte_val(pte));
473  }
474 #endif
475 #endif
476 
477  /* Invalidate the packet buffer. */
478  if (!hash_default)
479  __inv_buffer(va, len);
480 
481  /* Skip two bytes to satisfy LIPP assumptions. */
482  /* Note that this aligns IP on a 16 byte boundary. */
483  /* ISSUE: Do this when the packet arrives? */
484  skb_reserve(skb, NET_IP_ALIGN);
485 
486  /* Save a back-pointer to 'skb'. */
487  skb_ptr = va - sizeof(*skb_ptr);
488  *skb_ptr = skb;
489 
490  /* Make sure "skb_ptr" has been flushed. */
491  __insn_mf();
492 
493  /* Provide the new buffer. */
494  tile_net_provide_linux_buffer(info, va, small);
495 
496  return true;
497 }
498 
499 
500 /*
501  * Provide linux buffers for LIPP.
502  */
503 static void tile_net_provide_needed_buffers(struct tile_net_cpu *info)
504 {
505  while (info->num_needed_small_buffers != 0) {
506  if (!tile_net_provide_needed_buffer(info, true))
507  goto oops;
508  info->num_needed_small_buffers--;
509  }
510 
511  while (info->num_needed_large_buffers != 0) {
512  if (!tile_net_provide_needed_buffer(info, false))
513  goto oops;
514  info->num_needed_large_buffers--;
515  }
516 
517  return;
518 
519 oops:
520 
521  /* Add a description to the page allocation failure dump. */
522  pr_notice("Could not provide a linux buffer to LIPP.\n");
523 }
524 
525 
526 /*
527  * Grab some LEPP completions, and store them in "comps", of size
528  * "comps_size", and return the number of completions which were
529  * stored, so the caller can free them.
530  */
531 static unsigned int tile_net_lepp_grab_comps(lepp_queue_t *eq,
532  struct sk_buff *comps[],
533  unsigned int comps_size,
534  unsigned int min_size)
535 {
536  unsigned int n = 0;
537 
538  unsigned int comp_head = eq->comp_head;
539  unsigned int comp_busy = eq->comp_busy;
540 
541  while (comp_head != comp_busy && n < comps_size) {
542  comps[n++] = eq->comps[comp_head];
543  LEPP_QINC(comp_head);
544  }
545 
546  if (n < min_size)
547  return 0;
548 
549  eq->comp_head = comp_head;
550 
551  return n;
552 }
553 
554 
555 /*
556  * Free some comps, and return true iff there are still some pending.
557  */
558 static bool tile_net_lepp_free_comps(struct net_device *dev, bool all)
559 {
560  struct tile_net_priv *priv = netdev_priv(dev);
561 
562  lepp_queue_t *eq = priv->eq;
563 
564  struct sk_buff *olds[64];
565  unsigned int wanted = 64;
566  unsigned int i, n;
567  bool pending;
568 
569  spin_lock(&priv->eq_lock);
570 
571  if (all)
572  eq->comp_busy = eq->comp_tail;
573 
574  n = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
575 
576  pending = (eq->comp_head != eq->comp_tail);
577 
578  spin_unlock(&priv->eq_lock);
579 
580  for (i = 0; i < n; i++)
581  kfree_skb(olds[i]);
582 
583  return pending;
584 }
585 
586 
587 /*
588  * Make sure the egress timer is scheduled.
589  *
590  * Note that we use "schedule if not scheduled" logic instead of the more
591  * obvious "reschedule" logic, because "reschedule" is fairly expensive.
592  */
593 static void tile_net_schedule_egress_timer(struct tile_net_cpu *info)
594 {
595  if (!info->egress_timer_scheduled) {
596  mod_timer_pinned(&info->egress_timer, jiffies + 1);
597  info->egress_timer_scheduled = true;
598  }
599 }
600 
601 
602 /*
603  * The "function" for "info->egress_timer".
604  *
605  * This timer will reschedule itself as long as there are any pending
606  * completions expected (on behalf of any tile).
607  *
608  * ISSUE: Realistically, will the timer ever stop scheduling itself?
609  *
610  * ISSUE: This timer is almost never actually needed, so just use a global
611  * timer that can run on any tile.
612  *
613  * ISSUE: Maybe instead track number of expected completions, and free
614  * only that many, resetting to zero if "pending" is ever false.
615  */
616 static void tile_net_handle_egress_timer(unsigned long arg)
617 {
618  struct tile_net_cpu *info = (struct tile_net_cpu *)arg;
619  struct net_device *dev = info->napi.dev;
620 
621  /* The timer is no longer scheduled. */
622  info->egress_timer_scheduled = false;
623 
624  /* Free comps, and reschedule timer if more are pending. */
625  if (tile_net_lepp_free_comps(dev, false))
626  tile_net_schedule_egress_timer(info);
627 }
628 
629 
630 #ifdef IGNORE_DUP_ACKS
631 
632 /*
633  * Help detect "duplicate" ACKs. These are sequential packets (for a
634  * given flow) which are exactly 66 bytes long, sharing everything but
635  * ID=2@0x12, Hsum=2@0x18, Ack=4@0x2a, WinSize=2@0x30, Csum=2@0x32,
636  * Tstamps=10@0x38. The ID's are +1, the Hsum's are -1, the Ack's are
637  * +N, and the Tstamps are usually identical.
638  *
639  * NOTE: Apparently truly duplicate acks (with identical "ack" values),
640  * should not be collapsed, as they are used for some kind of flow control.
641  */
642 static bool is_dup_ack(char *s1, char *s2, unsigned int len)
643 {
644  int i;
645 
646  unsigned long long ignorable = 0;
647 
648  /* Identification. */
649  ignorable |= (1ULL << 0x12);
650  ignorable |= (1ULL << 0x13);
651 
652  /* Header checksum. */
653  ignorable |= (1ULL << 0x18);
654  ignorable |= (1ULL << 0x19);
655 
656  /* ACK. */
657  ignorable |= (1ULL << 0x2a);
658  ignorable |= (1ULL << 0x2b);
659  ignorable |= (1ULL << 0x2c);
660  ignorable |= (1ULL << 0x2d);
661 
662  /* WinSize. */
663  ignorable |= (1ULL << 0x30);
664  ignorable |= (1ULL << 0x31);
665 
666  /* Checksum. */
667  ignorable |= (1ULL << 0x32);
668  ignorable |= (1ULL << 0x33);
669 
670  for (i = 0; i < len; i++, ignorable >>= 1) {
671 
672  if ((ignorable & 1) || (s1[i] == s2[i]))
673  continue;
674 
675 #ifdef TILE_NET_DEBUG
676  /* HACK: Mention non-timestamp diffs. */
677  if (i < 0x38 && i != 0x2f &&
678  net_ratelimit())
679  pr_info("Diff at 0x%x\n", i);
680 #endif
681 
682  return false;
683  }
684 
685 #ifdef TILE_NET_NO_SUPPRESS_DUP_ACKS
686  /* HACK: Do not suppress truly duplicate ACKs. */
687  /* ISSUE: Is this actually necessary or helpful? */
688  if (s1[0x2a] == s2[0x2a] &&
689  s1[0x2b] == s2[0x2b] &&
690  s1[0x2c] == s2[0x2c] &&
691  s1[0x2d] == s2[0x2d]) {
692  return false;
693  }
694 #endif
695 
696  return true;
697 }
698 
699 #endif
700 
701 
702 
703 static void tile_net_discard_aux(struct tile_net_cpu *info, int index)
704 {
705  struct tile_netio_queue *queue = &info->queue;
706  netio_queue_impl_t *qsp = queue->__system_part;
707  netio_queue_user_impl_t *qup = &queue->__user_part;
708 
709  int index2_aux = index + sizeof(netio_pkt_t);
710  int index2 =
711  ((index2_aux ==
713  0 : index2_aux);
714 
715  netio_pkt_t *pkt = (netio_pkt_t *)((unsigned long) &qsp[1] + index);
716 
717  /* Extract the "linux_buffer_t". */
718  unsigned int buffer = pkt->__packet.word;
719 
720  /* Convert "linux_buffer_t" to "va". */
721  void *va = __va((phys_addr_t)(buffer >> 1) << 7);
722 
723  /* Acquire the associated "skb". */
724  struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
725  struct sk_buff *skb = *skb_ptr;
726 
727  kfree_skb(skb);
728 
729  /* Consume this packet. */
731 }
732 
733 
734 /*
735  * Like "tile_net_poll()", but just discard packets.
736  */
737 static void tile_net_discard_packets(struct net_device *dev)
738 {
739  struct tile_net_priv *priv = netdev_priv(dev);
740  int my_cpu = smp_processor_id();
741  struct tile_net_cpu *info = priv->cpu[my_cpu];
742  struct tile_netio_queue *queue = &info->queue;
743  netio_queue_impl_t *qsp = queue->__system_part;
744  netio_queue_user_impl_t *qup = &queue->__user_part;
745 
746  while (qup->__packet_receive_read !=
748  int index = qup->__packet_receive_read;
749  tile_net_discard_aux(info, index);
750  }
751 }
752 
753 
754 /*
755  * Handle the next packet. Return true if "processed", false if "filtered".
756  */
757 static bool tile_net_poll_aux(struct tile_net_cpu *info, int index)
758 {
759  struct net_device *dev = info->napi.dev;
760 
761  struct tile_netio_queue *queue = &info->queue;
762  netio_queue_impl_t *qsp = queue->__system_part;
763  netio_queue_user_impl_t *qup = &queue->__user_part;
764  struct tile_net_stats_t *stats = &info->stats;
765 
766  int filter;
767 
768  int index2_aux = index + sizeof(netio_pkt_t);
769  int index2 =
770  ((index2_aux ==
772  0 : index2_aux);
773 
774  netio_pkt_t *pkt = (netio_pkt_t *)((unsigned long) &qsp[1] + index);
775 
776  netio_pkt_metadata_t *metadata = NETIO_PKT_METADATA(pkt);
777 
778  /* Extract the packet size. FIXME: Shouldn't the second line */
779  /* get subtracted? Mostly moot, since it should be "zero". */
780  unsigned long len =
781  (NETIO_PKT_CUSTOM_LENGTH(pkt) +
783 
784  /* Extract the "linux_buffer_t". */
785  unsigned int buffer = pkt->__packet.word;
786 
787  /* Extract "small" (vs "large"). */
788  bool small = ((buffer & 1) != 0);
789 
790  /* Convert "linux_buffer_t" to "va". */
791  void *va = __va((phys_addr_t)(buffer >> 1) << 7);
792 
793  /* Extract the packet data pointer. */
794  /* Compare to "NETIO_PKT_CUSTOM_DATA(pkt)". */
795  unsigned char *buf = va + NET_IP_ALIGN;
796 
797  /* Invalidate the packet buffer. */
798  if (!hash_default)
799  __inv_buffer(buf, len);
800 
801  /* ISSUE: Is this needed? */
802  dev->last_rx = jiffies;
803 
804 #ifdef TILE_NET_DUMP_PACKETS
805  dump_packet(buf, len, "rx");
806 #endif /* TILE_NET_DUMP_PACKETS */
807 
808 #ifdef TILE_NET_VERIFY_INGRESS
809  if (!NETIO_PKT_L4_CSUM_CORRECT_M(metadata, pkt) &&
810  NETIO_PKT_L4_CSUM_CALCULATED_M(metadata, pkt)) {
811  /* Bug 6624: Includes UDP packets with a "zero" checksum. */
812  pr_warning("Bad L4 checksum on %d byte packet.\n", len);
813  }
814  if (!NETIO_PKT_L3_CSUM_CORRECT_M(metadata, pkt) &&
815  NETIO_PKT_L3_CSUM_CALCULATED_M(metadata, pkt)) {
816  dump_packet(buf, len, "rx");
817  panic("Bad L3 checksum.");
818  }
819  switch (NETIO_PKT_STATUS_M(metadata, pkt)) {
821  if (len >= 64) {
822  dump_packet(buf, len, "rx");
823  panic("Unexpected OVERSIZE.");
824  }
825  break;
827  pr_warning("Unexpected BAD %ld byte packet.\n", len);
828  }
829 #endif
830 
831  filter = 0;
832 
833  /* ISSUE: Filter TCP packets with "bad" checksums? */
834 
835  if (!(dev->flags & IFF_UP)) {
836  /* Filter packets received before we're up. */
837  filter = 1;
838  } else if (NETIO_PKT_STATUS_M(metadata, pkt) == NETIO_PKT_STATUS_BAD) {
839  /* Filter "truncated" packets. */
840  filter = 1;
841  } else if (!(dev->flags & IFF_PROMISC)) {
842  /* FIXME: Implement HW multicast filter. */
843  if (!is_multicast_ether_addr(buf)) {
844  /* Filter packets not for our address. */
845  const u8 *mine = dev->dev_addr;
846  filter = !ether_addr_equal(mine, buf);
847  }
848  }
849 
850  if (filter) {
851 
852  /* ISSUE: Update "drop" statistics? */
853 
854  tile_net_provide_linux_buffer(info, va, small);
855 
856  } else {
857 
858  /* Acquire the associated "skb". */
859  struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
860  struct sk_buff *skb = *skb_ptr;
861 
862  /* Paranoia. */
863  if (skb->data != buf)
864  panic("Corrupt linux buffer from LIPP! "
865  "VA=%p, skb=%p, skb->data=%p\n",
866  va, skb, skb->data);
867 
868  /* Encode the actual packet length. */
869  skb_put(skb, len);
870 
871  /* NOTE: This call also sets "skb->dev = dev". */
872  skb->protocol = eth_type_trans(skb, dev);
873 
874  /* Avoid recomputing "good" TCP/UDP checksums. */
875  if (NETIO_PKT_L4_CSUM_CORRECT_M(metadata, pkt))
877 
878  netif_receive_skb(skb);
879 
880  stats->rx_packets++;
881  stats->rx_bytes += len;
882  }
883 
884  /* ISSUE: It would be nice to defer this until the packet has */
885  /* actually been processed. */
886  tile_net_return_credit(info);
887 
888  /* Consume this packet. */
890 
891  return !filter;
892 }
893 
894 
895 /*
896  * Handle some packets for the given device on the current CPU.
897  *
898  * If "tile_net_stop()" is called on some other tile while this
899  * function is running, we will return, hopefully before that
900  * other tile asks us to call "napi_disable()".
901  *
902  * The "rotting packet" race condition occurs if a packet arrives
903  * during the extremely narrow window between the queue appearing to
904  * be empty, and the ingress interrupt being re-enabled. This happens
905  * a LOT under heavy network load.
906  */
907 static int tile_net_poll(struct napi_struct *napi, int budget)
908 {
909  struct net_device *dev = napi->dev;
910  struct tile_net_priv *priv = netdev_priv(dev);
911  int my_cpu = smp_processor_id();
912  struct tile_net_cpu *info = priv->cpu[my_cpu];
913  struct tile_netio_queue *queue = &info->queue;
914  netio_queue_impl_t *qsp = queue->__system_part;
915  netio_queue_user_impl_t *qup = &queue->__user_part;
916 
917  unsigned int work = 0;
918 
919  while (priv->active) {
920  int index = qup->__packet_receive_read;
921  if (index == qsp->__packet_receive_queue.__packet_write)
922  break;
923 
924  if (tile_net_poll_aux(info, index)) {
925  if (++work >= budget)
926  goto done;
927  }
928  }
929 
930  napi_complete(&info->napi);
931 
932  if (!priv->active)
933  goto done;
934 
935  /* Re-enable the ingress interrupt. */
936  enable_percpu_irq(priv->intr_id, 0);
937 
938  /* HACK: Avoid the "rotting packet" problem (see above). */
939  if (qup->__packet_receive_read !=
941  /* ISSUE: Sometimes this returns zero, presumably */
942  /* because an interrupt was handled for this tile. */
943  (void)napi_reschedule(&info->napi);
944  }
945 
946 done:
947 
948  if (priv->active)
949  tile_net_provide_needed_buffers(info);
950 
951  return work;
952 }
953 
954 
955 /*
956  * Handle an ingress interrupt for the given device on the current cpu.
957  *
958  * ISSUE: Sometimes this gets called after "disable_percpu_irq()" has
959  * been called! This is probably due to "pending hypervisor downcalls".
960  *
961  * ISSUE: Is there any race condition between the "napi_schedule()" here
962  * and the "napi_complete()" call above?
963  */
964 static irqreturn_t tile_net_handle_ingress_interrupt(int irq, void *dev_ptr)
965 {
966  struct net_device *dev = (struct net_device *)dev_ptr;
967  struct tile_net_priv *priv = netdev_priv(dev);
968  int my_cpu = smp_processor_id();
969  struct tile_net_cpu *info = priv->cpu[my_cpu];
970 
971  /* Disable the ingress interrupt. */
973 
974  /* Ignore unwanted interrupts. */
975  if (!priv->active)
976  return IRQ_HANDLED;
977 
978  /* ISSUE: Sometimes "info->napi_enabled" is false here. */
979 
980  napi_schedule(&info->napi);
981 
982  return IRQ_HANDLED;
983 }
984 
985 
986 /*
987  * One time initialization per interface.
988  */
989 static int tile_net_open_aux(struct net_device *dev)
990 {
991  struct tile_net_priv *priv = netdev_priv(dev);
992 
993  int ret;
994  int dummy;
995  unsigned int epp_lotar;
996 
997  /*
998  * Find out where EPP memory should be homed.
999  */
1000  ret = hv_dev_pread(priv->hv_devhdl, 0,
1001  (HV_VirtAddr)&epp_lotar, sizeof(epp_lotar),
1003  if (ret < 0) {
1004  pr_err("could not read epp_shm_queue lotar.\n");
1005  return -EIO;
1006  }
1007 
1008  /*
1009  * Home the page on the EPP.
1010  */
1011  {
1012  int epp_home = hv_lotar_to_cpu(epp_lotar);
1013  homecache_change_page_home(priv->eq_pages, EQ_ORDER, epp_home);
1014  }
1015 
1016  /*
1017  * Register the EPP shared memory queue.
1018  */
1019  {
1021  .va = 0,
1022  .pa = __pa(priv->eq),
1023  .pte = hv_pte(0),
1024  .size = EQ_SIZE,
1025  };
1026  ea.pte = hv_pte_set_lotar(ea.pte, epp_lotar);
1027  ea.pte = hv_pte_set_mode(ea.pte, HV_PTE_MODE_CACHE_TILE_L3);
1028  ret = hv_dev_pwrite(priv->hv_devhdl, 0,
1029  (HV_VirtAddr)&ea,
1030  sizeof(ea),
1032  if (ret < 0)
1033  return -EIO;
1034  }
1035 
1036  /*
1037  * Start LIPP/LEPP.
1038  */
1039  if (hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1040  sizeof(dummy), NETIO_IPP_START_SHIM_OFF) < 0) {
1041  pr_warning("Failed to start LIPP/LEPP.\n");
1042  return -EIO;
1043  }
1044 
1045  return 0;
1046 }
1047 
1048 
1049 /*
1050  * Register with hypervisor on the current CPU.
1051  *
1052  * Strangely, this function does important things even if it "fails",
1053  * which is especially common if the link is not up yet. Hopefully
1054  * these things are all "harmless" if done twice!
1055  */
1056 static void tile_net_register(void *dev_ptr)
1057 {
1058  struct net_device *dev = (struct net_device *)dev_ptr;
1059  struct tile_net_priv *priv = netdev_priv(dev);
1060  int my_cpu = smp_processor_id();
1061  struct tile_net_cpu *info;
1062 
1063  struct tile_netio_queue *queue;
1064 
1065  /* Only network cpus can receive packets. */
1066  int queue_id =
1067  cpumask_test_cpu(my_cpu, &priv->network_cpus_map) ? 0 : 255;
1068 
1070  .flags = 0,
1071  .num_receive_packets = priv->network_cpus_credits,
1072  .queue_id = queue_id
1073  };
1074 
1075  int ret = 0;
1076  netio_queue_impl_t *queuep;
1077 
1078  PDEBUG("tile_net_register(queue_id %d)\n", queue_id);
1079 
1080  if (!strcmp(dev->name, "xgbe0"))
1081  info = &__get_cpu_var(hv_xgbe0);
1082  else if (!strcmp(dev->name, "xgbe1"))
1083  info = &__get_cpu_var(hv_xgbe1);
1084  else if (!strcmp(dev->name, "gbe0"))
1085  info = &__get_cpu_var(hv_gbe0);
1086  else if (!strcmp(dev->name, "gbe1"))
1087  info = &__get_cpu_var(hv_gbe1);
1088  else
1089  BUG();
1090 
1091  /* Initialize the egress timer. */
1092  init_timer(&info->egress_timer);
1093  info->egress_timer.data = (long)info;
1094  info->egress_timer.function = tile_net_handle_egress_timer;
1095 
1096  priv->cpu[my_cpu] = info;
1097 
1098  /*
1099  * Register ourselves with LIPP. This does a lot of stuff,
1100  * including invoking the LIPP registration code.
1101  */
1102  ret = hv_dev_pwrite(priv->hv_devhdl, 0,
1103  (HV_VirtAddr)&config,
1104  sizeof(netio_input_config_t),
1106  PDEBUG("hv_dev_pwrite(NETIO_IPP_INPUT_REGISTER_OFF) returned %d\n",
1107  ret);
1108  if (ret < 0) {
1109  if (ret != NETIO_LINK_DOWN) {
1110  printk(KERN_DEBUG "hv_dev_pwrite "
1111  "NETIO_IPP_INPUT_REGISTER_OFF failure %d\n",
1112  ret);
1113  }
1114  info->link_down = (ret == NETIO_LINK_DOWN);
1115  return;
1116  }
1117 
1118  /*
1119  * Get the pointer to our queue's system part.
1120  */
1121 
1122  ret = hv_dev_pread(priv->hv_devhdl, 0,
1123  (HV_VirtAddr)&queuep,
1124  sizeof(netio_queue_impl_t *),
1126  PDEBUG("hv_dev_pread(NETIO_IPP_INPUT_REGISTER_OFF) returned %d\n",
1127  ret);
1128  PDEBUG("queuep %p\n", queuep);
1129  if (ret <= 0) {
1130  /* ISSUE: Shouldn't this be a fatal error? */
1131  pr_err("hv_dev_pread NETIO_IPP_INPUT_REGISTER_OFF failure\n");
1132  return;
1133  }
1134 
1135  queue = &info->queue;
1136 
1137  queue->__system_part = queuep;
1138 
1139  memset(&queue->__user_part, 0, sizeof(netio_queue_user_impl_t));
1140 
1141  /* This is traditionally "config.num_receive_packets / 2". */
1145 
1146  /*
1147  * Get a fastio index from the hypervisor.
1148  * ISSUE: Shouldn't this check the result?
1149  */
1150  ret = hv_dev_pread(priv->hv_devhdl, 0,
1152  sizeof(queue->__user_part.__fastio_index),
1154  PDEBUG("hv_dev_pread(NETIO_IPP_GET_FASTIO_OFF) returned %d\n", ret);
1155 
1156  /* Now we are registered. */
1157  info->registered = true;
1158 }
1159 
1160 
1161 /*
1162  * Deregister with hypervisor on the current CPU.
1163  *
1164  * This simply discards all our credits, so no more packets will be
1165  * delivered to this tile. There may still be packets in our queue.
1166  *
1167  * Also, disable the ingress interrupt.
1168  */
1169 static void tile_net_deregister(void *dev_ptr)
1170 {
1171  struct net_device *dev = (struct net_device *)dev_ptr;
1172  struct tile_net_priv *priv = netdev_priv(dev);
1173  int my_cpu = smp_processor_id();
1174  struct tile_net_cpu *info = priv->cpu[my_cpu];
1175 
1176  /* Disable the ingress interrupt. */
1177  disable_percpu_irq(priv->intr_id);
1178 
1179  /* Do nothing else if not registered. */
1180  if (info == NULL || !info->registered)
1181  return;
1182 
1183  {
1184  struct tile_netio_queue *queue = &info->queue;
1185  netio_queue_user_impl_t *qup = &queue->__user_part;
1186 
1187  /* Discard all our credits. */
1189  }
1190 }
1191 
1192 
1193 /*
1194  * Unregister with hypervisor on the current CPU.
1195  *
1196  * Also, disable the ingress interrupt.
1197  */
1198 static void tile_net_unregister(void *dev_ptr)
1199 {
1200  struct net_device *dev = (struct net_device *)dev_ptr;
1201  struct tile_net_priv *priv = netdev_priv(dev);
1202  int my_cpu = smp_processor_id();
1203  struct tile_net_cpu *info = priv->cpu[my_cpu];
1204 
1205  int ret;
1206  int dummy = 0;
1207 
1208  /* Disable the ingress interrupt. */
1209  disable_percpu_irq(priv->intr_id);
1210 
1211  /* Do nothing else if not registered. */
1212  if (info == NULL || !info->registered)
1213  return;
1214 
1215  /* Unregister ourselves with LIPP/LEPP. */
1216  ret = hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1217  sizeof(dummy), NETIO_IPP_INPUT_UNREGISTER_OFF);
1218  if (ret < 0)
1219  panic("Failed to unregister with LIPP/LEPP!\n");
1220 
1221  /* Discard all packets still in our NetIO queue. */
1222  tile_net_discard_packets(dev);
1223 
1224  /* Reset state. */
1225  info->num_needed_small_buffers = 0;
1226  info->num_needed_large_buffers = 0;
1227 
1228  /* Cancel egress timer. */
1229  del_timer(&info->egress_timer);
1230  info->egress_timer_scheduled = false;
1231 }
1232 
1233 
1234 /*
1235  * Helper function for "tile_net_stop()".
1236  *
1237  * Also used to handle registration failure in "tile_net_open_inner()",
1238  * when the various extra steps in "tile_net_stop()" are not necessary.
1239  */
1240 static void tile_net_stop_aux(struct net_device *dev)
1241 {
1242  struct tile_net_priv *priv = netdev_priv(dev);
1243  int i;
1244 
1245  int dummy = 0;
1246 
1247  /*
1248  * Unregister all tiles, so LIPP will stop delivering packets.
1249  * Also, delete all the "napi" objects (sequentially, to protect
1250  * "dev->napi_list").
1251  */
1252  on_each_cpu(tile_net_unregister, (void *)dev, 1);
1253  for_each_online_cpu(i) {
1254  struct tile_net_cpu *info = priv->cpu[i];
1255  if (info != NULL && info->registered) {
1256  netif_napi_del(&info->napi);
1257  info->registered = false;
1258  }
1259  }
1260 
1261  /* Stop LIPP/LEPP. */
1262  if (hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1263  sizeof(dummy), NETIO_IPP_STOP_SHIM_OFF) < 0)
1264  panic("Failed to stop LIPP/LEPP!\n");
1265 
1266  priv->partly_opened = false;
1267 }
1268 
1269 
1270 /*
1271  * Disable NAPI for the given device on the current cpu.
1272  */
1273 static void tile_net_stop_disable(void *dev_ptr)
1274 {
1275  struct net_device *dev = (struct net_device *)dev_ptr;
1276  struct tile_net_priv *priv = netdev_priv(dev);
1277  int my_cpu = smp_processor_id();
1278  struct tile_net_cpu *info = priv->cpu[my_cpu];
1279 
1280  /* Disable NAPI if needed. */
1281  if (info != NULL && info->napi_enabled) {
1282  napi_disable(&info->napi);
1283  info->napi_enabled = false;
1284  }
1285 }
1286 
1287 
1288 /*
1289  * Enable NAPI and the ingress interrupt for the given device
1290  * on the current cpu.
1291  *
1292  * ISSUE: Only do this for "network cpus"?
1293  */
1294 static void tile_net_open_enable(void *dev_ptr)
1295 {
1296  struct net_device *dev = (struct net_device *)dev_ptr;
1297  struct tile_net_priv *priv = netdev_priv(dev);
1298  int my_cpu = smp_processor_id();
1299  struct tile_net_cpu *info = priv->cpu[my_cpu];
1300 
1301  /* Enable NAPI. */
1302  napi_enable(&info->napi);
1303  info->napi_enabled = true;
1304 
1305  /* Enable the ingress interrupt. */
1306  enable_percpu_irq(priv->intr_id, 0);
1307 }
1308 
1309 
1310 /*
1311  * tile_net_open_inner does most of the work of bringing up the interface.
1312  * It's called from tile_net_open(), and also from tile_net_retry_open().
1313  * The return value is 0 if the interface was brought up, < 0 if
1314  * tile_net_open() should return the return value as an error, and > 0 if
1315  * tile_net_open() should return success and schedule a work item to
1316  * periodically retry the bringup.
1317  */
1318 static int tile_net_open_inner(struct net_device *dev)
1319 {
1320  struct tile_net_priv *priv = netdev_priv(dev);
1321  int my_cpu = smp_processor_id();
1322  struct tile_net_cpu *info;
1323  struct tile_netio_queue *queue;
1324  int result = 0;
1325  int i;
1326  int dummy = 0;
1327 
1328  /*
1329  * First try to register just on the local CPU, and handle any
1330  * semi-expected "link down" failure specially. Note that we
1331  * do NOT call "tile_net_stop_aux()", unlike below.
1332  */
1333  tile_net_register(dev);
1334  info = priv->cpu[my_cpu];
1335  if (!info->registered) {
1336  if (info->link_down)
1337  return 1;
1338  return -EAGAIN;
1339  }
1340 
1341  /*
1342  * Now register everywhere else. If any registration fails,
1343  * even for "link down" (which might not be possible), we
1344  * clean up using "tile_net_stop_aux()". Also, add all the
1345  * "napi" objects (sequentially, to protect "dev->napi_list").
1346  * ISSUE: Only use "netif_napi_add()" for "network cpus"?
1347  */
1348  smp_call_function(tile_net_register, (void *)dev, 1);
1349  for_each_online_cpu(i) {
1350  struct tile_net_cpu *info = priv->cpu[i];
1351  if (info->registered)
1352  netif_napi_add(dev, &info->napi, tile_net_poll, 64);
1353  else
1354  result = -EAGAIN;
1355  }
1356  if (result != 0) {
1357  tile_net_stop_aux(dev);
1358  return result;
1359  }
1360 
1361  queue = &info->queue;
1362 
1363  if (priv->intr_id == 0) {
1364  unsigned int irq;
1365 
1366  /*
1367  * Acquire the irq allocated by the hypervisor. Every
1368  * queue gets the same irq. The "__intr_id" field is
1369  * "1 << irq", so we use "__ffs()" to extract "irq".
1370  */
1371  priv->intr_id = queue->__system_part->__intr_id;
1372  BUG_ON(priv->intr_id == 0);
1373  irq = __ffs(priv->intr_id);
1374 
1375  /*
1376  * Register the ingress interrupt handler for this
1377  * device, permanently.
1378  *
1379  * We used to call "free_irq()" in "tile_net_stop()",
1380  * and then re-register the handler here every time,
1381  * but that caused DNP errors in "handle_IRQ_event()"
1382  * because "desc->action" was NULL. See bug 9143.
1383  */
1385  BUG_ON(request_irq(irq, tile_net_handle_ingress_interrupt,
1386  0, dev->name, (void *)dev) != 0);
1387  }
1388 
1389  {
1390  /* Allocate initial buffers. */
1391 
1392  int max_buffers =
1394 
1395  info->num_needed_small_buffers =
1396  min(LIPP_SMALL_BUFFERS, max_buffers);
1397 
1398  info->num_needed_large_buffers =
1399  min(LIPP_LARGE_BUFFERS, max_buffers);
1400 
1401  tile_net_provide_needed_buffers(info);
1402 
1403  if (info->num_needed_small_buffers != 0 ||
1404  info->num_needed_large_buffers != 0)
1405  panic("Insufficient memory for buffer stack!");
1406  }
1407 
1408  /* We are about to be active. */
1409  priv->active = true;
1410 
1411  /* Make sure "active" is visible to all tiles. */
1412  mb();
1413 
1414  /* On each tile, enable NAPI and the ingress interrupt. */
1415  on_each_cpu(tile_net_open_enable, (void *)dev, 1);
1416 
1417  /* Start LIPP/LEPP and activate "ingress" at the shim. */
1418  if (hv_dev_pwrite(priv->hv_devhdl, 0, (HV_VirtAddr)&dummy,
1419  sizeof(dummy), NETIO_IPP_INPUT_INIT_OFF) < 0)
1420  panic("Failed to activate the LIPP Shim!\n");
1421 
1422  /* Start our transmit queue. */
1423  netif_start_queue(dev);
1424 
1425  return 0;
1426 }
1427 
1428 
1429 /*
1430  * Called periodically to retry bringing up the NetIO interface,
1431  * if it doesn't come up cleanly during tile_net_open().
1432  */
1433 static void tile_net_open_retry(struct work_struct *w)
1434 {
1435  struct delayed_work *dw =
1436  container_of(w, struct delayed_work, work);
1437 
1438  struct tile_net_priv *priv =
1439  container_of(dw, struct tile_net_priv, retry_work);
1440 
1441  /*
1442  * Try to bring the NetIO interface up. If it fails, reschedule
1443  * ourselves to try again later; otherwise, tell Linux we now have
1444  * a working link. ISSUE: What if the return value is negative?
1445  */
1446  if (tile_net_open_inner(priv->dev) != 0)
1449  else
1450  netif_carrier_on(priv->dev);
1451 }
1452 
1453 
1454 /*
1455  * Called when a network interface is made active.
1456  *
1457  * Returns 0 on success, negative value on failure.
1458  *
1459  * The open entry point is called when a network interface is made
1460  * active by the system (IFF_UP). At this point all resources needed
1461  * for transmit and receive operations are allocated, the interrupt
1462  * handler is registered with the OS (if needed), the watchdog timer
1463  * is started, and the stack is notified that the interface is ready.
1464  *
1465  * If the actual link is not available yet, then we tell Linux that
1466  * we have no carrier, and we keep checking until the link comes up.
1467  */
1468 static int tile_net_open(struct net_device *dev)
1469 {
1470  int ret = 0;
1471  struct tile_net_priv *priv = netdev_priv(dev);
1472 
1473  /*
1474  * We rely on priv->partly_opened to tell us if this is the
1475  * first time this interface is being brought up. If it is
1476  * set, the IPP was already initialized and should not be
1477  * initialized again.
1478  */
1479  if (!priv->partly_opened) {
1480 
1481  int count;
1482  int credits;
1483 
1484  /* Initialize LIPP/LEPP, and start the Shim. */
1485  ret = tile_net_open_aux(dev);
1486  if (ret < 0) {
1487  pr_err("tile_net_open_aux failed: %d\n", ret);
1488  return ret;
1489  }
1490 
1491  /* Analyze the network cpus. */
1492 
1493  if (network_cpus_used)
1494  cpumask_copy(&priv->network_cpus_map,
1495  &network_cpus_map);
1496  else
1497  cpumask_copy(&priv->network_cpus_map, cpu_online_mask);
1498 
1499 
1500  count = cpumask_weight(&priv->network_cpus_map);
1501 
1502  /* Limit credits to available buffers, and apply min. */
1503  credits = max(16, (LIPP_LARGE_BUFFERS / count) & ~1);
1504 
1505  /* Apply "GBE" max limit. */
1506  /* ISSUE: Use higher limit for XGBE? */
1507  credits = min(NETIO_MAX_RECEIVE_PKTS, credits);
1508 
1509  priv->network_cpus_count = count;
1510  priv->network_cpus_credits = credits;
1511 
1512 #ifdef TILE_NET_DEBUG
1513  pr_info("Using %d network cpus, with %d credits each\n",
1515 #endif
1516 
1517  priv->partly_opened = true;
1518 
1519  } else {
1520  /* FIXME: Is this possible? */
1521  /* printk("Already partly opened.\n"); */
1522  }
1523 
1524  /*
1525  * Attempt to bring up the link.
1526  */
1527  ret = tile_net_open_inner(dev);
1528  if (ret <= 0) {
1529  if (ret == 0)
1530  netif_carrier_on(dev);
1531  return ret;
1532  }
1533 
1534  /*
1535  * We were unable to bring up the NetIO interface, but we want to
1536  * try again in a little bit. Tell Linux that we have no carrier
1537  * so it doesn't try to use the interface before the link comes up
1538  * and then remember to try again later.
1539  */
1540  netif_carrier_off(dev);
1542 
1543  return 0;
1544 }
1545 
1546 
1547 static int tile_net_drain_lipp_buffers(struct tile_net_priv *priv)
1548 {
1549  int n = 0;
1550 
1551  /* Drain all the LIPP buffers. */
1552  while (true) {
1553  unsigned int buffer;
1554 
1555  /* NOTE: This should never fail. */
1556  if (hv_dev_pread(priv->hv_devhdl, 0, (HV_VirtAddr)&buffer,
1557  sizeof(buffer), NETIO_IPP_DRAIN_OFF) < 0)
1558  break;
1559 
1560  /* Stop when done. */
1561  if (buffer == 0)
1562  break;
1563 
1564  {
1565  /* Convert "linux_buffer_t" to "va". */
1566  void *va = __va((phys_addr_t)(buffer >> 1) << 7);
1567 
1568  /* Acquire the associated "skb". */
1569  struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
1570  struct sk_buff *skb = *skb_ptr;
1571 
1572  kfree_skb(skb);
1573  }
1574 
1575  n++;
1576  }
1577 
1578  return n;
1579 }
1580 
1581 
1582 /*
1583  * Disables a network interface.
1584  *
1585  * Returns 0, this is not allowed to fail.
1586  *
1587  * The close entry point is called when an interface is de-activated
1588  * by the OS. The hardware is still under the drivers control, but
1589  * needs to be disabled. A global MAC reset is issued to stop the
1590  * hardware, and all transmit and receive resources are freed.
1591  *
1592  * ISSUE: How closely does "netif_running(dev)" mirror "priv->active"?
1593  *
1594  * Before we are called by "__dev_close()", "netif_running()" will
1595  * have been cleared, so no NEW calls to "tile_net_poll()" will be
1596  * made by "netpoll_poll_dev()".
1597  *
1598  * Often, this can cause some tiles to still have packets in their
1599  * queues, so we must call "tile_net_discard_packets()" later.
1600  *
1601  * Note that some other tile may still be INSIDE "tile_net_poll()",
1602  * and in fact, many will be, if there is heavy network load.
1603  *
1604  * Calling "on_each_cpu(tile_net_stop_disable, (void *)dev, 1)" when
1605  * any tile is still "napi_schedule()"'d will induce a horrible crash
1606  * when "msleep()" is called. This includes tiles which are inside
1607  * "tile_net_poll()" which have not yet called "napi_complete()".
1608  *
1609  * So, we must first try to wait long enough for other tiles to finish
1610  * with any current "tile_net_poll()" call, and, hopefully, to clear
1611  * the "scheduled" flag. ISSUE: It is unclear what happens to tiles
1612  * which have called "napi_schedule()" but which had not yet tried to
1613  * call "tile_net_poll()", or which exhausted their budget inside
1614  * "tile_net_poll()" just before this function was called.
1615  */
1616 static int tile_net_stop(struct net_device *dev)
1617 {
1618  struct tile_net_priv *priv = netdev_priv(dev);
1619 
1620  PDEBUG("tile_net_stop()\n");
1621 
1622  /* Start discarding packets. */
1623  priv->active = false;
1624 
1625  /* Make sure "active" is visible to all tiles. */
1626  mb();
1627 
1628  /*
1629  * On each tile, make sure no NEW packets get delivered, and
1630  * disable the ingress interrupt.
1631  *
1632  * Note that the ingress interrupt can fire AFTER this,
1633  * presumably due to packets which were recently delivered,
1634  * but it will have no effect.
1635  */
1636  on_each_cpu(tile_net_deregister, (void *)dev, 1);
1637 
1638  /* Optimistically drain LIPP buffers. */
1639  (void)tile_net_drain_lipp_buffers(priv);
1640 
1641  /* ISSUE: Only needed if not yet fully open. */
1643 
1644  /* Can't transmit any more. */
1645  netif_stop_queue(dev);
1646 
1647  /* Disable NAPI on each tile. */
1648  on_each_cpu(tile_net_stop_disable, (void *)dev, 1);
1649 
1650  /*
1651  * Drain any remaining LIPP buffers. NOTE: This "printk()"
1652  * has never been observed, but in theory it could happen.
1653  */
1654  if (tile_net_drain_lipp_buffers(priv) != 0)
1655  printk("Had to drain some extra LIPP buffers!\n");
1656 
1657  /* Stop LIPP/LEPP. */
1658  tile_net_stop_aux(dev);
1659 
1660  /*
1661  * ISSUE: It appears that, in practice anyway, by the time we
1662  * get here, there are no pending completions, but just in case,
1663  * we free (all of) them anyway.
1664  */
1665  while (tile_net_lepp_free_comps(dev, true))
1666  /* loop */;
1667 
1668  /* Wipe the EPP queue, and wait till the stores hit the EPP. */
1669  memset(priv->eq, 0, sizeof(lepp_queue_t));
1670  mb();
1671 
1672  return 0;
1673 }
1674 
1675 
1676 /*
1677  * Prepare the "frags" info for the resulting LEPP command.
1678  *
1679  * If needed, flush the memory used by the frags.
1680  */
1681 static unsigned int tile_net_tx_frags(lepp_frag_t *frags,
1682  struct sk_buff *skb,
1683  void *b_data, unsigned int b_len)
1684 {
1685  unsigned int i, n = 0;
1686 
1687  struct skb_shared_info *sh = skb_shinfo(skb);
1688 
1689  phys_addr_t cpa;
1690 
1691  if (b_len != 0) {
1692 
1693  if (!hash_default)
1694  finv_buffer_remote(b_data, b_len, 0);
1695 
1696  cpa = __pa(b_data);
1697  frags[n].cpa_lo = cpa;
1698  frags[n].cpa_hi = cpa >> 32;
1699  frags[n].length = b_len;
1700  frags[n].hash_for_home = hash_default;
1701  n++;
1702  }
1703 
1704  for (i = 0; i < sh->nr_frags; i++) {
1705 
1706  skb_frag_t *f = &sh->frags[i];
1707  unsigned long pfn = page_to_pfn(skb_frag_page(f));
1708 
1709  /* FIXME: Compute "hash_for_home" properly. */
1710  /* ISSUE: The hypervisor checks CHIP_HAS_REV1_DMA_PACKETS(). */
1711  int hash_for_home = hash_default;
1712 
1713  /* FIXME: Hmmm. */
1714  if (!hash_default) {
1715  void *va = pfn_to_kaddr(pfn) + f->page_offset;
1716  BUG_ON(PageHighMem(skb_frag_page(f)));
1717  finv_buffer_remote(va, skb_frag_size(f), 0);
1718  }
1719 
1720  cpa = ((phys_addr_t)pfn << PAGE_SHIFT) + f->page_offset;
1721  frags[n].cpa_lo = cpa;
1722  frags[n].cpa_hi = cpa >> 32;
1723  frags[n].length = skb_frag_size(f);
1724  frags[n].hash_for_home = hash_for_home;
1725  n++;
1726  }
1727 
1728  return n;
1729 }
1730 
1731 
1732 /*
1733  * This function takes "skb", consisting of a header template and a
1734  * payload, and hands it to LEPP, to emit as one or more segments,
1735  * each consisting of a possibly modified header, plus a piece of the
1736  * payload, via a process known as "tcp segmentation offload".
1737  *
1738  * Usually, "data" will contain the header template, of size "sh_len",
1739  * and "sh->frags" will contain "skb->data_len" bytes of payload, and
1740  * there will be "sh->gso_segs" segments.
1741  *
1742  * Sometimes, if "sendfile()" requires copying, we will be called with
1743  * "data" containing the header and payload, with "frags" being empty.
1744  *
1745  * Sometimes, for example when using NFS over TCP, a single segment can
1746  * span 3 fragments, which must be handled carefully in LEPP.
1747  *
1748  * See "emulate_large_send_offload()" for some reference code, which
1749  * does not handle checksumming.
1750  *
1751  * ISSUE: How do we make sure that high memory DMA does not migrate?
1752  */
1753 static int tile_net_tx_tso(struct sk_buff *skb, struct net_device *dev)
1754 {
1755  struct tile_net_priv *priv = netdev_priv(dev);
1756  int my_cpu = smp_processor_id();
1757  struct tile_net_cpu *info = priv->cpu[my_cpu];
1758  struct tile_net_stats_t *stats = &info->stats;
1759 
1760  struct skb_shared_info *sh = skb_shinfo(skb);
1761 
1762  unsigned char *data = skb->data;
1763 
1764  /* The ip header follows the ethernet header. */
1765  struct iphdr *ih = ip_hdr(skb);
1766  unsigned int ih_len = ih->ihl * 4;
1767 
1768  /* Note that "nh == ih", by definition. */
1769  unsigned char *nh = skb_network_header(skb);
1770  unsigned int eh_len = nh - data;
1771 
1772  /* The tcp header follows the ip header. */
1773  struct tcphdr *th = (struct tcphdr *)(nh + ih_len);
1774  unsigned int th_len = th->doff * 4;
1775 
1776  /* The total number of header bytes. */
1777  /* NOTE: This may be less than skb_headlen(skb). */
1778  unsigned int sh_len = eh_len + ih_len + th_len;
1779 
1780  /* The number of payload bytes at "skb->data + sh_len". */
1781  /* This is non-zero for sendfile() without HIGHDMA. */
1782  unsigned int b_len = skb_headlen(skb) - sh_len;
1783 
1784  /* The total number of payload bytes. */
1785  unsigned int d_len = b_len + skb->data_len;
1786 
1787  /* The maximum payload size. */
1788  unsigned int p_len = sh->gso_size;
1789 
1790  /* The total number of segments. */
1791  unsigned int num_segs = sh->gso_segs;
1792 
1793  /* The temporary copy of the command. */
1794  u32 cmd_body[(LEPP_MAX_CMD_SIZE + 3) / 4];
1795  lepp_tso_cmd_t *cmd = (lepp_tso_cmd_t *)cmd_body;
1796 
1797  /* Analyze the "frags". */
1798  unsigned int num_frags =
1799  tile_net_tx_frags(cmd->frags, skb, data + sh_len, b_len);
1800 
1801  /* The size of the command, including frags and header. */
1802  size_t cmd_size = LEPP_TSO_CMD_SIZE(num_frags, sh_len);
1803 
1804  /* The command header. */
1805  lepp_tso_cmd_t cmd_init = {
1806  .tso = true,
1807  .header_size = sh_len,
1808  .ip_offset = eh_len,
1809  .tcp_offset = eh_len + ih_len,
1810  .payload_size = p_len,
1811  .num_frags = num_frags,
1812  };
1813 
1814  unsigned long irqflags;
1815 
1816  lepp_queue_t *eq = priv->eq;
1817 
1818  struct sk_buff *olds[8];
1819  unsigned int wanted = 8;
1820  unsigned int i, nolds = 0;
1821 
1822  unsigned int cmd_head, cmd_tail, cmd_next;
1823  unsigned int comp_tail;
1824 
1825 
1826  /* Paranoia. */
1827  BUG_ON(skb->protocol != htons(ETH_P_IP));
1828  BUG_ON(ih->protocol != IPPROTO_TCP);
1830  BUG_ON(num_frags > LEPP_MAX_FRAGS);
1831  /*--BUG_ON(num_segs != (d_len + (p_len - 1)) / p_len); */
1832  BUG_ON(num_segs <= 1);
1833 
1834 
1835  /* Finish preparing the command. */
1836 
1837  /* Copy the command header. */
1838  *cmd = cmd_init;
1839 
1840  /* Copy the "header". */
1841  memcpy(&cmd->frags[num_frags], data, sh_len);
1842 
1843 
1844  /* Prefetch and wait, to minimize time spent holding the spinlock. */
1845  prefetch_L1(&eq->comp_tail);
1846  prefetch_L1(&eq->cmd_tail);
1847  mb();
1848 
1849 
1850  /* Enqueue the command. */
1851 
1852  spin_lock_irqsave(&priv->eq_lock, irqflags);
1853 
1854  /* Handle completions if needed to make room. */
1855  /* NOTE: Return NETDEV_TX_BUSY if there is still no room. */
1856  if (lepp_num_free_comp_slots(eq) == 0) {
1857  nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
1858  if (nolds == 0) {
1859 busy:
1860  spin_unlock_irqrestore(&priv->eq_lock, irqflags);
1861  return NETDEV_TX_BUSY;
1862  }
1863  }
1864 
1865  cmd_head = eq->cmd_head;
1866  cmd_tail = eq->cmd_tail;
1867 
1868  /* Prepare to advance, detecting full queue. */
1869  /* NOTE: Return NETDEV_TX_BUSY if the queue is full. */
1870  cmd_next = cmd_tail + cmd_size;
1871  if (cmd_tail < cmd_head && cmd_next >= cmd_head)
1872  goto busy;
1873  if (cmd_next > LEPP_CMD_LIMIT) {
1874  cmd_next = 0;
1875  if (cmd_next == cmd_head)
1876  goto busy;
1877  }
1878 
1879  /* Copy the command. */
1880  memcpy(&eq->cmds[cmd_tail], cmd, cmd_size);
1881 
1882  /* Advance. */
1883  cmd_tail = cmd_next;
1884 
1885  /* Record "skb" for eventual freeing. */
1886  comp_tail = eq->comp_tail;
1887  eq->comps[comp_tail] = skb;
1888  LEPP_QINC(comp_tail);
1889  eq->comp_tail = comp_tail;
1890 
1891  /* Flush before allowing LEPP to handle the command. */
1892  /* ISSUE: Is this the optimal location for the flush? */
1893  __insn_mf();
1894 
1895  eq->cmd_tail = cmd_tail;
1896 
1897  /* NOTE: Using "4" here is more efficient than "0" or "2", */
1898  /* and, strangely, more efficient than pre-checking the number */
1899  /* of available completions, and comparing it to 4. */
1900  if (nolds == 0)
1901  nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 4);
1902 
1903  spin_unlock_irqrestore(&priv->eq_lock, irqflags);
1904 
1905  /* Handle completions. */
1906  for (i = 0; i < nolds; i++)
1907  kfree_skb(olds[i]);
1908 
1909  /* Update stats. */
1910  stats->tx_packets += num_segs;
1911  stats->tx_bytes += (num_segs * sh_len) + d_len;
1912 
1913  /* Make sure the egress timer is scheduled. */
1914  tile_net_schedule_egress_timer(info);
1915 
1916  return NETDEV_TX_OK;
1917 }
1918 
1919 
1920 /*
1921  * Transmit a packet (called by the kernel via "hard_start_xmit" hook).
1922  */
1923 static int tile_net_tx(struct sk_buff *skb, struct net_device *dev)
1924 {
1925  struct tile_net_priv *priv = netdev_priv(dev);
1926  int my_cpu = smp_processor_id();
1927  struct tile_net_cpu *info = priv->cpu[my_cpu];
1928  struct tile_net_stats_t *stats = &info->stats;
1929 
1930  unsigned long irqflags;
1931 
1932  struct skb_shared_info *sh = skb_shinfo(skb);
1933 
1934  unsigned int len = skb->len;
1935  unsigned char *data = skb->data;
1936 
1937  unsigned int csum_start = skb_checksum_start_offset(skb);
1938 
1939  lepp_frag_t frags[LEPP_MAX_FRAGS];
1940 
1941  unsigned int num_frags;
1942 
1943  lepp_queue_t *eq = priv->eq;
1944 
1945  struct sk_buff *olds[8];
1946  unsigned int wanted = 8;
1947  unsigned int i, nolds = 0;
1948 
1949  unsigned int cmd_size = sizeof(lepp_cmd_t);
1950 
1951  unsigned int cmd_head, cmd_tail, cmd_next;
1952  unsigned int comp_tail;
1953 
1955 
1956 
1957  /*
1958  * This is paranoia, since we think that if the link doesn't come
1959  * up, telling Linux we have no carrier will keep it from trying
1960  * to transmit. If it does, though, we can't execute this routine,
1961  * since data structures we depend on aren't set up yet.
1962  */
1963  if (!info->registered)
1964  return NETDEV_TX_BUSY;
1965 
1966 
1967  /* Save the timestamp. */
1968  dev->trans_start = jiffies;
1969 
1970 
1971 #ifdef TILE_NET_PARANOIA
1972 #if CHIP_HAS_CBOX_HOME_MAP()
1973  if (hash_default) {
1974  HV_PTE pte = *virt_to_pte(current->mm, (unsigned long)data);
1975  if (hv_pte_get_mode(pte) != HV_PTE_MODE_CACHE_HASH_L3)
1976  panic("Non-HFH egress buffer! VA=%p Mode=%d PTE=%llx",
1977  data, hv_pte_get_mode(pte), hv_pte_val(pte));
1978  }
1979 #endif
1980 #endif
1981 
1982 
1983 #ifdef TILE_NET_DUMP_PACKETS
1984  /* ISSUE: Does not dump the "frags". */
1985  dump_packet(data, skb_headlen(skb), "tx");
1986 #endif /* TILE_NET_DUMP_PACKETS */
1987 
1988 
1989  if (sh->gso_size != 0)
1990  return tile_net_tx_tso(skb, dev);
1991 
1992 
1993  /* Prepare the commands. */
1994 
1995  num_frags = tile_net_tx_frags(frags, skb, data, skb_headlen(skb));
1996 
1997  for (i = 0; i < num_frags; i++) {
1998 
1999  bool final = (i == num_frags - 1);
2000 
2001  lepp_cmd_t cmd = {
2002  .cpa_lo = frags[i].cpa_lo,
2003  .cpa_hi = frags[i].cpa_hi,
2004  .length = frags[i].length,
2005  .hash_for_home = frags[i].hash_for_home,
2006  .send_completion = final,
2007  .end_of_packet = final
2008  };
2009 
2010  if (i == 0 && skb->ip_summed == CHECKSUM_PARTIAL) {
2011  cmd.compute_checksum = 1;
2012  cmd.checksum_data.bits.start_byte = csum_start;
2013  cmd.checksum_data.bits.count = len - csum_start;
2014  cmd.checksum_data.bits.destination_byte =
2015  csum_start + skb->csum_offset;
2016  }
2017 
2018  cmds[i] = cmd;
2019  }
2020 
2021 
2022  /* Prefetch and wait, to minimize time spent holding the spinlock. */
2023  prefetch_L1(&eq->comp_tail);
2024  prefetch_L1(&eq->cmd_tail);
2025  mb();
2026 
2027 
2028  /* Enqueue the commands. */
2029 
2030  spin_lock_irqsave(&priv->eq_lock, irqflags);
2031 
2032  /* Handle completions if needed to make room. */
2033  /* NOTE: Return NETDEV_TX_BUSY if there is still no room. */
2034  if (lepp_num_free_comp_slots(eq) == 0) {
2035  nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
2036  if (nolds == 0) {
2037 busy:
2038  spin_unlock_irqrestore(&priv->eq_lock, irqflags);
2039  return NETDEV_TX_BUSY;
2040  }
2041  }
2042 
2043  cmd_head = eq->cmd_head;
2044  cmd_tail = eq->cmd_tail;
2045 
2046  /* Copy the commands, or fail. */
2047  /* NOTE: Return NETDEV_TX_BUSY if the queue is full. */
2048  for (i = 0; i < num_frags; i++) {
2049 
2050  /* Prepare to advance, detecting full queue. */
2051  cmd_next = cmd_tail + cmd_size;
2052  if (cmd_tail < cmd_head && cmd_next >= cmd_head)
2053  goto busy;
2054  if (cmd_next > LEPP_CMD_LIMIT) {
2055  cmd_next = 0;
2056  if (cmd_next == cmd_head)
2057  goto busy;
2058  }
2059 
2060  /* Copy the command. */
2061  *(lepp_cmd_t *)&eq->cmds[cmd_tail] = cmds[i];
2062 
2063  /* Advance. */
2064  cmd_tail = cmd_next;
2065  }
2066 
2067  /* Record "skb" for eventual freeing. */
2068  comp_tail = eq->comp_tail;
2069  eq->comps[comp_tail] = skb;
2070  LEPP_QINC(comp_tail);
2071  eq->comp_tail = comp_tail;
2072 
2073  /* Flush before allowing LEPP to handle the command. */
2074  /* ISSUE: Is this the optimal location for the flush? */
2075  __insn_mf();
2076 
2077  eq->cmd_tail = cmd_tail;
2078 
2079  /* NOTE: Using "4" here is more efficient than "0" or "2", */
2080  /* and, strangely, more efficient than pre-checking the number */
2081  /* of available completions, and comparing it to 4. */
2082  if (nolds == 0)
2083  nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 4);
2084 
2085  spin_unlock_irqrestore(&priv->eq_lock, irqflags);
2086 
2087  /* Handle completions. */
2088  for (i = 0; i < nolds; i++)
2089  kfree_skb(olds[i]);
2090 
2091  /* HACK: Track "expanded" size for short packets (e.g. 42 < 60). */
2092  stats->tx_packets++;
2093  stats->tx_bytes += ((len >= ETH_ZLEN) ? len : ETH_ZLEN);
2094 
2095  /* Make sure the egress timer is scheduled. */
2096  tile_net_schedule_egress_timer(info);
2097 
2098  return NETDEV_TX_OK;
2099 }
2100 
2101 
2102 /*
2103  * Deal with a transmit timeout.
2104  */
2105 static void tile_net_tx_timeout(struct net_device *dev)
2106 {
2107  PDEBUG("tile_net_tx_timeout()\n");
2108  PDEBUG("Transmit timeout at %ld, latency %ld\n", jiffies,
2109  jiffies - dev->trans_start);
2110 
2111  /* XXX: ISSUE: This doesn't seem useful for us. */
2112  netif_wake_queue(dev);
2113 }
2114 
2115 
2116 /*
2117  * Ioctl commands.
2118  */
2119 static int tile_net_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
2120 {
2121  return -EOPNOTSUPP;
2122 }
2123 
2124 
2125 /*
2126  * Get System Network Statistics.
2127  *
2128  * Returns the address of the device statistics structure.
2129  */
2130 static struct net_device_stats *tile_net_get_stats(struct net_device *dev)
2131 {
2132  struct tile_net_priv *priv = netdev_priv(dev);
2133  u32 rx_packets = 0;
2134  u32 tx_packets = 0;
2135  u32 rx_bytes = 0;
2136  u32 tx_bytes = 0;
2137  int i;
2138 
2139  for_each_online_cpu(i) {
2140  if (priv->cpu[i]) {
2141  rx_packets += priv->cpu[i]->stats.rx_packets;
2142  rx_bytes += priv->cpu[i]->stats.rx_bytes;
2143  tx_packets += priv->cpu[i]->stats.tx_packets;
2144  tx_bytes += priv->cpu[i]->stats.tx_bytes;
2145  }
2146  }
2147 
2148  priv->stats.rx_packets = rx_packets;
2149  priv->stats.rx_bytes = rx_bytes;
2150  priv->stats.tx_packets = tx_packets;
2151  priv->stats.tx_bytes = tx_bytes;
2152 
2153  return &priv->stats;
2154 }
2155 
2156 
2157 /*
2158  * Change the "mtu".
2159  *
2160  * The "change_mtu" method is usually not needed.
2161  * If you need it, it must be like this.
2162  */
2163 static int tile_net_change_mtu(struct net_device *dev, int new_mtu)
2164 {
2165  PDEBUG("tile_net_change_mtu()\n");
2166 
2167  /* Check ranges. */
2168  if ((new_mtu < 68) || (new_mtu > 1500))
2169  return -EINVAL;
2170 
2171  /* Accept the value. */
2172  dev->mtu = new_mtu;
2173 
2174  return 0;
2175 }
2176 
2177 
2178 /*
2179  * Change the Ethernet Address of the NIC.
2180  *
2181  * The hypervisor driver does not support changing MAC address. However,
2182  * the IPP does not do anything with the MAC address, so the address which
2183  * gets used on outgoing packets, and which is accepted on incoming packets,
2184  * is completely up to the NetIO program or kernel driver which is actually
2185  * handling them.
2186  *
2187  * Returns 0 on success, negative on failure.
2188  */
2189 static int tile_net_set_mac_address(struct net_device *dev, void *p)
2190 {
2191  struct sockaddr *addr = p;
2192 
2193  if (!is_valid_ether_addr(addr->sa_data))
2194  return -EADDRNOTAVAIL;
2195 
2196  /* ISSUE: Note that "dev_addr" is now a pointer. */
2197  memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2199 
2200  return 0;
2201 }
2202 
2203 
2204 /*
2205  * Obtain the MAC address from the hypervisor.
2206  * This must be done before opening the device.
2207  */
2208 static int tile_net_get_mac(struct net_device *dev)
2209 {
2210  struct tile_net_priv *priv = netdev_priv(dev);
2211 
2212  char hv_dev_name[32];
2213  int len;
2214 
2216 
2217  int ret;
2218 
2219  /* For example, "xgbe0". */
2220  strcpy(hv_dev_name, dev->name);
2221  len = strlen(hv_dev_name);
2222 
2223  /* For example, "xgbe/0". */
2224  hv_dev_name[len] = hv_dev_name[len - 1];
2225  hv_dev_name[len - 1] = '/';
2226  len++;
2227 
2228  /* For example, "xgbe/0/native_hash". */
2229  strcpy(hv_dev_name + len, hash_default ? "/native_hash" : "/native");
2230 
2231  /* Get the hypervisor handle for this device. */
2232  priv->hv_devhdl = hv_dev_open((HV_VirtAddr)hv_dev_name, 0);
2233  PDEBUG("hv_dev_open(%s) returned %d %p\n",
2234  hv_dev_name, priv->hv_devhdl, &priv->hv_devhdl);
2235  if (priv->hv_devhdl < 0) {
2236  if (priv->hv_devhdl == HV_ENODEV)
2237  printk(KERN_DEBUG "Ignoring unconfigured device %s\n",
2238  hv_dev_name);
2239  else
2240  printk(KERN_DEBUG "hv_dev_open(%s) returned %d\n",
2241  hv_dev_name, priv->hv_devhdl);
2242  return -1;
2243  }
2244 
2245  /*
2246  * Read the hardware address from the hypervisor.
2247  * ISSUE: Note that "dev_addr" is now a pointer.
2248  */
2249  offset.bits.class = NETIO_PARAM;
2250  offset.bits.addr = NETIO_PARAM_MAC;
2251  ret = hv_dev_pread(priv->hv_devhdl, 0,
2252  (HV_VirtAddr)dev->dev_addr, dev->addr_len,
2253  offset.word);
2254  PDEBUG("hv_dev_pread(NETIO_PARAM_MAC) returned %d\n", ret);
2255  if (ret <= 0) {
2256  printk(KERN_DEBUG "hv_dev_pread(NETIO_PARAM_MAC) %s failed\n",
2257  dev->name);
2258  /*
2259  * Since the device is configured by the hypervisor but we
2260  * can't get its MAC address, we are most likely running
2261  * the simulator, so let's generate a random MAC address.
2262  */
2263  eth_hw_addr_random(dev);
2264  }
2265 
2266  return 0;
2267 }
2268 
2269 
2270 #ifdef CONFIG_NET_POLL_CONTROLLER
2271 /*
2272  * Polling 'interrupt' - used by things like netconsole to send skbs
2273  * without having to re-enable interrupts. It's not called while
2274  * the interrupt routine is executing.
2275  */
2276 static void tile_net_netpoll(struct net_device *dev)
2277 {
2278  struct tile_net_priv *priv = netdev_priv(dev);
2279  disable_percpu_irq(priv->intr_id);
2280  tile_net_handle_ingress_interrupt(priv->intr_id, dev);
2281  enable_percpu_irq(priv->intr_id, 0);
2282 }
2283 #endif
2284 
2285 
2286 static const struct net_device_ops tile_net_ops = {
2287  .ndo_open = tile_net_open,
2288  .ndo_stop = tile_net_stop,
2289  .ndo_start_xmit = tile_net_tx,
2290  .ndo_do_ioctl = tile_net_ioctl,
2291  .ndo_get_stats = tile_net_get_stats,
2292  .ndo_change_mtu = tile_net_change_mtu,
2293  .ndo_tx_timeout = tile_net_tx_timeout,
2294  .ndo_set_mac_address = tile_net_set_mac_address,
2295 #ifdef CONFIG_NET_POLL_CONTROLLER
2296  .ndo_poll_controller = tile_net_netpoll,
2297 #endif
2298 };
2299 
2300 
2301 /*
2302  * The setup function.
2303  *
2304  * This uses ether_setup() to assign various fields in dev, including
2305  * setting IFF_BROADCAST and IFF_MULTICAST, then sets some extra fields.
2306  */
2307 static void tile_net_setup(struct net_device *dev)
2308 {
2309  PDEBUG("tile_net_setup()\n");
2310 
2311  ether_setup(dev);
2312 
2313  dev->netdev_ops = &tile_net_ops;
2314 
2316 
2317  /* We want lockless xmit. */
2318  dev->features |= NETIF_F_LLTX;
2319 
2320  /* We support hardware tx checksums. */
2321  dev->features |= NETIF_F_HW_CSUM;
2322 
2323  /* We support scatter/gather. */
2324  dev->features |= NETIF_F_SG;
2325 
2326  /* We support TSO. */
2327  dev->features |= NETIF_F_TSO;
2328 
2329 #ifdef TILE_NET_GSO
2330  /* We support GSO. */
2331  dev->features |= NETIF_F_GSO;
2332 #endif
2333 
2334  if (hash_default)
2335  dev->features |= NETIF_F_HIGHDMA;
2336 
2337  /* ISSUE: We should support NETIF_F_UFO. */
2338 
2340 
2341  dev->mtu = TILE_NET_MTU;
2342 }
2343 
2344 
2345 /*
2346  * Allocate the device structure, register the device, and obtain the
2347  * MAC address from the hypervisor.
2348  */
2349 static struct net_device *tile_net_dev_init(const char *name)
2350 {
2351  int ret;
2352  struct net_device *dev;
2353  struct tile_net_priv *priv;
2354 
2355  /*
2356  * Allocate the device structure. This allocates "priv", calls
2357  * tile_net_setup(), and saves "name". Normally, "name" is a
2358  * template, instantiated by register_netdev(), but not for us.
2359  */
2360  dev = alloc_netdev(sizeof(*priv), name, tile_net_setup);
2361  if (!dev) {
2362  pr_err("alloc_netdev(%s) failed\n", name);
2363  return NULL;
2364  }
2365 
2366  priv = netdev_priv(dev);
2367 
2368  /* Initialize "priv". */
2369 
2370  memset(priv, 0, sizeof(*priv));
2371 
2372  /* Save "dev" for "tile_net_open_retry()". */
2373  priv->dev = dev;
2374 
2375  INIT_DELAYED_WORK(&priv->retry_work, tile_net_open_retry);
2376 
2377  spin_lock_init(&priv->eq_lock);
2378 
2379  /* Allocate "eq". */
2381  if (!priv->eq_pages) {
2382  free_netdev(dev);
2383  return NULL;
2384  }
2385  priv->eq = page_address(priv->eq_pages);
2386 
2387  /* Register the network device. */
2388  ret = register_netdev(dev);
2389  if (ret) {
2390  pr_err("register_netdev %s failed %d\n", dev->name, ret);
2391  __free_pages(priv->eq_pages, EQ_ORDER);
2392  free_netdev(dev);
2393  return NULL;
2394  }
2395 
2396  /* Get the MAC address. */
2397  ret = tile_net_get_mac(dev);
2398  if (ret < 0) {
2399  unregister_netdev(dev);
2400  __free_pages(priv->eq_pages, EQ_ORDER);
2401  free_netdev(dev);
2402  return NULL;
2403  }
2404 
2405  return dev;
2406 }
2407 
2408 
2409 /*
2410  * Module cleanup.
2411  *
2412  * FIXME: If compiled as a module, this module cannot be "unloaded",
2413  * because the "ingress interrupt handler" is registered permanently.
2414  */
2415 static void tile_net_cleanup(void)
2416 {
2417  int i;
2418 
2419  for (i = 0; i < TILE_NET_DEVS; i++) {
2420  if (tile_net_devs[i]) {
2421  struct net_device *dev = tile_net_devs[i];
2422  struct tile_net_priv *priv = netdev_priv(dev);
2423  unregister_netdev(dev);
2424  finv_buffer_remote(priv->eq, EQ_SIZE, 0);
2425  __free_pages(priv->eq_pages, EQ_ORDER);
2426  free_netdev(dev);
2427  }
2428  }
2429 }
2430 
2431 
2432 /*
2433  * Module initialization.
2434  */
2435 static int tile_net_init_module(void)
2436 {
2437  pr_info("Tilera Network Driver\n");
2438 
2439  tile_net_devs[0] = tile_net_dev_init("xgbe0");
2440  tile_net_devs[1] = tile_net_dev_init("xgbe1");
2441  tile_net_devs[2] = tile_net_dev_init("gbe0");
2442  tile_net_devs[3] = tile_net_dev_init("gbe1");
2443 
2444  return 0;
2445 }
2446 
2447 
2448 module_init(tile_net_init_module);
2449 module_exit(tile_net_cleanup);
2450 
2451 
2452 #ifndef MODULE
2453 
2454 /*
2455  * The "network_cpus" boot argument specifies the cpus that are dedicated
2456  * to handle ingress packets.
2457  *
2458  * The parameter should be in the form "network_cpus=m-n[,x-y]", where
2459  * m, n, x, y are integer numbers that represent the cpus that can be
2460  * neither a dedicated cpu nor a dataplane cpu.
2461  */
2462 static int __init network_cpus_setup(char *str)
2463 {
2464  int rc = cpulist_parse_crop(str, &network_cpus_map);
2465  if (rc != 0) {
2466  pr_warning("network_cpus=%s: malformed cpu list\n",
2467  str);
2468  } else {
2469 
2470  /* Remove dedicated cpus. */
2471  cpumask_and(&network_cpus_map, &network_cpus_map,
2472  cpu_possible_mask);
2473 
2474 
2475  if (cpumask_empty(&network_cpus_map)) {
2476  pr_warning("Ignoring network_cpus='%s'.\n",
2477  str);
2478  } else {
2479  char buf[1024];
2480  cpulist_scnprintf(buf, sizeof(buf), &network_cpus_map);
2481  pr_info("Linux network CPUs: %s\n", buf);
2482  network_cpus_used = true;
2483  }
2484  }
2485 
2486  return 0;
2487 }
2488 __setup("network_cpus=", network_cpus_setup);
2489 
2490 #endif