15 #include <linux/module.h>
18 #include <linux/sched.h>
19 #include <linux/kernel.h>
20 #include <linux/slab.h>
21 #include <linux/errno.h>
22 #include <linux/types.h>
25 #include <linux/netdevice.h>
31 #include <linux/in6.h>
34 #include <asm/checksum.h>
44 #include <linux/tcp.h>
89 #define TILE_NET_MTU 1500
102 #define TILE_NET_TX_QUEUE_LEN 0
115 #define TILE_NET_TIMEOUT (5 * HZ)
118 #define TILE_NET_RETRY_INTERVAL (5 * HZ)
121 #define TILE_NET_DEVS 4
126 #if NET_IP_ALIGN != LIPP_PACKET_PADDING
127 #error "NET_IP_ALIGN must match LIPP_PACKET_PADDING."
132 #ifdef TILE_NET_DEBUG
133 #define PDEBUG(fmt, args...) net_printk(fmt, ## args)
135 #define PDEBUG(fmt, args...)
230 #define EQ_ORDER get_order(sizeof(lepp_queue_t))
232 #define EQ_SIZE (1 << (PAGE_SHIFT + EQ_ORDER))
251 static bool network_cpus_used;
256 static struct cpumask network_cpus_map;
260 #ifdef TILE_NET_DEBUG
266 static void net_printk(
char *
fmt, ...)
271 static char buf[256];
275 i =
vscnprintf(buf + len,
sizeof(buf) - len - 1, fmt, args);
283 #ifdef TILE_NET_DUMP_PACKETS
287 static void dump_packet(
unsigned char *
data,
unsigned long length,
char *
s)
294 static unsigned int count;
296 pr_info(
"dump_packet(data %p, length 0x%lx s %s count 0x%x)\n",
297 data, length, s, count++);
301 for (i = 0; i <
length; i++) {
303 sprintf(buf,
"[%02d] %8.8lx:", my_cpu, i);
305 if ((i & 0xf) == 0xf || i == length - 1) {
330 long result, clobber_r1, clobber_r10;
331 asm volatile(
"swint2"
333 "=R01" (clobber_r1),
"=R10" (clobber_r10)
334 :
"R10" (fastio_index),
"R01" (arg0)
335 :
"memory",
"r2",
"r3",
"r4",
336 "r5",
"r6",
"r7",
"r8",
"r9",
337 "r11",
"r12",
"r13",
"r14",
338 "r15",
"r16",
"r17",
"r18",
"r19",
339 "r20",
"r21",
"r22",
"r23",
"r24",
340 "r25",
"r26",
"r27",
"r28",
"r29");
363 static void tile_net_provide_linux_buffer(
struct tile_net_cpu *info,
364 void *
va,
bool small)
369 unsigned int buffer = ((
unsigned int)(
__pa(va) >> 7) << 1) + small;
425 static bool tile_net_provide_needed_buffer(
struct tile_net_cpu *info,
428 #if TILE_NET_MTU <= 1536
451 skb = netdev_alloc_skb(info->
napi.dev, len + padding);
456 align = -(
long)skb->
data & (128 - 1);
458 skb_reserve(skb, align);
466 #ifdef TILE_NET_PARANOIA
467 #if CHIP_HAS_CBOX_HOME_MAP()
471 panic(
"Non-HFH ingress buffer! VA=%p Mode=%d PTE=%llx",
479 __inv_buffer(va, len);
487 skb_ptr = va -
sizeof(*skb_ptr);
494 tile_net_provide_linux_buffer(info, va, small);
503 static void tile_net_provide_needed_buffers(
struct tile_net_cpu *info)
506 if (!tile_net_provide_needed_buffer(info,
true))
512 if (!tile_net_provide_needed_buffer(info,
false))
522 pr_notice(
"Could not provide a linux buffer to LIPP.\n");
531 static unsigned int tile_net_lepp_grab_comps(
lepp_queue_t *eq,
533 unsigned int comps_size,
534 unsigned int min_size)
541 while (comp_head != comp_busy && n < comps_size) {
542 comps[n++] = eq->comps[comp_head];
558 static bool tile_net_lepp_free_comps(
struct net_device *
dev,
bool all)
574 n = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
580 for (i = 0; i <
n; i++)
593 static void tile_net_schedule_egress_timer(
struct tile_net_cpu *info)
616 static void tile_net_handle_egress_timer(
unsigned long arg)
625 if (tile_net_lepp_free_comps(dev,
false))
626 tile_net_schedule_egress_timer(info);
630 #ifdef IGNORE_DUP_ACKS
642 static bool is_dup_ack(
char *
s1,
char *
s2,
unsigned int len)
646 unsigned long long ignorable = 0;
649 ignorable |= (1ULL << 0x12);
650 ignorable |= (1ULL << 0x13);
653 ignorable |= (1ULL << 0x18);
654 ignorable |= (1ULL << 0x19);
657 ignorable |= (1ULL << 0x2a);
658 ignorable |= (1ULL << 0x2b);
659 ignorable |= (1ULL << 0x2c);
660 ignorable |= (1ULL << 0x2d);
663 ignorable |= (1ULL << 0x30);
664 ignorable |= (1ULL << 0x31);
667 ignorable |= (1ULL << 0x32);
668 ignorable |= (1ULL << 0x33);
670 for (i = 0; i < len; i++, ignorable >>= 1) {
672 if ((ignorable & 1) || (s1[
i] == s2[
i]))
675 #ifdef TILE_NET_DEBUG
677 if (i < 0x38 && i != 0x2f &&
685 #ifdef TILE_NET_NO_SUPPRESS_DUP_ACKS
688 if (s1[0x2a] == s2[0x2a] &&
689 s1[0x2b] == s2[0x2b] &&
690 s1[0x2c] == s2[0x2c] &&
691 s1[0x2d] == s2[0x2d]) {
724 struct sk_buff **skb_ptr = va -
sizeof(*skb_ptr);
737 static void tile_net_discard_packets(
struct net_device *dev)
749 tile_net_discard_aux(info, index);
757 static bool tile_net_poll_aux(
struct tile_net_cpu *info,
int index)
781 (NETIO_PKT_CUSTOM_LENGTH(pkt) +
788 bool small = ((buffer & 1) != 0);
799 __inv_buffer(buf, len);
804 #ifdef TILE_NET_DUMP_PACKETS
805 dump_packet(buf, len,
"rx");
808 #ifdef TILE_NET_VERIFY_INGRESS
809 if (!NETIO_PKT_L4_CSUM_CORRECT_M(metadata, pkt) &&
810 NETIO_PKT_L4_CSUM_CALCULATED_M(metadata, pkt)) {
812 pr_warning(
"Bad L4 checksum on %d byte packet.\n", len);
814 if (!NETIO_PKT_L3_CSUM_CORRECT_M(metadata, pkt) &&
815 NETIO_PKT_L3_CSUM_CALCULATED_M(metadata, pkt)) {
816 dump_packet(buf, len,
"rx");
817 panic(
"Bad L3 checksum.");
819 switch (NETIO_PKT_STATUS_M(metadata, pkt)) {
822 dump_packet(buf, len,
"rx");
823 panic(
"Unexpected OVERSIZE.");
827 pr_warning(
"Unexpected BAD %ld byte packet.\n", len);
843 if (!is_multicast_ether_addr(buf)) {
846 filter = !ether_addr_equal(mine, buf);
854 tile_net_provide_linux_buffer(info, va, small);
859 struct sk_buff **skb_ptr = va -
sizeof(*skb_ptr);
863 if (skb->
data != buf)
864 panic(
"Corrupt linux buffer from LIPP! "
865 "VA=%p, skb=%p, skb->data=%p\n",
875 if (NETIO_PKT_L4_CSUM_CORRECT_M(metadata, pkt))
886 tile_net_return_credit(info);
917 unsigned int work = 0;
924 if (tile_net_poll_aux(info, index)) {
925 if (++work >= budget)
949 tile_net_provide_needed_buffers(info);
964 static irqreturn_t tile_net_handle_ingress_interrupt(
int irq,
void *dev_ptr)
980 napi_schedule(&info->
napi);
989 static int tile_net_open_aux(
struct net_device *dev)
995 unsigned int epp_lotar;
1004 pr_err(
"could not read epp_shm_queue lotar.\n");
1012 int epp_home = hv_lotar_to_cpu(epp_lotar);
1026 ea.
pte = hv_pte_set_lotar(ea.
pte, epp_lotar);
1056 static void tile_net_register(
void *dev_ptr)
1072 .queue_id = queue_id
1078 PDEBUG(
"tile_net_register(queue_id %d)\n", queue_id);
1094 info->
egress_timer.function = tile_net_handle_egress_timer;
1106 PDEBUG(
"hv_dev_pwrite(NETIO_IPP_INPUT_REGISTER_OFF) returned %d\n",
1111 "NETIO_IPP_INPUT_REGISTER_OFF failure %d\n",
1126 PDEBUG(
"hv_dev_pread(NETIO_IPP_INPUT_REGISTER_OFF) returned %d\n",
1128 PDEBUG(
"queuep %p\n", queuep);
1131 pr_err(
"hv_dev_pread NETIO_IPP_INPUT_REGISTER_OFF failure\n");
1135 queue = &info->
queue;
1154 PDEBUG(
"hv_dev_pread(NETIO_IPP_GET_FASTIO_OFF) returned %d\n", ret);
1169 static void tile_net_deregister(
void *dev_ptr)
1198 static void tile_net_unregister(
void *dev_ptr)
1219 panic(
"Failed to unregister with LIPP/LEPP!\n");
1222 tile_net_discard_packets(dev);
1240 static void tile_net_stop_aux(
struct net_device *dev)
1264 panic(
"Failed to stop LIPP/LEPP!\n");
1273 static void tile_net_stop_disable(
void *dev_ptr)
1282 napi_disable(&info->
napi);
1294 static void tile_net_open_enable(
void *dev_ptr)
1302 napi_enable(&info->
napi);
1318 static int tile_net_open_inner(
struct net_device *dev)
1333 tile_net_register(dev);
1334 info = priv->
cpu[my_cpu];
1357 tile_net_stop_aux(dev);
1361 queue = &info->
queue;
1386 0, dev->
name, (
void *)dev) != 0);
1401 tile_net_provide_needed_buffers(info);
1405 panic(
"Insufficient memory for buffer stack!");
1415 on_each_cpu(tile_net_open_enable, (
void *)dev, 1);
1420 panic(
"Failed to activate the LIPP Shim!\n");
1423 netif_start_queue(dev);
1446 if (tile_net_open_inner(priv->
dev) != 0)
1468 static int tile_net_open(
struct net_device *dev)
1485 ret = tile_net_open_aux(dev);
1487 pr_err(
"tile_net_open_aux failed: %d\n", ret);
1493 if (network_cpus_used)
1512 #ifdef TILE_NET_DEBUG
1513 pr_info(
"Using %d network cpus, with %d credits each\n",
1527 ret = tile_net_open_inner(dev);
1547 static int tile_net_drain_lipp_buffers(
struct tile_net_priv *priv)
1569 struct sk_buff **skb_ptr = va -
sizeof(*skb_ptr);
1616 static int tile_net_stop(
struct net_device *dev)
1620 PDEBUG(
"tile_net_stop()\n");
1639 (
void)tile_net_drain_lipp_buffers(priv);
1645 netif_stop_queue(dev);
1648 on_each_cpu(tile_net_stop_disable, (
void *)dev, 1);
1654 if (tile_net_drain_lipp_buffers(priv) != 0)
1655 printk(
"Had to drain some extra LIPP buffers!\n");
1658 tile_net_stop_aux(dev);
1665 while (tile_net_lepp_free_comps(dev,
true))
1681 static unsigned int tile_net_tx_frags(
lepp_frag_t *frags,
1683 void *b_data,
unsigned int b_len)
1685 unsigned int i, n = 0;
1704 for (i = 0; i < sh->
nr_frags; i++) {
1707 unsigned long pfn =
page_to_pfn(skb_frag_page(f));
1716 BUG_ON(PageHighMem(skb_frag_page(f)));
1723 frags[
n].
length = skb_frag_size(f);
1762 unsigned char *data = skb->
data;
1765 struct iphdr *ih = ip_hdr(skb);
1766 unsigned int ih_len = ih->ihl * 4;
1769 unsigned char *nh = skb_network_header(skb);
1770 unsigned int eh_len = nh -
data;
1774 unsigned int th_len = th->doff * 4;
1778 unsigned int sh_len = eh_len + ih_len + th_len;
1782 unsigned int b_len = skb_headlen(skb) - sh_len;
1785 unsigned int d_len = b_len + skb->
data_len;
1791 unsigned int num_segs = sh->
gso_segs;
1798 unsigned int num_frags =
1799 tile_net_tx_frags(cmd->frags, skb, data + sh_len, b_len);
1807 .header_size = sh_len,
1808 .ip_offset = eh_len,
1809 .tcp_offset = eh_len + ih_len,
1810 .payload_size =
p_len,
1811 .num_frags = num_frags,
1814 unsigned long irqflags;
1819 unsigned int wanted = 8;
1820 unsigned int i, nolds = 0;
1822 unsigned int cmd_head, cmd_tail, cmd_next;
1823 unsigned int comp_tail;
1841 memcpy(&cmd->frags[num_frags], data, sh_len);
1856 if (lepp_num_free_comp_slots(eq) == 0) {
1857 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
1860 spin_unlock_irqrestore(&priv->
eq_lock, irqflags);
1871 if (cmd_tail < cmd_head && cmd_next >= cmd_head)
1875 if (cmd_next == cmd_head)
1880 memcpy(&eq->cmds[cmd_tail], cmd, cmd_size);
1883 cmd_tail = cmd_next;
1887 eq->comps[comp_tail] =
skb;
1901 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 4);
1903 spin_unlock_irqrestore(&priv->
eq_lock, irqflags);
1906 for (i = 0; i < nolds; i++)
1911 stats->
tx_bytes += (num_segs * sh_len) + d_len;
1914 tile_net_schedule_egress_timer(info);
1930 unsigned long irqflags;
1934 unsigned int len = skb->
len;
1935 unsigned char *data = skb->
data;
1937 unsigned int csum_start = skb_checksum_start_offset(skb);
1941 unsigned int num_frags;
1946 unsigned int wanted = 8;
1947 unsigned int i, nolds = 0;
1951 unsigned int cmd_head, cmd_tail, cmd_next;
1952 unsigned int comp_tail;
1971 #ifdef TILE_NET_PARANOIA
1972 #if CHIP_HAS_CBOX_HOME_MAP()
1976 panic(
"Non-HFH egress buffer! VA=%p Mode=%d PTE=%llx",
1977 data, hv_pte_get_mode(pte),
hv_pte_val(pte));
1983 #ifdef TILE_NET_DUMP_PACKETS
1985 dump_packet(data, skb_headlen(skb),
"tx");
1990 return tile_net_tx_tso(skb, dev);
1995 num_frags = tile_net_tx_frags(frags, skb, data, skb_headlen(skb));
1997 for (i = 0; i < num_frags; i++) {
1999 bool final = (i == num_frags - 1);
2006 .send_completion =
final,
2007 .end_of_packet =
final
2034 if (lepp_num_free_comp_slots(eq) == 0) {
2035 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 0);
2038 spin_unlock_irqrestore(&priv->
eq_lock, irqflags);
2048 for (i = 0; i < num_frags; i++) {
2052 if (cmd_tail < cmd_head && cmd_next >= cmd_head)
2056 if (cmd_next == cmd_head)
2061 *(
lepp_cmd_t *)&eq->cmds[cmd_tail] = cmds[i];
2064 cmd_tail = cmd_next;
2069 eq->comps[comp_tail] =
skb;
2083 nolds = tile_net_lepp_grab_comps(eq, olds, wanted, 4);
2085 spin_unlock_irqrestore(&priv->
eq_lock, irqflags);
2088 for (i = 0; i < nolds; i++)
2096 tile_net_schedule_egress_timer(info);
2105 static void tile_net_tx_timeout(
struct net_device *dev)
2107 PDEBUG(
"tile_net_tx_timeout()\n");
2108 PDEBUG(
"Transmit timeout at %ld, latency %ld\n", jiffies,
2112 netif_wake_queue(dev);
2141 rx_packets += priv->
cpu[
i]->stats.rx_packets;
2142 rx_bytes += priv->
cpu[
i]->stats.rx_bytes;
2143 tx_packets += priv->
cpu[
i]->stats.tx_packets;
2144 tx_bytes += priv->
cpu[
i]->stats.tx_bytes;
2153 return &priv->
stats;
2163 static int tile_net_change_mtu(
struct net_device *dev,
int new_mtu)
2165 PDEBUG(
"tile_net_change_mtu()\n");
2168 if ((new_mtu < 68) || (new_mtu > 1500))
2189 static int tile_net_set_mac_address(
struct net_device *dev,
void *
p)
2193 if (!is_valid_ether_addr(addr->
sa_data))
2208 static int tile_net_get_mac(
struct net_device *dev)
2212 char hv_dev_name[32];
2221 len =
strlen(hv_dev_name);
2224 hv_dev_name[len] = hv_dev_name[len - 1];
2225 hv_dev_name[len - 1] =
'/';
2233 PDEBUG(
"hv_dev_open(%s) returned %d %p\n",
2254 PDEBUG(
"hv_dev_pread(NETIO_PARAM_MAC) returned %d\n", ret);
2263 eth_hw_addr_random(dev);
2270 #ifdef CONFIG_NET_POLL_CONTROLLER
2276 static void tile_net_netpoll(
struct net_device *dev)
2280 tile_net_handle_ingress_interrupt(priv->
intr_id, dev);
2287 .ndo_open = tile_net_open,
2288 .ndo_stop = tile_net_stop,
2289 .ndo_start_xmit = tile_net_tx,
2290 .ndo_do_ioctl = tile_net_ioctl,
2291 .ndo_get_stats = tile_net_get_stats,
2292 .ndo_change_mtu = tile_net_change_mtu,
2293 .ndo_tx_timeout = tile_net_tx_timeout,
2294 .ndo_set_mac_address = tile_net_set_mac_address,
2295 #ifdef CONFIG_NET_POLL_CONTROLLER
2296 .ndo_poll_controller = tile_net_netpoll,
2307 static void tile_net_setup(
struct net_device *dev)
2309 PDEBUG(
"tile_net_setup()\n");
2360 dev =
alloc_netdev(
sizeof(*priv), name, tile_net_setup);
2362 pr_err(
"alloc_netdev(%s) failed\n", name);
2366 priv = netdev_priv(dev);
2370 memset(priv, 0,
sizeof(*priv));
2390 pr_err(
"register_netdev %s failed %d\n", dev->
name, ret);
2397 ret = tile_net_get_mac(dev);
2415 static void tile_net_cleanup(
void)
2420 if (tile_net_devs[i]) {
2435 static int tile_net_init_module(
void)
2437 pr_info(
"Tilera Network Driver\n");
2439 tile_net_devs[0] = tile_net_dev_init(
"xgbe0");
2440 tile_net_devs[1] = tile_net_dev_init(
"xgbe1");
2441 tile_net_devs[2] = tile_net_dev_init(
"gbe0");
2442 tile_net_devs[3] = tile_net_dev_init(
"gbe1");
2462 static int __init network_cpus_setup(
char *
str)
2466 pr_warning(
"network_cpus=%s: malformed cpu list\n",
2481 pr_info(
"Linux network CPUs: %s\n", buf);
2482 network_cpus_used =
true;
2488 __setup(
"network_cpus=", network_cpus_setup);