Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
sch_generic.c
Go to the documentation of this file.
1 /*
2  * net/sched/sch_generic.c Generic packet scheduler routines.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version
7  * 2 of the License, or (at your option) any later version.
8  *
9  * Authors: Alexey Kuznetsov, <[email protected]>
10  * Jamal Hadi Salim, <[email protected]> 990601
11  * - Ingress support
12  */
13 
14 #include <linux/bitops.h>
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/kernel.h>
18 #include <linux/sched.h>
19 #include <linux/string.h>
20 #include <linux/errno.h>
21 #include <linux/netdevice.h>
22 #include <linux/skbuff.h>
23 #include <linux/rtnetlink.h>
24 #include <linux/init.h>
25 #include <linux/rcupdate.h>
26 #include <linux/list.h>
27 #include <linux/slab.h>
28 #include <net/pkt_sched.h>
29 #include <net/dst.h>
30 
31 /* Main transmission queue. */
32 
33 /* Modifications to data participating in scheduling must be protected with
34  * qdisc_lock(qdisc) spinlock.
35  *
36  * The idea is the following:
37  * - enqueue, dequeue are serialized via qdisc root lock
38  * - ingress filtering is also serialized via qdisc root lock
39  * - updates to tree and tree walking are only done under the rtnl mutex.
40  */
41 
42 static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
43 {
44  skb_dst_force(skb);
45  q->gso_skb = skb;
46  q->qstats.requeues++;
47  q->q.qlen++; /* it's still part of the queue */
49 
50  return 0;
51 }
52 
53 static inline struct sk_buff *dequeue_skb(struct Qdisc *q)
54 {
55  struct sk_buff *skb = q->gso_skb;
56 
57  if (unlikely(skb)) {
58  struct net_device *dev = qdisc_dev(q);
59  struct netdev_queue *txq;
60 
61  /* check the reason of requeuing without tx lock first */
62  txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
63  if (!netif_xmit_frozen_or_stopped(txq)) {
64  q->gso_skb = NULL;
65  q->q.qlen--;
66  } else
67  skb = NULL;
68  } else {
69  skb = q->dequeue(q);
70  }
71 
72  return skb;
73 }
74 
75 static inline int handle_dev_cpu_collision(struct sk_buff *skb,
76  struct netdev_queue *dev_queue,
77  struct Qdisc *q)
78 {
79  int ret;
80 
81  if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) {
82  /*
83  * Same CPU holding the lock. It may be a transient
84  * configuration error, when hard_start_xmit() recurses. We
85  * detect it by checking xmit owner and drop the packet when
86  * deadloop is detected. Return OK to try the next skb.
87  */
88  kfree_skb(skb);
89  net_warn_ratelimited("Dead loop on netdevice %s, fix it urgently!\n",
90  dev_queue->dev->name);
91  ret = qdisc_qlen(q);
92  } else {
93  /*
94  * Another cpu is holding lock, requeue & delay xmits for
95  * some time.
96  */
98  ret = dev_requeue_skb(skb, q);
99  }
100 
101  return ret;
102 }
103 
104 /*
105  * Transmit one skb, and handle the return status as required. Holding the
106  * __QDISC_STATE_RUNNING bit guarantees that only one CPU can execute this
107  * function.
108  *
109  * Returns to the caller:
110  * 0 - queue is empty or throttled.
111  * >0 - queue is not empty.
112  */
113 int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
114  struct net_device *dev, struct netdev_queue *txq,
115  spinlock_t *root_lock)
116 {
117  int ret = NETDEV_TX_BUSY;
118 
119  /* And release qdisc */
120  spin_unlock(root_lock);
121 
122  HARD_TX_LOCK(dev, txq, smp_processor_id());
123  if (!netif_xmit_frozen_or_stopped(txq))
124  ret = dev_hard_start_xmit(skb, dev, txq);
125 
126  HARD_TX_UNLOCK(dev, txq);
127 
128  spin_lock(root_lock);
129 
130  if (dev_xmit_complete(ret)) {
131  /* Driver sent out skb successfully or skb was consumed */
132  ret = qdisc_qlen(q);
133  } else if (ret == NETDEV_TX_LOCKED) {
134  /* Driver try lock failed */
135  ret = handle_dev_cpu_collision(skb, txq, q);
136  } else {
137  /* Driver returned NETDEV_TX_BUSY - requeue skb */
138  if (unlikely(ret != NETDEV_TX_BUSY))
139  net_warn_ratelimited("BUG %s code %d qlen %d\n",
140  dev->name, ret, q->q.qlen);
141 
142  ret = dev_requeue_skb(skb, q);
143  }
144 
145  if (ret && netif_xmit_frozen_or_stopped(txq))
146  ret = 0;
147 
148  return ret;
149 }
150 
151 /*
152  * NOTE: Called under qdisc_lock(q) with locally disabled BH.
153  *
154  * __QDISC_STATE_RUNNING guarantees only one CPU can process
155  * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
156  * this queue.
157  *
158  * netif_tx_lock serializes accesses to device driver.
159  *
160  * qdisc_lock(q) and netif_tx_lock are mutually exclusive,
161  * if one is grabbed, another must be free.
162  *
163  * Note, that this procedure can be called by a watchdog timer
164  *
165  * Returns to the caller:
166  * 0 - queue is empty or throttled.
167  * >0 - queue is not empty.
168  *
169  */
170 static inline int qdisc_restart(struct Qdisc *q)
171 {
172  struct netdev_queue *txq;
173  struct net_device *dev;
174  spinlock_t *root_lock;
175  struct sk_buff *skb;
176 
177  /* Dequeue packet */
178  skb = dequeue_skb(q);
179  if (unlikely(!skb))
180  return 0;
181  WARN_ON_ONCE(skb_dst_is_noref(skb));
182  root_lock = qdisc_lock(q);
183  dev = qdisc_dev(q);
184  txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
185 
186  return sch_direct_xmit(skb, q, dev, txq, root_lock);
187 }
188 
189 void __qdisc_run(struct Qdisc *q)
190 {
191  int quota = weight_p;
192 
193  while (qdisc_restart(q)) {
194  /*
195  * Ordered by possible occurrence: Postpone processing if
196  * 1. we've exceeded packet quota
197  * 2. another process needs the CPU;
198  */
199  if (--quota <= 0 || need_resched()) {
200  __netif_schedule(q);
201  break;
202  }
203  }
204 
205  qdisc_run_end(q);
206 }
207 
208 unsigned long dev_trans_start(struct net_device *dev)
209 {
210  unsigned long val, res = dev->trans_start;
211  unsigned int i;
212 
213  for (i = 0; i < dev->num_tx_queues; i++) {
214  val = netdev_get_tx_queue(dev, i)->trans_start;
215  if (val && time_after(val, res))
216  res = val;
217  }
218  dev->trans_start = res;
219  return res;
220 }
222 
223 static void dev_watchdog(unsigned long arg)
224 {
225  struct net_device *dev = (struct net_device *)arg;
226 
227  netif_tx_lock(dev);
228  if (!qdisc_tx_is_noop(dev)) {
229  if (netif_device_present(dev) &&
230  netif_running(dev) &&
231  netif_carrier_ok(dev)) {
232  int some_queue_timedout = 0;
233  unsigned int i;
234  unsigned long trans_start;
235 
236  for (i = 0; i < dev->num_tx_queues; i++) {
237  struct netdev_queue *txq;
238 
239  txq = netdev_get_tx_queue(dev, i);
240  /*
241  * old device drivers set dev->trans_start
242  */
243  trans_start = txq->trans_start ? : dev->trans_start;
244  if (netif_xmit_stopped(txq) &&
245  time_after(jiffies, (trans_start +
246  dev->watchdog_timeo))) {
247  some_queue_timedout = 1;
248  txq->trans_timeout++;
249  break;
250  }
251  }
252 
253  if (some_queue_timedout) {
254  WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
255  dev->name, netdev_drivername(dev), i);
256  dev->netdev_ops->ndo_tx_timeout(dev);
257  }
258  if (!mod_timer(&dev->watchdog_timer,
259  round_jiffies(jiffies +
260  dev->watchdog_timeo)))
261  dev_hold(dev);
262  }
263  }
264  netif_tx_unlock(dev);
265 
266  dev_put(dev);
267 }
268 
270 {
271  if (dev->netdev_ops->ndo_tx_timeout) {
272  if (dev->watchdog_timeo <= 0)
273  dev->watchdog_timeo = 5*HZ;
274  if (!mod_timer(&dev->watchdog_timer,
276  dev_hold(dev);
277  }
278 }
279 
280 static void dev_watchdog_up(struct net_device *dev)
281 {
283 }
284 
285 static void dev_watchdog_down(struct net_device *dev)
286 {
287  netif_tx_lock_bh(dev);
288  if (del_timer(&dev->watchdog_timer))
289  dev_put(dev);
290  netif_tx_unlock_bh(dev);
291 }
292 
299 void netif_carrier_on(struct net_device *dev)
300 {
302  if (dev->reg_state == NETREG_UNINITIALIZED)
303  return;
305  if (netif_running(dev))
307  }
308 }
310 
317 void netif_carrier_off(struct net_device *dev)
318 {
320  if (dev->reg_state == NETREG_UNINITIALIZED)
321  return;
323  }
324 }
326 
327 /* "NOOP" scheduler: the best scheduler, recommended for all interfaces
328  under all circumstances. It is difficult to invent anything faster or
329  cheaper.
330  */
331 
332 static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
333 {
334  kfree_skb(skb);
335  return NET_XMIT_CN;
336 }
337 
338 static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
339 {
340  return NULL;
341 }
342 
344  .id = "noop",
345  .priv_size = 0,
346  .enqueue = noop_enqueue,
347  .dequeue = noop_dequeue,
348  .peek = noop_dequeue,
349  .owner = THIS_MODULE,
350 };
351 
352 static struct netdev_queue noop_netdev_queue = {
353  .qdisc = &noop_qdisc,
354  .qdisc_sleeping = &noop_qdisc,
355 };
356 
357 struct Qdisc noop_qdisc = {
358  .enqueue = noop_enqueue,
359  .dequeue = noop_dequeue,
360  .flags = TCQ_F_BUILTIN,
361  .ops = &noop_qdisc_ops,
362  .list = LIST_HEAD_INIT(noop_qdisc.list),
363  .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
364  .dev_queue = &noop_netdev_queue,
365  .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
366 };
367 EXPORT_SYMBOL(noop_qdisc);
368 
369 static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
370  .id = "noqueue",
371  .priv_size = 0,
372  .enqueue = noop_enqueue,
373  .dequeue = noop_dequeue,
374  .peek = noop_dequeue,
375  .owner = THIS_MODULE,
376 };
377 
378 static struct Qdisc noqueue_qdisc;
379 static struct netdev_queue noqueue_netdev_queue = {
380  .qdisc = &noqueue_qdisc,
381  .qdisc_sleeping = &noqueue_qdisc,
382 };
383 
384 static struct Qdisc noqueue_qdisc = {
385  .enqueue = NULL,
386  .dequeue = noop_dequeue,
387  .flags = TCQ_F_BUILTIN,
388  .ops = &noqueue_qdisc_ops,
389  .list = LIST_HEAD_INIT(noqueue_qdisc.list),
390  .q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock),
391  .dev_queue = &noqueue_netdev_queue,
392  .busylock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.busylock),
393 };
394 
395 
396 static const u8 prio2band[TC_PRIO_MAX + 1] = {
397  1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1
398 };
399 
400 /* 3-band FIFO queue: old style, but should be a bit faster than
401  generic prio+fifo combination.
402  */
403 
404 #define PFIFO_FAST_BANDS 3
405 
406 /*
407  * Private data for a pfifo_fast scheduler containing:
408  * - queues for the three band
409  * - bitmap indicating which of the bands contain skbs
410  */
414 };
415 
416 /*
417  * Convert a bitmap to the first band number where an skb is queued, where:
418  * bitmap=0 means there are no skbs on any band.
419  * bitmap=1 means there is an skb on band 0.
420  * bitmap=7 means there are skbs on all 3 bands, etc.
421  */
422 static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
423 
424 static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
425  int band)
426 {
427  return priv->q + band;
428 }
429 
430 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc)
431 {
432  if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) {
433  int band = prio2band[skb->priority & TC_PRIO_MAX];
434  struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
435  struct sk_buff_head *list = band2list(priv, band);
436 
437  priv->bitmap |= (1 << band);
438  qdisc->q.qlen++;
439  return __qdisc_enqueue_tail(skb, qdisc, list);
440  }
441 
442  return qdisc_drop(skb, qdisc);
443 }
444 
445 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
446 {
447  struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
448  int band = bitmap2band[priv->bitmap];
449 
450  if (likely(band >= 0)) {
451  struct sk_buff_head *list = band2list(priv, band);
452  struct sk_buff *skb = __qdisc_dequeue_head(qdisc, list);
453 
454  qdisc->q.qlen--;
455  if (skb_queue_empty(list))
456  priv->bitmap &= ~(1 << band);
457 
458  return skb;
459  }
460 
461  return NULL;
462 }
463 
464 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
465 {
466  struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
467  int band = bitmap2band[priv->bitmap];
468 
469  if (band >= 0) {
470  struct sk_buff_head *list = band2list(priv, band);
471 
472  return skb_peek(list);
473  }
474 
475  return NULL;
476 }
477 
478 static void pfifo_fast_reset(struct Qdisc *qdisc)
479 {
480  int prio;
481  struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
482 
483  for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
484  __qdisc_reset_queue(qdisc, band2list(priv, prio));
485 
486  priv->bitmap = 0;
487  qdisc->qstats.backlog = 0;
488  qdisc->q.qlen = 0;
489 }
490 
491 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
492 {
493  struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
494 
495  memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
496  if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
497  goto nla_put_failure;
498  return skb->len;
499 
500 nla_put_failure:
501  return -1;
502 }
503 
504 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
505 {
506  int prio;
507  struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
508 
509  for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
510  skb_queue_head_init(band2list(priv, prio));
511 
512  /* Can by-pass the queue discipline */
513  qdisc->flags |= TCQ_F_CAN_BYPASS;
514  return 0;
515 }
516 
517 struct Qdisc_ops pfifo_fast_ops __read_mostly = {
518  .id = "pfifo_fast",
519  .priv_size = sizeof(struct pfifo_fast_priv),
520  .enqueue = pfifo_fast_enqueue,
521  .dequeue = pfifo_fast_dequeue,
522  .peek = pfifo_fast_peek,
523  .init = pfifo_fast_init,
524  .reset = pfifo_fast_reset,
525  .dump = pfifo_fast_dump,
526  .owner = THIS_MODULE,
527 };
528 EXPORT_SYMBOL(pfifo_fast_ops);
529 
530 static struct lock_class_key qdisc_tx_busylock;
531 
532 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
533  struct Qdisc_ops *ops)
534 {
535  void *p;
536  struct Qdisc *sch;
537  unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size;
538  int err = -ENOBUFS;
539  struct net_device *dev = dev_queue->dev;
540 
541  p = kzalloc_node(size, GFP_KERNEL,
542  netdev_queue_numa_node_read(dev_queue));
543 
544  if (!p)
545  goto errout;
546  sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
547  /* if we got non aligned memory, ask more and do alignment ourself */
548  if (sch != p) {
549  kfree(p);
550  p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL,
551  netdev_queue_numa_node_read(dev_queue));
552  if (!p)
553  goto errout;
554  sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
555  sch->padded = (char *) sch - (char *) p;
556  }
557  INIT_LIST_HEAD(&sch->list);
558  skb_queue_head_init(&sch->q);
559 
560  spin_lock_init(&sch->busylock);
562  dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
563 
564  sch->ops = ops;
565  sch->enqueue = ops->enqueue;
566  sch->dequeue = ops->dequeue;
567  sch->dev_queue = dev_queue;
568  dev_hold(dev);
569  atomic_set(&sch->refcnt, 1);
570 
571  return sch;
572 errout:
573  return ERR_PTR(err);
574 }
575 
576 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
577  struct Qdisc_ops *ops, unsigned int parentid)
578 {
579  struct Qdisc *sch;
580 
581  sch = qdisc_alloc(dev_queue, ops);
582  if (IS_ERR(sch))
583  goto errout;
584  sch->parent = parentid;
585 
586  if (!ops->init || ops->init(sch, NULL) == 0)
587  return sch;
588 
589  qdisc_destroy(sch);
590 errout:
591  return NULL;
592 }
594 
595 /* Under qdisc_lock(qdisc) and BH! */
596 
597 void qdisc_reset(struct Qdisc *qdisc)
598 {
599  const struct Qdisc_ops *ops = qdisc->ops;
600 
601  if (ops->reset)
602  ops->reset(qdisc);
603 
604  if (qdisc->gso_skb) {
605  kfree_skb(qdisc->gso_skb);
606  qdisc->gso_skb = NULL;
607  qdisc->q.qlen = 0;
608  }
609 }
611 
612 static void qdisc_rcu_free(struct rcu_head *head)
613 {
614  struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
615 
616  kfree((char *) qdisc - qdisc->padded);
617 }
618 
619 void qdisc_destroy(struct Qdisc *qdisc)
620 {
621  const struct Qdisc_ops *ops = qdisc->ops;
622 
623  if (qdisc->flags & TCQ_F_BUILTIN ||
624  !atomic_dec_and_test(&qdisc->refcnt))
625  return;
626 
627 #ifdef CONFIG_NET_SCHED
628  qdisc_list_del(qdisc);
629 
631 #endif
632  gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
633  if (ops->reset)
634  ops->reset(qdisc);
635  if (ops->destroy)
636  ops->destroy(qdisc);
637 
638  module_put(ops->owner);
639  dev_put(qdisc_dev(qdisc));
640 
641  kfree_skb(qdisc->gso_skb);
642  /*
643  * gen_estimator est_timer() might access qdisc->q.lock,
644  * wait a RCU grace period before freeing qdisc.
645  */
646  call_rcu(&qdisc->rcu_head, qdisc_rcu_free);
647 }
649 
650 /* Attach toplevel qdisc to device queue. */
651 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
652  struct Qdisc *qdisc)
653 {
654  struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
655  spinlock_t *root_lock;
656 
657  root_lock = qdisc_lock(oqdisc);
658  spin_lock_bh(root_lock);
659 
660  /* Prune old scheduler */
661  if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
662  qdisc_reset(oqdisc);
663 
664  /* ... and graft new one */
665  if (qdisc == NULL)
666  qdisc = &noop_qdisc;
667  dev_queue->qdisc_sleeping = qdisc;
668  rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
669 
670  spin_unlock_bh(root_lock);
671 
672  return oqdisc;
673 }
675 
676 static void attach_one_default_qdisc(struct net_device *dev,
677  struct netdev_queue *dev_queue,
678  void *_unused)
679 {
680  struct Qdisc *qdisc = &noqueue_qdisc;
681 
682  if (dev->tx_queue_len) {
683  qdisc = qdisc_create_dflt(dev_queue,
684  &pfifo_fast_ops, TC_H_ROOT);
685  if (!qdisc) {
686  netdev_info(dev, "activation failed\n");
687  return;
688  }
689  }
690  dev_queue->qdisc_sleeping = qdisc;
691 }
692 
693 static void attach_default_qdiscs(struct net_device *dev)
694 {
695  struct netdev_queue *txq;
696  struct Qdisc *qdisc;
697 
698  txq = netdev_get_tx_queue(dev, 0);
699 
700  if (!netif_is_multiqueue(dev) || dev->tx_queue_len == 0) {
701  netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
702  dev->qdisc = txq->qdisc_sleeping;
703  atomic_inc(&dev->qdisc->refcnt);
704  } else {
705  qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT);
706  if (qdisc) {
707  qdisc->ops->attach(qdisc);
708  dev->qdisc = qdisc;
709  }
710  }
711 }
712 
713 static void transition_one_qdisc(struct net_device *dev,
714  struct netdev_queue *dev_queue,
715  void *_need_watchdog)
716 {
717  struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
718  int *need_watchdog_p = _need_watchdog;
719 
720  if (!(new_qdisc->flags & TCQ_F_BUILTIN))
722 
723  rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
724  if (need_watchdog_p && new_qdisc != &noqueue_qdisc) {
725  dev_queue->trans_start = 0;
726  *need_watchdog_p = 1;
727  }
728 }
729 
730 void dev_activate(struct net_device *dev)
731 {
732  int need_watchdog;
733 
734  /* No queueing discipline is attached to device;
735  create default one i.e. pfifo_fast for devices,
736  which need queueing and noqueue_qdisc for
737  virtual interfaces
738  */
739 
740  if (dev->qdisc == &noop_qdisc)
741  attach_default_qdiscs(dev);
742 
743  if (!netif_carrier_ok(dev))
744  /* Delay activation until next carrier-on event */
745  return;
746 
747  need_watchdog = 0;
748  netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
749  if (dev_ingress_queue(dev))
750  transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
751 
752  if (need_watchdog) {
753  dev->trans_start = jiffies;
754  dev_watchdog_up(dev);
755  }
756 }
758 
759 static void dev_deactivate_queue(struct net_device *dev,
760  struct netdev_queue *dev_queue,
761  void *_qdisc_default)
762 {
763  struct Qdisc *qdisc_default = _qdisc_default;
764  struct Qdisc *qdisc;
765 
766  qdisc = dev_queue->qdisc;
767  if (qdisc) {
768  spin_lock_bh(qdisc_lock(qdisc));
769 
770  if (!(qdisc->flags & TCQ_F_BUILTIN))
772 
773  rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
774  qdisc_reset(qdisc);
775 
776  spin_unlock_bh(qdisc_lock(qdisc));
777  }
778 }
779 
780 static bool some_qdisc_is_busy(struct net_device *dev)
781 {
782  unsigned int i;
783 
784  for (i = 0; i < dev->num_tx_queues; i++) {
785  struct netdev_queue *dev_queue;
786  spinlock_t *root_lock;
787  struct Qdisc *q;
788  int val;
789 
790  dev_queue = netdev_get_tx_queue(dev, i);
791  q = dev_queue->qdisc_sleeping;
792  root_lock = qdisc_lock(q);
793 
794  spin_lock_bh(root_lock);
795 
796  val = (qdisc_is_running(q) ||
798 
799  spin_unlock_bh(root_lock);
800 
801  if (val)
802  return true;
803  }
804  return false;
805 }
806 
814 void dev_deactivate_many(struct list_head *head)
815 {
816  struct net_device *dev;
817  bool sync_needed = false;
818 
819  list_for_each_entry(dev, head, unreg_list) {
820  netdev_for_each_tx_queue(dev, dev_deactivate_queue,
821  &noop_qdisc);
822  if (dev_ingress_queue(dev))
823  dev_deactivate_queue(dev, dev_ingress_queue(dev),
824  &noop_qdisc);
825 
826  dev_watchdog_down(dev);
827  sync_needed |= !dev->dismantle;
828  }
829 
830  /* Wait for outstanding qdisc-less dev_queue_xmit calls.
831  * This is avoided if all devices are in dismantle phase :
832  * Caller will call synchronize_net() for us
833  */
834  if (sync_needed)
835  synchronize_net();
836 
837  /* Wait for outstanding qdisc_run calls. */
838  list_for_each_entry(dev, head, unreg_list)
839  while (some_qdisc_is_busy(dev))
840  yield();
841 }
842 
843 void dev_deactivate(struct net_device *dev)
844 {
845  LIST_HEAD(single);
846 
847  list_add(&dev->unreg_list, &single);
848  dev_deactivate_many(&single);
849  list_del(&single);
850 }
852 
853 static void dev_init_scheduler_queue(struct net_device *dev,
854  struct netdev_queue *dev_queue,
855  void *_qdisc)
856 {
857  struct Qdisc *qdisc = _qdisc;
858 
859  dev_queue->qdisc = qdisc;
860  dev_queue->qdisc_sleeping = qdisc;
861 }
862 
864 {
865  dev->qdisc = &noop_qdisc;
866  netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
867  if (dev_ingress_queue(dev))
868  dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
869 
870  setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
871 }
872 
873 static void shutdown_scheduler_queue(struct net_device *dev,
874  struct netdev_queue *dev_queue,
875  void *_qdisc_default)
876 {
877  struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
878  struct Qdisc *qdisc_default = _qdisc_default;
879 
880  if (qdisc) {
881  rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
882  dev_queue->qdisc_sleeping = qdisc_default;
883 
884  qdisc_destroy(qdisc);
885  }
886 }
887 
888 void dev_shutdown(struct net_device *dev)
889 {
890  netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
891  if (dev_ingress_queue(dev))
892  shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
893  qdisc_destroy(dev->qdisc);
894  dev->qdisc = &noop_qdisc;
895 
896  WARN_ON(timer_pending(&dev->watchdog_timer));
897 }