Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
skbuff.h
Go to the documentation of this file.
1 /*
2  * Definitions for the 'struct sk_buff' memory handlers.
3  *
4  * Authors:
5  * Alan Cox, <[email protected]>
6  * Florian La Roche, <[email protected]>
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License
10  * as published by the Free Software Foundation; either version
11  * 2 of the License, or (at your option) any later version.
12  */
13 
14 #ifndef _LINUX_SKBUFF_H
15 #define _LINUX_SKBUFF_H
16 
17 #include <linux/kernel.h>
18 #include <linux/kmemcheck.h>
19 #include <linux/compiler.h>
20 #include <linux/time.h>
21 #include <linux/bug.h>
22 #include <linux/cache.h>
23 
24 #include <linux/atomic.h>
25 #include <asm/types.h>
26 #include <linux/spinlock.h>
27 #include <linux/net.h>
28 #include <linux/textsearch.h>
29 #include <net/checksum.h>
30 #include <linux/rcupdate.h>
31 #include <linux/dmaengine.h>
32 #include <linux/hrtimer.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/netdev_features.h>
35 
36 /* Don't change this without changing skb_csum_unnecessary! */
37 #define CHECKSUM_NONE 0
38 #define CHECKSUM_UNNECESSARY 1
39 #define CHECKSUM_COMPLETE 2
40 #define CHECKSUM_PARTIAL 3
41 
42 #define SKB_DATA_ALIGN(X) (((X) + (SMP_CACHE_BYTES - 1)) & \
43  ~(SMP_CACHE_BYTES - 1))
44 #define SKB_WITH_OVERHEAD(X) \
45  ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
46 #define SKB_MAX_ORDER(X, ORDER) \
47  SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
48 #define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0))
49 #define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2))
50 
51 /* return minimum truesize of one skb containing X bytes of data */
52 #define SKB_TRUESIZE(X) ((X) + \
53  SKB_DATA_ALIGN(sizeof(struct sk_buff)) + \
54  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
55 
56 /* A. Checksumming of received packets by device.
57  *
58  * NONE: device failed to checksum this packet.
59  * skb->csum is undefined.
60  *
61  * UNNECESSARY: device parsed packet and wouldbe verified checksum.
62  * skb->csum is undefined.
63  * It is bad option, but, unfortunately, many of vendors do this.
64  * Apparently with secret goal to sell you new device, when you
65  * will add new protocol to your host. F.e. IPv6. 8)
66  *
67  * COMPLETE: the most generic way. Device supplied checksum of _all_
68  * the packet as seen by netif_rx in skb->csum.
69  * NOTE: Even if device supports only some protocols, but
70  * is able to produce some skb->csum, it MUST use COMPLETE,
71  * not UNNECESSARY.
72  *
73  * PARTIAL: identical to the case for output below. This may occur
74  * on a packet received directly from another Linux OS, e.g.,
75  * a virtualised Linux kernel on the same host. The packet can
76  * be treated in the same way as UNNECESSARY except that on
77  * output (i.e., forwarding) the checksum must be filled in
78  * by the OS or the hardware.
79  *
80  * B. Checksumming on output.
81  *
82  * NONE: skb is checksummed by protocol or csum is not required.
83  *
84  * PARTIAL: device is required to csum packet as seen by hard_start_xmit
85  * from skb->csum_start to the end and to record the checksum
86  * at skb->csum_start + skb->csum_offset.
87  *
88  * Device must show its capabilities in dev->features, set
89  * at device setup time.
90  * NETIF_F_HW_CSUM - it is clever device, it is able to checksum
91  * everything.
92  * NETIF_F_IP_CSUM - device is dumb. It is able to csum only
93  * TCP/UDP over IPv4. Sigh. Vendors like this
94  * way by an unknown reason. Though, see comment above
95  * about CHECKSUM_UNNECESSARY. 8)
96  * NETIF_F_IPV6_CSUM about as dumb as the last one but does IPv6 instead.
97  *
98  * UNNECESSARY: device will do per protocol specific csum. Protocol drivers
99  * that do not want net to perform the checksum calculation should use
100  * this flag in their outgoing skbs.
101  * NETIF_F_FCOE_CRC this indicates the device can do FCoE FC CRC
102  * offload. Correspondingly, the FCoE protocol driver
103  * stack should use CHECKSUM_UNNECESSARY.
104  *
105  * Any questions? No questions, good. --ANK
106  */
107 
108 struct net_device;
109 struct scatterlist;
110 struct pipe_inode_info;
111 
112 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
113 struct nf_conntrack {
114  atomic_t use;
115 };
116 #endif
117 
118 #ifdef CONFIG_BRIDGE_NETFILTER
119 struct nf_bridge_info {
120  atomic_t use;
121  unsigned int mask;
122  struct net_device *physindev;
123  struct net_device *physoutdev;
124  unsigned long data[32 / sizeof(unsigned long)];
125 };
126 #endif
127 
128 struct sk_buff_head {
129  /* These two members must be first. */
130  struct sk_buff *next;
131  struct sk_buff *prev;
132 
135 };
136 
137 struct sk_buff;
138 
139 /* To allow 64K frame to be packed as single skb without frag_list we
140  * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
141  * buffers which do not start on a page boundary.
142  *
143  * Since GRO uses frags we allocate at least 16 regardless of page
144  * size.
145  */
146 #if (65536/PAGE_SIZE + 1) < 16
147 #define MAX_SKB_FRAGS 16UL
148 #else
149 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
150 #endif
151 
153 
155  struct {
156  struct page *p;
157  } page;
158 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
160  __u32 size;
161 #else
164 #endif
165 };
166 
167 static inline unsigned int skb_frag_size(const skb_frag_t *frag)
168 {
169  return frag->size;
170 }
171 
172 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
173 {
174  frag->size = size;
175 }
176 
177 static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
178 {
179  frag->size += delta;
180 }
181 
182 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
183 {
184  frag->size -= delta;
185 }
186 
187 #define HAVE_HW_TIME_STAMP
188 
215 };
216 
217 /* Definitions for tx_flags in struct skb_shared_info */
218 enum {
219  /* generate hardware time stamp */
220  SKBTX_HW_TSTAMP = 1 << 0,
221 
222  /* generate software time stamp */
223  SKBTX_SW_TSTAMP = 1 << 1,
224 
225  /* device driver is going to provide hardware time stamp */
227 
228  /* device driver supports TX zero-copy buffers */
230 
231  /* generate wifi status information (where possible) */
233 };
234 
235 /*
236  * The callback notifies userspace to release buffers when skb DMA is done in
237  * lower device, the skb last reference should be 0 when calling this.
238  * The ctx field is used to track device context.
239  * The desc field is used to track userspace buffer index.
240  */
241 struct ubuf_info {
242  void (*callback)(struct ubuf_info *);
243  void *ctx;
244  unsigned long desc;
245 };
246 
247 /* This data is invariant across clones and lives at
248  * the end of the header data, ie. at skb->end.
249  */
251  unsigned char nr_frags;
253  unsigned short gso_size;
254  /* Warning: this field is not always filled in (UFO)! */
255  unsigned short gso_segs;
256  unsigned short gso_type;
260 
261  /*
262  * Warning : all fields before dataref are cleared in __alloc_skb()
263  */
265 
266  /* Intermediate layers must ensure that destructor_arg
267  * remains valid until skb destructor */
269 
270  /* must be last field, see pskb_expand_head() */
272 };
273 
274 /* We divide dataref into two halves. The higher 16 bits hold references
275  * to the payload part of skb->data. The lower 16 bits hold references to
276  * the entire skb->data. A clone of a headerless skb holds the length of
277  * the header in skb->hdr_len.
278  *
279  * All users must obey the rule that the skb->data reference count must be
280  * greater than or equal to the payload reference count.
281  *
282  * Holding a reference to the payload part means that the user does not
283  * care about modifications to the header part of skb->data.
284  */
285 #define SKB_DATAREF_SHIFT 16
286 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
287 
288 
289 enum {
293 };
294 
295 enum {
296  SKB_GSO_TCPV4 = 1 << 0,
297  SKB_GSO_UDP = 1 << 1,
298 
299  /* This indicates the skb is from an untrusted source. */
300  SKB_GSO_DODGY = 1 << 2,
301 
302  /* This indicates the tcp segment has CWR set. */
303  SKB_GSO_TCP_ECN = 1 << 3,
304 
305  SKB_GSO_TCPV6 = 1 << 4,
306 
307  SKB_GSO_FCOE = 1 << 5,
308 };
309 
310 #if BITS_PER_LONG > 32
311 #define NET_SKBUFF_DATA_USES_OFFSET 1
312 #endif
313 
314 #ifdef NET_SKBUFF_DATA_USES_OFFSET
315 typedef unsigned int sk_buff_data_t;
316 #else
317 typedef unsigned char *sk_buff_data_t;
318 #endif
319 
320 #if defined(CONFIG_NF_DEFRAG_IPV4) || defined(CONFIG_NF_DEFRAG_IPV4_MODULE) || \
321  defined(CONFIG_NF_DEFRAG_IPV6) || defined(CONFIG_NF_DEFRAG_IPV6_MODULE)
322 #define NET_SKBUFF_NF_DEFRAG_NEEDED 1
323 #endif
324 
388 struct sk_buff {
389  /* These two members must be first. */
390  struct sk_buff *next;
391  struct sk_buff *prev;
392 
394 
395  struct sock *sk;
396  struct net_device *dev;
397 
398  /*
399  * This is the control buffer. It is free to use for every
400  * layer. Please put your private variables there. If you
401  * want to keep them across layers you have to do a skb_clone()
402  * first. This is owned by whoever has the skb queued ATM.
403  */
404  char cb[48] __aligned(8);
405 
406  unsigned long _skb_refdst;
407 #ifdef CONFIG_XFRM
408  struct sec_path *sp;
409 #endif
410  unsigned int len,
411  data_len;
413  hdr_len;
414  union {
416  struct {
419  };
420  };
424  cloned:1,
425  ip_summed:2,
426  nohdr:1,
427  nfctinfo:3;
429  fclone:2,
430  ipvs_property:1,
431  peeked:1,
432  nf_trace:1;
435 
436  void (*destructor)(struct sk_buff *skb);
437 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
438  struct nf_conntrack *nfct;
439 #endif
440 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
441  struct sk_buff *nfct_reasm;
442 #endif
443 #ifdef CONFIG_BRIDGE_NETFILTER
444  struct nf_bridge_info *nf_bridge;
445 #endif
446 
447  int skb_iif;
448 
450 
452 
453 #ifdef CONFIG_NET_SCHED
454  __u16 tc_index; /* traffic control index */
455 #ifdef CONFIG_NET_CLS_ACT
456  __u16 tc_verd; /* traffic control verdict */
457 #endif
458 #endif
459 
462 #ifdef CONFIG_IPV6_NDISC_NODETYPE
463  __u8 ndisc_nodetype:2;
464 #endif
472  /* 8/10 bit hole (depending on ndisc_nodetype presence) */
474 
475 #ifdef CONFIG_NET_DMA
476  dma_cookie_t dma_cookie;
477 #endif
478 #ifdef CONFIG_NETWORK_SECMARK
479  __u32 secmark;
480 #endif
481  union {
485  };
486 
490  /* These elements must be at the end, see alloc_skb() for details. */
493  unsigned char *head,
494  *data;
495  unsigned int truesize;
497 };
498 
499 #ifdef __KERNEL__
500 /*
501  * Handling routines are only of interest to the kernel
502  */
503 #include <linux/slab.h>
504 
505 
506 #define SKB_ALLOC_FCLONE 0x01
507 #define SKB_ALLOC_RX 0x02
508 
509 /* Returns true if the skb was allocated from PFMEMALLOC reserves */
510 static inline bool skb_pfmemalloc(const struct sk_buff *skb)
511 {
512  return unlikely(skb->pfmemalloc);
513 }
514 
515 /*
516  * skb might have a dst pointer attached, refcounted or not.
517  * _skb_refdst low order bit is set if refcount was _not_ taken
518  */
519 #define SKB_DST_NOREF 1UL
520 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
521 
528 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
529 {
530  /* If refdst was not refcounted, check we still are in a
531  * rcu_read_lock section
532  */
533  WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
534  !rcu_read_lock_held() &&
535  !rcu_read_lock_bh_held());
536  return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
537 }
538 
547 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
548 {
549  skb->_skb_refdst = (unsigned long)dst;
550 }
551 
552 extern void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst);
553 
558 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
559 {
560  return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
561 }
562 
563 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
564 {
565  return (struct rtable *)skb_dst(skb);
566 }
567 
568 extern void kfree_skb(struct sk_buff *skb);
569 extern void consume_skb(struct sk_buff *skb);
570 extern void __kfree_skb(struct sk_buff *skb);
571 extern struct kmem_cache *skbuff_head_cache;
572 
573 extern void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
574 extern bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
575  bool *fragstolen, int *delta_truesize);
576 
577 extern struct sk_buff *__alloc_skb(unsigned int size,
578  gfp_t priority, int flags, int node);
579 extern struct sk_buff *build_skb(void *data, unsigned int frag_size);
580 static inline struct sk_buff *alloc_skb(unsigned int size,
581  gfp_t priority)
582 {
583  return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
584 }
585 
586 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
587  gfp_t priority)
588 {
589  return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
590 }
591 
592 extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
593 extern int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
594 extern struct sk_buff *skb_clone(struct sk_buff *skb,
595  gfp_t priority);
596 extern struct sk_buff *skb_copy(const struct sk_buff *skb,
597  gfp_t priority);
598 extern struct sk_buff *__pskb_copy(struct sk_buff *skb,
599  int headroom, gfp_t gfp_mask);
600 
601 extern int pskb_expand_head(struct sk_buff *skb,
602  int nhead, int ntail,
603  gfp_t gfp_mask);
604 extern struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
605  unsigned int headroom);
606 extern struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
607  int newheadroom, int newtailroom,
608  gfp_t priority);
609 extern int skb_to_sgvec(struct sk_buff *skb,
610  struct scatterlist *sg, int offset,
611  int len);
612 extern int skb_cow_data(struct sk_buff *skb, int tailbits,
613  struct sk_buff **trailer);
614 extern int skb_pad(struct sk_buff *skb, int pad);
615 #define dev_kfree_skb(a) consume_skb(a)
616 
617 extern int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
618  int getfrag(void *from, char *to, int offset,
619  int len,int odd, struct sk_buff *skb),
620  void *from, int length);
621 
622 struct skb_seq_state {
623  __u32 lower_offset;
624  __u32 upper_offset;
625  __u32 frag_idx;
626  __u32 stepped_offset;
627  struct sk_buff *root_skb;
628  struct sk_buff *cur_skb;
629  __u8 *frag_data;
630 };
631 
632 extern void skb_prepare_seq_read(struct sk_buff *skb,
633  unsigned int from, unsigned int to,
634  struct skb_seq_state *st);
635 extern unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
636  struct skb_seq_state *st);
637 extern void skb_abort_seq_read(struct skb_seq_state *st);
638 
639 extern unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
640  unsigned int to, struct ts_config *config,
641  struct ts_state *state);
642 
643 extern void __skb_get_rxhash(struct sk_buff *skb);
644 static inline __u32 skb_get_rxhash(struct sk_buff *skb)
645 {
646  if (!skb->rxhash)
647  __skb_get_rxhash(skb);
648 
649  return skb->rxhash;
650 }
651 
652 #ifdef NET_SKBUFF_DATA_USES_OFFSET
653 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
654 {
655  return skb->head + skb->end;
656 }
657 
658 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
659 {
660  return skb->end;
661 }
662 #else
663 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
664 {
665  return skb->end;
666 }
667 
668 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
669 {
670  return skb->end - skb->head;
671 }
672 #endif
673 
674 /* Internal */
675 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
676 
677 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
678 {
679  return &skb_shinfo(skb)->hwtstamps;
680 }
681 
688 static inline int skb_queue_empty(const struct sk_buff_head *list)
689 {
690  return list->next == (struct sk_buff *)list;
691 }
692 
700 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
701  const struct sk_buff *skb)
702 {
703  return skb->next == (struct sk_buff *)list;
704 }
705 
713 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
714  const struct sk_buff *skb)
715 {
716  return skb->prev == (struct sk_buff *)list;
717 }
718 
727 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
728  const struct sk_buff *skb)
729 {
730  /* This BUG_ON may seem severe, but if we just return then we
731  * are going to dereference garbage.
732  */
733  BUG_ON(skb_queue_is_last(list, skb));
734  return skb->next;
735 }
736 
745 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
746  const struct sk_buff *skb)
747 {
748  /* This BUG_ON may seem severe, but if we just return then we
749  * are going to dereference garbage.
750  */
751  BUG_ON(skb_queue_is_first(list, skb));
752  return skb->prev;
753 }
754 
762 static inline struct sk_buff *skb_get(struct sk_buff *skb)
763 {
764  atomic_inc(&skb->users);
765  return skb;
766 }
767 
768 /*
769  * If users == 1, we are the only owner and are can avoid redundant
770  * atomic change.
771  */
772 
781 static inline int skb_cloned(const struct sk_buff *skb)
782 {
783  return skb->cloned &&
784  (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
785 }
786 
794 static inline int skb_header_cloned(const struct sk_buff *skb)
795 {
796  int dataref;
797 
798  if (!skb->cloned)
799  return 0;
800 
801  dataref = atomic_read(&skb_shinfo(skb)->dataref);
802  dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
803  return dataref != 1;
804 }
805 
814 static inline void skb_header_release(struct sk_buff *skb)
815 {
816  BUG_ON(skb->nohdr);
817  skb->nohdr = 1;
818  atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
819 }
820 
828 static inline int skb_shared(const struct sk_buff *skb)
829 {
830  return atomic_read(&skb->users) != 1;
831 }
832 
846 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
847 {
848  might_sleep_if(pri & __GFP_WAIT);
849  if (skb_shared(skb)) {
850  struct sk_buff *nskb = skb_clone(skb, pri);
851 
852  if (likely(nskb))
853  consume_skb(skb);
854  else
855  kfree_skb(skb);
856  skb = nskb;
857  }
858  return skb;
859 }
860 
861 /*
862  * Copy shared buffers into a new sk_buff. We effectively do COW on
863  * packets to handle cases where we have a local reader and forward
864  * and a couple of other messy ones. The normal one is tcpdumping
865  * a packet thats being forwarded.
866  */
867 
881 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
882  gfp_t pri)
883 {
884  might_sleep_if(pri & __GFP_WAIT);
885  if (skb_cloned(skb)) {
886  struct sk_buff *nskb = skb_copy(skb, pri);
887  kfree_skb(skb); /* Free our shared copy */
888  skb = nskb;
889  }
890  return skb;
891 }
892 
906 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
907 {
908  struct sk_buff *skb = list_->next;
909 
910  if (skb == (struct sk_buff *)list_)
911  skb = NULL;
912  return skb;
913 }
914 
924 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
925  const struct sk_buff_head *list_)
926 {
927  struct sk_buff *next = skb->next;
928 
929  if (next == (struct sk_buff *)list_)
930  next = NULL;
931  return next;
932 }
933 
947 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
948 {
949  struct sk_buff *skb = list_->prev;
950 
951  if (skb == (struct sk_buff *)list_)
952  skb = NULL;
953  return skb;
954 
955 }
956 
963 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
964 {
965  return list_->qlen;
966 }
967 
978 static inline void __skb_queue_head_init(struct sk_buff_head *list)
979 {
980  list->prev = list->next = (struct sk_buff *)list;
981  list->qlen = 0;
982 }
983 
984 /*
985  * This function creates a split out lock class for each invocation;
986  * this is needed for now since a whole lot of users of the skb-queue
987  * infrastructure in drivers have different locking usage (in hardirq)
988  * than the networking core (in softirq only). In the long run either the
989  * network layer or drivers should need annotation to consolidate the
990  * main types of usage into 3 classes.
991  */
992 static inline void skb_queue_head_init(struct sk_buff_head *list)
993 {
994  spin_lock_init(&list->lock);
995  __skb_queue_head_init(list);
996 }
997 
998 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
999  struct lock_class_key *class)
1000 {
1001  skb_queue_head_init(list);
1002  lockdep_set_class(&list->lock, class);
1003 }
1004 
1005 /*
1006  * Insert an sk_buff on a list.
1007  *
1008  * The "__skb_xxxx()" functions are the non-atomic ones that
1009  * can only be called with interrupts disabled.
1010  */
1011 extern void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list);
1012 static inline void __skb_insert(struct sk_buff *newsk,
1013  struct sk_buff *prev, struct sk_buff *next,
1014  struct sk_buff_head *list)
1015 {
1016  newsk->next = next;
1017  newsk->prev = prev;
1018  next->prev = prev->next = newsk;
1019  list->qlen++;
1020 }
1021 
1022 static inline void __skb_queue_splice(const struct sk_buff_head *list,
1023  struct sk_buff *prev,
1024  struct sk_buff *next)
1025 {
1026  struct sk_buff *first = list->next;
1027  struct sk_buff *last = list->prev;
1028 
1029  first->prev = prev;
1030  prev->next = first;
1031 
1032  last->next = next;
1033  next->prev = last;
1034 }
1035 
1041 static inline void skb_queue_splice(const struct sk_buff_head *list,
1042  struct sk_buff_head *head)
1043 {
1044  if (!skb_queue_empty(list)) {
1045  __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1046  head->qlen += list->qlen;
1047  }
1048 }
1049 
1057 static inline void skb_queue_splice_init(struct sk_buff_head *list,
1058  struct sk_buff_head *head)
1059 {
1060  if (!skb_queue_empty(list)) {
1061  __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1062  head->qlen += list->qlen;
1063  __skb_queue_head_init(list);
1064  }
1065 }
1066 
1072 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1073  struct sk_buff_head *head)
1074 {
1075  if (!skb_queue_empty(list)) {
1076  __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1077  head->qlen += list->qlen;
1078  }
1079 }
1080 
1089 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1090  struct sk_buff_head *head)
1091 {
1092  if (!skb_queue_empty(list)) {
1093  __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1094  head->qlen += list->qlen;
1095  __skb_queue_head_init(list);
1096  }
1097 }
1098 
1110 static inline void __skb_queue_after(struct sk_buff_head *list,
1111  struct sk_buff *prev,
1112  struct sk_buff *newsk)
1113 {
1114  __skb_insert(newsk, prev, prev->next, list);
1115 }
1116 
1117 extern void skb_append(struct sk_buff *old, struct sk_buff *newsk,
1118  struct sk_buff_head *list);
1119 
1120 static inline void __skb_queue_before(struct sk_buff_head *list,
1121  struct sk_buff *next,
1122  struct sk_buff *newsk)
1123 {
1124  __skb_insert(newsk, next->prev, next, list);
1125 }
1126 
1137 extern void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
1138 static inline void __skb_queue_head(struct sk_buff_head *list,
1139  struct sk_buff *newsk)
1140 {
1141  __skb_queue_after(list, (struct sk_buff *)list, newsk);
1142 }
1143 
1154 extern void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
1155 static inline void __skb_queue_tail(struct sk_buff_head *list,
1156  struct sk_buff *newsk)
1157 {
1158  __skb_queue_before(list, (struct sk_buff *)list, newsk);
1159 }
1160 
1161 /*
1162  * remove sk_buff from list. _Must_ be called atomically, and with
1163  * the list known..
1164  */
1165 extern void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
1166 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1167 {
1168  struct sk_buff *next, *prev;
1169 
1170  list->qlen--;
1171  next = skb->next;
1172  prev = skb->prev;
1173  skb->next = skb->prev = NULL;
1174  next->prev = prev;
1175  prev->next = next;
1176 }
1177 
1186 extern struct sk_buff *skb_dequeue(struct sk_buff_head *list);
1187 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
1188 {
1189  struct sk_buff *skb = skb_peek(list);
1190  if (skb)
1191  __skb_unlink(skb, list);
1192  return skb;
1193 }
1194 
1203 extern struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
1204 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
1205 {
1206  struct sk_buff *skb = skb_peek_tail(list);
1207  if (skb)
1208  __skb_unlink(skb, list);
1209  return skb;
1210 }
1211 
1212 
1213 static inline bool skb_is_nonlinear(const struct sk_buff *skb)
1214 {
1215  return skb->data_len;
1216 }
1217 
1218 static inline unsigned int skb_headlen(const struct sk_buff *skb)
1219 {
1220  return skb->len - skb->data_len;
1221 }
1222 
1223 static inline int skb_pagelen(const struct sk_buff *skb)
1224 {
1225  int i, len = 0;
1226 
1227  for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--)
1228  len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
1229  return len + skb_headlen(skb);
1230 }
1231 
1245 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
1246  struct page *page, int off, int size)
1247 {
1248  skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1249 
1250  /*
1251  * Propagate page->pfmemalloc to the skb if we can. The problem is
1252  * that not all callers have unique ownership of the page. If
1253  * pfmemalloc is set, we check the mapping as a mapping implies
1254  * page->index is set (index and pfmemalloc share space).
1255  * If it's a valid mapping, we cannot use page->pfmemalloc but we
1256  * do not lose pfmemalloc information as the pages would not be
1257  * allocated using __GFP_MEMALLOC.
1258  */
1259  if (page->pfmemalloc && !page->mapping)
1260  skb->pfmemalloc = true;
1261  frag->page.p = page;
1262  frag->page_offset = off;
1263  skb_frag_size_set(frag, size);
1264 }
1265 
1280 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
1281  struct page *page, int off, int size)
1282 {
1283  __skb_fill_page_desc(skb, i, page, off, size);
1284  skb_shinfo(skb)->nr_frags = i + 1;
1285 }
1286 
1287 extern void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page,
1288  int off, int size, unsigned int truesize);
1289 
1290 #define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags)
1291 #define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb))
1292 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
1293 
1294 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1295 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1296 {
1297  return skb->head + skb->tail;
1298 }
1299 
1300 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1301 {
1302  skb->tail = skb->data - skb->head;
1303 }
1304 
1305 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1306 {
1307  skb_reset_tail_pointer(skb);
1308  skb->tail += offset;
1309 }
1310 #else /* NET_SKBUFF_DATA_USES_OFFSET */
1311 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1312 {
1313  return skb->tail;
1314 }
1315 
1316 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1317 {
1318  skb->tail = skb->data;
1319 }
1320 
1321 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1322 {
1323  skb->tail = skb->data + offset;
1324 }
1325 
1326 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
1327 
1328 /*
1329  * Add data to an sk_buff
1330  */
1331 extern unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
1332 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
1333 {
1334  unsigned char *tmp = skb_tail_pointer(skb);
1335  SKB_LINEAR_ASSERT(skb);
1336  skb->tail += len;
1337  skb->len += len;
1338  return tmp;
1339 }
1340 
1341 extern unsigned char *skb_push(struct sk_buff *skb, unsigned int len);
1342 static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
1343 {
1344  skb->data -= len;
1345  skb->len += len;
1346  return skb->data;
1347 }
1348 
1349 extern unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
1350 static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
1351 {
1352  skb->len -= len;
1353  BUG_ON(skb->len < skb->data_len);
1354  return skb->data += len;
1355 }
1356 
1357 static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
1358 {
1359  return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1360 }
1361 
1362 extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
1363 
1364 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
1365 {
1366  if (len > skb_headlen(skb) &&
1367  !__pskb_pull_tail(skb, len - skb_headlen(skb)))
1368  return NULL;
1369  skb->len -= len;
1370  return skb->data += len;
1371 }
1372 
1373 static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len)
1374 {
1375  return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
1376 }
1377 
1378 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
1379 {
1380  if (likely(len <= skb_headlen(skb)))
1381  return 1;
1382  if (unlikely(len > skb->len))
1383  return 0;
1384  return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
1385 }
1386 
1393 static inline unsigned int skb_headroom(const struct sk_buff *skb)
1394 {
1395  return skb->data - skb->head;
1396 }
1397 
1404 static inline int skb_tailroom(const struct sk_buff *skb)
1405 {
1406  return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
1407 }
1408 
1416 static inline int skb_availroom(const struct sk_buff *skb)
1417 {
1418  return skb_is_nonlinear(skb) ? 0 : skb->avail_size - skb->len;
1419 }
1420 
1429 static inline void skb_reserve(struct sk_buff *skb, int len)
1430 {
1431  skb->data += len;
1432  skb->tail += len;
1433 }
1434 
1435 static inline void skb_reset_mac_len(struct sk_buff *skb)
1436 {
1437  skb->mac_len = skb->network_header - skb->mac_header;
1438 }
1439 
1440 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1441 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1442 {
1443  return skb->head + skb->transport_header;
1444 }
1445 
1446 static inline void skb_reset_transport_header(struct sk_buff *skb)
1447 {
1448  skb->transport_header = skb->data - skb->head;
1449 }
1450 
1451 static inline void skb_set_transport_header(struct sk_buff *skb,
1452  const int offset)
1453 {
1454  skb_reset_transport_header(skb);
1455  skb->transport_header += offset;
1456 }
1457 
1458 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1459 {
1460  return skb->head + skb->network_header;
1461 }
1462 
1463 static inline void skb_reset_network_header(struct sk_buff *skb)
1464 {
1465  skb->network_header = skb->data - skb->head;
1466 }
1467 
1468 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1469 {
1470  skb_reset_network_header(skb);
1471  skb->network_header += offset;
1472 }
1473 
1474 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1475 {
1476  return skb->head + skb->mac_header;
1477 }
1478 
1479 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1480 {
1481  return skb->mac_header != ~0U;
1482 }
1483 
1484 static inline void skb_reset_mac_header(struct sk_buff *skb)
1485 {
1486  skb->mac_header = skb->data - skb->head;
1487 }
1488 
1489 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1490 {
1491  skb_reset_mac_header(skb);
1492  skb->mac_header += offset;
1493 }
1494 
1495 #else /* NET_SKBUFF_DATA_USES_OFFSET */
1496 
1497 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1498 {
1499  return skb->transport_header;
1500 }
1501 
1502 static inline void skb_reset_transport_header(struct sk_buff *skb)
1503 {
1504  skb->transport_header = skb->data;
1505 }
1506 
1507 static inline void skb_set_transport_header(struct sk_buff *skb,
1508  const int offset)
1509 {
1510  skb->transport_header = skb->data + offset;
1511 }
1512 
1513 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1514 {
1515  return skb->network_header;
1516 }
1517 
1518 static inline void skb_reset_network_header(struct sk_buff *skb)
1519 {
1520  skb->network_header = skb->data;
1521 }
1522 
1523 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1524 {
1525  skb->network_header = skb->data + offset;
1526 }
1527 
1528 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1529 {
1530  return skb->mac_header;
1531 }
1532 
1533 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1534 {
1535  return skb->mac_header != NULL;
1536 }
1537 
1538 static inline void skb_reset_mac_header(struct sk_buff *skb)
1539 {
1540  skb->mac_header = skb->data;
1541 }
1542 
1543 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1544 {
1545  skb->mac_header = skb->data + offset;
1546 }
1547 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
1548 
1549 static inline void skb_mac_header_rebuild(struct sk_buff *skb)
1550 {
1551  if (skb_mac_header_was_set(skb)) {
1552  const unsigned char *old_mac = skb_mac_header(skb);
1553 
1554  skb_set_mac_header(skb, -skb->mac_len);
1555  memmove(skb_mac_header(skb), old_mac, skb->mac_len);
1556  }
1557 }
1558 
1559 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
1560 {
1561  return skb->csum_start - skb_headroom(skb);
1562 }
1563 
1564 static inline int skb_transport_offset(const struct sk_buff *skb)
1565 {
1566  return skb_transport_header(skb) - skb->data;
1567 }
1568 
1569 static inline u32 skb_network_header_len(const struct sk_buff *skb)
1570 {
1571  return skb->transport_header - skb->network_header;
1572 }
1573 
1574 static inline int skb_network_offset(const struct sk_buff *skb)
1575 {
1576  return skb_network_header(skb) - skb->data;
1577 }
1578 
1579 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
1580 {
1581  return pskb_may_pull(skb, skb_network_offset(skb) + len);
1582 }
1583 
1584 /*
1585  * CPUs often take a performance hit when accessing unaligned memory
1586  * locations. The actual performance hit varies, it can be small if the
1587  * hardware handles it or large if we have to take an exception and fix it
1588  * in software.
1589  *
1590  * Since an ethernet header is 14 bytes network drivers often end up with
1591  * the IP header at an unaligned offset. The IP header can be aligned by
1592  * shifting the start of the packet by 2 bytes. Drivers should do this
1593  * with:
1594  *
1595  * skb_reserve(skb, NET_IP_ALIGN);
1596  *
1597  * The downside to this alignment of the IP header is that the DMA is now
1598  * unaligned. On some architectures the cost of an unaligned DMA is high
1599  * and this cost outweighs the gains made by aligning the IP header.
1600  *
1601  * Since this trade off varies between architectures, we allow NET_IP_ALIGN
1602  * to be overridden.
1603  */
1604 #ifndef NET_IP_ALIGN
1605 #define NET_IP_ALIGN 2
1606 #endif
1607 
1608 /*
1609  * The networking layer reserves some headroom in skb data (via
1610  * dev_alloc_skb). This is used to avoid having to reallocate skb data when
1611  * the header has to grow. In the default case, if the header has to grow
1612  * 32 bytes or less we avoid the reallocation.
1613  *
1614  * Unfortunately this headroom changes the DMA alignment of the resulting
1615  * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
1616  * on some architectures. An architecture can override this value,
1617  * perhaps setting it to a cacheline in size (since that will maintain
1618  * cacheline alignment of the DMA). It must be a power of 2.
1619  *
1620  * Various parts of the networking layer expect at least 32 bytes of
1621  * headroom, you should not reduce this.
1622  *
1623  * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
1624  * to reduce average number of cache lines per packet.
1625  * get_rps_cpus() for example only access one 64 bytes aligned block :
1626  * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
1627  */
1628 #ifndef NET_SKB_PAD
1629 #define NET_SKB_PAD max(32, L1_CACHE_BYTES)
1630 #endif
1631 
1632 extern int ___pskb_trim(struct sk_buff *skb, unsigned int len);
1633 
1634 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
1635 {
1636  if (unlikely(skb_is_nonlinear(skb))) {
1637  WARN_ON(1);
1638  return;
1639  }
1640  skb->len = len;
1641  skb_set_tail_pointer(skb, len);
1642 }
1643 
1644 extern void skb_trim(struct sk_buff *skb, unsigned int len);
1645 
1646 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
1647 {
1648  if (skb->data_len)
1649  return ___pskb_trim(skb, len);
1650  __skb_trim(skb, len);
1651  return 0;
1652 }
1653 
1654 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
1655 {
1656  return (len < skb->len) ? __pskb_trim(skb, len) : 0;
1657 }
1658 
1668 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
1669 {
1670  int err = pskb_trim(skb, len);
1671  BUG_ON(err);
1672 }
1673 
1682 static inline void skb_orphan(struct sk_buff *skb)
1683 {
1684  if (skb->destructor)
1685  skb->destructor(skb);
1686  skb->destructor = NULL;
1687  skb->sk = NULL;
1688 }
1689 
1699 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
1700 {
1701  if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY)))
1702  return 0;
1703  return skb_copy_ubufs(skb, gfp_mask);
1704 }
1705 
1714 extern void skb_queue_purge(struct sk_buff_head *list);
1715 static inline void __skb_queue_purge(struct sk_buff_head *list)
1716 {
1717  struct sk_buff *skb;
1718  while ((skb = __skb_dequeue(list)) != NULL)
1719  kfree_skb(skb);
1720 }
1721 
1722 extern void *netdev_alloc_frag(unsigned int fragsz);
1723 
1724 extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
1725  unsigned int length,
1726  gfp_t gfp_mask);
1727 
1741 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
1742  unsigned int length)
1743 {
1744  return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
1745 }
1746 
1747 /* legacy helper around __netdev_alloc_skb() */
1748 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
1749  gfp_t gfp_mask)
1750 {
1751  return __netdev_alloc_skb(NULL, length, gfp_mask);
1752 }
1753 
1754 /* legacy helper around netdev_alloc_skb() */
1755 static inline struct sk_buff *dev_alloc_skb(unsigned int length)
1756 {
1757  return netdev_alloc_skb(NULL, length);
1758 }
1759 
1760 
1761 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
1762  unsigned int length, gfp_t gfp)
1763 {
1764  struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
1765 
1766  if (NET_IP_ALIGN && skb)
1767  skb_reserve(skb, NET_IP_ALIGN);
1768  return skb;
1769 }
1770 
1771 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
1772  unsigned int length)
1773 {
1774  return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
1775 }
1776 
1777 /*
1778  * __skb_alloc_page - allocate pages for ps-rx on a skb and preserve pfmemalloc data
1779  * @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX
1780  * @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used
1781  * @order: size of the allocation
1782  *
1783  * Allocate a new page.
1784  *
1785  * %NULL is returned if there is no free memory.
1786 */
1787 static inline struct page *__skb_alloc_pages(gfp_t gfp_mask,
1788  struct sk_buff *skb,
1789  unsigned int order)
1790 {
1791  struct page *page;
1792 
1793  gfp_mask |= __GFP_COLD;
1794 
1795  if (!(gfp_mask & __GFP_NOMEMALLOC))
1796  gfp_mask |= __GFP_MEMALLOC;
1797 
1798  page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
1799  if (skb && page && page->pfmemalloc)
1800  skb->pfmemalloc = true;
1801 
1802  return page;
1803 }
1804 
1814 static inline struct page *__skb_alloc_page(gfp_t gfp_mask,
1815  struct sk_buff *skb)
1816 {
1817  return __skb_alloc_pages(gfp_mask, skb, 0);
1818 }
1819 
1825 static inline void skb_propagate_pfmemalloc(struct page *page,
1826  struct sk_buff *skb)
1827 {
1828  if (page && page->pfmemalloc)
1829  skb->pfmemalloc = true;
1830 }
1831 
1838 static inline struct page *skb_frag_page(const skb_frag_t *frag)
1839 {
1840  return frag->page.p;
1841 }
1842 
1849 static inline void __skb_frag_ref(skb_frag_t *frag)
1850 {
1851  get_page(skb_frag_page(frag));
1852 }
1853 
1861 static inline void skb_frag_ref(struct sk_buff *skb, int f)
1862 {
1863  __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
1864 }
1865 
1872 static inline void __skb_frag_unref(skb_frag_t *frag)
1873 {
1874  put_page(skb_frag_page(frag));
1875 }
1876 
1884 static inline void skb_frag_unref(struct sk_buff *skb, int f)
1885 {
1886  __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
1887 }
1888 
1896 static inline void *skb_frag_address(const skb_frag_t *frag)
1897 {
1898  return page_address(skb_frag_page(frag)) + frag->page_offset;
1899 }
1900 
1908 static inline void *skb_frag_address_safe(const skb_frag_t *frag)
1909 {
1910  void *ptr = page_address(skb_frag_page(frag));
1911  if (unlikely(!ptr))
1912  return NULL;
1913 
1914  return ptr + frag->page_offset;
1915 }
1916 
1924 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
1925 {
1926  frag->page.p = page;
1927 }
1928 
1937 static inline void skb_frag_set_page(struct sk_buff *skb, int f,
1938  struct page *page)
1939 {
1940  __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
1941 }
1942 
1954 static inline dma_addr_t skb_frag_dma_map(struct device *dev,
1955  const skb_frag_t *frag,
1956  size_t offset, size_t size,
1957  enum dma_data_direction dir)
1958 {
1959  return dma_map_page(dev, skb_frag_page(frag),
1960  frag->page_offset + offset, size, dir);
1961 }
1962 
1963 static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
1964  gfp_t gfp_mask)
1965 {
1966  return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
1967 }
1968 
1977 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
1978 {
1979  return !skb_header_cloned(skb) &&
1980  skb_headroom(skb) + len <= skb->hdr_len;
1981 }
1982 
1983 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
1984  int cloned)
1985 {
1986  int delta = 0;
1987 
1988  if (headroom > skb_headroom(skb))
1989  delta = headroom - skb_headroom(skb);
1990 
1991  if (delta || cloned)
1992  return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
1993  GFP_ATOMIC);
1994  return 0;
1995 }
1996 
2009 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
2010 {
2011  return __skb_cow(skb, headroom, skb_cloned(skb));
2012 }
2013 
2024 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
2025 {
2026  return __skb_cow(skb, headroom, skb_header_cloned(skb));
2027 }
2028 
2040 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
2041 {
2042  unsigned int size = skb->len;
2043  if (likely(size >= len))
2044  return 0;
2045  return skb_pad(skb, len - size);
2046 }
2047 
2048 static inline int skb_add_data(struct sk_buff *skb,
2049  char __user *from, int copy)
2050 {
2051  const int off = skb->len;
2052 
2053  if (skb->ip_summed == CHECKSUM_NONE) {
2054  int err = 0;
2055  __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy),
2056  copy, 0, &err);
2057  if (!err) {
2058  skb->csum = csum_block_add(skb->csum, csum, off);
2059  return 0;
2060  }
2061  } else if (!copy_from_user(skb_put(skb, copy), from, copy))
2062  return 0;
2063 
2064  __skb_trim(skb, off);
2065  return -EFAULT;
2066 }
2067 
2068 static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
2069  const struct page *page, int off)
2070 {
2071  if (i) {
2072  const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
2073 
2074  return page == skb_frag_page(frag) &&
2075  off == frag->page_offset + skb_frag_size(frag);
2076  }
2077  return false;
2078 }
2079 
2080 static inline int __skb_linearize(struct sk_buff *skb)
2081 {
2082  return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
2083 }
2084 
2092 static inline int skb_linearize(struct sk_buff *skb)
2093 {
2094  return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
2095 }
2096 
2104 static inline int skb_linearize_cow(struct sk_buff *skb)
2105 {
2106  return skb_is_nonlinear(skb) || skb_cloned(skb) ?
2107  __skb_linearize(skb) : 0;
2108 }
2109 
2121 static inline void skb_postpull_rcsum(struct sk_buff *skb,
2122  const void *start, unsigned int len)
2123 {
2124  if (skb->ip_summed == CHECKSUM_COMPLETE)
2125  skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
2126 }
2127 
2128 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
2129 
2139 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
2140 {
2141  if (likely(len >= skb->len))
2142  return 0;
2143  if (skb->ip_summed == CHECKSUM_COMPLETE)
2144  skb->ip_summed = CHECKSUM_NONE;
2145  return __pskb_trim(skb, len);
2146 }
2147 
2148 #define skb_queue_walk(queue, skb) \
2149  for (skb = (queue)->next; \
2150  skb != (struct sk_buff *)(queue); \
2151  skb = skb->next)
2152 
2153 #define skb_queue_walk_safe(queue, skb, tmp) \
2154  for (skb = (queue)->next, tmp = skb->next; \
2155  skb != (struct sk_buff *)(queue); \
2156  skb = tmp, tmp = skb->next)
2157 
2158 #define skb_queue_walk_from(queue, skb) \
2159  for (; skb != (struct sk_buff *)(queue); \
2160  skb = skb->next)
2161 
2162 #define skb_queue_walk_from_safe(queue, skb, tmp) \
2163  for (tmp = skb->next; \
2164  skb != (struct sk_buff *)(queue); \
2165  skb = tmp, tmp = skb->next)
2166 
2167 #define skb_queue_reverse_walk(queue, skb) \
2168  for (skb = (queue)->prev; \
2169  skb != (struct sk_buff *)(queue); \
2170  skb = skb->prev)
2171 
2172 #define skb_queue_reverse_walk_safe(queue, skb, tmp) \
2173  for (skb = (queue)->prev, tmp = skb->prev; \
2174  skb != (struct sk_buff *)(queue); \
2175  skb = tmp, tmp = skb->prev)
2176 
2177 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp) \
2178  for (tmp = skb->prev; \
2179  skb != (struct sk_buff *)(queue); \
2180  skb = tmp, tmp = skb->prev)
2181 
2182 static inline bool skb_has_frag_list(const struct sk_buff *skb)
2183 {
2184  return skb_shinfo(skb)->frag_list != NULL;
2185 }
2186 
2187 static inline void skb_frag_list_init(struct sk_buff *skb)
2188 {
2189  skb_shinfo(skb)->frag_list = NULL;
2190 }
2191 
2192 static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag)
2193 {
2194  frag->next = skb_shinfo(skb)->frag_list;
2195  skb_shinfo(skb)->frag_list = frag;
2196 }
2197 
2198 #define skb_walk_frags(skb, iter) \
2199  for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
2200 
2201 extern struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
2202  int *peeked, int *off, int *err);
2203 extern struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags,
2204  int noblock, int *err);
2205 extern unsigned int datagram_poll(struct file *file, struct socket *sock,
2206  struct poll_table_struct *wait);
2207 extern int skb_copy_datagram_iovec(const struct sk_buff *from,
2208  int offset, struct iovec *to,
2209  int size);
2210 extern int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb,
2211  int hlen,
2212  struct iovec *iov);
2213 extern int skb_copy_datagram_from_iovec(struct sk_buff *skb,
2214  int offset,
2215  const struct iovec *from,
2216  int from_offset,
2217  int len);
2218 extern int skb_copy_datagram_const_iovec(const struct sk_buff *from,
2219  int offset,
2220  const struct iovec *to,
2221  int to_offset,
2222  int size);
2223 extern void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
2224 extern void skb_free_datagram_locked(struct sock *sk,
2225  struct sk_buff *skb);
2226 extern int skb_kill_datagram(struct sock *sk, struct sk_buff *skb,
2227  unsigned int flags);
2228 extern __wsum skb_checksum(const struct sk_buff *skb, int offset,
2229  int len, __wsum csum);
2230 extern int skb_copy_bits(const struct sk_buff *skb, int offset,
2231  void *to, int len);
2232 extern int skb_store_bits(struct sk_buff *skb, int offset,
2233  const void *from, int len);
2234 extern __wsum skb_copy_and_csum_bits(const struct sk_buff *skb,
2235  int offset, u8 *to, int len,
2236  __wsum csum);
2237 extern int skb_splice_bits(struct sk_buff *skb,
2238  unsigned int offset,
2239  struct pipe_inode_info *pipe,
2240  unsigned int len,
2241  unsigned int flags);
2242 extern void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
2243 extern void skb_split(struct sk_buff *skb,
2244  struct sk_buff *skb1, const u32 len);
2245 extern int skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
2246  int shiftlen);
2247 
2248 extern struct sk_buff *skb_segment(struct sk_buff *skb,
2250 
2251 static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
2252  int len, void *buffer)
2253 {
2254  int hlen = skb_headlen(skb);
2255 
2256  if (hlen - offset >= len)
2257  return skb->data + offset;
2258 
2259  if (skb_copy_bits(skb, offset, buffer, len) < 0)
2260  return NULL;
2261 
2262  return buffer;
2263 }
2264 
2265 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
2266  void *to,
2267  const unsigned int len)
2268 {
2269  memcpy(to, skb->data, len);
2270 }
2271 
2272 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
2273  const int offset, void *to,
2274  const unsigned int len)
2275 {
2276  memcpy(to, skb->data + offset, len);
2277 }
2278 
2279 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
2280  const void *from,
2281  const unsigned int len)
2282 {
2283  memcpy(skb->data, from, len);
2284 }
2285 
2286 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
2287  const int offset,
2288  const void *from,
2289  const unsigned int len)
2290 {
2291  memcpy(skb->data + offset, from, len);
2292 }
2293 
2294 extern void skb_init(void);
2295 
2296 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
2297 {
2298  return skb->tstamp;
2299 }
2300 
2310 static inline void skb_get_timestamp(const struct sk_buff *skb,
2311  struct timeval *stamp)
2312 {
2313  *stamp = ktime_to_timeval(skb->tstamp);
2314 }
2315 
2316 static inline void skb_get_timestampns(const struct sk_buff *skb,
2317  struct timespec *stamp)
2318 {
2319  *stamp = ktime_to_timespec(skb->tstamp);
2320 }
2321 
2322 static inline void __net_timestamp(struct sk_buff *skb)
2323 {
2324  skb->tstamp = ktime_get_real();
2325 }
2326 
2327 static inline ktime_t net_timedelta(ktime_t t)
2328 {
2329  return ktime_sub(ktime_get_real(), t);
2330 }
2331 
2332 static inline ktime_t net_invalid_timestamp(void)
2333 {
2334  return ktime_set(0, 0);
2335 }
2336 
2337 extern void skb_timestamping_init(void);
2338 
2339 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
2340 
2341 extern void skb_clone_tx_timestamp(struct sk_buff *skb);
2342 extern bool skb_defer_rx_timestamp(struct sk_buff *skb);
2343 
2344 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
2345 
2346 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
2347 {
2348 }
2349 
2350 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
2351 {
2352  return false;
2353 }
2354 
2355 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
2356 
2369 void skb_complete_tx_timestamp(struct sk_buff *skb,
2370  struct skb_shared_hwtstamps *hwtstamps);
2371 
2383 extern void skb_tstamp_tx(struct sk_buff *orig_skb,
2384  struct skb_shared_hwtstamps *hwtstamps);
2385 
2386 static inline void sw_tx_timestamp(struct sk_buff *skb)
2387 {
2388  if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP &&
2389  !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
2390  skb_tstamp_tx(skb, NULL);
2391 }
2392 
2401 static inline void skb_tx_timestamp(struct sk_buff *skb)
2402 {
2404  sw_tx_timestamp(skb);
2405 }
2406 
2414 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
2415 
2416 extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
2417 extern __sum16 __skb_checksum_complete(struct sk_buff *skb);
2418 
2419 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
2420 {
2421  return skb->ip_summed & CHECKSUM_UNNECESSARY;
2422 }
2423 
2440 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
2441 {
2442  return skb_csum_unnecessary(skb) ?
2443  0 : __skb_checksum_complete(skb);
2444 }
2445 
2446 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2447 extern void nf_conntrack_destroy(struct nf_conntrack *nfct);
2448 static inline void nf_conntrack_put(struct nf_conntrack *nfct)
2449 {
2450  if (nfct && atomic_dec_and_test(&nfct->use))
2451  nf_conntrack_destroy(nfct);
2452 }
2453 static inline void nf_conntrack_get(struct nf_conntrack *nfct)
2454 {
2455  if (nfct)
2456  atomic_inc(&nfct->use);
2457 }
2458 #endif
2459 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2460 static inline void nf_conntrack_get_reasm(struct sk_buff *skb)
2461 {
2462  if (skb)
2463  atomic_inc(&skb->users);
2464 }
2465 static inline void nf_conntrack_put_reasm(struct sk_buff *skb)
2466 {
2467  if (skb)
2468  kfree_skb(skb);
2469 }
2470 #endif
2471 #ifdef CONFIG_BRIDGE_NETFILTER
2472 static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge)
2473 {
2474  if (nf_bridge && atomic_dec_and_test(&nf_bridge->use))
2475  kfree(nf_bridge);
2476 }
2477 static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
2478 {
2479  if (nf_bridge)
2480  atomic_inc(&nf_bridge->use);
2481 }
2482 #endif /* CONFIG_BRIDGE_NETFILTER */
2483 static inline void nf_reset(struct sk_buff *skb)
2484 {
2485 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2486  nf_conntrack_put(skb->nfct);
2487  skb->nfct = NULL;
2488 #endif
2489 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2490  nf_conntrack_put_reasm(skb->nfct_reasm);
2491  skb->nfct_reasm = NULL;
2492 #endif
2493 #ifdef CONFIG_BRIDGE_NETFILTER
2494  nf_bridge_put(skb->nf_bridge);
2495  skb->nf_bridge = NULL;
2496 #endif
2497 }
2498 
2499 /* Note: This doesn't put any conntrack and bridge info in dst. */
2500 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2501 {
2502 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2503  dst->nfct = src->nfct;
2504  nf_conntrack_get(src->nfct);
2505  dst->nfctinfo = src->nfctinfo;
2506 #endif
2507 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2508  dst->nfct_reasm = src->nfct_reasm;
2509  nf_conntrack_get_reasm(src->nfct_reasm);
2510 #endif
2511 #ifdef CONFIG_BRIDGE_NETFILTER
2512  dst->nf_bridge = src->nf_bridge;
2513  nf_bridge_get(src->nf_bridge);
2514 #endif
2515 }
2516 
2517 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2518 {
2519 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2520  nf_conntrack_put(dst->nfct);
2521 #endif
2522 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2523  nf_conntrack_put_reasm(dst->nfct_reasm);
2524 #endif
2525 #ifdef CONFIG_BRIDGE_NETFILTER
2526  nf_bridge_put(dst->nf_bridge);
2527 #endif
2528  __nf_copy(dst, src);
2529 }
2530 
2531 #ifdef CONFIG_NETWORK_SECMARK
2532 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2533 {
2534  to->secmark = from->secmark;
2535 }
2536 
2537 static inline void skb_init_secmark(struct sk_buff *skb)
2538 {
2539  skb->secmark = 0;
2540 }
2541 #else
2542 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2543 { }
2544 
2545 static inline void skb_init_secmark(struct sk_buff *skb)
2546 { }
2547 #endif
2548 
2549 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
2550 {
2552 }
2553 
2554 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
2555 {
2556  return skb->queue_mapping;
2557 }
2558 
2559 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
2560 {
2561  to->queue_mapping = from->queue_mapping;
2562 }
2563 
2564 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
2565 {
2566  skb->queue_mapping = rx_queue + 1;
2567 }
2568 
2569 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
2570 {
2571  return skb->queue_mapping - 1;
2572 }
2573 
2574 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
2575 {
2576  return skb->queue_mapping != 0;
2577 }
2578 
2579 extern u16 __skb_tx_hash(const struct net_device *dev,
2580  const struct sk_buff *skb,
2581  unsigned int num_tx_queues);
2582 
2583 #ifdef CONFIG_XFRM
2584 static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2585 {
2586  return skb->sp;
2587 }
2588 #else
2589 static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2590 {
2591  return NULL;
2592 }
2593 #endif
2594 
2595 static inline bool skb_is_gso(const struct sk_buff *skb)
2596 {
2597  return skb_shinfo(skb)->gso_size;
2598 }
2599 
2600 static inline bool skb_is_gso_v6(const struct sk_buff *skb)
2601 {
2602  return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
2603 }
2604 
2605 extern void __skb_warn_lro_forwarding(const struct sk_buff *skb);
2606 
2607 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
2608 {
2609  /* LRO sets gso_size but not gso_type, whereas if GSO is really
2610  * wanted then gso_type will be set. */
2611  const struct skb_shared_info *shinfo = skb_shinfo(skb);
2612 
2613  if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
2614  unlikely(shinfo->gso_type == 0)) {
2616  return true;
2617  }
2618  return false;
2619 }
2620 
2621 static inline void skb_forward_csum(struct sk_buff *skb)
2622 {
2623  /* Unfortunately we don't support this one. Any brave souls? */
2624  if (skb->ip_summed == CHECKSUM_COMPLETE)
2625  skb->ip_summed = CHECKSUM_NONE;
2626 }
2627 
2636 static inline void skb_checksum_none_assert(const struct sk_buff *skb)
2637 {
2638 #ifdef DEBUG
2639  BUG_ON(skb->ip_summed != CHECKSUM_NONE);
2640 #endif
2641 }
2642 
2643 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
2644 
2654 static inline bool skb_head_is_locked(const struct sk_buff *skb)
2655 {
2656  return !skb->head_frag || skb_cloned(skb);
2657 }
2658 #endif /* __KERNEL__ */
2659 #endif /* _LINUX_SKBUFF_H */