Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
nf_conntrack_core.c
Go to the documentation of this file.
1 /* Connection state tracking for netfilter. This is separated from,
2  but required by, the NAT layer; it can also be used by an iptables
3  extension. */
4 
5 /* (C) 1999-2001 Paul `Rusty' Russell
6  * (C) 2002-2006 Netfilter Core Team <[email protected]>
7  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  */
13 
14 #include <linux/types.h>
15 #include <linux/netfilter.h>
16 #include <linux/module.h>
17 #include <linux/sched.h>
18 #include <linux/skbuff.h>
19 #include <linux/proc_fs.h>
20 #include <linux/vmalloc.h>
21 #include <linux/stddef.h>
22 #include <linux/slab.h>
23 #include <linux/random.h>
24 #include <linux/jhash.h>
25 #include <linux/err.h>
26 #include <linux/percpu.h>
27 #include <linux/moduleparam.h>
28 #include <linux/notifier.h>
29 #include <linux/kernel.h>
30 #include <linux/netdevice.h>
31 #include <linux/socket.h>
32 #include <linux/mm.h>
33 #include <linux/nsproxy.h>
34 #include <linux/rculist_nulls.h>
35 
48 #include <net/netfilter/nf_nat.h>
50 
51 #define NF_CONNTRACK_VERSION "0.5.0"
52 
54  enum nf_nat_manip_type manip,
55  const struct nlattr *attr) __read_mostly;
57 
59  struct nf_conn *ct,
60  enum ip_conntrack_info ctinfo,
61  unsigned int protoff);
63 
64 DEFINE_SPINLOCK(nf_conntrack_lock);
65 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
66 
68 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
69 
70 unsigned int nf_conntrack_max __read_mostly;
71 EXPORT_SYMBOL_GPL(nf_conntrack_max);
72 
73 DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
74 EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
75 
77 EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
78 
79 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
80 {
81  unsigned int n;
82 
83  /* The direction must be ignored, so we hash everything up to the
84  * destination ports (which is a multiple of 4) and treat the last
85  * three bytes manually.
86  */
87  n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
88  return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
89  (((__force __u16)tuple->dst.u.all << 16) |
90  tuple->dst.protonum));
91 }
92 
93 static u32 __hash_bucket(u32 hash, unsigned int size)
94 {
95  return ((u64)hash * size) >> 32;
96 }
97 
98 static u32 hash_bucket(u32 hash, const struct net *net)
99 {
100  return __hash_bucket(hash, net->ct.htable_size);
101 }
102 
103 static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
104  u16 zone, unsigned int size)
105 {
106  return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
107 }
108 
109 static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
110  const struct nf_conntrack_tuple *tuple)
111 {
112  return __hash_conntrack(tuple, zone, net->ct.htable_size);
113 }
114 
115 bool
116 nf_ct_get_tuple(const struct sk_buff *skb,
117  unsigned int nhoff,
118  unsigned int dataoff,
119  u_int16_t l3num,
120  u_int8_t protonum,
121  struct nf_conntrack_tuple *tuple,
122  const struct nf_conntrack_l3proto *l3proto,
123  const struct nf_conntrack_l4proto *l4proto)
124 {
125  memset(tuple, 0, sizeof(*tuple));
126 
127  tuple->src.l3num = l3num;
128  if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
129  return false;
130 
131  tuple->dst.protonum = protonum;
132  tuple->dst.dir = IP_CT_DIR_ORIGINAL;
133 
134  return l4proto->pkt_to_tuple(skb, dataoff, tuple);
135 }
137 
138 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
139  u_int16_t l3num, struct nf_conntrack_tuple *tuple)
140 {
143  unsigned int protoff;
144  u_int8_t protonum;
145  int ret;
146 
147  rcu_read_lock();
148 
149  l3proto = __nf_ct_l3proto_find(l3num);
150  ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
151  if (ret != NF_ACCEPT) {
152  rcu_read_unlock();
153  return false;
154  }
155 
156  l4proto = __nf_ct_l4proto_find(l3num, protonum);
157 
158  ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
159  l3proto, l4proto);
160 
161  rcu_read_unlock();
162  return ret;
163 }
165 
166 bool
168  const struct nf_conntrack_tuple *orig,
169  const struct nf_conntrack_l3proto *l3proto,
170  const struct nf_conntrack_l4proto *l4proto)
171 {
172  memset(inverse, 0, sizeof(*inverse));
173 
174  inverse->src.l3num = orig->src.l3num;
175  if (l3proto->invert_tuple(inverse, orig) == 0)
176  return false;
177 
178  inverse->dst.dir = !orig->dst.dir;
179 
180  inverse->dst.protonum = orig->dst.protonum;
181  return l4proto->invert_tuple(inverse, orig);
182 }
184 
185 static void
186 clean_from_lists(struct nf_conn *ct)
187 {
188  pr_debug("clean_from_lists(%p)\n", ct);
189  hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
190  hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
191 
192  /* Destroy all pending expectations */
194 }
195 
196 static void
197 destroy_conntrack(struct nf_conntrack *nfct)
198 {
199  struct nf_conn *ct = (struct nf_conn *)nfct;
200  struct net *net = nf_ct_net(ct);
202 
203  pr_debug("destroy_conntrack(%p)\n", ct);
204  NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
205  NF_CT_ASSERT(!timer_pending(&ct->timeout));
206 
207  /* To make sure we don't get any weird locking issues here:
208  * destroy_conntrack() MUST NOT be called with a write lock
209  * to nf_conntrack_lock!!! -HW */
210  rcu_read_lock();
211  l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
212  if (l4proto && l4proto->destroy)
213  l4proto->destroy(ct);
214 
215  rcu_read_unlock();
216 
217  spin_lock_bh(&nf_conntrack_lock);
218  /* Expectations will have been removed in clean_from_lists,
219  * except TFTP can create an expectation on the first packet,
220  * before connection is in the list, so we need to clean here,
221  * too. */
223 
224  /* We overload first tuple to link into unconfirmed list. */
225  if (!nf_ct_is_confirmed(ct)) {
226  BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
227  hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
228  }
229 
230  NF_CT_STAT_INC(net, delete);
231  spin_unlock_bh(&nf_conntrack_lock);
232 
233  if (ct->master)
234  nf_ct_put(ct->master);
235 
236  pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
237  nf_conntrack_free(ct);
238 }
239 
241 {
242  struct net *net = nf_ct_net(ct);
243 
245  spin_lock_bh(&nf_conntrack_lock);
246  /* Inside lock so preempt is disabled on module removal path.
247  * Otherwise we can get spurious warnings. */
248  NF_CT_STAT_INC(net, delete_list);
249  clean_from_lists(ct);
250  spin_unlock_bh(&nf_conntrack_lock);
251 }
253 
254 static void death_by_event(unsigned long ul_conntrack)
255 {
256  struct nf_conn *ct = (void *)ul_conntrack;
257  struct net *net = nf_ct_net(ct);
258  struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
259 
260  BUG_ON(ecache == NULL);
261 
262  if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
263  /* bad luck, let's retry again */
264  ecache->timeout.expires = jiffies +
265  (random32() % net->ct.sysctl_events_retry_timeout);
266  add_timer(&ecache->timeout);
267  return;
268  }
269  /* we've got the event delivered, now it's dying */
271  spin_lock(&nf_conntrack_lock);
272  hlist_nulls_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
273  spin_unlock(&nf_conntrack_lock);
274  nf_ct_put(ct);
275 }
276 
278 {
279  struct net *net = nf_ct_net(ct);
280  struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
281 
282  BUG_ON(ecache == NULL);
283 
284  /* add this conntrack to the dying list */
285  spin_lock_bh(&nf_conntrack_lock);
286  hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
287  &net->ct.dying);
288  spin_unlock_bh(&nf_conntrack_lock);
289  /* set a new timer to retry event delivery */
290  setup_timer(&ecache->timeout, death_by_event, (unsigned long)ct);
291  ecache->timeout.expires = jiffies +
292  (random32() % net->ct.sysctl_events_retry_timeout);
293  add_timer(&ecache->timeout);
294 }
296 
297 static void death_by_timeout(unsigned long ul_conntrack)
298 {
299  struct nf_conn *ct = (void *)ul_conntrack;
300  struct nf_conn_tstamp *tstamp;
301 
302  tstamp = nf_conn_tstamp_find(ct);
303  if (tstamp && tstamp->stop == 0)
304  tstamp->stop = ktime_to_ns(ktime_get_real());
305 
306  if (!test_bit(IPS_DYING_BIT, &ct->status) &&
307  unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
308  /* destroy event was not delivered */
311  return;
312  }
315  nf_ct_put(ct);
316 }
317 
318 /*
319  * Warning :
320  * - Caller must take a reference on returned object
321  * and recheck nf_ct_tuple_equal(tuple, &h->tuple)
322  * OR
323  * - Caller must lock nf_conntrack_lock before calling this function
324  */
325 static struct nf_conntrack_tuple_hash *
326 ____nf_conntrack_find(struct net *net, u16 zone,
327  const struct nf_conntrack_tuple *tuple, u32 hash)
328 {
329  struct nf_conntrack_tuple_hash *h;
330  struct hlist_nulls_node *n;
331  unsigned int bucket = hash_bucket(hash, net);
332 
333  /* Disable BHs the entire time since we normally need to disable them
334  * at least once for the stats anyway.
335  */
337 begin:
338  hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
339  if (nf_ct_tuple_equal(tuple, &h->tuple) &&
340  nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
341  NF_CT_STAT_INC(net, found);
342  local_bh_enable();
343  return h;
344  }
345  NF_CT_STAT_INC(net, searched);
346  }
347  /*
348  * if the nulls value we got at the end of this lookup is
349  * not the expected one, we must restart lookup.
350  * We probably met an item that was moved to another chain.
351  */
352  if (get_nulls_value(n) != bucket) {
353  NF_CT_STAT_INC(net, search_restart);
354  goto begin;
355  }
356  local_bh_enable();
357 
358  return NULL;
359 }
360 
362 __nf_conntrack_find(struct net *net, u16 zone,
363  const struct nf_conntrack_tuple *tuple)
364 {
365  return ____nf_conntrack_find(net, zone, tuple,
366  hash_conntrack_raw(tuple, zone));
367 }
369 
370 /* Find a connection corresponding to a tuple. */
371 static struct nf_conntrack_tuple_hash *
372 __nf_conntrack_find_get(struct net *net, u16 zone,
373  const struct nf_conntrack_tuple *tuple, u32 hash)
374 {
375  struct nf_conntrack_tuple_hash *h;
376  struct nf_conn *ct;
377 
378  rcu_read_lock();
379 begin:
380  h = ____nf_conntrack_find(net, zone, tuple, hash);
381  if (h) {
382  ct = nf_ct_tuplehash_to_ctrack(h);
383  if (unlikely(nf_ct_is_dying(ct) ||
384  !atomic_inc_not_zero(&ct->ct_general.use)))
385  h = NULL;
386  else {
387  if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
388  nf_ct_zone(ct) != zone)) {
389  nf_ct_put(ct);
390  goto begin;
391  }
392  }
393  }
394  rcu_read_unlock();
395 
396  return h;
397 }
398 
400 nf_conntrack_find_get(struct net *net, u16 zone,
401  const struct nf_conntrack_tuple *tuple)
402 {
403  return __nf_conntrack_find_get(net, zone, tuple,
404  hash_conntrack_raw(tuple, zone));
405 }
407 
408 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
409  unsigned int hash,
410  unsigned int repl_hash)
411 {
412  struct net *net = nf_ct_net(ct);
413 
414  hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
415  &net->ct.hash[hash]);
416  hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
417  &net->ct.hash[repl_hash]);
418 }
419 
420 int
422 {
423  struct net *net = nf_ct_net(ct);
424  unsigned int hash, repl_hash;
425  struct nf_conntrack_tuple_hash *h;
426  struct hlist_nulls_node *n;
427  u16 zone;
428 
429  zone = nf_ct_zone(ct);
430  hash = hash_conntrack(net, zone,
431  &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
432  repl_hash = hash_conntrack(net, zone,
433  &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
434 
435  spin_lock_bh(&nf_conntrack_lock);
436 
437  /* See if there's one in the list already, including reverse */
438  hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
439  if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
440  &h->tuple) &&
441  zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
442  goto out;
443  hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
444  if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
445  &h->tuple) &&
446  zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
447  goto out;
448 
449  add_timer(&ct->timeout);
450  nf_conntrack_get(&ct->ct_general);
451  __nf_conntrack_hash_insert(ct, hash, repl_hash);
452  NF_CT_STAT_INC(net, insert);
453  spin_unlock_bh(&nf_conntrack_lock);
454 
455  return 0;
456 
457 out:
458  NF_CT_STAT_INC(net, insert_failed);
459  spin_unlock_bh(&nf_conntrack_lock);
460  return -EEXIST;
461 }
463 
464 /* Confirm a connection given skb; places it in hash table */
465 int
467 {
468  unsigned int hash, repl_hash;
469  struct nf_conntrack_tuple_hash *h;
470  struct nf_conn *ct;
471  struct nf_conn_help *help;
472  struct nf_conn_tstamp *tstamp;
473  struct hlist_nulls_node *n;
474  enum ip_conntrack_info ctinfo;
475  struct net *net;
476  u16 zone;
477 
478  ct = nf_ct_get(skb, &ctinfo);
479  net = nf_ct_net(ct);
480 
481  /* ipt_REJECT uses nf_conntrack_attach to attach related
482  ICMP/TCP RST packets in other direction. Actual packet
483  which created connection will be IP_CT_NEW or for an
484  expected connection, IP_CT_RELATED. */
485  if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
486  return NF_ACCEPT;
487 
488  zone = nf_ct_zone(ct);
489  /* reuse the hash saved before */
490  hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
491  hash = hash_bucket(hash, net);
492  repl_hash = hash_conntrack(net, zone,
493  &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
494 
495  /* We're not in hash table, and we refuse to set up related
496  connections for unconfirmed conns. But packet copies and
497  REJECT will give spurious warnings here. */
498  /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
499 
500  /* No external references means no one else could have
501  confirmed us. */
502  NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
503  pr_debug("Confirming conntrack %p\n", ct);
504 
505  spin_lock_bh(&nf_conntrack_lock);
506 
507  /* We have to check the DYING flag inside the lock to prevent
508  a race against nf_ct_get_next_corpse() possibly called from
509  user context, else we insert an already 'dead' hash, blocking
510  further use of that particular connection -JM */
511 
512  if (unlikely(nf_ct_is_dying(ct))) {
513  spin_unlock_bh(&nf_conntrack_lock);
514  return NF_ACCEPT;
515  }
516 
517  /* See if there's one in the list already, including reverse:
518  NAT could have grabbed it without realizing, since we're
519  not in the hash. If there is, we lost race. */
520  hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
521  if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
522  &h->tuple) &&
523  zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
524  goto out;
525  hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
526  if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
527  &h->tuple) &&
528  zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
529  goto out;
530 
531  /* Remove from unconfirmed list */
532  hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
533 
534  /* Timer relative to confirmation time, not original
535  setting time, otherwise we'd get timer wrap in
536  weird delay cases. */
537  ct->timeout.expires += jiffies;
538  add_timer(&ct->timeout);
539  atomic_inc(&ct->ct_general.use);
540  ct->status |= IPS_CONFIRMED;
541 
542  /* set conntrack timestamp, if enabled. */
543  tstamp = nf_conn_tstamp_find(ct);
544  if (tstamp) {
545  if (skb->tstamp.tv64 == 0)
546  __net_timestamp(skb);
547 
548  tstamp->start = ktime_to_ns(skb->tstamp);
549  }
550  /* Since the lookup is lockless, hash insertion must be done after
551  * starting the timer and setting the CONFIRMED bit. The RCU barriers
552  * guarantee that no other CPU can find the conntrack before the above
553  * stores are visible.
554  */
555  __nf_conntrack_hash_insert(ct, hash, repl_hash);
556  NF_CT_STAT_INC(net, insert);
557  spin_unlock_bh(&nf_conntrack_lock);
558 
559  help = nfct_help(ct);
560  if (help && help->helper)
561  nf_conntrack_event_cache(IPCT_HELPER, ct);
562 
563  nf_conntrack_event_cache(master_ct(ct) ?
564  IPCT_RELATED : IPCT_NEW, ct);
565  return NF_ACCEPT;
566 
567 out:
568  NF_CT_STAT_INC(net, insert_failed);
569  spin_unlock_bh(&nf_conntrack_lock);
570  return NF_DROP;
571 }
573 
574 /* Returns true if a connection correspondings to the tuple (required
575  for NAT). */
576 int
578  const struct nf_conn *ignored_conntrack)
579 {
580  struct net *net = nf_ct_net(ignored_conntrack);
581  struct nf_conntrack_tuple_hash *h;
582  struct hlist_nulls_node *n;
583  struct nf_conn *ct;
584  u16 zone = nf_ct_zone(ignored_conntrack);
585  unsigned int hash = hash_conntrack(net, zone, tuple);
586 
587  /* Disable BHs the entire time since we need to disable them at
588  * least once for the stats anyway.
589  */
590  rcu_read_lock_bh();
591  hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
592  ct = nf_ct_tuplehash_to_ctrack(h);
593  if (ct != ignored_conntrack &&
594  nf_ct_tuple_equal(tuple, &h->tuple) &&
595  nf_ct_zone(ct) == zone) {
596  NF_CT_STAT_INC(net, found);
597  rcu_read_unlock_bh();
598  return 1;
599  }
600  NF_CT_STAT_INC(net, searched);
601  }
602  rcu_read_unlock_bh();
603 
604  return 0;
605 }
607 
608 #define NF_CT_EVICTION_RANGE 8
609 
610 /* There's a small race here where we may free a just-assured
611  connection. Too bad: we're in trouble anyway. */
612 static noinline int early_drop(struct net *net, unsigned int hash)
613 {
614  /* Use oldest entry, which is roughly LRU */
615  struct nf_conntrack_tuple_hash *h;
616  struct nf_conn *ct = NULL, *tmp;
617  struct hlist_nulls_node *n;
618  unsigned int i, cnt = 0;
619  int dropped = 0;
620 
621  rcu_read_lock();
622  for (i = 0; i < net->ct.htable_size; i++) {
623  hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
624  hnnode) {
625  tmp = nf_ct_tuplehash_to_ctrack(h);
626  if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
627  ct = tmp;
628  cnt++;
629  }
630 
631  if (ct != NULL) {
632  if (likely(!nf_ct_is_dying(ct) &&
633  atomic_inc_not_zero(&ct->ct_general.use)))
634  break;
635  else
636  ct = NULL;
637  }
638 
639  if (cnt >= NF_CT_EVICTION_RANGE)
640  break;
641 
642  hash = (hash + 1) % net->ct.htable_size;
643  }
644  rcu_read_unlock();
645 
646  if (!ct)
647  return dropped;
648 
649  if (del_timer(&ct->timeout)) {
650  death_by_timeout((unsigned long)ct);
651  /* Check if we indeed killed this entry. Reliable event
652  delivery may have inserted it into the dying list. */
653  if (test_bit(IPS_DYING_BIT, &ct->status)) {
654  dropped = 1;
655  NF_CT_STAT_INC_ATOMIC(net, early_drop);
656  }
657  }
658  nf_ct_put(ct);
659  return dropped;
660 }
661 
663 {
664  unsigned int rand;
665 
666  /*
667  * Why not initialize nf_conntrack_rnd in a "init()" function ?
668  * Because there isn't enough entropy when system initializing,
669  * and we initialize it as late as possible.
670  */
671  do {
672  get_random_bytes(&rand, sizeof(rand));
673  } while (!rand);
674  cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
675 }
676 
677 static struct nf_conn *
678 __nf_conntrack_alloc(struct net *net, u16 zone,
679  const struct nf_conntrack_tuple *orig,
680  const struct nf_conntrack_tuple *repl,
681  gfp_t gfp, u32 hash)
682 {
683  struct nf_conn *ct;
684 
685  if (unlikely(!nf_conntrack_hash_rnd)) {
687  /* recompute the hash as nf_conntrack_hash_rnd is initialized */
688  hash = hash_conntrack_raw(orig, zone);
689  }
690 
691  /* We don't want any race condition at early drop stage */
692  atomic_inc(&net->ct.count);
693 
694  if (nf_conntrack_max &&
695  unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
696  if (!early_drop(net, hash_bucket(hash, net))) {
697  atomic_dec(&net->ct.count);
698  net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
699  return ERR_PTR(-ENOMEM);
700  }
701  }
702 
703  /*
704  * Do not use kmem_cache_zalloc(), as this cache uses
705  * SLAB_DESTROY_BY_RCU.
706  */
707  ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
708  if (ct == NULL) {
709  atomic_dec(&net->ct.count);
710  return ERR_PTR(-ENOMEM);
711  }
712  /*
713  * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
714  * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
715  */
716  memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
717  offsetof(struct nf_conn, proto) -
719  spin_lock_init(&ct->lock);
720  ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
721  ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
722  ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
723  /* save hash for reusing when confirming */
724  *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
725  /* Don't set timer yet: wait for confirmation */
726  setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
727  write_pnet(&ct->ct_net, net);
728 #ifdef CONFIG_NF_CONNTRACK_ZONES
729  if (zone) {
730  struct nf_conntrack_zone *nf_ct_zone;
731 
732  nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
733  if (!nf_ct_zone)
734  goto out_free;
735  nf_ct_zone->id = zone;
736  }
737 #endif
738  /*
739  * changes to lookup keys must be done before setting refcnt to 1
740  */
741  smp_wmb();
742  atomic_set(&ct->ct_general.use, 1);
743  return ct;
744 
745 #ifdef CONFIG_NF_CONNTRACK_ZONES
746 out_free:
747  atomic_dec(&net->ct.count);
748  kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
749  return ERR_PTR(-ENOMEM);
750 #endif
751 }
752 
753 struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
754  const struct nf_conntrack_tuple *orig,
755  const struct nf_conntrack_tuple *repl,
756  gfp_t gfp)
757 {
758  return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
759 }
761 
762 void nf_conntrack_free(struct nf_conn *ct)
763 {
764  struct net *net = nf_ct_net(ct);
765 
766  nf_ct_ext_destroy(ct);
767  atomic_dec(&net->ct.count);
768  nf_ct_ext_free(ct);
769  kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
770 }
772 
773 /* Allocate a new conntrack: we return -ENOMEM if classification
774  failed due to stress. Otherwise it really is unclassifiable. */
775 static struct nf_conntrack_tuple_hash *
776 init_conntrack(struct net *net, struct nf_conn *tmpl,
777  const struct nf_conntrack_tuple *tuple,
778  struct nf_conntrack_l3proto *l3proto,
779  struct nf_conntrack_l4proto *l4proto,
780  struct sk_buff *skb,
781  unsigned int dataoff, u32 hash)
782 {
783  struct nf_conn *ct;
784  struct nf_conn_help *help;
785  struct nf_conntrack_tuple repl_tuple;
786  struct nf_conntrack_ecache *ecache;
787  struct nf_conntrack_expect *exp;
788  u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
789  struct nf_conn_timeout *timeout_ext;
790  unsigned int *timeouts;
791 
792  if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
793  pr_debug("Can't invert tuple.\n");
794  return NULL;
795  }
796 
797  ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
798  hash);
799  if (IS_ERR(ct))
800  return (struct nf_conntrack_tuple_hash *)ct;
801 
802  timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
803  if (timeout_ext)
804  timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
805  else
806  timeouts = l4proto->get_timeouts(net);
807 
808  if (!l4proto->new(ct, skb, dataoff, timeouts)) {
809  nf_conntrack_free(ct);
810  pr_debug("init conntrack: can't track with proto module\n");
811  return NULL;
812  }
813 
814  if (timeout_ext)
815  nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
816 
817  nf_ct_acct_ext_add(ct, GFP_ATOMIC);
818  nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
819 
820  ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
821  nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
822  ecache ? ecache->expmask : 0,
823  GFP_ATOMIC);
824 
825  spin_lock_bh(&nf_conntrack_lock);
826  exp = nf_ct_find_expectation(net, zone, tuple);
827  if (exp) {
828  pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
829  ct, exp);
830  /* Welcome, Mr. Bond. We've been expecting you... */
832  ct->master = exp->master;
833  if (exp->helper) {
834  help = nf_ct_helper_ext_add(ct, exp->helper,
835  GFP_ATOMIC);
836  if (help)
837  rcu_assign_pointer(help->helper, exp->helper);
838  }
839 
840 #ifdef CONFIG_NF_CONNTRACK_MARK
841  ct->mark = exp->master->mark;
842 #endif
843 #ifdef CONFIG_NF_CONNTRACK_SECMARK
844  ct->secmark = exp->master->secmark;
845 #endif
846  nf_conntrack_get(&ct->master->ct_general);
847  NF_CT_STAT_INC(net, expect_new);
848  } else {
850  NF_CT_STAT_INC(net, new);
851  }
852 
853  /* Overload tuple linked list to put us in unconfirmed list. */
854  hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
855  &net->ct.unconfirmed);
856 
857  spin_unlock_bh(&nf_conntrack_lock);
858 
859  if (exp) {
860  if (exp->expectfn)
861  exp->expectfn(ct, exp);
862  nf_ct_expect_put(exp);
863  }
864 
865  return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
866 }
867 
868 /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
869 static inline struct nf_conn *
870 resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
871  struct sk_buff *skb,
872  unsigned int dataoff,
873  u_int16_t l3num,
874  u_int8_t protonum,
875  struct nf_conntrack_l3proto *l3proto,
876  struct nf_conntrack_l4proto *l4proto,
877  int *set_reply,
878  enum ip_conntrack_info *ctinfo)
879 {
880  struct nf_conntrack_tuple tuple;
881  struct nf_conntrack_tuple_hash *h;
882  struct nf_conn *ct;
883  u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
884  u32 hash;
885 
886  if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
887  dataoff, l3num, protonum, &tuple, l3proto,
888  l4proto)) {
889  pr_debug("resolve_normal_ct: Can't get tuple\n");
890  return NULL;
891  }
892 
893  /* look for tuple match */
894  hash = hash_conntrack_raw(&tuple, zone);
895  h = __nf_conntrack_find_get(net, zone, &tuple, hash);
896  if (!h) {
897  h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
898  skb, dataoff, hash);
899  if (!h)
900  return NULL;
901  if (IS_ERR(h))
902  return (void *)h;
903  }
904  ct = nf_ct_tuplehash_to_ctrack(h);
905 
906  /* It exists; we have (non-exclusive) reference. */
907  if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
908  *ctinfo = IP_CT_ESTABLISHED_REPLY;
909  /* Please set reply bit if this packet OK */
910  *set_reply = 1;
911  } else {
912  /* Once we've had two way comms, always ESTABLISHED. */
913  if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
914  pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
915  *ctinfo = IP_CT_ESTABLISHED;
916  } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
917  pr_debug("nf_conntrack_in: related packet for %p\n",
918  ct);
919  *ctinfo = IP_CT_RELATED;
920  } else {
921  pr_debug("nf_conntrack_in: new packet for %p\n", ct);
922  *ctinfo = IP_CT_NEW;
923  }
924  *set_reply = 0;
925  }
926  skb->nfct = &ct->ct_general;
927  skb->nfctinfo = *ctinfo;
928  return ct;
929 }
930 
931 unsigned int
932 nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
933  struct sk_buff *skb)
934 {
935  struct nf_conn *ct, *tmpl = NULL;
936  enum ip_conntrack_info ctinfo;
939  unsigned int *timeouts;
940  unsigned int dataoff;
941  u_int8_t protonum;
942  int set_reply = 0;
943  int ret;
944 
945  if (skb->nfct) {
946  /* Previously seen (loopback or untracked)? Ignore. */
947  tmpl = (struct nf_conn *)skb->nfct;
948  if (!nf_ct_is_template(tmpl)) {
949  NF_CT_STAT_INC_ATOMIC(net, ignore);
950  return NF_ACCEPT;
951  }
952  skb->nfct = NULL;
953  }
954 
955  /* rcu_read_lock()ed by nf_hook_slow */
956  l3proto = __nf_ct_l3proto_find(pf);
957  ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
958  &dataoff, &protonum);
959  if (ret <= 0) {
960  pr_debug("not prepared to track yet or error occurred\n");
963  ret = -ret;
964  goto out;
965  }
966 
967  l4proto = __nf_ct_l4proto_find(pf, protonum);
968 
969  /* It may be an special packet, error, unclean...
970  * inverse of the return code tells to the netfilter
971  * core what to do with the packet. */
972  if (l4proto->error != NULL) {
973  ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
974  pf, hooknum);
975  if (ret <= 0) {
978  ret = -ret;
979  goto out;
980  }
981  /* ICMP[v6] protocol trackers may assign one conntrack. */
982  if (skb->nfct)
983  goto out;
984  }
985 
986  ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
987  l3proto, l4proto, &set_reply, &ctinfo);
988  if (!ct) {
989  /* Not valid part of a connection */
991  ret = NF_ACCEPT;
992  goto out;
993  }
994 
995  if (IS_ERR(ct)) {
996  /* Too stressed to deal. */
997  NF_CT_STAT_INC_ATOMIC(net, drop);
998  ret = NF_DROP;
999  goto out;
1000  }
1001 
1002  NF_CT_ASSERT(skb->nfct);
1003 
1004  /* Decide what timeout policy we want to apply to this flow. */
1005  timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1006 
1007  ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1008  if (ret <= 0) {
1009  /* Invalid: inverse of the return code tells
1010  * the netfilter core what to do */
1011  pr_debug("nf_conntrack_in: Can't track with proto module\n");
1012  nf_conntrack_put(skb->nfct);
1013  skb->nfct = NULL;
1015  if (ret == -NF_DROP)
1016  NF_CT_STAT_INC_ATOMIC(net, drop);
1017  ret = -ret;
1018  goto out;
1019  }
1020 
1021  if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1022  nf_conntrack_event_cache(IPCT_REPLY, ct);
1023 out:
1024  if (tmpl) {
1025  /* Special case: we have to repeat this hook, assign the
1026  * template again to this packet. We assume that this packet
1027  * has no conntrack assigned. This is used by nf_ct_tcp. */
1028  if (ret == NF_REPEAT)
1029  skb->nfct = (struct nf_conntrack *)tmpl;
1030  else
1031  nf_ct_put(tmpl);
1032  }
1033 
1034  return ret;
1035 }
1037 
1039  const struct nf_conntrack_tuple *orig)
1040 {
1041  bool ret;
1042 
1043  rcu_read_lock();
1044  ret = nf_ct_invert_tuple(inverse, orig,
1045  __nf_ct_l3proto_find(orig->src.l3num),
1046  __nf_ct_l4proto_find(orig->src.l3num,
1047  orig->dst.protonum));
1048  rcu_read_unlock();
1049  return ret;
1050 }
1052 
1053 /* Alter reply tuple (maybe alter helper). This is for NAT, and is
1054  implicitly racy: see __nf_conntrack_confirm */
1056  const struct nf_conntrack_tuple *newreply)
1057 {
1058  struct nf_conn_help *help = nfct_help(ct);
1059 
1060  /* Should be unconfirmed, so not in hash table yet */
1061  NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1062 
1063  pr_debug("Altering reply tuple of %p to ", ct);
1064  nf_ct_dump_tuple(newreply);
1065 
1066  ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1067  if (ct->master || (help && !hlist_empty(&help->expectations)))
1068  return;
1069 
1070  rcu_read_lock();
1072  rcu_read_unlock();
1073 }
1075 
1076 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1078  enum ip_conntrack_info ctinfo,
1079  const struct sk_buff *skb,
1080  unsigned long extra_jiffies,
1081  int do_acct)
1082 {
1083  NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1084  NF_CT_ASSERT(skb);
1085 
1086  /* Only update if this is not a fixed timeout */
1087  if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1088  goto acct;
1089 
1090  /* If not in hash table, timer will not be active yet */
1091  if (!nf_ct_is_confirmed(ct)) {
1092  ct->timeout.expires = extra_jiffies;
1093  } else {
1094  unsigned long newtime = jiffies + extra_jiffies;
1095 
1096  /* Only update the timeout if the new timeout is at least
1097  HZ jiffies from the old timeout. Need del_timer for race
1098  avoidance (may already be dying). */
1099  if (newtime - ct->timeout.expires >= HZ)
1100  mod_timer_pending(&ct->timeout, newtime);
1101  }
1102 
1103 acct:
1104  if (do_acct) {
1105  struct nf_conn_counter *acct;
1106 
1107  acct = nf_conn_acct_find(ct);
1108  if (acct) {
1109  atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1110  atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);
1111  }
1112  }
1113 }
1115 
1116 bool __nf_ct_kill_acct(struct nf_conn *ct,
1117  enum ip_conntrack_info ctinfo,
1118  const struct sk_buff *skb,
1119  int do_acct)
1120 {
1121  if (do_acct) {
1122  struct nf_conn_counter *acct;
1123 
1124  acct = nf_conn_acct_find(ct);
1125  if (acct) {
1126  atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1127  atomic64_add(skb->len - skb_network_offset(skb),
1128  &acct[CTINFO2DIR(ctinfo)].bytes);
1129  }
1130  }
1131 
1132  if (del_timer(&ct->timeout)) {
1133  ct->timeout.function((unsigned long)ct);
1134  return true;
1135  }
1136  return false;
1137 }
1139 
1140 #ifdef CONFIG_NF_CONNTRACK_ZONES
1141 static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
1142  .len = sizeof(struct nf_conntrack_zone),
1143  .align = __alignof__(struct nf_conntrack_zone),
1144  .id = NF_CT_EXT_ZONE,
1145 };
1146 #endif
1147 
1148 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1149 
1150 #include <linux/netfilter/nfnetlink.h>
1152 #include <linux/mutex.h>
1153 
1154 /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1155  * in ip_conntrack_core, since we don't want the protocols to autoload
1156  * or depend on ctnetlink */
1157 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1158  const struct nf_conntrack_tuple *tuple)
1159 {
1160  if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1161  nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1162  goto nla_put_failure;
1163  return 0;
1164 
1165 nla_put_failure:
1166  return -1;
1167 }
1169 
1171  [CTA_PROTO_SRC_PORT] = { .type = NLA_U16 },
1172  [CTA_PROTO_DST_PORT] = { .type = NLA_U16 },
1173 };
1174 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1175 
1176 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1177  struct nf_conntrack_tuple *t)
1178 {
1179  if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1180  return -EINVAL;
1181 
1182  t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1183  t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1184 
1185  return 0;
1186 }
1188 
1190 {
1191  return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1192 }
1194 #endif
1195 
1196 /* Used by ipt_REJECT and ip6t_REJECT. */
1197 static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
1198 {
1199  struct nf_conn *ct;
1200  enum ip_conntrack_info ctinfo;
1201 
1202  /* This ICMP is in reverse direction to the packet which caused it */
1203  ct = nf_ct_get(skb, &ctinfo);
1204  if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1205  ctinfo = IP_CT_RELATED_REPLY;
1206  else
1207  ctinfo = IP_CT_RELATED;
1208 
1209  /* Attach to new skbuff, and increment count */
1210  nskb->nfct = &ct->ct_general;
1211  nskb->nfctinfo = ctinfo;
1212  nf_conntrack_get(nskb->nfct);
1213 }
1214 
1215 /* Bring out ya dead! */
1216 static struct nf_conn *
1217 get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1218  void *data, unsigned int *bucket)
1219 {
1220  struct nf_conntrack_tuple_hash *h;
1221  struct nf_conn *ct;
1222  struct hlist_nulls_node *n;
1223 
1224  spin_lock_bh(&nf_conntrack_lock);
1225  for (; *bucket < net->ct.htable_size; (*bucket)++) {
1226  hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
1228  continue;
1229  ct = nf_ct_tuplehash_to_ctrack(h);
1230  if (iter(ct, data))
1231  goto found;
1232  }
1233  }
1234  hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
1235  ct = nf_ct_tuplehash_to_ctrack(h);
1236  if (iter(ct, data))
1237  set_bit(IPS_DYING_BIT, &ct->status);
1238  }
1239  spin_unlock_bh(&nf_conntrack_lock);
1240  return NULL;
1241 found:
1242  atomic_inc(&ct->ct_general.use);
1243  spin_unlock_bh(&nf_conntrack_lock);
1244  return ct;
1245 }
1246 
1247 void nf_ct_iterate_cleanup(struct net *net,
1248  int (*iter)(struct nf_conn *i, void *data),
1249  void *data)
1250 {
1251  struct nf_conn *ct;
1252  unsigned int bucket = 0;
1253 
1254  while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1255  /* Time to push up daises... */
1256  if (del_timer(&ct->timeout))
1257  death_by_timeout((unsigned long)ct);
1258  /* ... else the timer will get him soon. */
1259 
1260  nf_ct_put(ct);
1261  }
1262 }
1264 
1267  int report;
1268 };
1269 
1270 static int kill_report(struct nf_conn *i, void *data)
1271 {
1272  struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
1273  struct nf_conn_tstamp *tstamp;
1274 
1275  tstamp = nf_conn_tstamp_find(i);
1276  if (tstamp && tstamp->stop == 0)
1277  tstamp->stop = ktime_to_ns(ktime_get_real());
1278 
1279  /* If we fail to deliver the event, death_by_timeout() will retry */
1280  if (nf_conntrack_event_report(IPCT_DESTROY, i,
1281  fr->pid, fr->report) < 0)
1282  return 1;
1283 
1284  /* Avoid the delivery of the destroy event in death_by_timeout(). */
1286  return 1;
1287 }
1288 
1289 static int kill_all(struct nf_conn *i, void *data)
1290 {
1291  return 1;
1292 }
1293 
1294 void nf_ct_free_hashtable(void *hash, unsigned int size)
1295 {
1296  if (is_vmalloc_addr(hash))
1297  vfree(hash);
1298  else
1299  free_pages((unsigned long)hash,
1300  get_order(sizeof(struct hlist_head) * size));
1301 }
1303 
1304 void nf_conntrack_flush_report(struct net *net, u32 pid, int report)
1305 {
1306  struct __nf_ct_flush_report fr = {
1307  .pid = pid,
1308  .report = report,
1309  };
1310  nf_ct_iterate_cleanup(net, kill_report, &fr);
1311 }
1313 
1314 static void nf_ct_release_dying_list(struct net *net)
1315 {
1316  struct nf_conntrack_tuple_hash *h;
1317  struct nf_conn *ct;
1318  struct hlist_nulls_node *n;
1319 
1320  spin_lock_bh(&nf_conntrack_lock);
1321  hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
1322  ct = nf_ct_tuplehash_to_ctrack(h);
1323  /* never fails to remove them, no listeners at this point */
1324  nf_ct_kill(ct);
1325  }
1326  spin_unlock_bh(&nf_conntrack_lock);
1327 }
1328 
1329 static int untrack_refs(void)
1330 {
1331  int cnt = 0, cpu;
1332 
1334  struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1335 
1336  cnt += atomic_read(&ct->ct_general.use) - 1;
1337  }
1338  return cnt;
1339 }
1340 
1341 static void nf_conntrack_cleanup_init_net(void)
1342 {
1343  while (untrack_refs() > 0)
1344  schedule();
1345 
1346 #ifdef CONFIG_NF_CONNTRACK_ZONES
1347  nf_ct_extend_unregister(&nf_ct_zone_extend);
1348 #endif
1349 }
1350 
1351 static void nf_conntrack_cleanup_net(struct net *net)
1352 {
1353  i_see_dead_people:
1354  nf_ct_iterate_cleanup(net, kill_all, NULL);
1355  nf_ct_release_dying_list(net);
1356  if (atomic_read(&net->ct.count) != 0) {
1357  schedule();
1358  goto i_see_dead_people;
1359  }
1360 
1361  nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1368  kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1369  kfree(net->ct.slabname);
1370  free_percpu(net->ct.stat);
1371 }
1372 
1373 /* Mishearing the voices in his head, our hero wonders how he's
1374  supposed to kill the mall. */
1375 void nf_conntrack_cleanup(struct net *net)
1376 {
1377  if (net_eq(net, &init_net))
1378  RCU_INIT_POINTER(ip_ct_attach, NULL);
1379 
1380  /* This makes sure all current packets have passed through
1381  netfilter framework. Roll on, two-stage module
1382  delete... */
1383  synchronize_net();
1385  nf_conntrack_cleanup_net(net);
1386 
1387  if (net_eq(net, &init_net)) {
1388  RCU_INIT_POINTER(nf_ct_destroy, NULL);
1389  nf_conntrack_cleanup_init_net();
1390  }
1391 }
1392 
1393 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1394 {
1395  struct hlist_nulls_head *hash;
1396  unsigned int nr_slots, i;
1397  size_t sz;
1398 
1399  BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1400  nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1401  sz = nr_slots * sizeof(struct hlist_nulls_head);
1403  get_order(sz));
1404  if (!hash) {
1405  printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
1406  hash = vzalloc(sz);
1407  }
1408 
1409  if (hash && nulls)
1410  for (i = 0; i < nr_slots; i++)
1411  INIT_HLIST_NULLS_HEAD(&hash[i], i);
1412 
1413  return hash;
1414 }
1416 
1417 int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1418 {
1419  int i, bucket;
1420  unsigned int hashsize, old_size;
1421  struct hlist_nulls_head *hash, *old_hash;
1422  struct nf_conntrack_tuple_hash *h;
1423  struct nf_conn *ct;
1424 
1425  if (current->nsproxy->net_ns != &init_net)
1426  return -EOPNOTSUPP;
1427 
1428  /* On boot, we can set this without any fancy locking. */
1430  return param_set_uint(val, kp);
1431 
1432  hashsize = simple_strtoul(val, NULL, 0);
1433  if (!hashsize)
1434  return -EINVAL;
1435 
1436  hash = nf_ct_alloc_hashtable(&hashsize, 1);
1437  if (!hash)
1438  return -ENOMEM;
1439 
1440  /* Lookups in the old hash might happen in parallel, which means we
1441  * might get false negatives during connection lookup. New connections
1442  * created because of a false negative won't make it into the hash
1443  * though since that required taking the lock.
1444  */
1445  spin_lock_bh(&nf_conntrack_lock);
1446  for (i = 0; i < init_net.ct.htable_size; i++) {
1447  while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
1448  h = hlist_nulls_entry(init_net.ct.hash[i].first,
1449  struct nf_conntrack_tuple_hash, hnnode);
1450  ct = nf_ct_tuplehash_to_ctrack(h);
1451  hlist_nulls_del_rcu(&h->hnnode);
1452  bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
1453  hashsize);
1454  hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1455  }
1456  }
1457  old_size = init_net.ct.htable_size;
1458  old_hash = init_net.ct.hash;
1459 
1460  init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
1461  init_net.ct.hash = hash;
1462  spin_unlock_bh(&nf_conntrack_lock);
1463 
1464  nf_ct_free_hashtable(old_hash, old_size);
1465  return 0;
1466 }
1468 
1470  &nf_conntrack_htable_size, 0600);
1471 
1472 void nf_ct_untracked_status_or(unsigned long bits)
1473 {
1474  int cpu;
1475 
1477  per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1478 }
1480 
1481 static int nf_conntrack_init_init_net(void)
1482 {
1483  int max_factor = 8;
1484  int ret, cpu;
1485 
1486  /* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB
1487  * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
1488  if (!nf_conntrack_htable_size) {
1490  = (((totalram_pages << PAGE_SHIFT) / 16384)
1491  / sizeof(struct hlist_head));
1492  if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1493  nf_conntrack_htable_size = 16384;
1494  if (nf_conntrack_htable_size < 32)
1496 
1497  /* Use a max. factor of four by default to get the same max as
1498  * with the old struct list_heads. When a table size is given
1499  * we use the old value of 8 to avoid reducing the max.
1500  * entries. */
1501  max_factor = 4;
1502  }
1504 
1505  printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1506  NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1507  nf_conntrack_max);
1508 #ifdef CONFIG_NF_CONNTRACK_ZONES
1509  ret = nf_ct_extend_register(&nf_ct_zone_extend);
1510  if (ret < 0)
1511  goto err_extend;
1512 #endif
1513  /* Set up fake conntrack: to never be deleted, not in any hashes */
1514  for_each_possible_cpu(cpu) {
1515  struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1516  write_pnet(&ct->ct_net, &init_net);
1517  atomic_set(&ct->ct_general.use, 1);
1518  }
1519  /* - and look it like as a confirmed connection */
1521  return 0;
1522 
1523 #ifdef CONFIG_NF_CONNTRACK_ZONES
1524 err_extend:
1525 #endif
1526  return ret;
1527 }
1528 
1529 /*
1530  * We need to use special "null" values, not used in hash table
1531  */
1532 #define UNCONFIRMED_NULLS_VAL ((1<<30)+0)
1533 #define DYING_NULLS_VAL ((1<<30)+1)
1534 
1535 static int nf_conntrack_init_net(struct net *net)
1536 {
1537  int ret;
1538 
1539  atomic_set(&net->ct.count, 0);
1540  INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
1541  INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
1542  net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1543  if (!net->ct.stat) {
1544  ret = -ENOMEM;
1545  goto err_stat;
1546  }
1547 
1548  net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
1549  if (!net->ct.slabname) {
1550  ret = -ENOMEM;
1551  goto err_slabname;
1552  }
1553 
1554  net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
1555  sizeof(struct nf_conn), 0,
1557  if (!net->ct.nf_conntrack_cachep) {
1558  printk(KERN_ERR "Unable to create nf_conn slab cache\n");
1559  ret = -ENOMEM;
1560  goto err_cache;
1561  }
1562 
1563  net->ct.htable_size = nf_conntrack_htable_size;
1564  net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
1565  if (!net->ct.hash) {
1566  ret = -ENOMEM;
1567  printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
1568  goto err_hash;
1569  }
1570  ret = nf_conntrack_expect_init(net);
1571  if (ret < 0)
1572  goto err_expect;
1573  ret = nf_conntrack_acct_init(net);
1574  if (ret < 0)
1575  goto err_acct;
1576  ret = nf_conntrack_tstamp_init(net);
1577  if (ret < 0)
1578  goto err_tstamp;
1579  ret = nf_conntrack_ecache_init(net);
1580  if (ret < 0)
1581  goto err_ecache;
1582  ret = nf_conntrack_timeout_init(net);
1583  if (ret < 0)
1584  goto err_timeout;
1585  ret = nf_conntrack_helper_init(net);
1586  if (ret < 0)
1587  goto err_helper;
1588  return 0;
1589 err_helper:
1591 err_timeout:
1593 err_ecache:
1595 err_tstamp:
1597 err_acct:
1599 err_expect:
1600  nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1601 err_hash:
1602  kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1603 err_cache:
1604  kfree(net->ct.slabname);
1605 err_slabname:
1606  free_percpu(net->ct.stat);
1607 err_stat:
1608  return ret;
1609 }
1610 
1611 s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
1612  enum ip_conntrack_dir dir,
1613  u32 seq);
1615 
1616 int nf_conntrack_init(struct net *net)
1617 {
1618  int ret;
1619 
1620  if (net_eq(net, &init_net)) {
1621  ret = nf_conntrack_init_init_net();
1622  if (ret < 0)
1623  goto out_init_net;
1624  }
1625  ret = nf_conntrack_proto_init(net);
1626  if (ret < 0)
1627  goto out_proto;
1628  ret = nf_conntrack_init_net(net);
1629  if (ret < 0)
1630  goto out_net;
1631 
1632  if (net_eq(net, &init_net)) {
1633  /* For use by REJECT target */
1634  RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1635  RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1636 
1637  /* Howto get NAT offsets */
1639  }
1640  return 0;
1641 
1642 out_net:
1644 out_proto:
1645  if (net_eq(net, &init_net))
1646  nf_conntrack_cleanup_init_net();
1647 out_init_net:
1648  return ret;
1649 }