#include <linux/res_counter.h>
#include <linux/memcontrol.h>
#include <linux/cgroup.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/smp.h>
#include <linux/page-flags.h>
#include <linux/backing-dev.h>
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
#include <linux/limits.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include <linux/rbtree.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/spinlock.h>
#include <linux/eventfd.h>
#include <linux/sort.h>
#include <linux/fs.h>
#include <linux/seq_file.h>
#include <linux/vmalloc.h>
#include <linux/mm_inline.h>
#include <linux/page_cgroup.h>
#include <linux/cpu.h>
#include <linux/oom.h>
#include "internal.h"
#include <net/sock.h>
#include <net/ip.h>
#include <net/tcp_memcontrol.h>
#include <asm/uaccess.h>
#include <trace/events/vmscan.h>

Data Structures
struct	mem_cgroup_stat_cpu

struct	mem_cgroup_reclaim_iter

struct	mem_cgroup_per_zone

struct	mem_cgroup_per_node

struct	mem_cgroup_lru_info

struct	mem_cgroup_tree_per_zone

struct	mem_cgroup_tree_per_node

struct	mem_cgroup_tree

struct	mem_cgroup_threshold

struct	mem_cgroup_threshold_ary

struct	mem_cgroup_thresholds

struct	mem_cgroup_eventfd_list

struct	mem_cgroup

struct	move_charge_struct

struct	oom_wait_info

struct	memcg_stock_pcp

Macros
#define	MEM_CGROUP_RECLAIM_RETRIES 5

#define	do_swap_account 0

#define	THRESHOLDS_EVENTS_TARGET 128

#define	SOFTLIMIT_EVENTS_TARGET 1024

#define	NUMAINFO_EVENTS_TARGET 1024

#define	MEM_CGROUP_MAX_RECLAIM_LOOPS 100

#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2

#define	_MEM (0)

#define	_MEMSWAP (1)

#define	_OOM_TYPE (2)

#define	MEMFILE_PRIVATE(x, val) ((x) << 16 \| (val))

#define	MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)

#define	MEMFILE_ATTR(val) ((val) & 0xffff)

#define	OOM_CONTROL (0)

#define	MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0

#define	MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)

#define	MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1

#define	MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)

#define	for_each_mem_cgroup_tree(iter, root)

#define	for_each_mem_cgroup(iter)

#define	mem_cgroup_from_res_counter(counter, member) container_of(counter, struct mem_cgroup, member)

#define	CHARGE_BATCH 32U

#define	FLUSHING_CACHED_CHARGE 0

Enumerations
enum	mem_cgroup_stat_index { MEM_CGROUP_STAT_CACHE, MEM_CGROUP_STAT_RSS, MEM_CGROUP_STAT_FILE_MAPPED, MEM_CGROUP_STAT_SWAP, MEM_CGROUP_STAT_NSTATS }

enum	mem_cgroup_events_index { MEM_CGROUP_EVENTS_PGPGIN, MEM_CGROUP_EVENTS_PGPGOUT, MEM_CGROUP_EVENTS_PGFAULT, MEM_CGROUP_EVENTS_PGMAJFAULT, MEM_CGROUP_EVENTS_NSTATS }

enum	mem_cgroup_events_target { MEM_CGROUP_TARGET_THRESH, MEM_CGROUP_TARGET_SOFTLIMIT, MEM_CGROUP_TARGET_NUMAINFO, MEM_CGROUP_NTARGETS }

enum	move_type { MOVE_DOWN, MOVE_UP_FROM_LEFT, MOVE_UP_FROM_RIGHT, MOVE_CHARGE_TYPE_ANON, MOVE_CHARGE_TYPE_FILE, NR_MOVE_TYPE }

enum	charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_ANON, MEM_CGROUP_CHARGE_TYPE_SWAPOUT, MEM_CGROUP_CHARGE_TYPE_DROP, NR_CHARGE_TYPE }

enum	{ CHARGE_OK, CHARGE_RETRY, CHARGE_NOMEM, CHARGE_WOULDBLOCK, CHARGE_OOM_DIE }

Functions
struct cgroup_subsys_state *	mem_cgroup_css (struct mem_cgroup *memcg)

unsigned long	mem_cgroup_get_lru_size (struct lruvec *lruvec, enum lru_list lru)

struct mem_cgroup *	mem_cgroup_from_cont (struct cgroup *cont)

struct mem_cgroup *	mem_cgroup_from_task (struct task_struct *p)

struct mem_cgroup *	try_get_mem_cgroup_from_mm (struct mm_struct *mm)

struct mem_cgroup *	mem_cgroup_iter (struct mem_cgroup root, struct mem_cgroup prev, struct mem_cgroup_reclaim_cookie *reclaim)

void	mem_cgroup_iter_break (struct mem_cgroup root, struct mem_cgroup prev)

void	mem_cgroup_count_vm_event (struct mm_struct *mm, enum vm_event_item idx)

	EXPORT_SYMBOL (mem_cgroup_count_vm_event)

struct lruvec *	mem_cgroup_zone_lruvec (struct zone zone, struct mem_cgroup memcg)

struct lruvec *	mem_cgroup_page_lruvec (struct page page, struct zone zone)

void	mem_cgroup_update_lru_size (struct lruvec *lruvec, enum lru_list lru, int nr_pages)

bool	__mem_cgroup_same_or_subtree (const struct mem_cgroup root_memcg, struct mem_cgroup memcg)

int	task_in_mem_cgroup (struct task_struct task, const struct mem_cgroup memcg)

int	mem_cgroup_inactive_anon_is_low (struct lruvec *lruvec)

int	mem_cgroup_inactive_file_is_low (struct lruvec *lruvec)

int	mem_cgroup_swappiness (struct mem_cgroup *memcg)

void	mem_cgroup_print_oom_info (struct mem_cgroup memcg, struct task_struct p)

void	mem_cgroup_out_of_memory (struct mem_cgroup *memcg, gfp_t gfp_mask, int order)

int	mem_cgroup_select_victim_node (struct mem_cgroup *memcg)

void	__mem_cgroup_begin_update_page_stat (struct page page, bool locked, unsigned long *flags)

void	__mem_cgroup_end_update_page_stat (struct page page, unsigned long flags)

void	mem_cgroup_update_page_stat (struct page *page, enum mem_cgroup_page_stat_item idx, int val)

struct mem_cgroup *	try_get_mem_cgroup_from_page (struct page *page)

int	mem_cgroup_newpage_charge (struct page page, struct mm_struct mm, gfp_t gfp_mask)

int	mem_cgroup_try_charge_swapin (struct mm_struct mm, struct page page, gfp_t gfp_mask, struct mem_cgroup **memcgp)

void	mem_cgroup_cancel_charge_swapin (struct mem_cgroup *memcg)

void	mem_cgroup_commit_charge_swapin (struct page page, struct mem_cgroup memcg)

int	mem_cgroup_cache_charge (struct page page, struct mm_struct mm, gfp_t gfp_mask)

void	mem_cgroup_uncharge_page (struct page *page)

void	mem_cgroup_uncharge_cache_page (struct page *page)

void	mem_cgroup_uncharge_start (void)

void	mem_cgroup_uncharge_end (void)

void	mem_cgroup_prepare_migration (struct page page, struct page newpage, struct mem_cgroup **memcgp)

void	mem_cgroup_end_migration (struct mem_cgroup memcg, struct page oldpage, struct page *newpage, bool migration_ok)

void	mem_cgroup_replace_page_cache (struct page oldpage, struct page newpage)

unsigned long	mem_cgroup_soft_limit_reclaim (struct zone zone, int order, gfp_t gfp_mask, unsigned long total_scanned)

struct mem_cgroup *	parent_mem_cgroup (struct mem_cgroup *memcg)

	EXPORT_SYMBOL (parent_mem_cgroup)

Variables
struct cgroup_subsys mem_cgroup_subsys	__read_mostly

struct cgroup_subsys	mem_cgroup_subsys

Macro Definition Documentation

#define _MEM (0)

Definition at line 389 of file memcontrol.c.

#define _MEMSWAP (1)

Definition at line 390 of file memcontrol.c.

#define _OOM_TYPE (2)

Definition at line 391 of file memcontrol.c.

#define CHARGE_BATCH 32U

Definition at line 2053 of file memcontrol.c.

#define do_swap_account 0

Definition at line 77 of file memcontrol.c.

#define FLUSHING_CACHED_CHARGE 0

Definition at line 2059 of file memcontrol.c.

#define for_each_mem_cgroup ( iter )

Value:

for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
         iter != NULL;              \
         iter = mem_cgroup_iter(NULL, iter, NULL))

Definition at line 1013 of file memcontrol.c.

#define for_each_mem_cgroup_tree	(	iter,
		root
	)

Value:

for (iter = mem_cgroup_iter(root, NULL, NULL);  \
         iter != NULL;              \
         iter = mem_cgroup_iter(root, iter, NULL))

Definition at line 1008 of file memcontrol.c.

#define mem_cgroup_from_res_counter	(	counter,
		member
	)	container_of(counter, struct mem_cgroup, member)

Definition at line 1251 of file memcontrol.c.

#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100

Definition at line 377 of file memcontrol.c.

#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2

Definition at line 378 of file memcontrol.c.

#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)

Definition at line 402 of file memcontrol.c.

#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0

Definition at line 401 of file memcontrol.c.

#define MEM_CGROUP_RECLAIM_RETRIES 5

Definition at line 62 of file memcontrol.c.

#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)

Definition at line 404 of file memcontrol.c.

#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1

Definition at line 403 of file memcontrol.c.

#define MEMFILE_ATTR ( val ) ((val) & 0xffff)

Definition at line 394 of file memcontrol.c.

#define MEMFILE_PRIVATE	(	x,
		val
	)	((x) << 16 \| (val))

Definition at line 392 of file memcontrol.c.

#define MEMFILE_TYPE ( val ) ((val) >> 16 & 0xffff)

Definition at line 393 of file memcontrol.c.

#define NUMAINFO_EVENTS_TARGET 1024

Definition at line 131 of file memcontrol.c.

#define OOM_CONTROL (0)

Definition at line 396 of file memcontrol.c.

#define SOFTLIMIT_EVENTS_TARGET 1024

Definition at line 130 of file memcontrol.c.

#define THRESHOLDS_EVENTS_TARGET 128

Definition at line 129 of file memcontrol.c.

Enumeration Type Documentation

anonymous enum

Enumerator:

CHARGE_OK
CHARGE_RETRY
CHARGE_NOMEM
CHARGE_WOULDBLOCK
CHARGE_OOM_DIE

Definition at line 2245 of file memcontrol.c.

enum charge_type

Enumerator:

MEM_CGROUP_CHARGE_TYPE_CACHE
MEM_CGROUP_CHARGE_TYPE_ANON
MEM_CGROUP_CHARGE_TYPE_SWAPOUT
MEM_CGROUP_CHARGE_TYPE_DROP
NR_CHARGE_TYPE

Definition at line 380 of file memcontrol.c.

enum mem_cgroup_events_index

Enumerator:

MEM_CGROUP_EVENTS_PGPGIN
MEM_CGROUP_EVENTS_PGPGOUT
MEM_CGROUP_EVENTS_PGFAULT
MEM_CGROUP_EVENTS_PGMAJFAULT
MEM_CGROUP_EVENTS_NSTATS

Definition at line 102 of file memcontrol.c.

enum mem_cgroup_events_target

Enumerator:

MEM_CGROUP_TARGET_THRESH
MEM_CGROUP_TARGET_SOFTLIMIT
MEM_CGROUP_TARGET_NUMAINFO
MEM_CGROUP_NTARGETS

Definition at line 123 of file memcontrol.c.

enum mem_cgroup_stat_index

Enumerator:

MEM_CGROUP_STAT_CACHE
MEM_CGROUP_STAT_RSS
MEM_CGROUP_STAT_FILE_MAPPED
MEM_CGROUP_STAT_SWAP
MEM_CGROUP_STAT_NSTATS

Definition at line 84 of file memcontrol.c.

enum move_type

Enumerator:

MOVE_DOWN
MOVE_UP_FROM_LEFT
MOVE_UP_FROM_RIGHT
MOVE_CHARGE_TYPE_ANON
MOVE_CHARGE_TYPE_FILE
NR_MOVE_TYPE

Definition at line 340 of file memcontrol.c.

Function Documentation

void __mem_cgroup_begin_update_page_stat	(	struct page *	page,
		bool *	locked,
		unsigned long *	flags
	)

Definition at line 1984 of file memcontrol.c.

void __mem_cgroup_end_update_page_stat	(	struct page *	page,
		unsigned long *	flags
	)

Definition at line 2012 of file memcontrol.c.

bool __mem_cgroup_same_or_subtree	(	const struct mem_cgroup *	root_memcg,
		struct mem_cgroup *	memcg
	)

Definition at line 1165 of file memcontrol.c.

EXPORT_SYMBOL ( mem_cgroup_count_vm_event )

EXPORT_SYMBOL ( parent_mem_cgroup )

int mem_cgroup_cache_charge	(	struct page *	page,
		struct mm_struct *	mm,
		gfp_t	gfp_mask
	)

Definition at line 2936 of file memcontrol.c.

void mem_cgroup_cancel_charge_swapin ( struct mem_cgroup * memcg )

Definition at line 2890 of file memcontrol.c.

void mem_cgroup_commit_charge_swapin	(	struct page *	page,
		struct mem_cgroup *	memcg
	)

Definition at line 2929 of file memcontrol.c.

void mem_cgroup_count_vm_event	(	struct mm_struct *	mm,
		enum vm_event_item	idx
	)

Definition at line 1018 of file memcontrol.c.

struct cgroup_subsys_state* mem_cgroup_css ( struct mem_cgroup * memcg )

read

Definition at line 496 of file memcontrol.c.

void mem_cgroup_end_migration	(	struct mem_cgroup *	memcg,
		struct page *	oldpage,
		struct page *	newpage,
		bool	migration_ok
	)

Definition at line 3365 of file memcontrol.c.

struct mem_cgroup* mem_cgroup_from_cont ( struct cgroup * cont )

read

Definition at line 868 of file memcontrol.c.

struct mem_cgroup* mem_cgroup_from_task ( struct task_struct * p )

read

Definition at line 874 of file memcontrol.c.

unsigned long mem_cgroup_get_lru_size	(	struct lruvec *	lruvec,
		enum lru_list	lru
	)

Definition at line 758 of file memcontrol.c.

int mem_cgroup_inactive_anon_is_low ( struct lruvec * lruvec )

Definition at line 1221 of file memcontrol.c.

int mem_cgroup_inactive_file_is_low ( struct lruvec * lruvec )

Definition at line 1240 of file memcontrol.c.

struct mem_cgroup* mem_cgroup_iter	(	struct mem_cgroup *	root,
		struct mem_cgroup *	prev,
		struct mem_cgroup_reclaim_cookie *	reclaim
	)

read

mem_cgroup_iter - iterate over memory cgroup hierarchy : hierarchy root : previously returned memcg, NULL on first invocation : cookie for shared reclaim walks, NULL for full walks

Returns references to children of the hierarchy below , or itself, or NULL after a full round-trip.

Caller must pass the return value in on subsequent invocations for reference counting, or use mem_cgroup_iter_break() to cancel a hierarchy walk before the round-trip is complete.

Reclaimers can specify a zone and a priority level in to divide up the memcgs in the hierarchy among all concurrent reclaimers operating on the same zone and priority.

Definition at line 925 of file memcontrol.c.

void mem_cgroup_iter_break	(	struct mem_cgroup *	root,
		struct mem_cgroup *	prev
	)

mem_cgroup_iter_break - abort a hierarchy walk prematurely : hierarchy root : last visited hierarchy member as returned by mem_cgroup_iter()

Definition at line 994 of file memcontrol.c.

int mem_cgroup_newpage_charge	(	struct page *	page,
		struct mm_struct *	mm,
		gfp_t	gfp_mask
	)

Definition at line 2812 of file memcontrol.c.

void mem_cgroup_out_of_memory	(	struct mem_cgroup *	memcg,
		gfp_t	gfp_mask,
		int	order
	)

Definition at line 1501 of file memcontrol.c.

struct lruvec* mem_cgroup_page_lruvec	(	struct page *	page,
		struct zone *	zone
	)

read

Definition at line 1097 of file memcontrol.c.

void mem_cgroup_prepare_migration	(	struct page *	page,
		struct page *	newpage,
		struct mem_cgroup **	memcgp
	)

Definition at line 3287 of file memcontrol.c.

void mem_cgroup_print_oom_info	(	struct mem_cgroup *	memcg,
		struct task_struct *	p
	)

mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode. : The memory cgroup that went over limit : Task that is going to be killed

NOTE: and 's mem_cgroup can be different when hierarchy is enabled

Definition at line 1401 of file memcontrol.c.

void mem_cgroup_replace_page_cache	(	struct page *	oldpage,
		struct page *	newpage
	)

Definition at line 3423 of file memcontrol.c.

int mem_cgroup_select_victim_node ( struct mem_cgroup * memcg )

Definition at line 1730 of file memcontrol.c.

unsigned long mem_cgroup_soft_limit_reclaim	(	struct zone *	zone,
		int	order,
		gfp_t	gfp_mask,
		unsigned long *	total_scanned
	)

Definition at line 3623 of file memcontrol.c.

int mem_cgroup_swappiness ( struct mem_cgroup * memcg )

Definition at line 1271 of file memcontrol.c.

int mem_cgroup_try_charge_swapin	(	struct mm_struct *	mm,
		struct page *	page,
		gfp_t	gfp_mask,
		struct mem_cgroup **	memcgp
	)

Definition at line 2867 of file memcontrol.c.

void mem_cgroup_uncharge_cache_page ( struct page * page )

Definition at line 3129 of file memcontrol.c.

void mem_cgroup_uncharge_end ( void )

Definition at line 3155 of file memcontrol.c.

void mem_cgroup_uncharge_page ( struct page * page )

Definition at line 3118 of file memcontrol.c.

void mem_cgroup_uncharge_start ( void )

Definition at line 3144 of file memcontrol.c.

void mem_cgroup_update_lru_size	(	struct lruvec *	lruvec,
		enum lru_list	lru,
		int	nr_pages
	)

mem_cgroup_update_lru_size - account for adding or removing an lru page : mem_cgroup per zone lru vector : index of lru list the page is sitting on : positive when adding or negative when removing

This function must be called when a page is added to or removed from an lru list.

Definition at line 1146 of file memcontrol.c.

void mem_cgroup_update_page_stat	(	struct page *	page,
		enum mem_cgroup_page_stat_item	idx,
		int	val
	)

Definition at line 2024 of file memcontrol.c.

struct lruvec* mem_cgroup_zone_lruvec	(	struct zone *	zone,
		struct mem_cgroup *	memcg
	)

read

mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg : zone of the wanted lruvec : memcg of the wanted lruvec

Returns the lru list vector holding pages for the given and . This can be the global zone lruvec, if the memory controller is disabled.

Definition at line 1054 of file memcontrol.c.

struct mem_cgroup* parent_mem_cgroup ( struct mem_cgroup * memcg )

read

Definition at line 4900 of file memcontrol.c.

int task_in_mem_cgroup	(	struct task_struct *	task,
		const struct mem_cgroup *	memcg
	)

Definition at line 1186 of file memcontrol.c.

struct mem_cgroup* try_get_mem_cgroup_from_mm ( struct mm_struct * mm )

read

Definition at line 887 of file memcontrol.c.

struct mem_cgroup* try_get_mem_cgroup_from_page ( struct page * page )

read

Definition at line 2530 of file memcontrol.c.

Variable Documentation

atomic_t memcg_moving __read_mostly

Definition at line 61 of file memcontrol.c.

struct cgroup_subsys mem_cgroup_subsys

Initial value:

= {
    .name = "memory",
    .subsys_id = mem_cgroup_subsys_id,
    .create = mem_cgroup_create,
    .pre_destroy = mem_cgroup_pre_destroy,
    .destroy = mem_cgroup_destroy,
    .can_attach = mem_cgroup_can_attach,
    .cancel_attach = mem_cgroup_cancel_attach,
    .attach = mem_cgroup_move_task,
    .base_cftypes = mem_cgroup_files,
    .early_init = 0,
    .use_id = 1,
    .__DEPRECATED_clear_css_refs = true,
}

Definition at line 5631 of file memcontrol.c.

Data Structures

Macros

Enumerations

Functions

Variables

Macro Definition Documentation

Enumeration Type Documentation

Function Documentation

Variable Documentation