cfq-iosched.c File Reference

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/elevator.h>
#include <linux/jiffies.h>
#include <linux/rbtree.h>
#include <linux/ioprio.h>
#include <linux/blktrace_api.h>
#include "blk.h"
#include "blk-cgroup.h"

Go to the source code of this file.

Data Structures
struct	cfq_ttime
struct	cfq_rb_root
struct	cfq_queue
struct	cfqg_stats
struct	cfq_group
struct	cfq_io_cq
struct	cfq_data

Macros
#define	CFQ_IDLE_DELAY   (HZ / 5)
#define	CFQ_MIN_TT   (2)
#define	CFQ_SLICE_SCALE   (5)
#define	CFQ_HW_QUEUE_MIN   (5)
#define	CFQ_SERVICE_SHIFT   12
#define	CFQQ_SEEK_THR   (sector_t)(8 * 100)
#define	CFQQ_CLOSE_THR   (sector_t)(8 * 1024)
#define	CFQQ_SECT_THR_NONROT   (sector_t)(2 * 32)
#define	CFQQ_SEEKY(cfqq)   (hweight32(cfqq->seek_history) > 32/8)
#define	RQ_CIC(rq)   icq_to_cic((rq)->elv.icq)
#define	RQ_CFQQ(rq)   (struct cfq_queue *) ((rq)->elv.priv[0])
#define	RQ_CFQG(rq)   (struct cfq_group *) ((rq)->elv.priv[1])
#define	CFQ_PRIO_LISTS   IOPRIO_BE_NR
#define	cfq_class_idle(cfqq)   ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
#define	cfq_class_rt(cfqq)   ((cfqq)->ioprio_class == IOPRIO_CLASS_RT)
#define	sample_valid(samples)   ((samples) > 80)
#define	rb_entry_cfqg(node)   rb_entry((node), struct cfq_group, rb_node)
#define	CFQ_RB_ROOT
#define	CFQ_CFQQ_FNS(name)
#define	cfq_log_cfqq(cfqd, cfqq, fmt, args...)   blk_add_trace_msg((cfqd)->queue, "cfq%d " fmt, (cfqq)->pid, ##args)
#define	cfq_log_cfqg(cfqd, cfqg, fmt, args...)   do {} while (0)
#define	cfq_log(cfqd, fmt, args...)   blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args)
#define	for_each_cfqg_st(cfqg, i, j, st)
#define	CFQ_RQ1_WRAP   0x01 /* request 1 wraps */
#define	CFQ_RQ2_WRAP   0x02 /* request 2 wraps */
#define	SHOW_FUNCTION(__FUNC, __VAR, __CONV)
#define	STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV)
#define	CFQ_ATTR(name)   __ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store)

Enumerations
enum	wl_prio_t { BE_WORKLOAD = 0, RT_WORKLOAD = 1, IDLE_WORKLOAD = 2, CFQ_PRIO_NR }
enum	wl_type_t { ASYNC_WORKLOAD = 0, SYNC_NOIDLE_WORKLOAD = 1, SYNC_WORKLOAD = 2 }
enum	cfqq_state_flags {   CFQ_CFQQ_FLAG_on_rr = 0, CFQ_CFQQ_FLAG_wait_request, CFQ_CFQQ_FLAG_must_dispatch, CFQ_CFQQ_FLAG_must_alloc_slice,   CFQ_CFQQ_FLAG_fifo_expire, CFQ_CFQQ_FLAG_idle_window, CFQ_CFQQ_FLAG_prio_changed, CFQ_CFQQ_FLAG_slice_new,   CFQ_CFQQ_FLAG_sync, CFQ_CFQQ_FLAG_coop, CFQ_CFQQ_FLAG_split_coop, CFQ_CFQQ_FLAG_deep,   CFQ_CFQQ_FLAG_wait_busy }

Functions
	CFQ_CFQQ_FNS (on_rr)
	CFQ_CFQQ_FNS (wait_request)
	CFQ_CFQQ_FNS (must_dispatch)
	CFQ_CFQQ_FNS (must_alloc_slice)
	CFQ_CFQQ_FNS (fifo_expire)
	CFQ_CFQQ_FNS (idle_window)
	CFQ_CFQQ_FNS (prio_changed)
	CFQ_CFQQ_FNS (slice_new)
	CFQ_CFQQ_FNS (sync)
	CFQ_CFQQ_FNS (coop)
	CFQ_CFQQ_FNS (split_coop)
	CFQ_CFQQ_FNS (deep)
	CFQ_CFQQ_FNS (wait_busy)
	SHOW_FUNCTION (cfq_quantum_show, cfqd->cfq_quantum, 0)
	SHOW_FUNCTION (cfq_fifo_expire_sync_show, cfqd->cfq_fifo_expire[1], 1)
	SHOW_FUNCTION (cfq_fifo_expire_async_show, cfqd->cfq_fifo_expire[0], 1)
	SHOW_FUNCTION (cfq_back_seek_max_show, cfqd->cfq_back_max, 0)
	SHOW_FUNCTION (cfq_back_seek_penalty_show, cfqd->cfq_back_penalty, 0)
	SHOW_FUNCTION (cfq_slice_idle_show, cfqd->cfq_slice_idle, 1)
	SHOW_FUNCTION (cfq_group_idle_show, cfqd->cfq_group_idle, 1)
	SHOW_FUNCTION (cfq_slice_sync_show, cfqd->cfq_slice[1], 1)
	SHOW_FUNCTION (cfq_slice_async_show, cfqd->cfq_slice[0], 1)
	SHOW_FUNCTION (cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0)
	SHOW_FUNCTION (cfq_low_latency_show, cfqd->cfq_latency, 0)
	SHOW_FUNCTION (cfq_target_latency_show, cfqd->cfq_target_latency, 1)
	STORE_FUNCTION (cfq_quantum_store,&cfqd->cfq_quantum, 1, UINT_MAX, 0)
	STORE_FUNCTION (cfq_fifo_expire_sync_store,&cfqd->cfq_fifo_expire[1], 1, UINT_MAX, 1)
	STORE_FUNCTION (cfq_fifo_expire_async_store,&cfqd->cfq_fifo_expire[0], 1, UINT_MAX, 1)
	STORE_FUNCTION (cfq_back_seek_max_store,&cfqd->cfq_back_max, 0, UINT_MAX, 0)
	STORE_FUNCTION (cfq_back_seek_penalty_store,&cfqd->cfq_back_penalty, 1, UINT_MAX, 0)
	STORE_FUNCTION (cfq_slice_idle_store,&cfqd->cfq_slice_idle, 0, UINT_MAX, 1)
	STORE_FUNCTION (cfq_group_idle_store,&cfqd->cfq_group_idle, 0, UINT_MAX, 1)
	STORE_FUNCTION (cfq_slice_sync_store,&cfqd->cfq_slice[1], 1, UINT_MAX, 1)
	STORE_FUNCTION (cfq_slice_async_store,&cfqd->cfq_slice[0], 1, UINT_MAX, 1)
	STORE_FUNCTION (cfq_slice_async_rq_store,&cfqd->cfq_slice_async_rq, 1, UINT_MAX, 0)
	STORE_FUNCTION (cfq_low_latency_store,&cfqd->cfq_latency, 0, 1, 0)
	STORE_FUNCTION (cfq_target_latency_store,&cfqd->cfq_target_latency, 1, UINT_MAX, 1)
	module_init (cfq_init)
	module_exit (cfq_exit)
	MODULE_AUTHOR ("Jens Axboe")
	MODULE_LICENSE ("GPL")
	MODULE_DESCRIPTION ("Completely Fair Queueing IO scheduler")

Macro Definition Documentation

#define CFQ_ATTR

(

name

)

__ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store)

Definition at line 4125 of file cfq-iosched.c.

#define CFQ_CFQQ_FNS

(

name

)

Value:

static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq)     \
{                                   \
    (cfqq)->flags |= (1 << CFQ_CFQQ_FLAG_##name);           \
}                                   \
static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq)    \
{                                   \
    (cfqq)->flags &= ~(1 << CFQ_CFQQ_FLAG_##name);          \
}                                   \
static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq)     \
{                                   \
    return ((cfqq)->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0;  \
}

Definition at line 385 of file cfq-iosched.c.

#define cfq_class_idle

(

cfqq

)

((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)

Definition at line 64 of file cfq-iosched.c.

#define cfq_class_rt

(

cfqq

)

((cfqq)->ioprio_class == IOPRIO_CLASS_RT)

Definition at line 65 of file cfq-iosched.c.

#define CFQ_HW_QUEUE_MIN   (5)

Definition at line 49 of file cfq-iosched.c.

#define CFQ_IDLE_DELAY   (HZ / 5)

Definition at line 41 of file cfq-iosched.c.

#define cfq_log	(		cfqd,
			fmt,
			args...
	)		blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args)

Definition at line 694 of file cfq-iosched.c.

#define cfq_log_cfqg	(		cfqd,
			cfqg,
			fmt,
			args...
	)		do {} while (0)

Definition at line 679 of file cfq-iosched.c.

#define cfq_log_cfqq	(		cfqd,
			cfqq,
			fmt,
			args...
	)		blk_add_trace_msg((cfqd)->queue, "cfq%d " fmt, (cfqq)->pid, ##args)

Definition at line 677 of file cfq-iosched.c.

#define CFQ_MIN_TT   (2)

Definition at line 46 of file cfq-iosched.c.

#define CFQ_PRIO_LISTS   IOPRIO_BE_NR

Definition at line 63 of file cfq-iosched.c.

#define CFQ_RB_ROOT

Value:

(struct cfq_rb_root) { .rb = RB_ROOT, \
            .ttime = {.last_end_request = jiffies,},}

Definition at line 92 of file cfq-iosched.c.

#define CFQ_RQ1_WRAP   0x01 /* request 1 wraps */

#define CFQ_RQ2_WRAP   0x02 /* request 2 wraps */

#define CFQ_SERVICE_SHIFT   12

Definition at line 50 of file cfq-iosched.c.

#define CFQ_SLICE_SCALE   (5)

Definition at line 48 of file cfq-iosched.c.

#define CFQQ_CLOSE_THR   (sector_t)(8 * 1024)

Definition at line 53 of file cfq-iosched.c.

#define CFQQ_SECT_THR_NONROT   (sector_t)(2 * 32)

Definition at line 54 of file cfq-iosched.c.

#define CFQQ_SEEK_THR   (sector_t)(8 * 100)

Definition at line 52 of file cfq-iosched.c.

#define CFQQ_SEEKY

(

cfqq

)

(hweight32(cfqq->seek_history) > 32/8)

Definition at line 55 of file cfq-iosched.c.

#define for_each_cfqg_st	(		cfqg,
			i,
			j,
			st
	)

Value:

for (i = 0; i <= IDLE_WORKLOAD; i++) \
        for (j = 0, st = i < IDLE_WORKLOAD ? &cfqg->service_trees[i][j]\
            : &cfqg->service_tree_idle; \
            (i < IDLE_WORKLOAD && j <= SYNC_WORKLOAD) || \
            (i == IDLE_WORKLOAD && j == 0); \
            j++, st = i < IDLE_WORKLOAD ? \
            &cfqg->service_trees[i][j]: NULL) \

Definition at line 698 of file cfq-iosched.c.

#define rb_entry_cfqg

(

node

)

rb_entry((node), struct cfq_group, rb_node)

Definition at line 68 of file cfq-iosched.c.

#define RQ_CFQG

(

rq

)

(struct cfq_group *) ((rq)->elv.priv[1])

Definition at line 59 of file cfq-iosched.c.

#define RQ_CFQQ

(

rq

)

(struct cfq_queue *) ((rq)->elv.priv[0])

Definition at line 58 of file cfq-iosched.c.

#define RQ_CIC

(

rq

)

icq_to_cic((rq)->elv.icq)

Definition at line 57 of file cfq-iosched.c.

#define sample_valid

(

samples

)

((samples) > 80)

Definition at line 67 of file cfq-iosched.c.

#define SHOW_FUNCTION	(		__FUNC,
			__VAR,
			__CONV
	)

Value:

static ssize_t __FUNC(struct elevator_queue *e, char *page)     \
{                                   \
	struct cfq_data *cfqd = e->elevator_data;          \
    unsigned int __data = __VAR;                    \
    if (__CONV)                         \
        __data = jiffies_to_msecs(__data);          \
    return cfq_var_show(__data, (page));                \
}

Definition at line 4068 of file cfq-iosched.c.

#define STORE_FUNCTION	(		__FUNC,
			__PTR,
			MIN,
			MAX,
			__CONV
	)

Value:

static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \
{                                   \
	struct cfq_data *cfqd = e->elevator_data;          \
    unsigned int __data;                        \
    int ret = cfq_var_store(&__data, (page), count);        \
    if (__data < (MIN))                     \
        __data = (MIN);                     \
    else if (__data > (MAX))                    \
        __data = (MAX);                     \
    if (__CONV)                         \
        *(__PTR) = msecs_to_jiffies(__data);            \
    else                                \
        *(__PTR) = __data;                  \
    return ret;                         \
}

Definition at line 4091 of file cfq-iosched.c.

Enumeration Type Documentation

enum cfqq_state_flags

Enumerator:

CFQ_CFQQ_FLAG_on_rr
CFQ_CFQQ_FLAG_wait_request
CFQ_CFQQ_FLAG_must_dispatch
CFQ_CFQQ_FLAG_must_alloc_slice
CFQ_CFQQ_FLAG_fifo_expire
CFQ_CFQQ_FLAG_idle_window
CFQ_CFQQ_FLAG_prio_changed
CFQ_CFQQ_FLAG_slice_new
CFQ_CFQQ_FLAG_sync
CFQ_CFQQ_FLAG_coop
CFQ_CFQQ_FLAG_split_coop
CFQ_CFQQ_FLAG_deep
CFQ_CFQQ_FLAG_wait_busy

Definition at line 369 of file cfq-iosched.c.

enum wl_prio_t

Enumerator:

BE_WORKLOAD
RT_WORKLOAD
IDLE_WORKLOAD
CFQ_PRIO_NR

Definition at line 158 of file cfq-iosched.c.

enum wl_type_t

Enumerator:

ASYNC_WORKLOAD
SYNC_NOIDLE_WORKLOAD
SYNC_WORKLOAD

Definition at line 168 of file cfq-iosched.c.

Function Documentation

CFQ_CFQQ_FNS

(

on_rr

)

CFQ_CFQQ_FNS

(

wait_request

)

CFQ_CFQQ_FNS

(

must_dispatch

)

CFQ_CFQQ_FNS

(

must_alloc_slice

)

CFQ_CFQQ_FNS

(

fifo_expire

)

CFQ_CFQQ_FNS

(

idle_window

)

CFQ_CFQQ_FNS

(

prio_changed

)

CFQ_CFQQ_FNS

(

slice_new

)

CFQ_CFQQ_FNS

(

sync

)

CFQ_CFQQ_FNS

(

coop

)

CFQ_CFQQ_FNS

(

split_coop

)

CFQ_CFQQ_FNS

(

deep

)

CFQ_CFQQ_FNS

(

wait_busy

)

MODULE_AUTHOR

(

"Jens Axboe"

)

MODULE_DESCRIPTION

(

"Completely Fair Queueing IO scheduler"

)

module_exit

(

cfq_exit

)

module_init

(

cfq_init

)

MODULE_LICENSE

(

"GPL"

)

SHOW_FUNCTION	(	cfq_quantum_show	,
		cfqd->	cfq_quantum,
		0
	)

SHOW_FUNCTION	(	cfq_fifo_expire_sync_show	,
		cfqd->	cfq_fifo_expire[1],
		1
	)

SHOW_FUNCTION	(	cfq_fifo_expire_async_show	,
		cfqd->	cfq_fifo_expire[0],
		1
	)

SHOW_FUNCTION	(	cfq_back_seek_max_show	,
		cfqd->	cfq_back_max,
		0
	)

SHOW_FUNCTION	(	cfq_back_seek_penalty_show	,
		cfqd->	cfq_back_penalty,
		0
	)

SHOW_FUNCTION	(	cfq_slice_idle_show	,
		cfqd->	cfq_slice_idle,
		1
	)

SHOW_FUNCTION	(	cfq_group_idle_show	,
		cfqd->	cfq_group_idle,
		1
	)

SHOW_FUNCTION	(	cfq_slice_sync_show	,
		cfqd->	cfq_slice[1],
		1
	)

SHOW_FUNCTION	(	cfq_slice_async_show	,
		cfqd->	cfq_slice[0],
		1
	)

SHOW_FUNCTION	(	cfq_slice_async_rq_show	,
		cfqd->	cfq_slice_async_rq,
		0
	)

SHOW_FUNCTION	(	cfq_low_latency_show	,
		cfqd->	cfq_latency,
		0
	)

SHOW_FUNCTION	(	cfq_target_latency_show	,
		cfqd->	cfq_target_latency,
		1
	)

STORE_FUNCTION	(	cfq_quantum_store	,
		&cfqd->	cfq_quantum,
		1	,
		UINT_MAX	,
		0
	)

STORE_FUNCTION	(	cfq_fifo_expire_sync_store	,
		&cfqd->	cfq_fifo_expire[1],
		1	,
		UINT_MAX	,
		1
	)

STORE_FUNCTION	(	cfq_fifo_expire_async_store	,
		&cfqd->	cfq_fifo_expire[0],
		1	,
		UINT_MAX	,
		1
	)

STORE_FUNCTION	(	cfq_back_seek_max_store	,
		&cfqd->	cfq_back_max,
		0	,
		UINT_MAX	,
		0
	)

STORE_FUNCTION	(	cfq_back_seek_penalty_store	,
		&cfqd->	cfq_back_penalty,
		1	,
		UINT_MAX	,
		0
	)

STORE_FUNCTION	(	cfq_slice_idle_store	,
		&cfqd->	cfq_slice_idle,
		0	,
		UINT_MAX	,
		1
	)

STORE_FUNCTION	(	cfq_group_idle_store	,
		&cfqd->	cfq_group_idle,
		0	,
		UINT_MAX	,
		1
	)

STORE_FUNCTION	(	cfq_slice_sync_store	,
		&cfqd->	cfq_slice[1],
		1	,
		UINT_MAX	,
		1
	)

STORE_FUNCTION	(	cfq_slice_async_store	,
		&cfqd->	cfq_slice[0],
		1	,
		UINT_MAX	,
		1
	)

STORE_FUNCTION	(	cfq_slice_async_rq_store	,
		&cfqd->	cfq_slice_async_rq,
		1	,
		UINT_MAX	,
		0
	)

STORE_FUNCTION	(	cfq_low_latency_store	,
		&cfqd->	cfq_latency,
		0	,
		1	,
		0
	)

STORE_FUNCTION	(	cfq_target_latency_store	,
		&cfqd->	cfq_target_latency,
		1	,
		UINT_MAX	,
		1
	)