11 #include <linux/kernel.h>
13 #include <linux/errno.h>
14 #include <linux/time.h>
16 #include <linux/export.h>
19 #include <linux/uio.h>
23 #include <linux/sched.h>
27 #include <linux/mman.h>
29 #include <linux/slab.h>
39 #include <asm/kmap_types.h>
40 #include <asm/uaccess.h>
43 #define dprintk printk
45 #define dprintk(x...) do { ; } while (0)
60 static void aio_queue_work(
struct kioctx *);
66 static int __init aio_setup(
void)
74 pr_debug(
"aio_setup: sizeof(struct page) = %d\n", (
int)
sizeof(
struct page));
80 static void aio_free_ring(
struct kioctx *
ctx)
99 static int aio_setup_ring(
struct kioctx *ctx)
111 size +=
sizeof(
struct io_event) * nr_events;
153 info->
nr = nr_events;
156 ring->
nr = nr_events;
172 #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
173 #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
174 #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
176 #define aio_ring_event(info, nr) ({ \
177 unsigned pos = (nr) + AIO_EVENTS_OFFSET; \
178 struct io_event *__event; \
179 __event = kmap_atomic( \
180 (info)->ring_pages[pos / AIO_EVENTS_PER_PAGE]); \
181 __event += pos % AIO_EVENTS_PER_PAGE; \
185 #define put_aio_ring_event(event) do { \
186 struct io_event *__event = (event); \
188 kunmap_atomic((void *)((unsigned long)__event & PAGE_MASK)); \
201 static void __put_ioctx(
struct kioctx *ctx)
211 spin_lock(&aio_nr_lock);
214 spin_unlock(&aio_nr_lock);
216 pr_debug(
"__put_ioctx: freeing %p\n", ctx);
235 static struct kioctx *ioctx_alloc(
unsigned nr_events)
242 if ((nr_events > (0x10000000U /
sizeof(
struct io_event))) ||
243 (nr_events > (0x10000000U /
sizeof(
struct kiocb)))) {
244 pr_debug(
"ENOMEM: nr_events too high\n");
248 if (!nr_events || (
unsigned long)nr_events >
aio_max_nr)
251 ctx = kmem_cache_zalloc(kioctx_cachep,
GFP_KERNEL);
268 if (aio_setup_ring(ctx) < 0)
272 spin_lock(&aio_nr_lock);
275 spin_unlock(&aio_nr_lock);
279 spin_unlock(&aio_nr_lock);
282 spin_lock(&mm->ioctx_lock);
283 hlist_add_head_rcu(&ctx->
list, &mm->ioctx_list);
284 spin_unlock(&mm->ioctx_lock);
286 dprintk(
"aio: allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
296 dprintk(
"aio: error allocating ioctx %d\n", err);
305 static void kill_ctx(
struct kioctx *ctx)
373 while (!hlist_empty(&mm->ioctx_list)) {
375 hlist_del_rcu(&ctx->
list);
381 "exit_aio:ioctx still alive: %d %d %d\n",
409 static struct kiocb *__aio_get_req(
struct kioctx *ctx)
436 #define KIOCB_BATCH_SIZE 32L
442 static void kiocb_batch_init(
struct kiocb_batch *batch,
long total)
444 INIT_LIST_HEAD(&batch->
head);
445 batch->
count = total;
448 static void kiocb_batch_free(
struct kioctx *ctx,
struct kiocb_batch *batch)
452 if (list_empty(&batch->
head))
471 static int kiocb_batch_refill(
struct kioctx *ctx,
struct kiocb_batch *batch)
473 unsigned short allocated, to_alloc;
479 for (allocated = 0; allocated < to_alloc; allocated++) {
480 req = __aio_get_req(ctx);
495 if (avail < allocated) {
500 if (--allocated <= avail)
505 batch->
count -= allocated;
518 static inline struct kiocb *aio_get_req(
struct kioctx *ctx,
523 if (list_empty(&batch->
head))
524 if (kiocb_batch_refill(ctx, batch) == 0)
531 static inline void really_put_req(
struct kioctx *ctx,
struct kiocb *req)
551 static int __aio_put_req(
struct kioctx *ctx,
struct kiocb *req)
554 req, atomic_long_read(&req->
ki_filp->f_count));
568 really_put_req(ctx, req);
578 struct kioctx *ctx = req->
ki_ctx;
581 ret = __aio_put_req(ctx, req);
587 static struct kioctx *lookup_ioctx(
unsigned long ctx_id)
595 hlist_for_each_entry_rcu(ctx, n, &mm->ioctx_list,
list) {
602 if (ctx->
user_id == ctx_id && !ctx->
dead && try_get_ioctx(ctx)){
621 static inline int __queue_kicked_iocb(
struct kiocb *iocb)
623 struct kioctx *ctx = iocb->
ki_ctx;
659 struct kioctx *ctx = iocb->
ki_ctx;
664 printk(
"aio_run_iocb: iocb->ki_retry = NULL\n");
738 __queue_kicked_iocb(iocb);
759 static int __aio_run_iocbs(
struct kioctx *ctx)
766 list_replace_init(&ctx->
run_list, &run_list);
767 while (!list_empty(&run_list)) {
776 __aio_put_req(ctx, iocb);
783 static void aio_queue_work(
struct kioctx * ctx)
791 if (waitqueue_active(&ctx->
wait))
805 static inline void aio_run_all_iocbs(
struct kioctx *ctx)
808 while (__aio_run_iocbs(ctx))
832 requeue =__aio_run_iocbs(ctx);
850 static void try_queue_kicked_iocb(
struct kiocb *iocb)
852 struct kioctx *ctx = iocb->
ki_ctx;
860 run = __queue_kicked_iocb(iocb);
861 spin_unlock_irqrestore(&ctx->
ctx_lock, flags);
877 if (is_sync_kiocb(iocb)) {
883 try_queue_kicked_iocb(iocb);
894 struct kioctx *ctx = iocb->
ki_ctx;
909 if (is_sync_kiocb(iocb)) {
941 if (++tail >= info->
nr)
949 dprintk(
"aio_complete: %p[%lu]: %p: %p %Lx %lx %lx\n",
964 pr_debug(
"added to ring %p at [%lu]\n", iocb, tail);
976 ret = __aio_put_req(ctx, iocb);
986 if (waitqueue_active(&ctx->
wait))
989 spin_unlock_irqrestore(&ctx->
ctx_lock, flags);
1000 static int aio_read_evt(
struct kioctx *ioctx,
struct io_event *
ent)
1008 dprintk(
"in aio_read_evt h%lu t%lu m%lu\n",
1009 (
unsigned long)ring->
head, (
unsigned long)ring->
tail,
1010 (
unsigned long)ring->
nr);
1017 head = ring->
head % info->
nr;
1018 if (head != ring->
tail) {
1021 head = (head + 1) % info->
nr;
1031 dprintk(
"leaving aio_read_evt: %d h%lu t%lu\n", ret,
1032 (
unsigned long)ring->
head, (
unsigned long)ring->
tail);
1042 static void timeout_func(
unsigned long data)
1050 static inline void init_timeout(
struct aio_timeout *to)
1057 static inline void set_timeout(
long start_jiffies,
struct aio_timeout *to,
1067 static inline void clear_timeout(
struct aio_timeout *to)
1072 static int read_events(
struct kioctx *ctx,
1073 long min_nr,
long nr,
1089 memset(&ent, 0,
sizeof(ent));
1093 ret = aio_read_evt(ctx, &ent);
1097 dprintk(
"read event: %Lx %Lx %Lx %Lx\n",
1103 dprintk(
"aio: lost an event due to EFAULT.\n");
1123 aio_run_all_iocbs(ctx);
1134 set_timeout(start_jiffies, &to, &ts);
1141 ret = aio_read_evt(ctx, &ent);
1158 if (signal_pending(tsk)) {
1173 dprintk(
"aio: lost an event due to EFAULT.\n");
1185 destroy_timer_on_stack(&to.
timer);
1198 spin_lock(&mm->ioctx_lock);
1199 was_dead = ioctx->
dead;
1201 hlist_del_rcu(&ioctx->
list);
1202 spin_unlock(&mm->ioctx_lock);
1204 dprintk(
"aio_release(%p)\n", ioctx);
1233 struct kioctx *ioctx =
NULL;
1242 if (
unlikely(ctx || nr_events == 0)) {
1243 pr_debug(
"EINVAL: io_setup: ctx %lu nr_events %u\n",
1248 ioctx = ioctx_alloc(nr_events);
1249 ret = PTR_ERR(ioctx);
1250 if (!IS_ERR(ioctx)) {
1269 struct kioctx *ioctx = lookup_ioctx(ctx);
1275 pr_debug(
"EINVAL: io_destroy: invalid context id\n");
1279 static void aio_advance_iovec(
struct kiocb *iocb,
ssize_t ret)
1308 unsigned long, loff_t);
1314 rw_op = file->
f_op->aio_read;
1317 rw_op = file->
f_op->aio_write;
1330 aio_advance_iovec(iocb, ret);
1334 }
while (ret > 0 && iocb->
ki_left > 0 &&
1340 if ((ret == 0) || (iocb->
ki_left == 0))
1355 struct file *file = iocb->
ki_filp;
1358 if (file->
f_op->aio_fsync)
1359 ret = file->
f_op->aio_fsync(iocb, 1);
1365 struct file *file = iocb->
ki_filp;
1368 if (file->
f_op->aio_fsync)
1369 ret = file->
f_op->aio_fsync(iocb, 0);
1377 #ifdef CONFIG_COMPAT
1380 (
struct compat_iovec __user *)kiocb->
ki_buf,
1407 static ssize_t aio_setup_single_vector(
int type,
struct file * file,
struct kiocb *kiocb)
1428 static ssize_t aio_setup_iocb(
struct kiocb *kiocb,
bool compat)
1430 struct file *file = kiocb->
ki_filp;
1442 ret = aio_setup_single_vector(
READ, file, kiocb);
1446 if (file->
f_op->aio_read)
1447 kiocb->
ki_retry = aio_rw_vect_retry;
1457 ret = aio_setup_single_vector(
WRITE, file, kiocb);
1461 if (file->
f_op->aio_write)
1462 kiocb->
ki_retry = aio_rw_vect_retry;
1468 ret = aio_setup_vectored_rw(
READ, kiocb, compat);
1472 if (file->
f_op->aio_read)
1473 kiocb->
ki_retry = aio_rw_vect_retry;
1479 ret = aio_setup_vectored_rw(
WRITE, kiocb, compat);
1483 if (file->
f_op->aio_write)
1484 kiocb->
ki_retry = aio_rw_vect_retry;
1488 if (file->
f_op->aio_fsync)
1493 if (file->
f_op->aio_fsync)
1497 dprintk(
"EINVAL: io_submit: no operation provided\n");
1507 static int io_submit_one(
struct kioctx *ctx,
struct iocb __user *user_iocb,
1517 pr_debug(
"EINVAL: io_submit: reserve field set\n");
1527 pr_debug(
"EINVAL: io_submit: overflow check\n");
1535 req = aio_get_req(ctx, batch);
1570 ret = aio_setup_iocb(req, compat);
1596 while (__aio_run_iocbs(ctx))
1611 struct iocb __user *__user *iocbpp,
bool compat)
1628 ctx = lookup_ioctx(ctx_id);
1630 pr_debug(
"EINVAL: io_submit: invalid context id\n");
1634 kiocb_batch_init(&batch, nr);
1642 for (i=0; i<
nr; i++) {
1643 struct iocb __user *user_iocb;
1656 ret = io_submit_one(ctx, user_iocb, &tmp, &batch, compat);
1662 kiocb_batch_free(ctx, &batch);
1680 struct iocb __user * __user *, iocbpp)
1688 static struct kiocb *lookup_kiocb(
struct kioctx *ctx,
struct iocb __user *iocb,
1697 struct kiocb *kiocb = list_kiocb(pos);
1719 struct kiocb *kiocb;
1723 ret =
get_user(key, &iocb->aio_key);
1727 ctx = lookup_ioctx(ctx_id);
1733 kiocb = lookup_kiocb(ctx, iocb, key);
1745 memset(&tmp, 0,
sizeof(tmp));
1748 ret =
cancel(kiocb, &tmp);
1783 struct kioctx *ioctx = lookup_ioctx(ctx_id);
1787 if (
likely(min_nr <= nr && min_nr >= 0))
1788 ret = read_events(ioctx, min_nr, nr,
events, timeout);