ore_raid.c

Go to the documentation of this file.

/*
 * Copyright (C) 2011
 * Boaz Harrosh <[email protected]>
 *
 * This file is part of the objects raid engine (ore).
 *
 * It is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * You should have received a copy of the GNU General Public License
 * along with "ore". If not, write to the Free Software Foundation, Inc:
 *  "Free Software Foundation <[email protected]>"
 */

#include <linux/gfp.h>
#include <linux/async_tx.h>

#include "ore_raid.h"

#undef ORE_DBGMSG2
#define ORE_DBGMSG2 ORE_DBGMSG

struct page *_raid_page_alloc(void)
{
    return alloc_page(GFP_KERNEL);
}

void _raid_page_free(struct page *p)
{
    __free_page(p);
}

/* This struct is forward declare in ore_io_state, but is private to here.
 * It is put on ios->sp2d for RAID5/6 writes only. See _gen_xor_unit.
 *
 * __stripe_pages_2d is a 2d array of pages, and it is also a corner turn.
 * Ascending page index access is sp2d(p-minor, c-major). But storage is
 * sp2d[p-minor][c-major], so it can be properlly presented to the async-xor
 * API.
 */
struct __stripe_pages_2d {
    /* Cache some hot path repeated calculations */
    unsigned parity;
    unsigned data_devs;
    unsigned pages_in_unit;

    bool needed ;

    /* Array size is pages_in_unit (layout->stripe_unit / PAGE_SIZE) */
    struct __1_page_stripe {
        bool alloc;
        unsigned write_count;
        struct async_submit_ctl submit;
        struct dma_async_tx_descriptor *tx;

        /* The size of this array is data_devs + parity */
        struct page **pages;
        struct page **scribble;
        /* bool array, size of this array is data_devs */
        char *page_is_read;
    } _1p_stripes[];
};

/* This can get bigger then a page. So support multiple page allocations
 * _sp2d_free should be called even if _sp2d_alloc fails (by returning
 * none-zero).
 */
static int _sp2d_alloc(unsigned pages_in_unit, unsigned group_width,
               unsigned parity, struct __stripe_pages_2d **psp2d)
{
    struct __stripe_pages_2d *sp2d;
    unsigned data_devs = group_width - parity;
    struct _alloc_all_bytes {
        struct __alloc_stripe_pages_2d {
            struct __stripe_pages_2d sp2d;
            struct __1_page_stripe _1p_stripes[pages_in_unit];
        } __asp2d;
        struct __alloc_1p_arrays {
            struct page *pages[group_width];
            struct page *scribble[group_width];
            char page_is_read[data_devs];
        } __a1pa[pages_in_unit];
    } *_aab;
    struct __alloc_1p_arrays *__a1pa;
    struct __alloc_1p_arrays *__a1pa_end;
    const unsigned sizeof__a1pa = sizeof(_aab->__a1pa[0]);
    unsigned num_a1pa, alloc_size, i;

    /* FIXME: check these numbers in ore_verify_layout */
    BUG_ON(sizeof(_aab->__asp2d) > PAGE_SIZE);
    BUG_ON(sizeof__a1pa > PAGE_SIZE);

    if (sizeof(*_aab) > PAGE_SIZE) {
        num_a1pa = (PAGE_SIZE - sizeof(_aab->__asp2d)) / sizeof__a1pa;
        alloc_size = sizeof(_aab->__asp2d) + sizeof__a1pa * num_a1pa;
    } else {
        num_a1pa = pages_in_unit;
        alloc_size = sizeof(*_aab);
    }

    _aab = kzalloc(alloc_size, GFP_KERNEL);
    if (unlikely(!_aab)) {
        ORE_DBGMSG("!! Failed to alloc sp2d size=%d\n", alloc_size);
        return -ENOMEM;
    }

    sp2d = &_aab->__asp2d.sp2d;
    *psp2d = sp2d; /* From here Just call _sp2d_free */

    __a1pa = _aab->__a1pa;
    __a1pa_end = __a1pa + num_a1pa;

    for (i = 0; i < pages_in_unit; ++i) {
        if (unlikely(__a1pa >= __a1pa_end)) {
            num_a1pa = min_t(unsigned, PAGE_SIZE / sizeof__a1pa,
                            pages_in_unit - i);

            __a1pa = kzalloc(num_a1pa * sizeof__a1pa, GFP_KERNEL);
            if (unlikely(!__a1pa)) {
                ORE_DBGMSG("!! Failed to _alloc_1p_arrays=%d\n",
                       num_a1pa);
                return -ENOMEM;
            }
            __a1pa_end = __a1pa + num_a1pa;
            /* First *pages is marked for kfree of the buffer */
            sp2d->_1p_stripes[i].alloc = true;
        }

        sp2d->_1p_stripes[i].pages = __a1pa->pages;
        sp2d->_1p_stripes[i].scribble = __a1pa->scribble ;
        sp2d->_1p_stripes[i].page_is_read = __a1pa->page_is_read;
        ++__a1pa;
    }

    sp2d->parity = parity;
    sp2d->data_devs = data_devs;
    sp2d->pages_in_unit = pages_in_unit;
    return 0;
}

static void _sp2d_reset(struct __stripe_pages_2d *sp2d,
            const struct _ore_r4w_op *r4w, void *priv)
{
    unsigned data_devs = sp2d->data_devs;
    unsigned group_width = data_devs + sp2d->parity;
    int p, c;

    if (!sp2d->needed)
        return;

    for (c = data_devs - 1; c >= 0; --c)
        for (p = sp2d->pages_in_unit - 1; p >= 0; --p) {
            struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];

            if (_1ps->page_is_read[c]) {
                struct page *page = _1ps->pages[c];

                r4w->put_page(priv, page);
                _1ps->page_is_read[c] = false;
            }
        }

    for (p = 0; p < sp2d->pages_in_unit; p++) {
        struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];

        memset(_1ps->pages, 0, group_width * sizeof(*_1ps->pages));
        _1ps->write_count = 0;
        _1ps->tx = NULL;
    }

    sp2d->needed = false;
}

static void _sp2d_free(struct __stripe_pages_2d *sp2d)
{
    unsigned i;

    if (!sp2d)
        return;

    for (i = 0; i < sp2d->pages_in_unit; ++i) {
        if (sp2d->_1p_stripes[i].alloc)
            kfree(sp2d->_1p_stripes[i].pages);
    }

    kfree(sp2d);
}

static unsigned _sp2d_min_pg(struct __stripe_pages_2d *sp2d)
{
    unsigned p;

    for (p = 0; p < sp2d->pages_in_unit; p++) {
        struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];

        if (_1ps->write_count)
            return p;
    }

    return ~0;
}

static unsigned _sp2d_max_pg(struct __stripe_pages_2d *sp2d)
{
    int p;

    for (p = sp2d->pages_in_unit - 1; p >= 0; --p) {
        struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];

        if (_1ps->write_count)
            return p;
    }

    return ~0;
}

static void _gen_xor_unit(struct __stripe_pages_2d *sp2d)
{
    unsigned p;
    for (p = 0; p < sp2d->pages_in_unit; p++) {
        struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];

        if (!_1ps->write_count)
            continue;

        init_async_submit(&_1ps->submit,
            ASYNC_TX_XOR_ZERO_DST | ASYNC_TX_ACK,
            NULL,
            NULL, NULL,
            (addr_conv_t *)_1ps->scribble);

        /* TODO: raid6 */
        _1ps->tx = async_xor(_1ps->pages[sp2d->data_devs], _1ps->pages,
                     0, sp2d->data_devs, PAGE_SIZE,
                     &_1ps->submit);
    }

    for (p = 0; p < sp2d->pages_in_unit; p++) {
        struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
        /* NOTE: We wait for HW synchronously (I don't have such HW
         * to test with.) Is parallelism needed with today's multi
         * cores?
         */
        async_tx_issue_pending(_1ps->tx);
    }
}

void _ore_add_stripe_page(struct __stripe_pages_2d *sp2d,
               struct ore_striping_info *si, struct page *page)
{
    struct __1_page_stripe *_1ps;

    sp2d->needed = true;

    _1ps = &sp2d->_1p_stripes[si->cur_pg];
    _1ps->pages[si->cur_comp] = page;
    ++_1ps->write_count;

    si->cur_pg = (si->cur_pg + 1) % sp2d->pages_in_unit;
    /* si->cur_comp is advanced outside at main loop */
}

void _ore_add_sg_seg(struct ore_per_dev_state *per_dev, unsigned cur_len,
             bool not_last)
{
    struct osd_sg_entry *sge;

    ORE_DBGMSG("dev=%d cur_len=0x%x not_last=%d cur_sg=%d "
             "offset=0x%llx length=0x%x last_sgs_total=0x%x\n",
             per_dev->dev, cur_len, not_last, per_dev->cur_sg,
             _LLU(per_dev->offset), per_dev->length,
             per_dev->last_sgs_total);

    if (!per_dev->cur_sg) {
        sge = per_dev->sglist;

        /* First time we prepare two entries */
        if (per_dev->length) {
            ++per_dev->cur_sg;
            sge->offset = per_dev->offset;
            sge->len = per_dev->length;
        } else {
            /* Here the parity is the first unit of this object.
             * This happens every time we reach a parity device on
             * the same stripe as the per_dev->offset. We need to
             * just skip this unit.
             */
            per_dev->offset += cur_len;
            return;
        }
    } else {
        /* finalize the last one */
        sge = &per_dev->sglist[per_dev->cur_sg - 1];
        sge->len = per_dev->length - per_dev->last_sgs_total;
    }

    if (not_last) {
        /* Partly prepare the next one */
        struct osd_sg_entry *next_sge = sge + 1;

        ++per_dev->cur_sg;
        next_sge->offset = sge->offset + sge->len + cur_len;
        /* Save cur len so we know how mutch was added next time */
        per_dev->last_sgs_total = per_dev->length;
        next_sge->len = 0;
    } else if (!sge->len) {
        /* Optimize for when the last unit is a parity */
        --per_dev->cur_sg;
    }
}

static int _alloc_read_4_write(struct ore_io_state *ios)
{
    struct ore_layout *layout = ios->layout;
    int ret;
    /* We want to only read those pages not in cache so worst case
     * is a stripe populated with every other page
     */
    unsigned sgs_per_dev = ios->sp2d->pages_in_unit + 2;

    ret = _ore_get_io_state(layout, ios->oc,
                layout->group_width * layout->mirrors_p1,
                sgs_per_dev, 0, &ios->ios_read_4_write);
    return ret;
}

/* @si contains info of the to-be-inserted page. Update of @si should be
 * maintained by caller. Specificaly si->dev, si->obj_offset, ...
 */
static int _add_to_r4w(struct ore_io_state *ios, struct ore_striping_info *si,
               struct page *page, unsigned pg_len)
{
    struct request_queue *q;
    struct ore_per_dev_state *per_dev;
    struct ore_io_state *read_ios;
    unsigned first_dev = si->dev - (si->dev %
              (ios->layout->group_width * ios->layout->mirrors_p1));
    unsigned comp = si->dev - first_dev;
    unsigned added_len;

    if (!ios->ios_read_4_write) {
        int ret = _alloc_read_4_write(ios);

        if (unlikely(ret))
            return ret;
    }

    read_ios = ios->ios_read_4_write;
    read_ios->numdevs = ios->layout->group_width * ios->layout->mirrors_p1;

    per_dev = &read_ios->per_dev[comp];
    if (!per_dev->length) {
        per_dev->bio = bio_kmalloc(GFP_KERNEL,
                       ios->sp2d->pages_in_unit);
        if (unlikely(!per_dev->bio)) {
            ORE_DBGMSG("Failed to allocate BIO size=%u\n",
                     ios->sp2d->pages_in_unit);
            return -ENOMEM;
        }
        per_dev->offset = si->obj_offset;
        per_dev->dev = si->dev;
    } else if (si->obj_offset != (per_dev->offset + per_dev->length)) {
        u64 gap = si->obj_offset - (per_dev->offset + per_dev->length);

        _ore_add_sg_seg(per_dev, gap, true);
    }
    q = osd_request_queue(ore_comp_dev(read_ios->oc, per_dev->dev));
    added_len = bio_add_pc_page(q, per_dev->bio, page, pg_len,
                    si->obj_offset % PAGE_SIZE);
    if (unlikely(added_len != pg_len)) {
        ORE_DBGMSG("Failed to bio_add_pc_page bi_vcnt=%d\n",
                  per_dev->bio->bi_vcnt);
        return -ENOMEM;
    }

    per_dev->length += pg_len;
    return 0;
}

/* read the beginning of an unaligned first page */
static int _add_to_r4w_first_page(struct ore_io_state *ios, struct page *page)
{
    struct ore_striping_info si;
    unsigned pg_len;

    ore_calc_stripe_info(ios->layout, ios->offset, 0, &si);

    pg_len = si.obj_offset % PAGE_SIZE;
    si.obj_offset -= pg_len;

    ORE_DBGMSG("offset=0x%llx len=0x%x index=0x%lx dev=%x\n",
           _LLU(si.obj_offset), pg_len, page->index, si.dev);

    return _add_to_r4w(ios, &si, page, pg_len);
}

/* read the end of an incomplete last page */
static int _add_to_r4w_last_page(struct ore_io_state *ios, u64 *offset)
{
    struct ore_striping_info si;
    struct page *page;
    unsigned pg_len, p, c;

    ore_calc_stripe_info(ios->layout, *offset, 0, &si);

    p = si.unit_off / PAGE_SIZE;
    c = _dev_order(ios->layout->group_width * ios->layout->mirrors_p1,
               ios->layout->mirrors_p1, si.par_dev, si.dev);
    page = ios->sp2d->_1p_stripes[p].pages[c];

    pg_len = PAGE_SIZE - (si.unit_off % PAGE_SIZE);
    *offset += pg_len;

    ORE_DBGMSG("p=%d, c=%d next-offset=0x%llx len=0x%x dev=%x par_dev=%d\n",
           p, c, _LLU(*offset), pg_len, si.dev, si.par_dev);

    BUG_ON(!page);

    return _add_to_r4w(ios, &si, page, pg_len);
}

static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
{
    struct bio_vec *bv;
    unsigned i, d;

    /* loop on all devices all pages */
    for (d = 0; d < ios->numdevs; d++) {
        struct bio *bio = ios->per_dev[d].bio;

        if (!bio)
            continue;

        __bio_for_each_segment(bv, bio, i, 0) {
            struct page *page = bv->bv_page;

            SetPageUptodate(page);
            if (PageError(page))
                ClearPageError(page);
        }
    }
}

/* read_4_write is hacked to read the start of the first stripe and/or
 * the end of the last stripe. If needed, with an sg-gap at each device/page.
 * It is assumed to be called after the to_be_written pages of the first stripe
 * are populating ios->sp2d[][]
 *
 * NOTE: We call ios->r4w->lock_fn for all pages needed for parity calculations
 * These pages are held at sp2d[p].pages[c] but with
 * sp2d[p].page_is_read[c] = true. At _sp2d_reset these pages are
 * ios->r4w->lock_fn(). The ios->r4w->lock_fn might signal that the page is
 * @uptodate=true, so we don't need to read it, only unlock, after IO.
 *
 * TODO: The read_4_write should calc a need_to_read_pages_count, if bigger then
 * to-be-written count, we should consider the xor-in-place mode.
 * need_to_read_pages_count is the actual number of pages not present in cache.
 * maybe "devs_in_group - ios->sp2d[p].write_count" is a good enough
 * approximation? In this mode the read pages are put in the empty places of
 * ios->sp2d[p][*], xor is calculated the same way. These pages are
 * allocated/freed and don't go through cache
 */
static int _read_4_write_first_stripe(struct ore_io_state *ios)
{
    struct ore_striping_info read_si;
    struct __stripe_pages_2d *sp2d = ios->sp2d;
    u64 offset = ios->si.first_stripe_start;
    unsigned c, p, min_p = sp2d->pages_in_unit, max_p = -1;

    if (offset == ios->offset) /* Go to start collect $200 */
        goto read_last_stripe;

    min_p = _sp2d_min_pg(sp2d);
    max_p = _sp2d_max_pg(sp2d);

    ORE_DBGMSG("stripe_start=0x%llx ios->offset=0x%llx min_p=%d max_p=%d\n",
           offset, ios->offset, min_p, max_p);

    for (c = 0; ; c++) {
        ore_calc_stripe_info(ios->layout, offset, 0, &read_si);
        read_si.obj_offset += min_p * PAGE_SIZE;
        offset += min_p * PAGE_SIZE;
        for (p = min_p; p <= max_p; p++) {
            struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];
            struct page **pp = &_1ps->pages[c];
            bool uptodate;

            if (*pp) {
                if (ios->offset % PAGE_SIZE)
                    /* Read the remainder of the page */
                    _add_to_r4w_first_page(ios, *pp);
                /* to-be-written pages start here */
                goto read_last_stripe;
            }

            *pp = ios->r4w->get_page(ios->private, offset,
                         &uptodate);
            if (unlikely(!*pp))
                return -ENOMEM;

            if (!uptodate)
                _add_to_r4w(ios, &read_si, *pp, PAGE_SIZE);

            /* Mark read-pages to be cache_released */
            _1ps->page_is_read[c] = true;
            read_si.obj_offset += PAGE_SIZE;
            offset += PAGE_SIZE;
        }
        offset += (sp2d->pages_in_unit - p) * PAGE_SIZE;
    }

read_last_stripe:
    return 0;
}

static int _read_4_write_last_stripe(struct ore_io_state *ios)
{
    struct ore_striping_info read_si;
    struct __stripe_pages_2d *sp2d = ios->sp2d;
    u64 offset;
    u64 last_stripe_end;
    unsigned bytes_in_stripe = ios->si.bytes_in_stripe;
    unsigned c, p, min_p = sp2d->pages_in_unit, max_p = -1;

    offset = ios->offset + ios->length;
    if (offset % PAGE_SIZE)
        _add_to_r4w_last_page(ios, &offset);
        /* offset will be aligned to next page */

    last_stripe_end = div_u64(offset + bytes_in_stripe - 1, bytes_in_stripe)
                 * bytes_in_stripe;
    if (offset == last_stripe_end) /* Optimize for the aligned case */
        goto read_it;

    ore_calc_stripe_info(ios->layout, offset, 0, &read_si);
    p = read_si.unit_off / PAGE_SIZE;
    c = _dev_order(ios->layout->group_width * ios->layout->mirrors_p1,
               ios->layout->mirrors_p1, read_si.par_dev, read_si.dev);

    if (min_p == sp2d->pages_in_unit) {
        /* Didn't do it yet */
        min_p = _sp2d_min_pg(sp2d);
        max_p = _sp2d_max_pg(sp2d);
    }

    ORE_DBGMSG("offset=0x%llx stripe_end=0x%llx min_p=%d max_p=%d\n",
           offset, last_stripe_end, min_p, max_p);

    while (offset < last_stripe_end) {
        struct __1_page_stripe *_1ps = &sp2d->_1p_stripes[p];

        if ((min_p <= p) && (p <= max_p)) {
            struct page *page;
            bool uptodate;

            BUG_ON(_1ps->pages[c]);
            page = ios->r4w->get_page(ios->private, offset,
                          &uptodate);
            if (unlikely(!page))
                return -ENOMEM;

            _1ps->pages[c] = page;
            /* Mark read-pages to be cache_released */
            _1ps->page_is_read[c] = true;
            if (!uptodate)
                _add_to_r4w(ios, &read_si, page, PAGE_SIZE);
        }

        offset += PAGE_SIZE;
        if (p == (sp2d->pages_in_unit - 1)) {
            ++c;
            p = 0;
            ore_calc_stripe_info(ios->layout, offset, 0, &read_si);
        } else {
            read_si.obj_offset += PAGE_SIZE;
            ++p;
        }
    }

read_it:
    return 0;
}

static int _read_4_write_execute(struct ore_io_state *ios)
{
    struct ore_io_state *ios_read;
    unsigned i;
    int ret;

    ios_read = ios->ios_read_4_write;
    if (!ios_read)
        return 0;

    /* FIXME: Ugly to signal _sbi_read_mirror that we have bio(s). Change
     * to check for per_dev->bio
     */
    ios_read->pages = ios->pages;

    /* Now read these devices */
    for (i = 0; i < ios_read->numdevs; i += ios_read->layout->mirrors_p1) {
        ret = _ore_read_mirror(ios_read, i);
        if (unlikely(ret))
            return ret;
    }

    ret = ore_io_execute(ios_read); /* Synchronus execution */
    if (unlikely(ret)) {
        ORE_DBGMSG("!! ore_io_execute => %d\n", ret);
        return ret;
    }

    _mark_read4write_pages_uptodate(ios_read, ret);
    ore_put_io_state(ios_read);
    ios->ios_read_4_write = NULL; /* Might need a reuse at last stripe */
    return 0;
}

/* In writes @cur_len means length left. .i.e cur_len==0 is the last parity U */
int _ore_add_parity_unit(struct ore_io_state *ios,
                struct ore_striping_info *si,
                struct ore_per_dev_state *per_dev,
                unsigned cur_len)
{
    if (ios->reading) {
        if (per_dev->cur_sg >= ios->sgs_per_dev) {
            ORE_DBGMSG("cur_sg(%d) >= sgs_per_dev(%d)\n" ,
                per_dev->cur_sg, ios->sgs_per_dev);
            return -ENOMEM;
        }
        _ore_add_sg_seg(per_dev, cur_len, true);
    } else {
        struct __stripe_pages_2d *sp2d = ios->sp2d;
        struct page **pages = ios->parity_pages + ios->cur_par_page;
        unsigned num_pages;
        unsigned array_start = 0;
        unsigned i;
        int ret;

        si->cur_pg = _sp2d_min_pg(sp2d);
        num_pages  = _sp2d_max_pg(sp2d) + 1 - si->cur_pg;

        if (!cur_len) /* If last stripe operate on parity comp */
            si->cur_comp = sp2d->data_devs;

        if (!per_dev->length) {
            per_dev->offset += si->cur_pg * PAGE_SIZE;
            /* If first stripe, Read in all read4write pages
             * (if needed) before we calculate the first parity.
             */
            _read_4_write_first_stripe(ios);
        }
        if (!cur_len) /* If last stripe r4w pages of last stripe */
            _read_4_write_last_stripe(ios);
        _read_4_write_execute(ios);

        for (i = 0; i < num_pages; i++) {
            pages[i] = _raid_page_alloc();
            if (unlikely(!pages[i]))
                return -ENOMEM;

            ++(ios->cur_par_page);
        }

        BUG_ON(si->cur_comp != sp2d->data_devs);
        BUG_ON(si->cur_pg + num_pages > sp2d->pages_in_unit);

        ret = _ore_add_stripe_unit(ios,  &array_start, 0, pages,
                       per_dev, num_pages * PAGE_SIZE);
        if (unlikely(ret))
            return ret;

        /* TODO: raid6 if (last_parity_dev) */
        _gen_xor_unit(sp2d);
        _sp2d_reset(sp2d, ios->r4w, ios->private);
    }
    return 0;
}

int _ore_post_alloc_raid_stuff(struct ore_io_state *ios)
{
    if (ios->parity_pages) {
        struct ore_layout *layout = ios->layout;
        unsigned pages_in_unit = layout->stripe_unit / PAGE_SIZE;

        if (_sp2d_alloc(pages_in_unit, layout->group_width,
                layout->parity, &ios->sp2d)) {
            return -ENOMEM;
        }
    }
    return 0;
}

void _ore_free_raid_stuff(struct ore_io_state *ios)
{
    if (ios->sp2d) { /* writing and raid */
        unsigned i;

        for (i = 0; i < ios->cur_par_page; i++) {
            struct page *page = ios->parity_pages[i];

            if (page)
                _raid_page_free(page);
        }
        if (ios->extra_part_alloc)
            kfree(ios->parity_pages);
        /* If IO returned an error pages might need unlocking */
        _sp2d_reset(ios->sp2d, ios->r4w, ios->private);
        _sp2d_free(ios->sp2d);
    } else {
        /* Will only be set if raid reading && sglist is big */
        if (ios->extra_part_alloc)
            kfree(ios->per_dev[0].sglist);
    }
    if (ios->ios_read_4_write)
        ore_put_io_state(ios->ios_read_4_write);
}