csum_partial_copy.c

Go to the documentation of this file.

/*
 * csum_partial_copy - do IP checksumming and copy
 *
 * (C) Copyright 1996 Linus Torvalds
 * accelerated versions (and 21264 assembly versions ) contributed by
 *  Rick Gorton <[email protected]>
 *
 * Don't look at this too closely - you'll go mad. The things
 * we do for performance..
 */

#include <linux/types.h>
#include <linux/string.h>
#include <asm/uaccess.h>


#define ldq_u(x,y) \
__asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))

#define stq_u(x,y) \
__asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))

#define extql(x,y,z) \
__asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))

#define extqh(x,y,z) \
__asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))

#define mskql(x,y,z) \
__asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))

#define mskqh(x,y,z) \
__asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))

#define insql(x,y,z) \
__asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))

#define insqh(x,y,z) \
__asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))


#define __get_user_u(x,ptr)             \
({                          \
    long __guu_err;                 \
    __asm__ __volatile__(               \
    "1: ldq_u %0,%2\n"              \
    "2:\n"                      \
    ".section __ex_table,\"a\"\n"           \
    "   .long 1b - .\n"             \
    "   lda %0,2b-1b(%1)\n"         \
    ".previous"                 \
        : "=r"(x), "=r"(__guu_err)      \
        : "m"(__m(ptr)), "1"(0));       \
    __guu_err;                  \
})

#define __put_user_u(x,ptr)             \
({                          \
    long __puu_err;                 \
    __asm__ __volatile__(               \
    "1: stq_u %2,%1\n"              \
    "2:\n"                      \
    ".section __ex_table,\"a\"\n"           \
    "   .long 1b - ."               \
    "   lda $31,2b-1b(%0)\n"            \
    ".previous"                 \
        : "=r"(__puu_err)           \
        : "m"(__m(addr)), "rJ"(x), "0"(0)); \
    __puu_err;                  \
})


static inline unsigned short from64to16(unsigned long x)
{
    /* Using extract instructions is a bit more efficient
       than the original shift/bitmask version.  */

    union {
        unsigned long   ul;
        unsigned int    ui[2];
        unsigned short  us[4];
    } in_v, tmp_v, out_v;

    in_v.ul = x;
    tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];

    /* Since the bits of tmp_v.sh[3] are going to always be zero,
       we don't have to bother to add that in.  */
    out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
            + (unsigned long) tmp_v.us[2];

    /* Similarly, out_v.us[2] is always zero for the final add.  */
    return out_v.us[0] + out_v.us[1];
}



/*
 * Ok. This isn't fun, but this is the EASY case.
 */
static inline unsigned long
csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
             long len, unsigned long checksum,
             int *errp)
{
    unsigned long carry = 0;
    int err = 0;

    while (len >= 0) {
        unsigned long word;
        err |= __get_user(word, src);
        checksum += carry;
        src++;
        checksum += word;
        len -= 8;
        carry = checksum < word;
        *dst = word;
        dst++;
    }
    len += 8;
    checksum += carry;
    if (len) {
        unsigned long word, tmp;
        err |= __get_user(word, src);
        tmp = *dst;
        mskql(word, len, word);
        checksum += word;
        mskqh(tmp, len, tmp);
        carry = checksum < word;
        *dst = word | tmp;
        checksum += carry;
    }
    if (err) *errp = err;
    return checksum;
}

/*
 * This is even less fun, but this is still reasonably
 * easy.
 */
static inline unsigned long
csum_partial_cfu_dest_aligned(const unsigned long __user *src,
                  unsigned long *dst,
                  unsigned long soff,
                  long len, unsigned long checksum,
                  int *errp)
{
    unsigned long first;
    unsigned long word, carry;
    unsigned long lastsrc = 7+len+(unsigned long)src;
    int err = 0;

    err |= __get_user_u(first,src);
    carry = 0;
    while (len >= 0) {
        unsigned long second;

        err |= __get_user_u(second, src+1);
        extql(first, soff, word);
        len -= 8;
        src++;
        extqh(second, soff, first);
        checksum += carry;
        word |= first;
        first = second;
        checksum += word;
        *dst = word;
        dst++;
        carry = checksum < word;
    }
    len += 8;
    checksum += carry;
    if (len) {
        unsigned long tmp;
        unsigned long second;
        err |= __get_user_u(second, lastsrc);
        tmp = *dst;
        extql(first, soff, word);
        extqh(second, soff, first);
        word |= first;
        mskql(word, len, word);
        checksum += word;
        mskqh(tmp, len, tmp);
        carry = checksum < word;
        *dst = word | tmp;
        checksum += carry;
    }
    if (err) *errp = err;
    return checksum;
}

/*
 * This is slightly less fun than the above..
 */
static inline unsigned long
csum_partial_cfu_src_aligned(const unsigned long __user *src,
                 unsigned long *dst,
                 unsigned long doff,
                 long len, unsigned long checksum,
                 unsigned long partial_dest,
                 int *errp)
{
    unsigned long carry = 0;
    unsigned long word;
    unsigned long second_dest;
    int err = 0;

    mskql(partial_dest, doff, partial_dest);
    while (len >= 0) {
        err |= __get_user(word, src);
        len -= 8;
        insql(word, doff, second_dest);
        checksum += carry;
        stq_u(partial_dest | second_dest, dst);
        src++;
        checksum += word;
        insqh(word, doff, partial_dest);
        carry = checksum < word;
        dst++;
    }
    len += 8;
    if (len) {
        checksum += carry;
        err |= __get_user(word, src);
        mskql(word, len, word);
        len -= 8;
        checksum += word;
        insql(word, doff, second_dest);
        len += doff;
        carry = checksum < word;
        partial_dest |= second_dest;
        if (len >= 0) {
            stq_u(partial_dest, dst);
            if (!len) goto out;
            dst++;
            insqh(word, doff, partial_dest);
        }
        doff = len;
    }
    ldq_u(second_dest, dst);
    mskqh(second_dest, doff, second_dest);
    stq_u(partial_dest | second_dest, dst);
out:
    checksum += carry;
    if (err) *errp = err;
    return checksum;
}

/*
 * This is so totally un-fun that it's frightening. Don't
 * look at this too closely, you'll go blind.
 */
static inline unsigned long
csum_partial_cfu_unaligned(const unsigned long __user * src,
               unsigned long * dst,
               unsigned long soff, unsigned long doff,
               long len, unsigned long checksum,
               unsigned long partial_dest,
               int *errp)
{
    unsigned long carry = 0;
    unsigned long first;
    unsigned long lastsrc;
    int err = 0;

    err |= __get_user_u(first, src);
    lastsrc = 7+len+(unsigned long)src;
    mskql(partial_dest, doff, partial_dest);
    while (len >= 0) {
        unsigned long second, word;
        unsigned long second_dest;

        err |= __get_user_u(second, src+1);
        extql(first, soff, word);
        checksum += carry;
        len -= 8;
        extqh(second, soff, first);
        src++;
        word |= first;
        first = second;
        insql(word, doff, second_dest);
        checksum += word;
        stq_u(partial_dest | second_dest, dst);
        carry = checksum < word;
        insqh(word, doff, partial_dest);
        dst++;
    }
    len += doff;
    checksum += carry;
    if (len >= 0) {
        unsigned long second, word;
        unsigned long second_dest;

        err |= __get_user_u(second, lastsrc);
        extql(first, soff, word);
        extqh(second, soff, first);
        word |= first;
        first = second;
        mskql(word, len-doff, word);
        checksum += word;
        insql(word, doff, second_dest);
        carry = checksum < word;
        stq_u(partial_dest | second_dest, dst);
        if (len) {
            ldq_u(second_dest, dst+1);
            insqh(word, doff, partial_dest);
            mskqh(second_dest, len, second_dest);
            stq_u(partial_dest | second_dest, dst+1);
        }
        checksum += carry;
    } else {
        unsigned long second, word;
        unsigned long second_dest;

        err |= __get_user_u(second, lastsrc);
        extql(first, soff, word);
        extqh(second, soff, first);
        word |= first;
        ldq_u(second_dest, dst);
        mskql(word, len-doff, word);
        checksum += word;
        mskqh(second_dest, len, second_dest);
        carry = checksum < word;
        insql(word, doff, word);
        stq_u(partial_dest | word | second_dest, dst);
        checksum += carry;
    }
    if (err) *errp = err;
    return checksum;
}

__wsum
csum_partial_copy_from_user(const void __user *src, void *dst, int len,
                   __wsum sum, int *errp)
{
    unsigned long checksum = (__force u32) sum;
    unsigned long soff = 7 & (unsigned long) src;
    unsigned long doff = 7 & (unsigned long) dst;

    if (len) {
        if (!doff) {
            if (!soff)
                checksum = csum_partial_cfu_aligned(
                    (const unsigned long __user *) src,
                    (unsigned long *) dst,
                    len-8, checksum, errp);
            else
                checksum = csum_partial_cfu_dest_aligned(
                    (const unsigned long __user *) src,
                    (unsigned long *) dst,
                    soff, len-8, checksum, errp);
        } else {
            unsigned long partial_dest;
            ldq_u(partial_dest, dst);
            if (!soff)
                checksum = csum_partial_cfu_src_aligned(
                    (const unsigned long __user *) src,
                    (unsigned long *) dst,
                    doff, len-8, checksum,
                    partial_dest, errp);
            else
                checksum = csum_partial_cfu_unaligned(
                    (const unsigned long __user *) src,
                    (unsigned long *) dst,
                    soff, doff, len-8, checksum,
                    partial_dest, errp);
        }
        checksum = from64to16 (checksum);
    }
    return (__force __wsum)checksum;
}

__wsum
csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
{
    return csum_partial_copy_from_user((__force const void __user *)src,
            dst, len, sum, NULL);
}