15 #include <linux/types.h>
16 #include <linux/string.h>
17 #include <linux/module.h>
21 #define word_t uint64_t
23 #if CHIP_L2_LINE_SIZE() != 64 && CHIP_L2_LINE_SIZE() != 128
24 #error "Assumes 64 or 128 byte line size"
28 #define PREFETCH_LINES_AHEAD 3
34 #define ST(p, v) (*(p) = (v))
47 void *
memcpy(
void *__restrict dstv,
const void *__restrict srcv,
size_t n)
54 int USERCOPY_FUNC(
void *__restrict dstv,
const void *__restrict srcv,
size_t n)
57 char *__restrict dst1 = (
char *)dstv;
58 const char *__restrict src1 = (
const char *)srcv;
59 const char *__restrict src1_end;
75 src1_end = src1 + n - 1;
80 __insn_prefetch(prefetch);
82 prefetch = (prefetch > src1_end) ? prefetch : src1;
101 const word_t *__restrict src8 =
108 a = __insn_dblalign(a, b, src1);
116 b = ((
const char *)src8 <= src1_end) ? *src8 : 0;
122 final = __insn_dblalign(a, b, src1);
142 __insn_prefetch(prefetch);
144 prefetch = (prefetch > src1_end) ? prefetch :
152 #define COPY_WORD(offset) ({ ST8(dst8+offset, LD8(src8+offset)); n -= 8; })
161 #if CHIP_L2_LINE_SIZE() == 128
170 #elif CHIP_L2_LINE_SIZE() != 64
171 # error Fix code that assumes particular L2 cache line sizes
182 if (__builtin_expect(n == 0, 1))
190 #ifndef __BIG_ENDIAN__