7 #include <linux/sched.h>
13 #include <asm/pgtable.h>
37 if (!page_cache_get_speculative(page))
46 }
while (ptep++, addr +=
PAGE_SIZE, addr != end);
51 static inline int gup_huge_pmd(
pmd_t *pmdp,
pmd_t pmd,
unsigned long addr,
52 unsigned long end,
int write,
struct page **pages,
int *nr)
60 if ((
pmd_val(pmd) & mask) != result)
76 if (!page_cache_add_speculative(head, refs)) {
94 get_huge_page_tail(tail);
102 static inline int gup_pmd_range(
pud_t *pudp,
pud_t pud,
unsigned long addr,
103 unsigned long end,
int write,
struct page **pages,
int *nr)
108 pmdp = (
pmd_t *) pudp;
110 if ((
pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
127 if (
pmd_none(pmd) || pmd_trans_splitting(pmd))
130 if (!gup_huge_pmd(pmdp, pmd, addr, next,
133 }
else if (!gup_pte_range(pmdp, pmd, addr, next,
136 }
while (pmdp++, addr = next, addr != end);
141 static inline int gup_pud_range(
pgd_t *pgdp,
pgd_t pgd,
unsigned long addr,
142 unsigned long end,
int write,
struct page **pages,
int *nr)
147 pudp = (
pud_t *) pgdp;
149 if ((
pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
159 if (!gup_pmd_range(pudp, pud, addr, next, write, pages, nr))
161 }
while (pudp++, addr = next, addr != end);
191 next = pgd_addr_end(addr, end);
194 if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
196 }
while (pgdp++, addr = next, addr != end);
246 next = pgd_addr_end(addr, end);
249 if (!gup_pud_range(pgdp, pgd, addr, next, write, pages, &nr))
251 }
while (pgdp++, addr = next, addr != end);
268 (end - start) >> PAGE_SHIFT, write, 0, pages,
NULL);