22 #define _LARGEFILE64_SOURCE
32 #include <sys/types.h>
33 #include <sys/errno.h>
34 #include <sys/fcntl.h>
35 #include <sys/mount.h>
36 #include <sys/statfs.h>
37 #include "../../include/uapi/linux/magic.h"
38 #include "../../include/uapi/linux/kernel-page-flags.h"
47 # define STR(x) _STR(x)
54 #define PM_ENTRY_BYTES sizeof(uint64_t)
55 #define PM_STATUS_BITS 3
56 #define PM_STATUS_OFFSET (64 - PM_STATUS_BITS)
57 #define PM_STATUS_MASK (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
58 #define PM_STATUS(nr) (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
59 #define PM_PSHIFT_BITS 6
60 #define PM_PSHIFT_OFFSET (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
61 #define PM_PSHIFT_MASK (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
62 #define PM_PSHIFT(x) (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
63 #define PM_PFRAME_MASK ((1LL << PM_PSHIFT_OFFSET) - 1)
64 #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK)
66 #define PM_PRESENT PM_STATUS(4LL)
67 #define PM_SWAP PM_STATUS(2LL)
75 #define PROC_KPAGEFLAGS "/proc/kpageflags"
78 #define KPF_RESERVED 32
79 #define KPF_MLOCKED 33
80 #define KPF_MAPPEDTODISK 34
81 #define KPF_PRIVATE 35
82 #define KPF_PRIVATE_2 36
83 #define KPF_OWNER_PRIVATE 37
85 #define KPF_UNCACHED 39
90 #define KPF_READAHEAD 48
91 #define KPF_SLOB_FREE 49
92 #define KPF_SLUB_FROZEN 50
93 #define KPF_SLUB_DEBUG 51
95 #define KPF_ALL_BITS ((uint64_t)~0ULL)
96 #define KPF_HACKERS_BITS (0xffffULL << 32)
97 #define KPF_OVERLOADED_BITS (0xffffULL << 48)
98 #define BIT(name) (1ULL << KPF_##name)
99 #define BITS_COMPOUND (BIT(COMPOUND_HEAD) | BIT(COMPOUND_TAIL))
101 static const char *
const page_flag_names[] = {
143 static const char *
const debugfs_known_mountpoints[] = {
155 static int opt_no_summary;
156 static pid_t opt_pid;
158 #define MAX_ADDR_RANGES 1024
159 static int nr_addr_ranges;
163 #define MAX_VMAS 10240
165 static unsigned long pg_start[
MAX_VMAS];
166 static unsigned long pg_end[
MAX_VMAS];
168 #define MAX_BIT_FILTERS 64
169 static int nr_bit_filters;
175 static int pagemap_fd;
176 static int kpageflags_fd;
178 static int opt_hwpoison;
179 static int opt_unpoison;
181 static char hwpoison_debug_fs[
MAX_PATH+1];
182 static int hwpoison_inject_fd;
183 static int hwpoison_forget_fd;
185 #define HASH_SHIFT 13
186 #define HASH_SIZE (1 << HASH_SHIFT)
187 #define HASH_MASK (HASH_SIZE - 1)
188 #define HASH_KEY(flags) (flags & HASH_MASK)
190 static unsigned long total_pages;
191 static unsigned long nr_pages[
HASH_SIZE];
199 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
201 #define min_t(type, x, y) ({ \
204 __min1 < __min2 ? __min1 : __min2; })
206 #define max_t(type, x, y) ({ \
209 __max1 > __max2 ? __max1 : __max2; })
211 static unsigned long pages2mb(
unsigned long pages)
216 static void fatal(
const char *
x, ...)
228 int fd =
open(pathname, flags);
242 static unsigned long do_u64_read(
int fd,
char *
name,
250 fatal(
"index overflow: %lu\n", index);
252 if (lseek(fd, index * 8,
SEEK_SET) < 0) {
257 bytes =
read(fd, buf, count * 8);
263 fatal(
"partial read: %lu bytes\n", bytes);
268 static unsigned long kpageflags_read(
uint64_t *buf,
275 static unsigned long pagemap_read(
uint64_t *buf,
279 return do_u64_read(pagemap_fd,
"/proc/pid/pagemap", buf, index, pages);
299 static char *page_flag_name(
uint64_t flags)
305 for (i = 0, j = 0; i <
ARRAY_SIZE(page_flag_names); i++) {
306 present = (flags >>
i) & 1;
307 if (!page_flag_names[i]) {
309 fatal(
"unknown flag bit %d\n", i);
312 buf[j++] = present ? page_flag_names[
i][0] :
'_';
318 static char *page_flag_longname(
uint64_t flags)
320 static char buf[1024];
323 for (i = 0, n = 0; i <
ARRAY_SIZE(page_flag_names); i++) {
324 if (!page_flag_names[i])
326 if ((flags >> i) & 1)
327 n +=
snprintf(buf + n,
sizeof(buf) -
n,
"%s,",
328 page_flag_names[
i] + 2);
342 static void show_page_range(
unsigned long voffset,
346 static unsigned long voff;
347 static unsigned long index;
348 static unsigned long count;
350 if (flags == flags0 && offset == index + count &&
351 (!opt_pid || voffset == voff + count)) {
360 index, count, page_flag_name(flags0));
369 static void show_page(
unsigned long voffset,
370 unsigned long offset,
uint64_t flags)
374 printf(
"%lx\t%s\n", offset, page_flag_name(flags));
377 static void show_summary(
void)
381 printf(
" flags\tpage-count MB"
382 " symbolic-flags\t\t\tlong-symbolic-flags\n");
386 printf(
"0x%016llx\t%10lu %8lu %s\t%s\n",
387 (
unsigned long long)page_flags[i],
389 pages2mb(nr_pages[i]),
390 page_flag_name(page_flags[i]),
391 page_flag_longname(page_flags[i]));
394 printf(
" total\t%10lu %8lu\n",
395 total_pages, pages2mb(total_pages));
403 static int bit_mask_ok(
uint64_t flags)
407 for (i = 0; i < nr_bit_filters; i++) {
409 if ((flags & opt_mask[i]) == 0)
412 if ((flags & opt_mask[i]) != opt_bits[i])
423 if (flags &
BIT(SLAB)) {
424 if (flags &
BIT(PRIVATE))
425 flags ^=
BIT(PRIVATE) |
BIT(SLOB_FREE);
433 if ((flags & (
BIT(RECLAIM) |
BIT(WRITEBACK))) ==
BIT(RECLAIM))
434 flags ^=
BIT(RECLAIM) |
BIT(READAHEAD);
446 flags &= ~BITS_COMPOUND;
453 flags = expand_overloaded_flags(flags);
456 flags = well_known_flags(flags);
466 if (
statfs(debugfs, &st_fs) < 0)
477 const char *
const *
ptr;
481 ptr = debugfs_known_mountpoints;
484 strcpy(hwpoison_debug_fs, *ptr);
485 return hwpoison_debug_fs;
491 fp = fopen(
"/proc/mounts",
"r");
493 perror(
"Can't open /proc/mounts for read");
495 while (fscanf(fp,
"%*s %"
497 "s %99s %*s %*d %*d\n",
498 hwpoison_debug_fs, type) == 2) {
499 if (
strcmp(type,
"debugfs") == 0)
504 if (
strcmp(type,
"debugfs") != 0)
507 return hwpoison_debug_fs;
514 const char *
const *
ptr;
520 ptr = debugfs_known_mountpoints;
524 strcpy(hwpoison_debug_fs, *ptr);
531 perror(
"mount debugfs");
540 static void prepare_hwpoison_fd(
void)
546 if (opt_hwpoison && !hwpoison_inject_fd) {
549 hwpoison_inject_fd = checked_open(buf,
O_WRONLY);
552 if (opt_unpoison && !hwpoison_forget_fd) {
555 hwpoison_forget_fd = checked_open(buf,
O_WRONLY);
559 static int hwpoison_page(
unsigned long offset)
564 len =
sprintf(buf,
"0x%lx\n", offset);
565 len =
write(hwpoison_inject_fd, buf, len);
567 perror(
"hwpoison inject");
573 static int unpoison_page(
unsigned long offset)
578 len =
sprintf(buf,
"0x%lx\n", offset);
579 len =
write(hwpoison_forget_fd, buf, len);
581 perror(
"hwpoison forget");
591 static size_t hash_slot(
uint64_t flags)
603 for (i = 1; i <
ARRAY_SIZE(page_flags); i++, k++) {
606 if (page_flags[k] == 0) {
610 if (page_flags[k] == flags)
614 fatal(
"hash table full: bump up HASH_SHIFT?\n");
618 static void add_page(
unsigned long voffset,
619 unsigned long offset,
uint64_t flags)
621 flags = kpageflags_flags(flags);
623 if (!bit_mask_ok(flags))
627 hwpoison_page(offset);
629 unpoison_page(offset);
632 show_page_range(voffset, offset, flags);
633 else if (opt_list == 2)
634 show_page(voffset, offset, flags);
636 nr_pages[hash_slot(flags)]++;
640 #define KPAGEFLAGS_BATCH (64 << 10)
641 static void walk_pfn(
unsigned long voffset,
652 pages = kpageflags_read(buf, index, batch);
656 for (i = 0; i <
pages; i++)
657 add_page(voffset + i, index + i, buf[i]);
664 #define PAGEMAP_BATCH (64 << 10)
665 static void walk_vma(
unsigned long index,
unsigned long count)
675 pages = pagemap_read(buf, index, batch);
679 for (i = 0; i <
pages; i++) {
680 pfn = pagemap_pfn(buf[i]);
682 walk_pfn(index + i, pfn, 1);
690 static void walk_task(
unsigned long index,
unsigned long count)
692 const unsigned long end = index +
count;
696 while (index < end) {
698 while (pg_end[i] <= index)
701 if (pg_start[i] >= end)
704 start =
max_t(
unsigned long, pg_start[i], index);
705 index =
min_t(
unsigned long, pg_end[i], end);
708 walk_vma(start, index - start);
712 static void add_addr_range(
unsigned long offset,
unsigned long size)
715 fatal(
"too many addr ranges\n");
717 opt_offset[nr_addr_ranges] =
offset;
718 opt_size[nr_addr_ranges] =
min_t(
unsigned long, size,
ULONG_MAX-offset);
722 static void walk_addr_ranges(
void)
731 for (i = 0; i < nr_addr_ranges; i++)
733 walk_pfn(0, opt_offset[i], opt_size[i]);
735 walk_task(opt_offset[i], opt_size[i]);
737 close(kpageflags_fd);
754 static void usage(
void)
759 "page-types [options]\n"
760 " -r|--raw Raw mode, for kernel developers\n"
761 " -d|--describe flags Describe flags\n"
762 " -a|--addr addr-spec Walk a range of pages\n"
763 " -b|--bits bits-spec Walk pages with specified bits\n"
764 " -p|--pid pid Walk process address space\n"
766 " -f|--file filename Walk file address space\n"
768 " -l|--list Show page details in ranges\n"
769 " -L|--list-each Show page details one by one\n"
770 " -N|--no-summary Don't show summary info\n"
771 " -X|--hwpoison hwpoison pages\n"
772 " -x|--unpoison unpoison pages\n"
773 " -h|--help Show this usage message\n"
775 " 0x10 bitfield format, e.g.\n"
776 " anon bit-name, e.g.\n"
777 " 0x10,anon comma-separated list, e.g.\n"
779 " N one page at offset N (unit: pages)\n"
780 " N+M pages range from N to N+M-1\n"
781 " N,M pages range from N to M-1\n"
782 " N, pages range from N to end\n"
783 " ,M pages range from 0 to M-1\n"
785 " bit1,bit2 (flags & (bit1|bit2)) != 0\n"
786 " bit1,bit2=bit1 (flags & (bit1|bit2)) == bit1\n"
787 " bit1,~bit2 (flags & (bit1|bit2)) == bit1\n"
788 " =bit1,bit2 flags == (bit1|bit2)\n"
792 for (i = 0, j = 0; i <
ARRAY_SIZE(page_flag_names); i++) {
793 if (!page_flag_names[i])
795 printf(
"%16s%s", page_flag_names[i] + 2,
796 page_flag_type(1ULL << i));
803 "(r) raw mode bits (o) overloaded bits\n");
806 static unsigned long long parse_number(
const char *
str)
808 unsigned long long n;
810 n = strtoll(str,
NULL, 0);
812 if (n == 0 && str[0] !=
'0')
813 fatal(
"invalid name or number: %s\n", str);
818 static void parse_pid(
const char *str)
823 opt_pid = parse_number(str);
825 sprintf(buf,
"/proc/%d/pagemap", opt_pid);
826 pagemap_fd = checked_open(buf,
O_RDONLY);
828 sprintf(buf,
"/proc/%d/maps", opt_pid);
829 file = fopen(buf,
"r");
835 while (fgets(buf,
sizeof(buf), file) !=
NULL) {
836 unsigned long vm_start;
837 unsigned long vm_end;
838 unsigned long long pgoff;
844 n =
sscanf(buf,
"%lx-%lx %c%c%c%c %llx %x:%x %lu",
852 fprintf(stderr,
"unexpected line: %s\n", buf);
855 pg_start[nr_vmas] = vm_start /
page_size;
858 fprintf(stderr,
"too many VMAs\n");
865 static void parse_file(
const char *name)
869 static void parse_addr_range(
const char *optarg)
881 size = parse_number(p + 1);
883 offset = parse_number(optarg);
887 size = parse_number(p + 1);
890 fatal(
"invalid range: %lu,%lu\n",
896 offset = parse_number(optarg);
900 add_addr_range(offset, size);
906 fatal(
"too much bit filters\n");
908 opt_mask[nr_bit_filters] =
mask;
909 opt_bits[nr_bit_filters] =
bits;
913 static uint64_t parse_flag_name(
const char *str,
int len)
920 if (len <= 8 && !
strncmp(str,
"compound", len))
923 for (i = 0; i <
ARRAY_SIZE(page_flag_names); i++) {
924 if (!page_flag_names[i])
926 if (!
strncmp(str, page_flag_names[i] + 2, len))
930 return parse_number(str);
933 static uint64_t parse_flag_names(
const char *str,
int all)
939 if (*p ==
',' || *p ==
'=' || *p ==
'\0') {
940 if ((*str !=
'~') || (*str ==
'~' && all && *++str))
941 flags |= parse_flag_name(str, p - str);
952 static void parse_bits_mask(
const char *optarg)
961 bits = parse_flag_names(p + 1, 0);
963 mask = parse_flag_names(optarg, 0);
964 bits = parse_flag_names(p + 1, 0);
965 }
else if (
strchr(optarg,
'~')) {
966 mask = parse_flag_names(optarg, 1);
967 bits = parse_flag_names(optarg, 0);
969 mask = parse_flag_names(optarg, 0);
973 add_bits_filter(mask, bits);
976 static void describe_flags(
const char *optarg)
978 uint64_t flags = parse_flag_names(optarg, 0);
980 printf(
"0x%016llx\t%s\t%s\n",
981 (
unsigned long long)flags,
982 page_flag_name(flags),
983 page_flag_longname(flags));
987 {
"raw" , 0,
NULL,
'r' },
988 {
"pid" , 1,
NULL,
'p' },
989 {
"file" , 1,
NULL,
'f' },
990 {
"addr" , 1,
NULL,
'a' },
991 {
"bits" , 1,
NULL,
'b' },
992 {
"describe" , 1,
NULL,
'd' },
993 {
"list" , 0,
NULL,
'l' },
994 {
"list-each" , 0,
NULL,
'L' },
995 {
"no-summary", 0,
NULL,
'N' },
996 {
"hwpoison" , 0,
NULL,
'X' },
997 {
"unpoison" , 0,
NULL,
'x' },
998 {
"help" , 0,
NULL,
'h' },
1008 while ((c = getopt_long(argc, argv,
1009 "rp:f:a:b:d:lLNXxh", opts,
NULL)) != -1) {
1021 parse_addr_range(optarg);
1024 parse_bits_mask(optarg);
1027 describe_flags(optarg);
1040 prepare_hwpoison_fd();
1044 prepare_hwpoison_fd();
1055 if (opt_list && opt_pid)
1058 printf(
"offset\tlen\tflags\n");
1060 printf(
"offset\tflags\n");
1065 show_page_range(0, 0, 0);