28 #include <linux/slab.h>
344 static struct kmem_cache *ext4_free_data_cachep;
349 #define NR_GRPINFO_CACHES 8
353 "ext4_groupinfo_1k",
"ext4_groupinfo_2k",
"ext4_groupinfo_4k",
354 "ext4_groupinfo_8k",
"ext4_groupinfo_16k",
"ext4_groupinfo_32k",
355 "ext4_groupinfo_64k",
"ext4_groupinfo_128k"
365 static inline void *mb_correct_addr_and_bit(
int *
bit,
void *
addr)
367 #if BITS_PER_LONG == 64
368 *bit += ((
unsigned long) addr & 7
UL) << 3;
369 addr = (
void *) ((
unsigned long) addr & ~7
UL);
370 #elif BITS_PER_LONG == 32
371 *bit += ((
unsigned long) addr & 3UL) << 3;
372 addr = (
void *) ((
unsigned long) addr & ~3
UL);
374 #error "how many bits you are?!"
379 static inline int mb_test_bit(
int bit,
void *addr)
385 addr = mb_correct_addr_and_bit(&bit, addr);
389 static inline void mb_set_bit(
int bit,
void *addr)
391 addr = mb_correct_addr_and_bit(&bit, addr);
395 static inline void mb_clear_bit(
int bit,
void *addr)
397 addr = mb_correct_addr_and_bit(&bit, addr);
401 static inline int mb_find_next_zero_bit(
void *addr,
int max,
int start)
403 int fix = 0,
ret, tmpmax;
404 addr = mb_correct_addr_and_bit(&fix, addr);
414 static inline int mb_find_next_bit(
void *addr,
int max,
int start)
416 int fix = 0,
ret, tmpmax;
417 addr = mb_correct_addr_and_bit(&fix, addr);
427 static void *mb_find_buddy(
struct ext4_buddy *e4b,
int order,
int *max)
461 for (i = 0; i <
count; i++) {
462 if (!mb_test_bit(first + i, e4b->
bd_info->bb_bitmap)) {
465 blocknr = ext4_group_first_block_no(sb, e4b->
bd_group);
467 ext4_grp_locked_error(sb, e4b->
bd_group,
468 inode ? inode->
i_ino : 0,
470 "freeing block already freed "
474 mb_clear_bit(first + i, e4b->
bd_info->bb_bitmap);
478 static void mb_mark_used_double(
struct ext4_buddy *e4b,
int first,
int count)
485 for (i = 0; i <
count; i++) {
487 mb_set_bit(first + i, e4b->
bd_info->bb_bitmap);
494 unsigned char *b1, *b2;
496 b1 = (
unsigned char *) e4b->
bd_info->bb_bitmap;
497 b2 = (
unsigned char *)
bitmap;
498 for (i = 0; i < e4b->
bd_sb->s_blocksize; i++) {
499 if (b1[i] != b2[i]) {
501 "corruption in group %u "
502 "at byte %u(%u): %x in copy != %x "
504 e4b->
bd_group, i, i * 8, b1[i], b2[i]);
512 static inline void mb_free_blocks_double(
struct inode *inode,
517 static inline void mb_mark_used_double(
struct ext4_buddy *e4b,
518 int first,
int count)
522 static inline void mb_cmp_bitmaps(
struct ext4_buddy *e4b,
void *bitmap)
528 #ifdef AGGRESSIVE_CHECK
530 #define MB_CHECK_ASSERT(assert) \
534 "Assertion failure in %s() at %s:%d: \"%s\"\n", \
535 function, file, line, # assert); \
541 const char *
function,
int line)
551 struct ext4_group_info *grp;
559 static int mb_check_counter;
560 if (mb_check_counter++ % 100 != 0)
565 buddy = mb_find_buddy(e4b, order, &max);
566 MB_CHECK_ASSERT(buddy);
567 buddy2 = mb_find_buddy(e4b, order - 1, &max2);
568 MB_CHECK_ASSERT(buddy2);
569 MB_CHECK_ASSERT(buddy != buddy2);
570 MB_CHECK_ASSERT(max * 2 == max2);
573 for (i = 0; i <
max; i++) {
575 if (mb_test_bit(i, buddy)) {
577 if (!mb_test_bit(i << 1, buddy2)) {
579 mb_test_bit((i<<1)+1, buddy2));
580 }
else if (!mb_test_bit((i << 1) + 1, buddy2)) {
582 mb_test_bit(i << 1, buddy2));
588 MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2));
589 MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2));
591 for (j = 0; j < (1 <<
order); j++) {
592 k = (i * (1 <<
order)) +
j;
598 MB_CHECK_ASSERT(e4b->
bd_info->bb_counters[order] == count);
603 buddy = mb_find_buddy(e4b, 0, &max);
604 for (i = 0; i <
max; i++) {
605 if (!mb_test_bit(i, buddy)) {
606 MB_CHECK_ASSERT(i >= e4b->
bd_info->bb_first_free);
616 buddy2 = mb_find_buddy(e4b, j, &max2);
618 MB_CHECK_ASSERT(k < max2);
619 MB_CHECK_ASSERT(mb_test_bit(k, buddy2));
622 MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->
bd_info));
623 MB_CHECK_ASSERT(e4b->
bd_info->bb_fragments == fragments);
625 grp = ext4_get_group_info(sb, e4b->
bd_group);
631 MB_CHECK_ASSERT(groupnr == e4b->
bd_group);
632 for (i = 0; i < pa->
pa_len; i++)
633 MB_CHECK_ASSERT(mb_test_bit(k + i, buddy));
637 #undef MB_CHECK_ASSERT
638 #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \
639 __FILE__, __func__, __LINE__)
641 #define mb_check_buddy(e4b)
650 static void ext4_mb_mark_free_simple(
struct super_block *sb,
652 struct ext4_group_info *grp)
654 struct ext4_sb_info *sbi =
EXT4_SB(sb);
658 unsigned short border;
660 BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb));
666 max =
ffs(first | border) - 1;
676 grp->bb_counters[
min]++;
678 mb_clear_bit(first >> min,
679 buddy + sbi->s_mb_offsets[min]);
691 mb_set_largest_free_order(
struct super_block *sb,
struct ext4_group_info *grp)
696 grp->bb_largest_free_order = -1;
699 for (i = bits; i >= 0; i--) {
700 if (grp->bb_counters[i] > 0) {
701 grp->bb_largest_free_order =
i;
707 static noinline_for_stack
708 void ext4_mb_generate_buddy(
struct super_block *sb,
711 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
717 unsigned fragments = 0;
722 i = mb_find_next_zero_bit(bitmap, max, 0);
723 grp->bb_first_free =
i;
727 i = mb_find_next_bit(bitmap, max, i);
731 ext4_mb_mark_free_simple(sb, buddy, first, len, grp);
733 grp->bb_counters[0]++;
735 i = mb_find_next_zero_bit(bitmap, max, i);
737 grp->bb_fragments = fragments;
739 if (free != grp->bb_free) {
740 ext4_grp_locked_error(sb, group, 0, 0,
741 "%u clusters in bitmap, %u in gd",
749 mb_set_largest_free_order(sb, grp);
751 clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state));
754 spin_lock(&
EXT4_SB(sb)->s_bal_lock);
755 EXT4_SB(sb)->s_mb_buddies_generated++;
757 spin_unlock(&
EXT4_SB(sb)->s_bal_lock);
780 static int ext4_mb_init_cache(
struct page *
page,
char *incore)
791 struct buffer_head *bhs;
792 struct buffer_head **bh =
NULL;
796 struct ext4_group_info *grinfo;
802 ngroups = ext4_get_groups_count(sb);
806 groups_per_page = blocks_per_page >> 1;
807 if (groups_per_page == 0)
811 if (groups_per_page > 1) {
812 i =
sizeof(
struct buffer_head *) * groups_per_page;
821 first_group = page->
index * blocks_per_page / 2;
824 for (i = 0, group = first_group; i < groups_per_page; i++, group++) {
825 if (group >= ngroups)
828 grinfo = ext4_get_group_info(sb, group);
835 if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) {
843 mb_debug(1,
"read bitmap for group %u\n", group);
847 for (i = 0, group = first_group; i < groups_per_page; i++, group++) {
854 first_block = page->
index * blocks_per_page;
855 for (i = 0; i < blocks_per_page; i++) {
858 group = (first_block +
i) >> 1;
859 if (group >= ngroups)
862 if (!bh[group - first_group])
873 bitmap = bh[group - first_group]->b_data;
879 if ((first_block + i) & 1) {
882 mb_debug(1,
"put buddy for group %u in page %lu/%x\n",
883 group, page->
index, i * blocksize);
884 trace_ext4_mb_buddy_bitmap_load(sb, group);
885 grinfo = ext4_get_group_info(sb, group);
886 grinfo->bb_fragments = 0;
887 memset(grinfo->bb_counters, 0,
888 sizeof(*grinfo->bb_counters) *
893 ext4_lock_group(sb, group);
895 memset(data, 0xff, blocksize);
896 ext4_mb_generate_buddy(sb, data, incore, group);
897 ext4_unlock_group(sb, group);
902 mb_debug(1,
"put bitmap for group %u in page %lu/%x\n",
903 group, page->
index, i * blocksize);
904 trace_ext4_mb_bitmap_load(sb, group);
907 ext4_lock_group(sb, group);
908 memcpy(data, bitmap, blocksize);
911 ext4_mb_generate_from_pa(sb, data, group);
912 ext4_mb_generate_from_freelist(sb, data, group);
913 ext4_unlock_group(sb, group);
921 SetPageUptodate(page);
925 for (i = 0; i < groups_per_page; i++)
939 static int ext4_mb_get_buddy_page_lock(
struct super_block *sb,
942 struct inode *inode =
EXT4_SB(sb)->s_buddy_cache;
943 int block, pnum, poff;
957 pnum = block / blocks_per_page;
958 poff = block % blocks_per_page;
966 if (blocks_per_page >= 2) {
972 pnum = block / blocks_per_page;
981 static void ext4_mb_put_buddy_page_lock(
struct ext4_buddy *e4b)
998 static noinline_for_stack
1002 struct ext4_group_info *this_grp;
1007 mb_debug(1,
"init group %u\n", group);
1008 this_grp = ext4_get_group_info(sb, group);
1016 ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b);
1017 if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) {
1026 ret = ext4_mb_init_cache(page,
NULL);
1029 if (!PageUptodate(page)) {
1046 ret = ext4_mb_init_cache(page, e4b.
bd_bitmap);
1049 if (!PageUptodate(page)) {
1055 ext4_mb_put_buddy_page_lock(&e4b);
1064 static noinline_for_stack
int
1068 int blocks_per_page;
1074 struct ext4_group_info *grp;
1075 struct ext4_sb_info *sbi =
EXT4_SB(sb);
1076 struct inode *inode = sbi->s_buddy_cache;
1078 mb_debug(1,
"load group %u\n", group);
1081 grp = ext4_get_group_info(sb, group);
1090 if (
unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
1095 ret = ext4_mb_init_group(sb, group);
1106 pnum = block / blocks_per_page;
1107 poff = block % blocks_per_page;
1112 if (page ==
NULL || !PageUptodate(page)) {
1126 if (!PageUptodate(page)) {
1127 ret = ext4_mb_init_cache(page,
NULL);
1138 if (page ==
NULL || !PageUptodate(page)) {
1147 pnum = block / blocks_per_page;
1148 poff = block % blocks_per_page;
1151 if (page ==
NULL || !PageUptodate(page)) {
1157 if (!PageUptodate(page)) {
1158 ret = ext4_mb_init_cache(page, e4b->
bd_bitmap);
1167 if (page ==
NULL || !PageUptodate(page)) {
1192 static void ext4_mb_unload_buddy(
struct ext4_buddy *e4b)
1201 static int mb_find_order_for_block(
struct ext4_buddy *e4b,
int block)
1210 while (order <= e4b->bd_blkbits + 1) {
1212 if (!mb_test_bit(block, bb)) {
1222 static void mb_clear_bits(
void *bm,
int cur,
int len)
1228 if ((cur & 31) == 0 && (len - cur) >= 32) {
1230 addr = bm + (cur >> 3);
1235 mb_clear_bit(cur, bm);
1246 if ((cur & 31) == 0 && (len - cur) >= 32) {
1248 addr = bm + (cur >> 3);
1253 mb_set_bit(cur, bm);
1258 static void mb_free_blocks(
struct inode *inode,
struct ext4_buddy *e4b,
1259 int first,
int count)
1271 mb_free_blocks_double(inode, e4b, first, count);
1274 if (first < e4b->
bd_info->bb_first_free)
1279 block = !mb_test_bit(first - 1, e4b->
bd_bitmap);
1280 if (first + count <
EXT4_SB(sb)->s_mb_maxs[0])
1281 max = !mb_test_bit(first + count, e4b->
bd_bitmap);
1284 else if (!block && !max)
1288 while (count-- > 0) {
1292 if (!mb_test_bit(block, e4b->
bd_bitmap)) {
1295 blocknr = ext4_group_first_block_no(sb, e4b->
bd_group);
1297 ext4_grp_locked_error(sb, e4b->
bd_group,
1298 inode ? inode->
i_ino : 0,
1300 "freeing already freed block "
1307 buddy = mb_find_buddy(e4b, order, &max);
1311 if (mb_test_bit(block, buddy) ||
1312 mb_test_bit(block + 1, buddy))
1316 buddy2 = mb_find_buddy(e4b, order + 1, &max);
1324 mb_set_bit(block, buddy);
1325 mb_set_bit(block + 1, buddy);
1334 mb_clear_bit(block, buddy2);
1338 mb_set_largest_free_order(sb, e4b->
bd_info);
1342 static int mb_find_extent(
struct ext4_buddy *e4b,
int block,
1352 buddy = mb_find_buddy(e4b, 0, &max);
1355 if (mb_test_bit(block, buddy)) {
1363 order = mb_find_order_for_block(e4b, block);
1364 block = block >>
order;
1375 while (needed > ex->
fe_len &&
1376 (buddy = mb_find_buddy(e4b, order, &max))) {
1378 if (block + 1 >= max)
1381 next = (block + 1) * (1 << order);
1385 order = mb_find_order_for_block(e4b, next);
1387 block = next >>
order;
1407 BUG_ON(start + len > (e4b->
bd_sb->s_blocksize << 3));
1411 mb_mark_used_double(e4b, start, len);
1414 if (e4b->
bd_info->bb_first_free == start)
1415 e4b->
bd_info->bb_first_free += len;
1419 mlen = !mb_test_bit(start - 1, e4b->
bd_bitmap);
1421 max = !mb_test_bit(start + len, e4b->
bd_bitmap);
1424 else if (!mlen && !max)
1429 ord = mb_find_order_for_block(e4b, start);
1431 if (((start >> ord) << ord) == start && len >= (1 << ord)) {
1434 buddy = mb_find_buddy(e4b, ord, &max);
1435 BUG_ON((start >> ord) >= max);
1436 mb_set_bit(start >> ord, buddy);
1446 ret = len | (ord << 16);
1450 buddy = mb_find_buddy(e4b, ord, &max);
1451 mb_set_bit(start >> ord, buddy);
1455 cur = (start >>
ord) & ~1
U;
1456 buddy = mb_find_buddy(e4b, ord, &max);
1457 mb_clear_bit(cur, buddy);
1458 mb_clear_bit(cur + 1, buddy);
1484 ret = mb_mark_used(e4b, &ac->
ac_b_ex);
1507 spin_lock(&sbi->s_md_lock);
1508 sbi->s_mb_last_group = ac->
ac_f_ex.fe_group;
1509 sbi->s_mb_last_start = ac->
ac_f_ex.fe_start;
1510 spin_unlock(&sbi->s_md_lock);
1533 if (ac->
ac_found > sbi->s_mb_max_to_scan &&
1545 if ((finish_group || ac->
ac_found > sbi->s_mb_min_to_scan)
1551 if (max >= gex->
fe_len) {
1552 ext4_mb_use_best_found(ac, e4b);
1587 ext4_mb_use_best_found(ac, e4b);
1596 ext4_mb_use_best_found(ac, e4b);
1624 ext4_mb_check_limits(ac, e4b, 0);
1627 static noinline_for_stack
1637 err = ext4_mb_load_buddy(ac->
ac_sb, group, e4b);
1641 ext4_lock_group(ac->
ac_sb, group);
1646 ext4_mb_use_best_found(ac, e4b);
1649 ext4_unlock_group(ac->
ac_sb, group);
1650 ext4_mb_unload_buddy(e4b);
1655 static noinline_for_stack
1663 struct ext4_group_info *grp = ext4_get_group_info(ac->
ac_sb, group);
1668 if (grp->bb_free == 0)
1671 err = ext4_mb_load_buddy(ac->
ac_sb, group, e4b);
1675 ext4_lock_group(ac->
ac_sb, group);
1676 max = mb_find_extent(e4b, ac->
ac_g_ex.fe_start,
1679 if (max >= ac->
ac_g_ex.fe_len && ac->
ac_g_ex.fe_len == sbi->s_stripe) {
1682 start = ext4_group_first_block_no(ac->
ac_sb, e4b->
bd_group) +
1685 if (
do_div(start, sbi->s_stripe) == 0) {
1688 ext4_mb_use_best_found(ac, e4b);
1690 }
else if (max >= ac->
ac_g_ex.fe_len) {
1696 ext4_mb_use_best_found(ac, e4b);
1705 ext4_mb_use_best_found(ac, e4b);
1707 ext4_unlock_group(ac->
ac_sb, group);
1708 ext4_mb_unload_buddy(e4b);
1717 static noinline_for_stack
1722 struct ext4_group_info *grp = e4b->
bd_info;
1729 for (i = ac->
ac_2order; i <= sb->s_blocksize_bits + 1; i++) {
1730 if (grp->bb_counters[i] == 0)
1733 buddy = mb_find_buddy(e4b, i, &max);
1736 k = mb_find_next_zero_bit(buddy, max, 0);
1745 ext4_mb_use_best_found(ac, e4b);
1761 static noinline_for_stack
1774 i = e4b->
bd_info->bb_first_free;
1777 i = mb_find_next_zero_bit(bitmap,
1778 EXT4_CLUSTERS_PER_GROUP(sb), i);
1779 if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) {
1785 ext4_grp_locked_error(sb, e4b->
bd_group, 0, 0,
1786 "%d free clusters as per "
1787 "group info. But bitmap says 0",
1792 mb_find_extent(e4b, i, ac->
ac_g_ex.fe_len, &ex);
1795 ext4_grp_locked_error(sb, e4b->
bd_group, 0, 0,
1796 "%d free clusters as per "
1797 "group info. But got %d blocks",
1807 ext4_mb_measure_extent(ac, &ex, e4b);
1813 ext4_mb_check_limits(ac, e4b, 1);
1820 static noinline_for_stack
1825 struct ext4_sb_info *sbi =
EXT4_SB(sb);
1833 BUG_ON(sbi->s_stripe == 0);
1836 first_group_block = ext4_group_first_block_no(sb, e4b->
bd_group);
1838 a = first_group_block + sbi->s_stripe - 1;
1839 do_div(a, sbi->s_stripe);
1840 i = (a * sbi->s_stripe) - first_group_block;
1842 while (i < EXT4_CLUSTERS_PER_GROUP(sb)) {
1843 if (!mb_test_bit(i, bitmap)) {
1844 max = mb_find_extent(e4b, i, sbi->s_stripe, &ex);
1845 if (max >= sbi->s_stripe) {
1848 ext4_mb_use_best_found(ac, e4b);
1860 unsigned free, fragments;
1862 struct ext4_group_info *grp = ext4_get_group_info(ac->
ac_sb, group);
1864 BUG_ON(cr < 0 || cr >= 4);
1866 free = grp->bb_free;
1869 if (cr <= 2 && free < ac->ac_g_ex.fe_len)
1873 if (
unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
1874 int ret = ext4_mb_init_group(ac->
ac_sb, group);
1879 fragments = grp->bb_fragments;
1887 if (grp->bb_largest_free_order < ac->
ac_2order)
1893 ((group % flex_size) == 0))
1898 if ((free / fragments) >= ac->
ac_g_ex.fe_len)
1902 if (free >= ac->
ac_g_ex.fe_len)
1914 static noinline_for_stack
int
1920 struct ext4_sb_info *sbi;
1926 ngroups = ext4_get_groups_count(sb);
1929 ngroups = sbi->s_blockfile_groups;
1934 err = ext4_mb_find_by_goal(ac, &e4b);
1953 if (i >= sbi->s_mb_order2_reqs) {
1957 if ((ac->
ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)
1964 spin_lock(&sbi->s_md_lock);
1965 ac->
ac_g_ex.fe_group = sbi->s_mb_last_group;
1966 ac->
ac_g_ex.fe_start = sbi->s_mb_last_start;
1967 spin_unlock(&sbi->s_md_lock);
1985 for (i = 0; i < ngroups; group++, i++) {
1986 if (group == ngroups)
1990 if (!ext4_mb_good_group(ac, group, cr))
1993 err = ext4_mb_load_buddy(sb, group, &e4b);
1997 ext4_lock_group(sb, group);
2003 if (!ext4_mb_good_group(ac, group, cr)) {
2004 ext4_unlock_group(sb, group);
2005 ext4_mb_unload_buddy(&e4b);
2011 ext4_mb_simple_scan_group(ac, &e4b);
2012 else if (cr == 1 && sbi->s_stripe &&
2013 !(ac->
ac_g_ex.fe_len % sbi->s_stripe))
2014 ext4_mb_scan_aligned(ac, &e4b);
2016 ext4_mb_complex_scan_group(ac, &e4b);
2018 ext4_unlock_group(sb, group);
2019 ext4_mb_unload_buddy(&e4b);
2033 ext4_mb_try_best_found(ac, &e4b);
2055 static void *ext4_mb_seq_groups_start(
struct seq_file *seq, loff_t *
pos)
2060 if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
2063 return (
void *) ((
unsigned long) group);
2066 static void *ext4_mb_seq_groups_next(
struct seq_file *seq,
void *
v, loff_t *pos)
2072 if (*pos < 0 || *pos >= ext4_get_groups_count(sb))
2075 return (
void *) ((
unsigned long) group);
2078 static int ext4_mb_seq_groups_show(
struct seq_file *seq,
void *v)
2083 int err, buddy_loaded = 0;
2085 struct ext4_group_info *grinfo;
2087 struct ext4_group_info
info;
2094 "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s "
2095 "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n",
2096 "group",
"free",
"frags",
"first",
2097 "2^0",
"2^1",
"2^2",
"2^3",
"2^4",
"2^5",
"2^6",
2098 "2^7",
"2^8",
"2^9",
"2^10",
"2^11",
"2^12",
"2^13");
2101 sizeof(
struct ext4_group_info);
2102 grinfo = ext4_get_group_info(sb, group);
2104 if (
unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) {
2105 err = ext4_mb_load_buddy(sb, group, &e4b);
2107 seq_printf(seq,
"#%-5u: I/O error\n", group);
2113 memcpy(&
sg, ext4_get_group_info(sb, group), i);
2116 ext4_mb_unload_buddy(&e4b);
2118 seq_printf(seq,
"#%-5u: %-5u %-5u %-5u [", group,
sg.info.bb_free,
2119 sg.info.bb_fragments,
sg.info.bb_first_free);
2120 for (i = 0; i <= 13; i++)
2121 seq_printf(seq,
" %-5u", i <= sb->s_blocksize_bits + 1 ?
2122 sg.info.bb_counters[i] : 0);
2128 static void ext4_mb_seq_groups_stop(
struct seq_file *seq,
void *v)
2133 .start = ext4_mb_seq_groups_start,
2134 .next = ext4_mb_seq_groups_next,
2135 .stop = ext4_mb_seq_groups_stop,
2136 .show = ext4_mb_seq_groups_show,
2139 static int ext4_mb_seq_groups_open(
struct inode *inode,
struct file *file)
2144 rc =
seq_open(file, &ext4_mb_seq_groups_ops);
2155 .open = ext4_mb_seq_groups_open,
2161 static struct kmem_cache *get_groupinfo_cache(
int blocksize_bits)
2164 struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index];
2176 struct ext4_sb_info *sbi =
EXT4_SB(sb);
2178 struct ext4_group_info ***new_groupinfo;
2181 EXT4_DESC_PER_BLOCK_BITS(sb);
2182 if (size <= sbi->s_group_info_size)
2187 if (!new_groupinfo) {
2191 if (sbi->s_group_info) {
2192 memcpy(new_groupinfo, sbi->s_group_info,
2193 sbi->s_group_info_size *
sizeof(*sbi->s_group_info));
2196 sbi->s_group_info = new_groupinfo;
2197 sbi->s_group_info_size = size /
sizeof(*sbi->s_group_info);
2198 ext4_debug(
"allocated s_groupinfo array for %d meta_bg's\n",
2199 sbi->s_group_info_size);
2209 struct ext4_sb_info *sbi =
EXT4_SB(sb);
2210 struct ext4_group_info **meta_group_info;
2219 metalen =
sizeof(*meta_group_info) <<
2220 EXT4_DESC_PER_BLOCK_BITS(sb);
2222 if (meta_group_info ==
NULL) {
2224 "for a buddy group");
2225 goto exit_meta_group_info;
2227 sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] =
2232 sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)];
2235 meta_group_info[
i] = kmem_cache_zalloc(cachep,
GFP_KERNEL);
2236 if (meta_group_info[i] ==
NULL) {
2238 goto exit_group_info;
2240 set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT,
2241 &(meta_group_info[i]->bb_state));
2248 meta_group_info[
i]->bb_free =
2251 meta_group_info[
i]->bb_free =
2255 INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list);
2257 meta_group_info[
i]->bb_free_root =
RB_ROOT;
2258 meta_group_info[
i]->bb_largest_free_order = -1;
2262 struct buffer_head *bh;
2263 meta_group_info[
i]->bb_bitmap =
2265 BUG_ON(meta_group_info[i]->bb_bitmap ==
NULL);
2268 memcpy(meta_group_info[i]->bb_bitmap, bh->b_data,
2279 kfree(sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]);
2280 sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] =
NULL;
2282 exit_meta_group_info:
2286 static int ext4_mb_init_backend(
struct super_block *sb)
2290 struct ext4_sb_info *sbi =
EXT4_SB(sb);
2300 if (sbi->s_buddy_cache ==
NULL) {
2309 EXT4_I(sbi->s_buddy_cache)->i_disksize = 0;
2310 for (i = 0; i < ngroups; i++) {
2326 i = sbi->s_group_info_size;
2328 kfree(sbi->s_group_info[i]);
2329 iput(sbi->s_buddy_cache);
2335 static void ext4_groupinfo_destroy_slabs(
void)
2340 if (ext4_groupinfo_caches[i])
2342 ext4_groupinfo_caches[
i] =
NULL;
2346 static int ext4_groupinfo_create_slab(
size_t size)
2354 if (cache_index >= NR_GRPINFO_CACHES)
2361 if (ext4_groupinfo_caches[cache_index]) {
2366 slab_size =
offsetof(
struct ext4_group_info,
2367 bb_counters[blocksize_bits + 2]);
2373 ext4_groupinfo_caches[cache_index] = cachep;
2378 "EXT4-fs: no memory for groupinfo slab cache\n");
2387 struct ext4_sb_info *sbi =
EXT4_SB(sb);
2396 if (sbi->s_mb_offsets ==
NULL) {
2403 if (sbi->s_mb_maxs ==
NULL) {
2408 ret = ext4_groupinfo_create_slab(sb->
s_blocksize);
2414 sbi->s_mb_offsets[0] = 0;
2420 sbi->s_mb_offsets[
i] =
offset;
2421 sbi->s_mb_maxs[
i] =
max;
2425 }
while (i <= sb->s_blocksize_bits + 1);
2448 sbi->s_cluster_bits, 32);
2457 if (sbi->s_stripe > 1) {
2458 sbi->s_mb_group_prealloc =
roundup(
2459 sbi->s_mb_group_prealloc, sbi->s_stripe);
2463 if (sbi->s_locality_groups ==
NULL) {
2465 goto out_free_groupinfo_slab;
2477 ret = ext4_mb_init_backend(sb);
2479 goto out_free_locality_groups;
2483 &ext4_mb_seq_groups_fops, sb);
2487 out_free_locality_groups:
2489 sbi->s_locality_groups =
NULL;
2490 out_free_groupinfo_slab:
2491 ext4_groupinfo_destroy_slabs();
2493 kfree(sbi->s_mb_offsets);
2494 sbi->s_mb_offsets =
NULL;
2495 kfree(sbi->s_mb_maxs);
2496 sbi->s_mb_maxs =
NULL;
2501 static void ext4_mb_cleanup_pa(
struct ext4_group_info *grp)
2514 mb_debug(1,
"mballoc: %u PAs left\n", count);
2522 int num_meta_group_infos;
2523 struct ext4_group_info *grinfo;
2524 struct ext4_sb_info *sbi =
EXT4_SB(sb);
2530 if (sbi->s_group_info) {
2531 for (i = 0; i < ngroups; i++) {
2532 grinfo = ext4_get_group_info(sb, i);
2534 kfree(grinfo->bb_bitmap);
2536 ext4_lock_group(sb, i);
2537 ext4_mb_cleanup_pa(grinfo);
2538 ext4_unlock_group(sb, i);
2541 num_meta_group_infos = (ngroups +
2543 EXT4_DESC_PER_BLOCK_BITS(sb);
2544 for (i = 0; i < num_meta_group_infos; i++)
2545 kfree(sbi->s_group_info[i]);
2548 kfree(sbi->s_mb_offsets);
2549 kfree(sbi->s_mb_maxs);
2550 if (sbi->s_buddy_cache)
2551 iput(sbi->s_buddy_cache);
2552 if (sbi->s_mb_stats) {
2554 "mballoc: %u blocks %u reqs (%u success)",
2559 "mballoc: %u extents scanned, %u goal hits, "
2560 "%u 2^N hits, %u breaks, %u lost",
2567 "mballoc: %lu generated and it took %Lu",
2568 sbi->s_mb_buddies_generated,
2569 sbi->s_mb_generation_time);
2571 "mballoc: %u preallocated, %u discarded",
2581 static inline int ext4_issue_discard(
struct super_block *sb,
2587 ext4_group_first_block_no(sb, block_group));
2589 trace_ext4_discard_blocks(sb,
2590 (
unsigned long long) discard_block, count);
2591 return sb_issue_discard(sb, discard_block, count,
GFP_NOFS, 0);
2598 static void ext4_free_data_callback(
struct super_block *sb,
2604 struct ext4_group_info *db;
2607 mb_debug(1,
"gonna free %u blocks in group %u (0x%p):",
2611 ext4_issue_discard(sb, entry->
efd_group,
2614 err = ext4_mb_load_buddy(sb, entry->
efd_group, &e4b);
2635 EXT4_MB_GRP_CLEAR_TRIMMED(db);
2637 if (!db->bb_free_root.rb_node) {
2644 ext4_unlock_group(sb, entry->
efd_group);
2646 ext4_mb_unload_buddy(&e4b);
2648 mb_debug(1,
"freed %u blocks in %u structures\n", count,
count2);
2651 #ifdef CONFIG_EXT4_DEBUG
2654 static struct dentry *debugfs_dir;
2655 static struct dentry *debugfs_debug;
2657 static void __init ext4_create_debugfs_entry(
void)
2667 static void ext4_remove_debugfs_entry(
void)
2675 static void __init ext4_create_debugfs_entry(
void)
2679 static void ext4_remove_debugfs_entry(
void)
2689 if (ext4_pspace_cachep ==
NULL)
2694 if (ext4_ac_cachep ==
NULL) {
2701 if (ext4_free_data_cachep ==
NULL) {
2706 ext4_create_debugfs_entry();
2720 ext4_groupinfo_destroy_slabs();
2721 ext4_remove_debugfs_entry();
2729 static noinline_for_stack
int
2731 handle_t *
handle,
unsigned int reserv_clstrs)
2733 struct buffer_head *bitmap_bh =
NULL;
2735 struct buffer_head *gdp_bh;
2736 struct ext4_sb_info *sbi;
2768 block = ext4_grp_offs_to_block(sb, &ac->
ac_b_ex);
2772 ext4_error(sb,
"Allocating blocks %llu-%llu which overlap "
2773 "fs metadata", block, block+len);
2778 ext4_lock_group(sb, ac->
ac_b_ex.fe_group);
2781 ext4_unlock_group(sb, ac->
ac_b_ex.fe_group);
2788 ext4_lock_group(sb, ac->
ac_b_ex.fe_group);
2789 #ifdef AGGRESSIVE_CHECK
2792 for (i = 0; i < ac->
ac_b_ex.fe_len; i++) {
2794 bitmap_bh->b_data));
2811 ext4_unlock_group(sb, ac->
ac_b_ex.fe_group);
2812 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->
ac_b_ex.fe_len);
2818 percpu_counter_sub(&sbi->s_dirtyclusters_counter,
2821 if (sbi->s_log_groups_per_flex) {
2825 &sbi->s_flex_groups[flex_group].free_clusters);
2854 mb_debug(1,
"#%u: goal %u blocks for locality group\n",
2862 static noinline_for_stack
void
2869 loff_t
size, start_off;
2890 ext4_mb_normalize_group_request(ac);
2894 bsbits = ac->
ac_sb->s_blocksize_bits;
2899 size = size << bsbits;
2900 if (size < i_size_read(ac->
ac_inode))
2907 #define NRL_CHECK_SIZE(req, size, max, chunk_size) \
2908 (req <= (size) || max <= (chunk_size))
2913 if (size <= 16 * 1024) {
2915 }
else if (size <= 32 * 1024) {
2917 }
else if (size <= 64 * 1024) {
2919 }
else if (size <= 128 * 1024) {
2921 }
else if (size <= 256 * 1024) {
2923 }
else if (size <= 512 * 1024) {
2925 }
else if (size <= 1024 * 1024) {
2927 }
else if (
NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) {
2928 start_off = ((loff_t)ac->
ac_o_ex.fe_logical >>
2929 (21 - bsbits)) << 21;
2930 size = 2 * 1024 * 1024;
2931 }
else if (
NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) {
2932 start_off = ((loff_t)ac->
ac_o_ex.fe_logical >>
2933 (22 - bsbits)) << 22;
2934 size = 4 * 1024 * 1024;
2936 (8<<20)>>bsbits,
max, 8 * 1024)) {
2937 start_off = ((loff_t)ac->
ac_o_ex.fe_logical >>
2938 (23 - bsbits)) << 23;
2939 size = 8 * 1024 * 1024;
2941 start_off = (loff_t)ac->
ac_o_ex.fe_logical << bsbits;
2942 size = ac->
ac_o_ex.fe_len << bsbits;
2944 size = size >> bsbits;
2945 start = start_off >> bsbits;
2948 if (ar->
pleft && start <= ar->lleft) {
2950 start = ar->
lleft + 1;
2953 size -= start + size - ar->
lright;
2978 if (pa->
pa_lstart >= end || pa_end <= start) {
2985 if (pa_end <= ac->ac_o_ex.fe_logical) {
3012 if (start + size <= ac->ac_o_ex.fe_logical &&
3013 start > ac->
ac_o_ex.fe_logical) {
3015 "start %lu, size %lu, fe_logical %lu",
3016 (
unsigned long) start, (
unsigned long) size,
3017 (
unsigned long) ac->
ac_o_ex.fe_logical);
3019 BUG_ON(start + size <= ac->ac_o_ex.fe_logical &&
3020 start > ac->
ac_o_ex.fe_logical);
3021 BUG_ON(size <= 0 || size > EXT4_CLUSTERS_PER_GROUP(ac->
ac_sb));
3046 mb_debug(1,
"goal: %u(was %u) blocks at %u\n", (
unsigned) size,
3047 (
unsigned) orig_size, (
unsigned) start);
3054 if (sbi->s_mb_stats && ac->
ac_g_ex.fe_len > 1) {
3063 if (ac->
ac_found > sbi->s_mb_max_to_scan)
3068 trace_ext4_mballoc_alloc(ac);
3070 trace_ext4_mballoc_prealloc(ac);
3109 BUG_ON(start < pa->pa_pstart);
3114 mb_debug(1,
"use %llu/%u from inode pa %p\n", start, len, pa);
3123 unsigned int len = ac->
ac_o_ex.fe_len;
3161 if (cur_distance <= new_distance)
3173 static noinline_for_stack
int
3208 ext4_mb_use_inode_pa(ac, pa);
3226 order = fls(ac->
ac_o_ex.fe_len) - 1;
3231 goal_block = ext4_grp_offs_to_block(ac->
ac_sb, &ac->
ac_g_ex);
3244 cpa = ext4_mb_check_group_pa(goal_block,
3252 ext4_mb_use_group_pa(ac, cpa);
3265 static void ext4_mb_generate_from_freelist(
struct super_block *sb,
void *bitmap,
3269 struct ext4_group_info *grp;
3272 grp = ext4_get_group_info(sb, group);
3273 n =
rb_first(&(grp->bb_free_root));
3288 static noinline_for_stack
3289 void ext4_mb_generate_from_pa(
struct super_block *sb,
void *bitmap,
3292 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
3297 int preallocated = 0;
3317 BUG_ON(groupnr != group);
3319 preallocated += len;
3321 mb_debug(1,
"prellocated %u for group %u\n", preallocated, group);
3378 ext4_lock_group(sb, grp);
3380 ext4_unlock_group(sb, grp);
3386 call_rcu(&(pa)->
u.pa_rcu, ext4_mb_pa_callback);
3392 static noinline_for_stack
int
3396 struct ext4_sb_info *sbi =
EXT4_SB(sb);
3398 struct ext4_group_info *grp;
3431 win =
min(winl, wins);
3433 offs = ac->
ac_o_ex.fe_logical %
3435 if (offs && offs < win)
3459 mb_debug(1,
"new inode pa %p: %llu/%u for %u\n", pa,
3461 trace_ext4_mb_new_inode_pa(ac, pa);
3463 ext4_mb_use_inode_pa(ac, pa);
3467 grp = ext4_get_group_info(sb, ac->
ac_b_ex.fe_group);
3472 ext4_lock_group(sb, ac->
ac_b_ex.fe_group);
3474 ext4_unlock_group(sb, ac->
ac_b_ex.fe_group);
3486 static noinline_for_stack
int
3492 struct ext4_group_info *grp;
3519 mb_debug(1,
"new group pa %p: %llu/%u for %u\n", pa,
3521 trace_ext4_mb_new_group_pa(ac, pa);
3523 ext4_mb_use_group_pa(ac, pa);
3526 grp = ext4_get_group_info(sb, ac->
ac_b_ex.fe_group);
3533 ext4_lock_group(sb, ac->
ac_b_ex.fe_group);
3535 ext4_unlock_group(sb, ac->
ac_b_ex.fe_group);
3549 err = ext4_mb_new_group_pa(ac);
3551 err = ext4_mb_new_inode_pa(ac);
3563 static noinline_for_stack
int
3564 ext4_mb_release_inode_pa(
struct ext4_buddy *e4b,
struct buffer_head *bitmap_bh,
3568 struct ext4_sb_info *sbi =
EXT4_SB(sb);
3573 unsigned long long grp_blk_start;
3584 bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit);
3587 next = mb_find_next_bit(bitmap_bh->b_data, end, bit);
3588 mb_debug(1,
" free preallocated %u/%u in group %u\n",
3589 (
unsigned) ext4_group_first_block_no(sb, group) + bit,
3590 (
unsigned) next - bit, (
unsigned) group);
3593 trace_ext4_mballoc_discard(sb,
NULL, group, bit, next - bit);
3594 trace_ext4_mb_release_inode_pa(pa, (grp_blk_start +
3597 mb_free_blocks(pa->
pa_inode, e4b, bit, next - bit);
3602 "pa %p: logic %lu, phys. %lu, len %lu",
3605 (
unsigned long) pa->
pa_len);
3606 ext4_grp_locked_error(sb, group, 0, 0,
"free %u, pa_free %u",
3618 static noinline_for_stack
int
3619 ext4_mb_release_group_pa(
struct ext4_buddy *e4b,
3626 trace_ext4_mb_release_group_pa(sb, pa);
3632 trace_ext4_mballoc_discard(sb,
NULL, group, bit, pa->
pa_len);
3646 static noinline_for_stack
int
3647 ext4_mb_discard_group_preallocations(
struct super_block *sb,
3650 struct ext4_group_info *grp = ext4_get_group_info(sb, group);
3651 struct buffer_head *bitmap_bh =
NULL;
3659 mb_debug(1,
"discard preallocation for group %u\n", group);
3661 if (list_empty(&grp->bb_prealloc_list))
3665 if (bitmap_bh ==
NULL) {
3666 ext4_error(sb,
"Error reading block bitmap for %u", group);
3670 err = ext4_mb_load_buddy(sb, group, &e4b);
3672 ext4_error(sb,
"Error loading buddy information for %u", group);
3678 needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1;
3680 INIT_LIST_HEAD(&
list);
3682 ext4_lock_group(sb, group);
3684 &grp->bb_prealloc_list, pa_group_list) {
3709 if (free < needed && busy) {
3711 ext4_unlock_group(sb, group);
3721 if (list_empty(&
list)) {
3735 ext4_mb_release_group_pa(&e4b, pa);
3737 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
3740 call_rcu(&(pa)->
u.pa_rcu, ext4_mb_pa_callback);
3744 ext4_unlock_group(sb, group);
3745 ext4_mb_unload_buddy(&e4b);
3763 struct buffer_head *bitmap_bh =
NULL;
3775 mb_debug(1,
"discard preallocation for inode %lu\n", inode->
i_ino);
3776 trace_ext4_discard_preallocations(inode);
3778 INIT_LIST_HEAD(&list);
3794 "uh-oh! used pa while discarding");
3833 err = ext4_mb_load_buddy(sb, group, &e4b);
3835 ext4_error(sb,
"Error loading buddy information for %u",
3841 if (bitmap_bh ==
NULL) {
3842 ext4_error(sb,
"Error reading block bitmap for %u",
3844 ext4_mb_unload_buddy(&e4b);
3848 ext4_lock_group(sb, group);
3850 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
3851 ext4_unlock_group(sb, group);
3853 ext4_mb_unload_buddy(&e4b);
3857 call_rcu(&(pa)->
u.pa_rcu, ext4_mb_pa_callback);
3861 #ifdef CONFIG_EXT4_DEBUG
3867 if (!mb_enable_debug ||
3868 (
EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED))
3872 " Allocation context details:");
3876 "goal %lu/%lu/%lu@%lu, "
3877 "best %lu/%lu/%lu@%lu cr %d",
3878 (
unsigned long)ac->
ac_o_ex.fe_group,
3879 (
unsigned long)ac->
ac_o_ex.fe_start,
3880 (
unsigned long)ac->
ac_o_ex.fe_len,
3881 (
unsigned long)ac->
ac_o_ex.fe_logical,
3882 (
unsigned long)ac->
ac_g_ex.fe_group,
3883 (
unsigned long)ac->
ac_g_ex.fe_start,
3884 (
unsigned long)ac->
ac_g_ex.fe_len,
3885 (
unsigned long)ac->
ac_g_ex.fe_logical,
3886 (
unsigned long)ac->
ac_b_ex.fe_group,
3887 (
unsigned long)ac->
ac_b_ex.fe_start,
3888 (
unsigned long)ac->
ac_b_ex.fe_len,
3889 (
unsigned long)ac->
ac_b_ex.fe_logical,
3894 ngroups = ext4_get_groups_count(sb);
3895 for (i = 0; i < ngroups; i++) {
3896 struct ext4_group_info *grp = ext4_get_group_info(sb, i);
3900 ext4_lock_group(sb, i);
3911 ext4_unlock_group(sb, i);
3913 if (grp->bb_free == 0)
3916 i, grp->bb_free, grp->bb_fragments);
3937 int bsbits = ac->
ac_sb->s_blocksize_bits;
3947 isize = (i_size_read(ac->
ac_inode) + ac->
ac_sb->s_blocksize - 1)
3950 if ((size == isize) &&
3951 !ext4_fs_is_busy(sbi) &&
3957 if (sbi->s_mb_group_prealloc <= 0) {
3963 size =
max(size, isize);
3964 if (size > sbi->s_mb_stream_request) {
3984 static noinline_for_stack
int
3989 struct ext4_sb_info *sbi =
EXT4_SB(sb);
4000 if (len >= EXT4_CLUSTERS_PER_GROUP(sb) - 10)
4001 len = EXT4_CLUSTERS_PER_GROUP(sb) - 10;
4006 goal >= ext4_blocks_count(es))
4011 ac->
ac_b_ex.fe_logical = ar->
logical & ~(sbi->s_cluster_ratio - 1);
4024 ext4_mb_group_or_file(ac);
4026 mb_debug(1,
"init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, "
4027 "left: %u/%u, right %u/%u to %swritable\n",
4037 static noinline_for_stack
void
4038 ext4_mb_discard_lg_preallocations(
struct super_block *sb,
4040 int order,
int total_entries)
4047 mb_debug(1,
"discard locality group preallocation\n");
4049 INIT_LIST_HEAD(&discard_list);
4079 if (total_entries <= 5) {
4094 if (ext4_mb_load_buddy(sb, group, &e4b)) {
4095 ext4_error(sb,
"Error loading buddy information for %u",
4099 ext4_lock_group(sb, group);
4101 ext4_mb_release_group_pa(&e4b, pa);
4102 ext4_unlock_group(sb, group);
4104 ext4_mb_unload_buddy(&e4b);
4106 call_rcu(&(pa)->
u.pa_rcu, ext4_mb_pa_callback);
4121 int order, added = 0, lg_prealloc_count = 1;
4127 if (order > PREALLOC_TB_SIZE - 1)
4129 order = PREALLOC_TB_SIZE - 1;
4136 spin_unlock(&tmp_pa->
pa_lock);
4149 spin_unlock(&tmp_pa->
pa_lock);
4150 lg_prealloc_count++;
4158 if (lg_prealloc_count > 8) {
4159 ext4_mb_discard_lg_preallocations(sb, lg,
4160 order, lg_prealloc_count);
4195 ext4_mb_add_n_trim(ac);
4197 ext4_mb_put_pa(ac, ac->
ac_sb, pa);
4205 ext4_mb_collect_stats(ac);
4209 static int ext4_mb_discard_preallocations(
struct super_block *sb,
int needed)
4215 trace_ext4_mb_discard_preallocations(sb, needed);
4216 for (i = 0; i < ngroups && needed > 0; i++) {
4217 ret = ext4_mb_discard_group_preallocations(sb, i, needed);
4235 struct ext4_sb_info *sbi;
4238 unsigned int inquota = 0;
4239 unsigned int reserv_clstrs = 0;
4241 sb = ar->
inode->i_sb;
4244 trace_ext4_request_blocks(ar);
4255 if (ext4_test_inode_state(ar->
inode, EXT4_STATE_DELALLOC_RESERVED))
4273 reserv_clstrs = ar->
len;
4275 dquot_alloc_block_nofail(ar->
inode,
4279 dquot_alloc_block(ar->
inode,
4293 ac = kmem_cache_zalloc(ext4_ac_cachep,
GFP_NOFS);
4300 *errp = ext4_mb_initialize_context(ac, ar);
4307 if (!ext4_mb_use_preallocated(ac)) {
4309 ext4_mb_normalize_request(ac, ar);
4312 *errp = ext4_mb_regular_allocator(ac);
4321 ext4_mb_new_preallocation(ac);
4324 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs);
4330 ext4_mb_release_context(ac);
4338 ext4_discard_allocated_blocks(ac);
4340 block = ext4_grp_offs_to_block(sb, &ac->
ac_b_ex);
4344 freed = ext4_mb_discard_preallocations(sb, ac->
ac_o_ex.fe_len);
4353 ext4_mb_show_ac(ac);
4355 ext4_mb_release_context(ac);
4359 if (inquota && ar->
len < inquota)
4362 if (!ext4_test_inode_state(ar->
inode,
4363 EXT4_STATE_DELALLOC_RESERVED))
4365 percpu_counter_sub(&sbi->s_dirtyclusters_counter,
4369 trace_ext4_allocate_blocks(ar, (
unsigned long long)block);
4389 static noinline_for_stack
int
4390 ext4_mb_free_metadata(handle_t *handle,
struct ext4_buddy *e4b,
4396 struct ext4_group_info *db = e4b->
bd_info;
4398 struct ext4_sb_info *sbi =
EXT4_SB(sb);
4399 struct rb_node **n = &db->bb_free_root.rb_node, *
node;
4402 BUG_ON(!ext4_handle_valid(handle));
4421 if (cluster < entry->efd_start_cluster)
4426 ext4_grp_locked_error(sb, group, 0,
4427 ext4_group_first_block_no(sb, group) +
4429 "Block already on to-be-freed list");
4434 rb_link_node(new_node, parent, n);
4441 if (can_merge(entry, new_entry)) {
4445 ext4_journal_callback_del(handle, &entry->
efd_jce);
4453 if (can_merge(new_entry, entry)) {
4456 ext4_journal_callback_del(handle, &entry->
efd_jce);
4461 ext4_journal_callback_add(handle, ext4_free_data_callback,
4476 unsigned long count,
int flags)
4478 struct buffer_head *bitmap_bh =
NULL;
4481 unsigned long freed = 0;
4484 struct buffer_head *gd_bh;
4486 struct ext4_sb_info *sbi;
4488 unsigned int count_clusters;
4494 BUG_ON(block != bh->b_blocknr);
4496 block = bh->b_blocknr;
4502 ext4_error(sb,
"Freeing blocks not in datazone - "
4503 "block = %llu, count = %lu", block, count);
4508 trace_ext4_free_blocks(inode, block, count, flags);
4511 struct buffer_head *tbh = bh;
4514 BUG_ON(bh && (count > 1));
4516 for (i = 0; i <
count; i++) {
4518 tbh = sb_find_get_block(inode->
i_sb,
4523 inode, tbh, block + i);
4534 if (!ext4_should_writeback_data(inode))
4544 overflow = block & (sbi->s_cluster_ratio - 1);
4547 overflow = sbi->s_cluster_ratio -
overflow;
4549 if (count > overflow)
4558 overflow = count & (sbi->s_cluster_ratio - 1);
4561 if (count > overflow)
4566 count += sbi->s_cluster_ratio -
overflow;
4578 overflow =
EXT4_C2B(sbi, bit) + count -
4582 count_clusters =
EXT4_B2C(sbi, count);
4597 EXT4_SB(sb)->s_itb_per_group) ||
4599 EXT4_SB(sb)->s_itb_per_group)) {
4601 ext4_error(sb,
"Freeing blocks in system zone - "
4602 "Block = %llu, count = %lu", block, count);
4607 BUFFER_TRACE(bitmap_bh,
"getting write access");
4617 BUFFER_TRACE(gd_bh,
"get_write_access");
4621 #ifdef AGGRESSIVE_CHECK
4624 for (i = 0; i < count_clusters; i++)
4625 BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data));
4628 trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters);
4630 err = ext4_mb_load_buddy(sb, block_group, &e4b);
4642 ext4_mb_unload_buddy(&e4b);
4649 new_entry->
efd_tid = handle->h_transaction->t_tid;
4651 ext4_lock_group(sb, block_group);
4652 mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);
4653 ext4_mb_free_metadata(handle, &e4b, new_entry);
4660 ext4_issue_discard(sb, block_group, bit, count);
4661 ext4_lock_group(sb, block_group);
4662 mb_clear_bits(bitmap_bh->b_data, bit, count_clusters);
4663 mb_free_blocks(inode, &e4b, bit, count_clusters);
4670 ext4_unlock_group(sb, block_group);
4671 percpu_counter_add(&sbi->s_freeclusters_counter, count_clusters);
4673 if (sbi->s_log_groups_per_flex) {
4674 ext4_group_t flex_group = ext4_flex_group(sbi, block_group);
4676 &sbi->s_flex_groups[flex_group].free_clusters);
4679 ext4_mb_unload_buddy(&e4b);
4684 dquot_free_block(inode,
EXT4_C2B(sbi, count_clusters));
4687 BUFFER_TRACE(bitmap_bh,
"dirtied bitmap block");
4691 BUFFER_TRACE(gd_bh,
"dirtied group descriptor block");
4696 if (overflow && !err) {
4704 ext4_std_error(sb, err);
4720 struct buffer_head *bitmap_bh =
NULL;
4721 struct buffer_head *gd_bh;
4726 struct ext4_sb_info *sbi =
EXT4_SB(sb);
4728 int err = 0,
ret, blk_free_count;
4731 ext4_debug(
"Adding block(s) %llu-%llu\n", block, block + count - 1);
4742 ext4_warning(sb,
"too much blocks added to group %u\n",
4764 sbi->s_itb_per_group)) {
4765 ext4_error(sb,
"Adding blocks in system zones - "
4766 "Block = %llu, count = %lu",
4772 BUFFER_TRACE(bitmap_bh,
"getting write access");
4782 BUFFER_TRACE(gd_bh,
"get_write_access");
4787 for (i = 0, blocks_freed = 0; i <
count; i++) {
4788 BUFFER_TRACE(bitmap_bh,
"clear bit");
4789 if (!mb_test_bit(bit + i, bitmap_bh->b_data)) {
4790 ext4_error(sb,
"bit already cleared for block %llu",
4792 BUFFER_TRACE(bitmap_bh,
"bit already cleared");
4798 err = ext4_mb_load_buddy(sb, block_group, &e4b);
4807 ext4_lock_group(sb, block_group);
4808 mb_clear_bits(bitmap_bh->b_data, bit, count);
4809 mb_free_blocks(
NULL, &e4b, bit, count);
4814 ext4_unlock_group(sb, block_group);
4815 percpu_counter_add(&sbi->s_freeclusters_counter,
4818 if (sbi->s_log_groups_per_flex) {
4819 ext4_group_t flex_group = ext4_flex_group(sbi, block_group);
4821 &sbi->s_flex_groups[flex_group].free_clusters);
4824 ext4_mb_unload_buddy(&e4b);
4827 BUFFER_TRACE(bitmap_bh,
"dirtied bitmap block");
4831 BUFFER_TRACE(gd_bh,
"dirtied group descriptor block");
4838 ext4_std_error(sb, err);
4854 static void ext4_trim_extent(
struct super_block *sb,
int start,
int count,
4859 trace_ext4_trim_extent(sb, group, start, count);
4871 mb_mark_used(e4b, &ex);
4872 ext4_unlock_group(sb, group);
4873 ext4_issue_discard(sb, group, start, count);
4874 ext4_lock_group(sb, group);
4906 trace_ext4_trim_all_free(sb, group, start, max);
4908 ret = ext4_mb_load_buddy(sb, group, &e4b);
4910 ext4_error(sb,
"Error in loading buddy "
4911 "information for %u", group);
4916 ext4_lock_group(sb, group);
4917 if (EXT4_MB_GRP_WAS_TRIMMED(e4b.
bd_info) &&
4922 e4b.
bd_info->bb_first_free : start;
4924 while (start <= max) {
4925 start = mb_find_next_zero_bit(bitmap, max + 1, start);
4928 next = mb_find_next_bit(bitmap, max + 1, start);
4930 if ((next - start) >= minblocks) {
4931 ext4_trim_extent(sb, start,
4932 next - start, group, &e4b);
4933 count += next -
start;
4935 free_count += next -
start;
4938 if (fatal_signal_pending(
current)) {
4943 if (need_resched()) {
4944 ext4_unlock_group(sb, group);
4946 ext4_lock_group(sb, group);
4949 if ((e4b.
bd_info->bb_free - free_count) < minblocks)
4954 EXT4_MB_GRP_SET_TRIMMED(e4b.
bd_info);
4956 ext4_unlock_group(sb, group);
4957 ext4_mb_unload_buddy(&e4b);
4959 ext4_debug(
"trimmed %d blocks in the group %d\n",
4979 struct ext4_group_info *grp;
4993 if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) ||
4994 start >= max_blks ||
4997 if (end >= max_blks)
4999 if (end <= first_data_blk)
5001 if (start < first_data_blk)
5002 start = first_data_blk;
5006 &first_group, &first_cluster);
5008 &last_group, &last_cluster);
5011 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1;
5013 for (group = first_group; group <= last_group; group++) {
5014 grp = ext4_get_group_info(sb, group);
5016 if (
unlikely(EXT4_MB_GRP_NEED_INIT(grp))) {
5017 ret = ext4_mb_init_group(sb, group);
5028 if (group == last_group)
5031 if (grp->bb_free >= minlen) {
5032 cnt = ext4_trim_all_free(sb, group, first_cluster,