18 #include <linux/slab.h>
42 }
else if ((*path)[ext_depth(inode)].p_ext ==
NULL)
57 if (ext4_ext_is_uninitialized(src))
58 ext4_ext_mark_uninitialized(dest);
77 mext_next_extent(
struct inode *inode,
struct ext4_ext_path *path,
81 int ppos, leaf_ppos = path->
p_depth;
86 *extent = ++path[ppos].
p_ext;
87 path[ppos].
p_block = ext4_ext_pblock(path[ppos].p_ext);
98 path[ppos].
p_block = ext4_idx_pblock(path[ppos].p_idx);
99 if (path[ppos+1].p_bh)
100 brelse(path[ppos+1].p_bh);
103 if (!path[ppos+1].p_bh)
106 ext_block_hdr(path[ppos+1].p_bh);
109 while (++cur_ppos < leaf_ppos) {
110 path[cur_ppos].
p_idx =
113 ext4_idx_pblock(path[cur_ppos].p_idx);
114 if (path[cur_ppos+1].p_bh)
115 brelse(path[cur_ppos+1].p_bh);
116 path[cur_ppos+1].
p_bh = sb_bread(inode->
i_sb,
118 if (!path[cur_ppos+1].p_bh)
120 path[cur_ppos+1].
p_hdr =
121 ext_block_hdr(path[cur_ppos+1].p_bh);
126 eh = path[leaf_ppos].
p_hdr;
132 path[leaf_ppos].
p_ext = *extent =
135 ext4_ext_pblock(path[leaf_ppos].p_ext);
149 double_down_write_data_sem(
struct inode *
first,
struct inode *
second)
151 if (first < second) {
169 double_up_write_data_sem(
struct inode *orig_inode,
struct inode *donor_inode)
190 mext_insert_across_blocks(handle_t *
handle,
struct inode *orig_inode,
202 if (o_start == o_end) {
217 ext4_ext_store_pblock(o_end, ext4_ext_pblock(end_ext));
225 !end_ext->
ee_len && o_start == o_end) {
236 end_ext->
ee_len && o_start == o_end) {
244 ext4_ext_store_pblock(o_end, ext4_ext_pblock(end_ext));
255 ext4_debug(
"ext4 move extent: Unexpected insert case\n");
260 err = get_ext_path(orig_inode, eblock, &orig_path);
265 orig_path, new_ext, 0))
270 err = get_ext_path(orig_inode,
276 orig_path, end_ext, 0))
304 mext_insert_inside_block(
struct ext4_extent *o_start,
318 (
unsigned long)(o_end + 1);
319 memmove(o_end + 1 + range_to_move, o_end + 1, len);
328 o_start[
i] = *new_ext;
329 ext4_ext_store_pblock(&o_start[i++], ext4_ext_pblock(new_ext));
358 mext_insert_extents(handle_t *handle,
struct inode *orig_inode,
367 unsigned long need_slots, slots_range;
374 need_slots = (start_ext->
ee_len ? 1 : 0) + (end_ext->
ee_len ? 1 : 0) +
375 (new_ext->
ee_len ? 1 : 0);
378 slots_range = ((
unsigned long)(o_end + 1) - (
unsigned long)o_start + 1)
382 range_to_move = need_slots - slots_range;
385 eh = orig_path->
p_hdr;
395 if (range_to_move > 0 &&
399 ret = mext_insert_across_blocks(handle, orig_inode, o_start,
400 o_end, start_ext, new_ext, end_ext);
404 mext_insert_inside_block(o_start, o_end, start_ext, new_ext,
405 end_ext, eh, range_to_move);
439 mext_leaf_block(handle_t *handle,
struct inode *orig_inode,
443 struct ext4_extent *oext, *o_start, *o_end, *prev_ext;
446 int oext_alen, new_ext_alen, end_ext_alen;
447 int depth = ext_depth(orig_inode);
451 o_start = o_end = oext = orig_path[
depth].
p_ext;
452 oext_alen = ext4_ext_get_actual_len(oext);
456 ext4_ext_store_pblock(&new_ext, ext4_ext_pblock(dext));
458 new_ext_alen = ext4_ext_get_actual_len(&new_ext);
473 copy_extent_status(oext, &start_ext);
484 ext4_ext_get_actual_len(prev_ext) +
487 copy_extent_status(prev_ext, &start_ext);
499 "new_ext_end(%u) should be less than or equal to "
500 "oext->ee_block(%u) + oext_alen(%d) - 1",
517 oext_alen - 1 - new_ext_end);
518 copy_extent_status(oext, &end_ext);
519 end_ext_alen = ext4_ext_get_actual_len(&end_ext);
520 ext4_ext_store_pblock(&end_ext,
521 (ext4_ext_pblock(o_end) + oext_alen - end_ext_alen));
524 oext_alen - end_ext_alen);
527 ret = mext_insert_extents(handle, orig_inode, orig_path, o_start,
528 o_end, &start_ext, &new_ext, &end_ext);
545 mext_calc_swap_extents(
struct ext4_extent *tmp_dext,
553 BUG_ON(orig_off != donor_off);
558 ext4_ext_get_actual_len(tmp_oext) - 1 < orig_off)
563 ext4_ext_get_actual_len(tmp_dext) - 1 < orig_off)
566 dext_old = *tmp_dext;
567 oext_old = *tmp_oext;
572 ext4_ext_store_pblock(tmp_dext, ext4_ext_pblock(tmp_dext) + diff);
573 le32_add_cpu(&tmp_dext->
ee_block, diff);
574 le16_add_cpu(&tmp_dext->
ee_len, -diff);
576 if (max_count < ext4_ext_get_actual_len(tmp_dext))
580 ext4_ext_store_pblock(tmp_oext, ext4_ext_pblock(tmp_oext) + orig_diff);
583 if (ext4_ext_get_actual_len(tmp_dext) >
584 ext4_ext_get_actual_len(tmp_oext) - orig_diff)
590 copy_extent_status(&oext_old, tmp_dext);
591 copy_extent_status(&dext_old, tmp_oext);
609 int uninit,
int *err)
614 while (from < last) {
615 *err = get_ext_path(inode, from, &path);
618 ext = path[ext_depth(inode)].
p_ext;
623 if (uninit != ext4_ext_is_uninitialized(ext)) {
627 from += ext4_ext_get_actual_len(ext);
655 mext_replace_branches(handle_t *handle,
struct inode *orig_inode,
665 int replaced_count = 0;
669 *err = get_ext_path(orig_inode, orig_off, &orig_path);
674 *err = get_ext_path(donor_inode, donor_off, &donor_path);
677 depth = ext_depth(orig_inode);
681 depth = ext_depth(donor_inode);
685 *err = mext_calc_swap_extents(&tmp_dext, &tmp_oext, orig_off,
695 "The extent for donor must be found");
700 "Donor offset(%u) and the first block of donor "
701 "extent(%u) should be equal",
709 *err = mext_leaf_block(handle, orig_inode,
710 orig_path, &tmp_dext, &orig_off);
715 *err = mext_leaf_block(handle, donor_inode,
716 donor_path, &tmp_oext, &donor_off);
720 dext_alen = ext4_ext_get_actual_len(&tmp_dext);
721 replaced_count += dext_alen;
722 donor_off += dext_alen;
723 orig_off += dext_alen;
726 if (replaced_count >= count)
731 *err = get_ext_path(orig_inode, orig_off, &orig_path);
734 depth = ext_depth(orig_inode);
740 *err = get_ext_path(donor_inode, donor_off, &donor_path);
743 depth = ext_depth(donor_inode);
747 *err = mext_calc_swap_extents(&tmp_dext, &tmp_oext, orig_off,
748 donor_off, count - replaced_count);
763 ext4_ext_invalidate_cache(orig_inode);
764 ext4_ext_invalidate_cache(donor_inode);
766 return replaced_count;
780 mext_page_double_lock(
struct inode *inode1,
struct inode *inode2,
786 BUG_ON(!inode1 || !inode2);
787 if (inode1 < inode2) {
806 if (inode1 > inode2) {
817 mext_page_mkuptodate(
struct page *page,
unsigned from,
unsigned to)
819 struct inode *inode = page->
mapping->host;
821 struct buffer_head *bh, *
head, *arr[MAX_BUF_PER_PAGE];
822 unsigned int blocksize, block_start, block_end;
823 int i,
err,
nr = 0, partial = 0;
824 BUG_ON(!PageLocked(page));
825 BUG_ON(PageWriteback(page));
827 if (PageUptodate(page))
831 if (!page_has_buffers(page))
834 head = page_buffers(page);
836 for (bh = head, block_start = 0; bh != head || !block_start;
837 block++, block_start = block_end, bh = bh->b_this_page) {
838 block_end = block_start + blocksize;
839 if (block_end <= from || block_start >= to) {
840 if (!buffer_uptodate(bh))
844 if (buffer_uptodate(bh))
846 if (!buffer_mapped(bh)) {
853 if (!buffer_mapped(bh)) {
854 zero_user(page, block_start, blocksize);
856 set_buffer_uptodate(bh);
860 BUG_ON(nr >= MAX_BUF_PER_PAGE);
867 for (i = 0; i <
nr; i++) {
877 SetPageUptodate(page);
898 move_extent_per_page(
struct file *o_filp,
struct inode *donor_inode,
899 pgoff_t orig_page_offset,
int data_offset_in_page,
900 int block_len_in_page,
int uninit,
int *err)
902 struct inode *orig_inode = o_filp->f_dentry->d_inode;
903 struct page *pagep[2] = {
NULL,
NULL};
907 unsigned long blocksize = orig_inode->
i_sb->s_blocksize;
908 unsigned int w_flags = 0;
909 unsigned int tmp_data_size,
data_size, replaced_size;
910 int err2, jblocks,
retries = 0;
911 int replaced_count = 0;
912 int from = data_offset_in_page << orig_inode->
i_blkbits;
922 handle = ext4_journal_start(orig_inode, jblocks);
923 if (IS_ERR(handle)) {
924 *err = PTR_ERR(handle);
931 orig_blk_offset = orig_page_offset * blocks_per_page +
937 if ((orig_blk_offset + block_len_in_page - 1) ==
940 tmp_data_size = orig_inode->
i_size & (blocksize - 1);
945 if (tmp_data_size == 0)
946 tmp_data_size = blocksize;
948 data_size = tmp_data_size +
949 ((block_len_in_page - 1) << orig_inode->
i_blkbits);
951 data_size = block_len_in_page << orig_inode->
i_blkbits;
955 *err = mext_page_double_lock(orig_inode, donor_inode, orig_page_offset,
967 double_down_write_data_sem(orig_inode, donor_inode);
970 uninit = mext_check_coverage(orig_inode, orig_blk_offset,
971 block_len_in_page, 1, err);
975 uninit &= mext_check_coverage(donor_inode, orig_blk_offset,
976 block_len_in_page, 1, err);
981 double_up_write_data_sem(orig_inode, donor_inode);
984 if ((page_has_private(pagep[0]) &&
986 (page_has_private(pagep[1]) &&
991 replaced_count = mext_replace_branches(handle, orig_inode,
992 donor_inode, orig_blk_offset,
993 block_len_in_page, err);
995 double_up_write_data_sem(orig_inode, donor_inode);
999 *err = mext_page_mkuptodate(pagep[0], from, from + replaced_size);
1011 replaced_count = mext_replace_branches(handle, orig_inode, donor_inode,
1013 block_len_in_page, err);
1015 if (replaced_count) {
1016 block_len_in_page = replaced_count;
1018 block_len_in_page << orig_inode->
i_blkbits;
1030 goto repair_branches;
1034 *err = ext4_jbd2_file_inode(handle, orig_inode);
1048 return replaced_count;
1056 double_down_write_data_sem(orig_inode, donor_inode);
1057 replaced_count = mext_replace_branches(handle, donor_inode, orig_inode,
1059 block_len_in_page, &err2);
1060 double_up_write_data_sem(orig_inode, donor_inode);
1061 if (replaced_count != block_len_in_page) {
1063 "Unable to copy data block,"
1064 " data will be lost.");
1085 mext_check_arguments(
struct inode *orig_inode,
1086 struct inode *donor_inode,
__u64 orig_start,
1090 unsigned int blkbits = orig_inode->
i_blkbits;
1091 unsigned int blocksize = 1 << blkbits;
1094 ext4_debug(
"ext4 move extent: suid or sgid is set"
1095 " to donor file [ino:orig %lu, donor %lu]\n",
1105 ext4_debug(
"ext4 move extent: The argument files should "
1106 "not be swapfile [ino:orig %lu, donor %lu]\n",
1113 ext4_debug(
"ext4 move extent: orig file is not extents "
1114 "based file [ino:orig %lu]\n", orig_inode->
i_ino);
1117 ext4_debug(
"ext4 move extent: donor file is not extents "
1118 "based file [ino:donor %lu]\n", donor_inode->
i_ino);
1123 ext4_debug(
"ext4 move extent: File size is 0 byte\n");
1128 if (orig_start != donor_start) {
1129 ext4_debug(
"ext4 move extent: orig and donor's start "
1130 "offset are not same [ino:orig %lu, donor %lu]\n",
1138 ext4_debug(
"ext4 move extent: Can't handle over [%u] blocks "
1145 donor_blocks = (donor_inode->
i_size + blocksize - 1) >> blkbits;
1147 if (orig_start >= donor_blocks) {
1148 ext4_debug(
"ext4 move extent: orig start offset "
1149 "[%llu] should be less than donor file blocks "
1150 "[%u] [ino:orig %lu, donor %lu]\n",
1151 orig_start, donor_blocks,
1157 if (orig_start + *len > donor_blocks) {
1158 ext4_debug(
"ext4 move extent: End offset [%llu] should "
1159 "be less than donor file blocks [%u]."
1160 "So adjust length from %llu to %llu "
1161 "[ino:orig %lu, donor %lu]\n",
1162 orig_start + *len, donor_blocks,
1163 *len, donor_blocks - orig_start,
1165 *len = donor_blocks - orig_start;
1168 orig_blocks = (orig_inode->
i_size + blocksize - 1) >> blkbits;
1169 if (orig_start >= orig_blocks) {
1170 ext4_debug(
"ext4 move extent: start offset [%llu] "
1171 "should be less than original file blocks "
1172 "[%u] [ino:orig %lu, donor %lu]\n",
1173 orig_start, orig_blocks,
1178 if (orig_start + *len > orig_blocks) {
1179 ext4_debug(
"ext4 move extent: Adjust length "
1180 "from %llu to %llu. Because it should be "
1181 "less than original file blocks "
1182 "[ino:orig %lu, donor %lu]\n",
1183 *len, orig_blocks - orig_start,
1185 *len = orig_blocks - orig_start;
1190 ext4_debug(
"ext4 move extent: len should not be 0 "
1191 "[ino:orig %lu, donor %lu]\n", orig_inode->
i_ino,
1192 donor_inode->
i_ino);
1208 mext_inode_double_lock(
struct inode *inode1,
struct inode *inode2)
1210 BUG_ON(inode1 == inode2);
1211 if (inode1 < inode2) {
1229 mext_inode_double_unlock(
struct inode *inode1,
struct inode *inode2)
1281 struct inode *orig_inode = o_filp->f_dentry->d_inode;
1282 struct inode *donor_inode = d_filp->f_dentry->d_inode;
1284 struct ext4_extent *ext_prev, *ext_cur, *ext_dummy;
1286 ext4_lblk_t block_end, seq_start, add_blocks, file_end, seq_blocks = 0;
1288 pgoff_t orig_page_offset = 0, seq_end_page;
1291 int data_offset_in_page;
1292 int block_len_in_page;
1295 if (orig_inode->
i_sb != donor_inode->
i_sb) {
1296 ext4_debug(
"ext4 move extent: The argument files "
1297 "should be in same FS [ino:orig %lu, donor %lu]\n",
1303 if (orig_inode == donor_inode) {
1304 ext4_debug(
"ext4 move extent: The argument files should not "
1305 "be same inode [ino:orig %lu, donor %lu]\n",
1312 ext4_debug(
"ext4 move extent: The argument files should be "
1313 "regular file [ino:orig %lu, donor %lu]\n",
1319 if (ext4_should_journal_data(orig_inode) ||
1320 ext4_should_journal_data(donor_inode)) {
1324 mext_inode_double_lock(orig_inode, donor_inode);
1327 ext4_inode_block_unlocked_dio(orig_inode);
1328 ext4_inode_block_unlocked_dio(donor_inode);
1333 double_down_write_data_sem(orig_inode, donor_inode);
1335 ret = mext_check_arguments(orig_inode, donor_inode, orig_start,
1340 file_end = (i_size_read(orig_inode) - 1) >> orig_inode->
i_blkbits;
1341 block_end = block_start + len - 1;
1342 if (file_end < block_end)
1343 len -= block_end - file_end;
1345 ret = get_ext_path(orig_inode, block_start, &orig_path);
1350 ret = get_ext_path(orig_inode, block_start, &holecheck_path);
1354 depth = ext_depth(orig_inode);
1355 ext_cur = holecheck_path[
depth].p_ext;
1362 ext4_ext_get_actual_len(ext_cur) - 1 < block_start) {
1367 last_extent = mext_next_extent(orig_inode,
1368 holecheck_path, &ext_cur);
1369 if (last_extent < 0) {
1373 last_extent = mext_next_extent(orig_inode, orig_path,
1375 if (last_extent < 0) {
1384 seq_start = block_start;
1388 ext4_debug(
"ext4 move extent: The specified range of file "
1389 "may be the hole\n");
1396 ext4_ext_get_actual_len(ext_cur), block_end + 1) -
1400 seq_blocks += add_blocks;
1403 if (seq_start + seq_blocks - 1 > block_end)
1404 seq_blocks = block_end - seq_start + 1;
1407 last_extent = mext_next_extent(orig_inode, holecheck_path,
1409 if (last_extent < 0) {
1413 add_blocks = ext4_ext_get_actual_len(ext_cur);
1420 ext_prev, ext_cur) &&
1426 uninit = ext4_ext_is_uninitialized(ext_prev);
1428 data_offset_in_page = seq_start % blocks_per_page;
1434 if (data_offset_in_page + seq_blocks > blocks_per_page) {
1437 blocks_per_page - data_offset_in_page;
1440 block_len_in_page = seq_blocks;
1443 orig_page_offset = seq_start >>
1444 (PAGE_CACHE_SHIFT - orig_inode->
i_blkbits);
1445 seq_end_page = (seq_start + seq_blocks - 1) >>
1446 (PAGE_CACHE_SHIFT - orig_inode->
i_blkbits);
1448 rest_blocks = seq_blocks;
1457 double_up_write_data_sem(orig_inode, donor_inode);
1459 while (orig_page_offset <= seq_end_page) {
1462 block_len_in_page = move_extent_per_page(
1463 o_filp, donor_inode,
1465 data_offset_in_page,
1466 block_len_in_page, uninit,
1470 *moved_len += block_len_in_page;
1473 if (*moved_len > len) {
1475 "We replaced blocks too much! "
1476 "sum of replaced: %llu requested: %llu",
1483 data_offset_in_page = 0;
1484 rest_blocks -= block_len_in_page;
1485 if (rest_blocks > blocks_per_page)
1486 block_len_in_page = blocks_per_page;
1488 block_len_in_page = rest_blocks;
1491 double_down_write_data_sem(orig_inode, donor_inode);
1498 ret = get_ext_path(orig_inode, seq_start, &holecheck_path);
1501 depth = holecheck_path->p_depth;
1506 ret = get_ext_path(orig_inode, seq_start, &orig_path);
1510 ext_cur = holecheck_path[
depth].p_ext;
1511 add_blocks = ext4_ext_get_actual_len(ext_cur);
1525 if (holecheck_path) {
1527 kfree(holecheck_path);
1529 double_up_write_data_sem(orig_inode, donor_inode);
1530 ext4_inode_resume_unlocked_dio(orig_inode);
1531 ext4_inode_resume_unlocked_dio(donor_inode);
1532 mext_inode_double_unlock(orig_inode, donor_inode);