// SPDX-License-Identifier: GPL-2.0

/*

* Copyright (C) 2007 Oracle. All rights reserved.

*/

#include <linux/fs.h>

#include <linux/pagemap.h>

#include <linux/time.h>

#include <linux/init.h>

#include <linux/string.h>

#include <linux/backing-dev.h>

#include <linux/falloc.h>

#include <linux/filelock.h>

#include <linux/writeback.h>

#include <linux/compat.h>

#include <linux/slab.h>

#include <linux/btrfs.h>

#include <linux/uio.h>

#include <linux/iversion.h>

#include <linux/fsverity.h>

#include "ctree.h"

#include "direct-io.h"

#include "disk-io.h"

#include "transaction.h"

#include "btrfs_inode.h"

#include "tree-log.h"

#include "locking.h"

#include "qgroup.h"

#include "compression.h"

#include "delalloc-space.h"

#include "reflink.h"

#include "subpage.h"

#include "fs.h"

#include "accessors.h"

#include "extent-tree.h"

#include "file-item.h"

#include "ioctl.h"

#include "file.h"

#include "super.h"

#include "print-tree.h"

/*

* Unlock folio after btrfs_file_write() is done with it.

*/

static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio,

u64 pos, u64 copied)

{

u64 block_start = round_down(pos, fs_info->sectorsize);

u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start;

ASSERT(block_len <= U32_MAX);

/*

* Folio checked is some magic around finding folios that have been

* modified without going through btrfs_dirty_folio(). Clear it here.

* There should be no need to mark the pages accessed as

* prepare_one_folio() should have marked them accessed in

* prepare_one_folio() via find_or_create_page()

*/

btrfs_folio_clamp_clear_checked(fs_info, folio, block_start, block_len);

folio_unlock(folio);

folio_put(folio);

}

/*

* After copy_folio_from_iter_atomic(), update the following things for delalloc:

* - Mark newly dirtied folio as DELALLOC in the io tree.

* Used to advise which range is to be written back.

* - Mark modified folio as Uptodate/Dirty and not needing COW fixup

* - Update inode size for past EOF write

*/

int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos,

size_t write_bytes, struct extent_state **cached, bool noreserve)

{

struct btrfs_fs_info *fs_info = inode->root->fs_info;

int ret = 0;

u64 num_bytes;

u64 start_pos;

u64 end_of_last_block;

const u64 end_pos = pos + write_bytes;

loff_t isize = i_size_read(&inode->vfs_inode);

unsigned int extra_bits = 0;

if (write_bytes == 0)

return 0;

if (noreserve)

extra_bits |= EXTENT_NORESERVE;

start_pos = round_down(pos, fs_info->sectorsize);

num_bytes = round_up(end_pos - start_pos, fs_info->sectorsize);

ASSERT(num_bytes <= U32_MAX);

ASSERT(folio_pos(folio) <= pos && folio_next_pos(folio) >= end_pos);

end_of_last_block = start_pos + num_bytes - 1;

/*

* The pages may have already been dirty, clear out old accounting so

* we can set things up properly

*/

btrfs_clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block,

EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,

cached);

ret = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,

extra_bits, cached);

if (ret)

return ret;

btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes);

btrfs_folio_clamp_clear_checked(fs_info, folio, start_pos, num_bytes);

btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes);

/*

* we've only changed i_size in ram, and we haven't updated

* the disk i_size. There is no need to log the inode

* at this time.

*/

if (end_pos > isize)

i_size_write(&inode->vfs_inode, end_pos);

return 0;

}

/*

* this is very complex, but the basic idea is to drop all extents

* in the range start - end. hint_block is filled in with a block number

* that would be a good hint to the block allocator for this file.

*

* If an extent intersects the range but is not entirely inside the range

* it is either truncated or split. Anything entirely inside the range

* is deleted from the tree.

*

* Note: the VFS' inode number of bytes is not updated, it's up to the caller

* to deal with that. We set the field 'bytes_found' of the arguments structure

* with the number of allocated bytes found in the target range, so that the

* caller can update the inode's number of bytes in an atomic way when

* replacing extents in a range to avoid races with stat(2).

*/

int btrfs_drop_extents(struct btrfs_trans_handle *trans,

struct btrfs_root *root, struct btrfs_inode *inode,

struct btrfs_drop_extents_args *args)

{

struct btrfs_fs_info *fs_info = root->fs_info;

struct extent_buffer *leaf;

struct btrfs_file_extent_item *fi;

struct btrfs_key key;

struct btrfs_key new_key;

u64 ino = btrfs_ino(inode);

u64 search_start = args->start;

u64 disk_bytenr = 0;

u64 num_bytes = 0;

u64 extent_offset = 0;

u64 extent_end = 0;

u64 last_end = args->start;

int del_nr = 0;

int del_slot = 0;

int extent_type;

int recow;

int ret;

int modify_tree = -1;

int update_refs;

int found = 0;

struct btrfs_path *path = args->path;

args->bytes_found = 0;

args->extent_inserted = false;

/* Must always have a path if ->replace_extent is true */

ASSERT(!(args->replace_extent && !args->path));

if (!path) {

path = btrfs_alloc_path();

if (!path) {

ret = -ENOMEM;

goto out;

}

}

if (args->drop_cache)

btrfs_drop_extent_map_range(inode, args->start, args->end - 1, false);

if (data_race(args->start >= inode->disk_i_size) && !args->replace_extent)

modify_tree = 0;

update_refs = (btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID);

while (1) {

recow = 0;

ret = btrfs_lookup_file_extent(trans, root, path, ino,

search_start, modify_tree);

if (ret < 0)

break;

if (ret > 0 && path->slots[0] > 0 && search_start == args->start) {

leaf = path->nodes[0];

btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);

if (key.objectid == ino &&

key.type == BTRFS_EXTENT_DATA_KEY)

path->slots[0]--;

}

ret = 0;

next_slot:

leaf = path->nodes[0];

if (path->slots[0] >= btrfs_header_nritems(leaf)) {

if (WARN_ON(del_nr > 0)) {

btrfs_print_leaf(leaf);

ret = -EINVAL;

break;

}

ret = btrfs_next_leaf(root, path);

if (ret < 0)

break;

if (ret > 0) {

ret = 0;

break;

}

leaf = path->nodes[0];

recow = 1;

}

btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

if (key.objectid > ino)

break;

if (WARN_ON_ONCE(key.objectid < ino) ||

key.type < BTRFS_EXTENT_DATA_KEY) {

ASSERT(del_nr == 0);

path->slots[0]++;

goto next_slot;

}

if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end)

break;

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

extent_type = btrfs_file_extent_type(leaf, fi);

if (extent_type == BTRFS_FILE_EXTENT_REG ||

extent_type == BTRFS_FILE_EXTENT_PREALLOC) {

disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);

num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);

extent_offset = btrfs_file_extent_offset(leaf, fi);

extent_end = key.offset +

btrfs_file_extent_num_bytes(leaf, fi);

} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {

extent_end = key.offset +

btrfs_file_extent_ram_bytes(leaf, fi);

} else {

/* can't happen */

BUG();

}

/*

* Don't skip extent items representing 0 byte lengths. They

* used to be created (bug) if while punching holes we hit

* -ENOSPC condition. So if we find one here, just ensure we

* delete it, otherwise we would insert a new file extent item

* with the same key (offset) as that 0 bytes length file

* extent item in the call to setup_items_for_insert() later

* in this function.

*/

if (extent_end == key.offset && extent_end >= search_start) {

last_end = extent_end;

goto delete_extent_item;

}

if (extent_end <= search_start) {

path->slots[0]++;

goto next_slot;

}

found = 1;

search_start = max(key.offset, args->start);

if (recow || !modify_tree) {

modify_tree = -1;

btrfs_release_path(path);

continue;

}

/*

* | - range to drop - |

* | -------- extent -------- |

*/

if (args->start > key.offset && args->end < extent_end) {

if (WARN_ON(del_nr > 0)) {

btrfs_print_leaf(leaf);

ret = -EINVAL;

break;

}

if (extent_type == BTRFS_FILE_EXTENT_INLINE) {

ret = -EOPNOTSUPP;

break;

}

memcpy(&new_key, &key, sizeof(new_key));

new_key.offset = args->start;

ret = btrfs_duplicate_item(trans, root, path,

&new_key);

if (ret == -EAGAIN) {

btrfs_release_path(path);

continue;

}

if (ret < 0)

break;

leaf = path->nodes[0];

fi = btrfs_item_ptr(leaf, path->slots[0] - 1,

struct btrfs_file_extent_item);

btrfs_set_file_extent_num_bytes(leaf, fi,

args->start - key.offset);

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

extent_offset += args->start - key.offset;

btrfs_set_file_extent_offset(leaf, fi, extent_offset);

btrfs_set_file_extent_num_bytes(leaf, fi,

extent_end - args->start);

if (update_refs && disk_bytenr > 0) {

struct btrfs_ref ref = {

.action = BTRFS_ADD_DELAYED_REF,

.bytenr = disk_bytenr,

.num_bytes = num_bytes,

.parent = 0,

.owning_root = btrfs_root_id(root),

.ref_root = btrfs_root_id(root),

};

btrfs_init_data_ref(&ref, new_key.objectid,

args->start - extent_offset,

0, false);

ret = btrfs_inc_extent_ref(trans, &ref);

if (unlikely(ret)) {

btrfs_abort_transaction(trans, ret);

break;

}

}

key.offset = args->start;

}

/*

* From here on out we will have actually dropped something, so

* last_end can be updated.

*/

last_end = extent_end;

/*

* | ---- range to drop ----- |

* | -------- extent -------- |

*/

if (args->start <= key.offset && args->end < extent_end) {

if (extent_type == BTRFS_FILE_EXTENT_INLINE) {

ret = -EOPNOTSUPP;

break;

}

memcpy(&new_key, &key, sizeof(new_key));

new_key.offset = args->end;

btrfs_set_item_key_safe(trans, path, &new_key);

extent_offset += args->end - key.offset;

btrfs_set_file_extent_offset(leaf, fi, extent_offset);

btrfs_set_file_extent_num_bytes(leaf, fi,

extent_end - args->end);

if (update_refs && disk_bytenr > 0)

args->bytes_found += args->end - key.offset;

break;

}

search_start = extent_end;

/*

* | ---- range to drop ----- |

* | -------- extent -------- |

*/

if (args->start > key.offset && args->end >= extent_end) {

if (WARN_ON(del_nr > 0)) {

btrfs_print_leaf(leaf);

ret = -EINVAL;

break;

}

if (extent_type == BTRFS_FILE_EXTENT_INLINE) {

ret = -EOPNOTSUPP;

break;

}

btrfs_set_file_extent_num_bytes(leaf, fi,

args->start - key.offset);

if (update_refs && disk_bytenr > 0)

args->bytes_found += extent_end - args->start;

if (args->end == extent_end)

break;

path->slots[0]++;

goto next_slot;

}

/*

* | ---- range to drop ----- |

* | ------ extent ------ |

*/

if (args->start <= key.offset && args->end >= extent_end) {

delete_extent_item:

if (del_nr == 0) {

del_slot = path->slots[0];

del_nr = 1;

} else {

if (WARN_ON(del_slot + del_nr != path->slots[0])) {

btrfs_print_leaf(leaf);

ret = -EINVAL;

break;

}

del_nr++;

}

if (update_refs &&

extent_type == BTRFS_FILE_EXTENT_INLINE) {

args->bytes_found += extent_end - key.offset;

extent_end = ALIGN(extent_end,

fs_info->sectorsize);

} else if (update_refs && disk_bytenr > 0) {

struct btrfs_ref ref = {

.action = BTRFS_DROP_DELAYED_REF,

.bytenr = disk_bytenr,

.num_bytes = num_bytes,

.parent = 0,

.owning_root = btrfs_root_id(root),

.ref_root = btrfs_root_id(root),

};

btrfs_init_data_ref(&ref, key.objectid,

key.offset - extent_offset,

0, false);

ret = btrfs_free_extent(trans, &ref);

if (unlikely(ret)) {

btrfs_abort_transaction(trans, ret);

break;

}

args->bytes_found += extent_end - key.offset;

}

if (args->end == extent_end)

break;

if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {

path->slots[0]++;

goto next_slot;

}

ret = btrfs_del_items(trans, root, path, del_slot,

del_nr);

if (unlikely(ret)) {

btrfs_abort_transaction(trans, ret);

break;

}

del_nr = 0;

del_slot = 0;

btrfs_release_path(path);

continue;

}

BUG();

}

if (!ret && del_nr > 0) {

/*

* Set path->slots[0] to first slot, so that after the delete

* if items are move off from our leaf to its immediate left or

* right neighbor leafs, we end up with a correct and adjusted

* path->slots[0] for our insertion (if args->replace_extent).

*/

path->slots[0] = del_slot;

ret = btrfs_del_items(trans, root, path, del_slot, del_nr);

if (ret)

btrfs_abort_transaction(trans, ret);

}

leaf = path->nodes[0];

/*

* If btrfs_del_items() was called, it might have deleted a leaf, in

* which case it unlocked our path, so check path->locks[0] matches a

* write lock.

*/

if (!ret && args->replace_extent &&

path->locks[0] == BTRFS_WRITE_LOCK &&

btrfs_leaf_free_space(leaf) >=

sizeof(struct btrfs_item) + args->extent_item_size) {

key.objectid = ino;

key.type = BTRFS_EXTENT_DATA_KEY;

key.offset = args->start;

if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {

struct btrfs_key slot_key;

btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);

if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)

path->slots[0]++;

}

btrfs_setup_item_for_insert(trans, root, path, &key,

args->extent_item_size);

args->extent_inserted = true;

}

if (!args->path)

btrfs_free_path(path);

else if (!args->extent_inserted)

btrfs_release_path(path);

out:

args->drop_end = found ? min(args->end, last_end) : args->end;

return ret;

}

static bool extent_mergeable(struct extent_buffer *leaf, int slot, u64 objectid,

u64 bytenr, u64 orig_offset, u64 *start, u64 *end)

{

struct btrfs_file_extent_item *fi;

struct btrfs_key key;

u64 extent_end;

if (slot < 0 || slot >= btrfs_header_nritems(leaf))

return false;

btrfs_item_key_to_cpu(leaf, &key, slot);

if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)

return false;

fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);

if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||

btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||

btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||

btrfs_file_extent_compression(leaf, fi) ||

btrfs_file_extent_encryption(leaf, fi) ||

btrfs_file_extent_other_encoding(leaf, fi))

return false;

extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);

if ((*start && *start != key.offset) || (*end && *end != extent_end))

return false;

*start = key.offset;

*end = extent_end;

return true;

}

/*

* Mark extent in the range start - end as written.

*

* This changes extent type from 'pre-allocated' to 'regular'. If only

* part of extent is marked as written, the extent will be split into

* two or three.

*/

int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,

struct btrfs_inode *inode, u64 start, u64 end)

{

struct btrfs_root *root = inode->root;

struct extent_buffer *leaf;

BTRFS_PATH_AUTO_FREE(path);

struct btrfs_file_extent_item *fi;

struct btrfs_ref ref = { 0 };

struct btrfs_key key;

struct btrfs_key new_key;

u64 bytenr;

u64 num_bytes;

u64 extent_end;

u64 orig_offset;

u64 other_start;

u64 other_end;

u64 split;

int del_nr = 0;

int del_slot = 0;

int recow;

int ret;

u64 ino = btrfs_ino(inode);

path = btrfs_alloc_path();

if (!path)

return -ENOMEM;

again:

recow = 0;

split = start;

key.objectid = ino;

key.type = BTRFS_EXTENT_DATA_KEY;

key.offset = split;

ret = btrfs_search_slot(trans, root, &key, path, -1, 1);

if (ret < 0)

return ret;

if (ret > 0 && path->slots[0] > 0)

path->slots[0]--;

leaf = path->nodes[0];

btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);

if (unlikely(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)) {

ret = -EINVAL;

btrfs_abort_transaction(trans, ret);

return ret;

}

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

if (unlikely(btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC)) {

ret = -EINVAL;

btrfs_abort_transaction(trans, ret);

return ret;

}

extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);

if (unlikely(key.offset > start || extent_end < end)) {

ret = -EINVAL;

btrfs_abort_transaction(trans, ret);

return ret;

}

bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);

num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);

orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);

memcpy(&new_key, &key, sizeof(new_key));

if (start == key.offset && end < extent_end) {

other_start = 0;

other_end = start;

if (extent_mergeable(leaf, path->slots[0] - 1,

ino, bytenr, orig_offset,

&other_start, &other_end)) {

new_key.offset = end;

btrfs_set_item_key_safe(trans, path, &new_key);

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

btrfs_set_file_extent_generation(leaf, fi,

trans->transid);

btrfs_set_file_extent_num_bytes(leaf, fi,

extent_end - end);

btrfs_set_file_extent_offset(leaf, fi,

end - orig_offset);

fi = btrfs_item_ptr(leaf, path->slots[0] - 1,

struct btrfs_file_extent_item);

btrfs_set_file_extent_generation(leaf, fi,

trans->transid);

btrfs_set_file_extent_num_bytes(leaf, fi,

end - other_start);

return 0;

}

}

if (start > key.offset && end == extent_end) {

other_start = end;

other_end = 0;

if (extent_mergeable(leaf, path->slots[0] + 1,

ino, bytenr, orig_offset,

&other_start, &other_end)) {

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

btrfs_set_file_extent_num_bytes(leaf, fi,

start - key.offset);

btrfs_set_file_extent_generation(leaf, fi,

trans->transid);

path->slots[0]++;

new_key.offset = start;

btrfs_set_item_key_safe(trans, path, &new_key);

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

btrfs_set_file_extent_generation(leaf, fi,

trans->transid);

btrfs_set_file_extent_num_bytes(leaf, fi,

other_end - start);

btrfs_set_file_extent_offset(leaf, fi,

start - orig_offset);

return 0;

}

}

while (start > key.offset || end < extent_end) {

if (key.offset == start)

split = end;

new_key.offset = split;

ret = btrfs_duplicate_item(trans, root, path, &new_key);

if (ret == -EAGAIN) {

btrfs_release_path(path);

goto again;

}

if (unlikely(ret < 0)) {

btrfs_abort_transaction(trans, ret);

return ret;

}

leaf = path->nodes[0];

fi = btrfs_item_ptr(leaf, path->slots[0] - 1,

struct btrfs_file_extent_item);

btrfs_set_file_extent_generation(leaf, fi, trans->transid);

btrfs_set_file_extent_num_bytes(leaf, fi,

split - key.offset);

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

btrfs_set_file_extent_generation(leaf, fi, trans->transid);

btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);

btrfs_set_file_extent_num_bytes(leaf, fi,

extent_end - split);

ref.action = BTRFS_ADD_DELAYED_REF;

ref.bytenr = bytenr;

ref.num_bytes = num_bytes;

ref.parent = 0;

ref.owning_root = btrfs_root_id(root);

ref.ref_root = btrfs_root_id(root);

btrfs_init_data_ref(&ref, ino, orig_offset, 0, false);

ret = btrfs_inc_extent_ref(trans, &ref);

if (unlikely(ret)) {

btrfs_abort_transaction(trans, ret);

return ret;

}

if (split == start) {

key.offset = start;

} else {

if (unlikely(start != key.offset)) {

ret = -EINVAL;

btrfs_abort_transaction(trans, ret);

return ret;

}

path->slots[0]--;

extent_end = end;

}

recow = 1;

}

other_start = end;

other_end = 0;

ref.action = BTRFS_DROP_DELAYED_REF;

ref.bytenr = bytenr;

ref.num_bytes = num_bytes;

ref.parent = 0;

ref.owning_root = btrfs_root_id(root);

ref.ref_root = btrfs_root_id(root);

btrfs_init_data_ref(&ref, ino, orig_offset, 0, false);

if (extent_mergeable(leaf, path->slots[0] + 1,

ino, bytenr, orig_offset,

&other_start, &other_end)) {

if (recow) {

btrfs_release_path(path);

goto again;

}

extent_end = other_end;

del_slot = path->slots[0] + 1;

del_nr++;

ret = btrfs_free_extent(trans, &ref);

if (unlikely(ret)) {

btrfs_abort_transaction(trans, ret);

return ret;

}

}

other_start = 0;

other_end = start;

if (extent_mergeable(leaf, path->slots[0] - 1,

ino, bytenr, orig_offset,

&other_start, &other_end)) {

if (recow) {

btrfs_release_path(path);

goto again;

}

key.offset = other_start;

del_slot = path->slots[0];

del_nr++;

ret = btrfs_free_extent(trans, &ref);

if (unlikely(ret)) {

btrfs_abort_transaction(trans, ret);

return ret;

}

}

if (del_nr == 0) {

fi = btrfs_item_ptr(leaf, path->slots[0],

struct btrfs_file_extent_item);

btrfs_set_file_extent_type(leaf, fi,

BTRFS_FILE_EXTENT_REG);

btrfs_set_file_extent_generation(leaf, fi, trans->transid);

} else {

fi = btrfs_item_ptr(leaf, del_slot - 1,

struct btrfs_file_extent_item);

btrfs_set_file_extent_type(leaf, fi,

BTRFS_FILE_EXTENT_REG);

btrfs_set_file_extent_generation(leaf, fi, trans->transid);

btrfs_set_file_extent_num_bytes(leaf, fi,

extent_end - key.offset);

ret = btrfs_del_items(trans, root, path, del_slot, del_nr);

if (unlikely(ret < 0)) {

btrfs_abort_transaction(trans, ret);

return ret;

}

}

return 0;

}

/*

* On error return an unlocked folio and the error value

* On success return a locked folio and 0

*/

static int prepare_uptodate_folio(struct inode *inode, struct folio *folio, u64 pos,

u64 len)

{

u64 clamp_start = max_t(u64, pos, folio_pos(folio));

u64 clamp_end = min_t(u64, pos + len, folio_next_pos(folio));

const u32 blocksize = inode_to_fs_info(inode)->sectorsize;

int ret = 0;

if (folio_test_uptodate(folio))

return 0;

if (IS_ALIGNED(clamp_start, blocksize) &&

IS_ALIGNED(clamp_end, blocksize))

return 0;

ret = btrfs_read_folio(NULL, folio);

if (ret)

return ret;

folio_lock(folio);

if (unlikely(!folio_test_uptodate(folio))) {

folio_unlock(folio);

return -EIO;

}

/*

* Since btrfs_read_folio() will unlock the folio before it returns,

* there is a window where btrfs_release_folio() can be called to

* release the page. Here we check both inode mapping and page

* private to make sure the page was not released.

*

* The private flag check is essential for subpage as we need to store

* extra bitmap using folio private.

*/

if (folio->mapping != inode->i_mapping || !folio_test_private(folio)) {

folio_unlock(folio);

return -EAGAIN;

}

return 0;

}

static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait)

{

gfp_t gfp;

gfp = btrfs_alloc_write_mask(inode->i_mapping);

if (nowait) {

gfp &= ~__GFP_DIRECT_RECLAIM;

gfp |= GFP_NOWAIT;

}

return gfp;

}

/*

* Get folio into the page cache and lock it.

*/

static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_ret,

loff_t pos, size_t write_bytes,

bool nowait)

{

const pgoff_t index = pos >> PAGE_SHIFT;

gfp_t mask = get_prepare_gfp_flags(inode, nowait);

fgf_t fgp_flags = (nowait ? FGP_WRITEBEGIN | FGP_NOWAIT : FGP_WRITEBEGIN) |

fgf_set_order(write_bytes);

struct folio *folio;

int ret;

again:

folio = __filemap_get_folio(inode->i_mapping, index, fgp_flags, mask);

if (IS_ERR(folio))

return PTR_ERR(folio);

ret = set_folio_extent_mapped(folio);

if (ret < 0) {

folio_unlock(folio);

folio_put(folio);

return ret;

}

ret = prepare_uptodate_folio(inode, folio, pos, write_bytes);

if (ret) {

/* The folio is already unlocked. */

folio_put(folio);

if (!nowait && ret == -EAGAIN)

goto again;

return ret;

}

*folio_ret = folio;

return 0;

}

/*

* Locks the extent and properly waits for data=ordered extents to finish

* before allowing the folios to be modified if need.

*

* Return:

* 1 - the extent is locked

* 0 - the extent is not locked, and everything is OK

* -EAGAIN - need to prepare the folios again

*/

static noinline int

lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio,

loff_t pos, size_t write_bytes,

u64 *lockstart, u64 *lockend, bool nowait,

struct extent_state **cached_state)

{

struct btrfs_fs_info *fs_info = inode->root->fs_info;

u64 start_pos;

u64 last_pos;

int ret = 0;

start_pos = round_down(pos, fs_info->sectorsize);

last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;

if (start_pos < inode->vfs_inode.i_size) {

struct btrfs_ordered_extent *ordered;

if (nowait) {

if (!btrfs_try_lock_extent(&inode->io_tree, start_pos,

last_pos, cached_state)) {

folio_unlock(folio);

folio_put(folio);

return -EAGAIN;

}

} else {

btrfs_lock_extent(&inode->io_tree, start_pos, last_pos,

cached_state);

}

ordered = btrfs_lookup_ordered_range(inode, start_pos,

last_pos - start_pos + 1);

if (ordered &&

ordered->file_offset + ordered->num_bytes > start_pos &&

ordered->file_offset <= last_pos) {

btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos,

cached_state);

folio_unlock(folio);

folio_put(folio);

btrfs_start_ordered_extent(ordered);

btrfs_put_ordered_extent(ordered);

return -EAGAIN;

}

if (ordered)

btrfs_put_ordered_extent(ordered);

*lockstart = start_pos;

*lockend = last_pos;

ret = 1;

}

/*

* We should be called after prepare_one_folio() which should have locked

* all pages in the range.

*/

WARN_ON(!folio_test_locked(folio));

return ret;

}

/*

* Check if we can do nocow write into the range [@pos, @pos + @write_bytes)

*

* @pos: File offset.

* @write_bytes: The length to write, will be updated to the nocow writeable

* range.

* @nowait: Indicate if we can block or not (non-blocking IO context).

*

* This function will flush ordered extents in the range to ensure proper

* nocow checks.

*

* Return:

* > 0 If we can nocow, and updates @write_bytes.

* 0 If we can't do a nocow write.

* -EAGAIN If we can't do a nocow write because snapshotting of the inode's

* root is in progress or because we are in a non-blocking IO

* context and need to block (@nowait is true).

* < 0 If an error happened.

*

* NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0.

*/

int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,

size_t *write_bytes, bool nowait)

{

struct btrfs_fs_info *fs_info = inode->root->fs_info;

struct btrfs_root *root = inode->root;

struct extent_state *cached_state = NULL;

u64 lockstart, lockend;

u64 cur_offset;

int ret = 0;

if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))

return 0;

if (!btrfs_drew_try_write_lock(&root->snapshot_lock))

return -EAGAIN;

lockstart = round_down(pos, fs_info->sectorsize);

lockend = round_up(pos + *write_bytes,

fs_info->sectorsize) - 1;

if (nowait) {

if (!btrfs_try_lock_ordered_range(inode, lockstart, lockend,

&cached_state)) {

btrfs_drew_write_unlock(&root->snapshot_lock);