// SPDX-License-Identifier: GPL-2.0

/*

* Copyright (C) 2008 Oracle. All rights reserved.

*/

#include <linux/kernel.h>

#include <linux/bio.h>

#include <linux/file.h>

#include <linux/fs.h>

#include <linux/pagemap.h>

#include <linux/folio_batch.h>

#include <linux/highmem.h>

#include <linux/kthread.h>

#include <linux/time.h>

#include <linux/init.h>

#include <linux/string.h>

#include <linux/backing-dev.h>

#include <linux/writeback.h>

#include <linux/psi.h>

#include <linux/slab.h>

#include <linux/sched/mm.h>

#include <linux/log2.h>

#include <linux/shrinker.h>

#include "misc.h"

#include "ctree.h"

#include "fs.h"

#include "btrfs_inode.h"

#include "bio.h"

#include "ordered-data.h"

#include "compression.h"

#include "extent_io.h"

#include "extent_map.h"

#include "subpage.h"

#include "messages.h"

#include "super.h"

static struct bio_set btrfs_compressed_bioset;

static const char* const btrfs_compress_types[] = { "", "zlib", "lzo", "zstd" };

const char* btrfs_compress_type2str(enum btrfs_compression_type type)

{

switch (type) {

case BTRFS_COMPRESS_ZLIB:

case BTRFS_COMPRESS_LZO:

case BTRFS_COMPRESS_ZSTD:

case BTRFS_COMPRESS_NONE:

return btrfs_compress_types[type];

default:

break;

}

return NULL;

}

static inline struct compressed_bio *to_compressed_bio(struct btrfs_bio *bbio)

{

return container_of(bbio, struct compressed_bio, bbio);

}

static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode,

u64 start, blk_opf_t op,

btrfs_bio_end_io_t end_io)

{

struct btrfs_bio *bbio;

bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op,

GFP_NOFS, &btrfs_compressed_bioset));

btrfs_bio_init(bbio, inode, start, end_io, NULL);

return to_compressed_bio(bbio);

}

bool btrfs_compress_is_valid_type(const char *str, size_t len)

{

int i;

for (i = 1; i < ARRAY_SIZE(btrfs_compress_types); i++) {

size_t comp_len = strlen(btrfs_compress_types[i]);

if (len < comp_len)

continue;

if (!strncmp(btrfs_compress_types[i], str, comp_len))

return true;

}

return false;

}

static int compression_decompress_bio(struct list_head *ws,

struct compressed_bio *cb)

{

switch (cb->compress_type) {

case BTRFS_COMPRESS_ZLIB: return zlib_decompress_bio(ws, cb);

case BTRFS_COMPRESS_LZO: return lzo_decompress_bio(ws, cb);

case BTRFS_COMPRESS_ZSTD: return zstd_decompress_bio(ws, cb);

case BTRFS_COMPRESS_NONE:

default:

/*

* This can't happen, the type is validated several times

* before we get here.

*/

BUG();

}

}

static int compression_decompress(int type, struct list_head *ws,

const u8 *data_in, struct folio *dest_folio,

unsigned long dest_pgoff, size_t srclen, size_t destlen)

{

switch (type) {

case BTRFS_COMPRESS_ZLIB: return zlib_decompress(ws, data_in, dest_folio,

dest_pgoff, srclen, destlen);

case BTRFS_COMPRESS_LZO: return lzo_decompress(ws, data_in, dest_folio,

dest_pgoff, srclen, destlen);

case BTRFS_COMPRESS_ZSTD: return zstd_decompress(ws, data_in, dest_folio,

dest_pgoff, srclen, destlen);

case BTRFS_COMPRESS_NONE:

default:

/*

* This can't happen, the type is validated several times

* before we get here.

*/

BUG();

}

}

static int btrfs_decompress_bio(struct compressed_bio *cb);

/*

* Global cache of last unused pages for compression/decompression.

*/

static struct btrfs_compr_pool {

struct shrinker *shrinker;

spinlock_t lock;

struct list_head list;

int count;

int thresh;

} compr_pool;

static unsigned long btrfs_compr_pool_count(struct shrinker *sh, struct shrink_control *sc)

{

int ret;

/*

* We must not read the values more than once if 'ret' gets expanded in

* the return statement so we don't accidentally return a negative

* number, even if the first condition finds it positive.

*/

ret = READ_ONCE(compr_pool.count) - READ_ONCE(compr_pool.thresh);

return ret > 0 ? ret : 0;

}

static unsigned long btrfs_compr_pool_scan(struct shrinker *sh, struct shrink_control *sc)

{

LIST_HEAD(remove);

struct list_head *tmp, *next;

int freed;

if (compr_pool.count == 0)

return SHRINK_STOP;

/* For now, just simply drain the whole list. */

spin_lock(&compr_pool.lock);

list_splice_init(&compr_pool.list, &remove);

freed = compr_pool.count;

compr_pool.count = 0;

spin_unlock(&compr_pool.lock);

list_for_each_safe(tmp, next, &remove) {

struct page *page = list_entry(tmp, struct page, lru);

ASSERT(page_ref_count(page) == 1);

put_page(page);

}

return freed;

}

/*

* Common wrappers for page allocation from compression wrappers

*/

struct folio *btrfs_alloc_compr_folio(struct btrfs_fs_info *fs_info, gfp_t gfp)

{

struct folio *folio = NULL;

/* For bs > ps cases, no cached folio pool for now. */

if (fs_info->block_min_order)

goto alloc;

spin_lock(&compr_pool.lock);

if (compr_pool.count > 0) {

folio = list_first_entry(&compr_pool.list, struct folio, lru);

list_del_init(&folio->lru);

compr_pool.count--;

}

spin_unlock(&compr_pool.lock);

if (folio)

return folio;

alloc:

return folio_alloc(gfp, fs_info->block_min_order);

}

void btrfs_free_compr_folio(struct folio *folio)

{

bool do_free = false;

/* The folio is from bs > ps fs, no cached pool for now. */

if (folio_order(folio))

goto free;

spin_lock(&compr_pool.lock);

if (compr_pool.count > compr_pool.thresh) {

do_free = true;

} else {

list_add(&folio->lru, &compr_pool.list);

compr_pool.count++;

}

spin_unlock(&compr_pool.lock);

if (!do_free)

return;

free:

ASSERT(folio_ref_count(folio) == 1);

folio_put(folio);

}

static void end_bbio_compressed_read(struct btrfs_bio *bbio)

{

struct compressed_bio *cb = to_compressed_bio(bbio);

blk_status_t status = bbio->bio.bi_status;

struct folio_iter fi;

if (!status)

status = errno_to_blk_status(btrfs_decompress_bio(cb));

btrfs_bio_end_io(cb->orig_bbio, status);

bio_for_each_folio_all(fi, &bbio->bio)

btrfs_free_compr_folio(fi.folio);

bio_put(&bbio->bio);

}

/*

* Clear the writeback bits on all of the file

* pages for a compressed write

*/

static noinline void end_compressed_writeback(const struct compressed_bio *cb)

{

struct inode *inode = &cb->bbio.inode->vfs_inode;

struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);

pgoff_t index = cb->start >> PAGE_SHIFT;

const pgoff_t end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT;

struct folio_batch fbatch;

int i;

int ret;

ret = blk_status_to_errno(cb->bbio.bio.bi_status);

if (ret)

mapping_set_error(inode->i_mapping, ret);

folio_batch_init(&fbatch);

while (index <= end_index) {

ret = filemap_get_folios(inode->i_mapping, &index, end_index,

&fbatch);

if (ret == 0)

return;

for (i = 0; i < ret; i++) {

struct folio *folio = fbatch.folios[i];

btrfs_folio_clamp_clear_writeback(fs_info, folio,

cb->start, cb->len);

}

folio_batch_release(&fbatch);

}

/* the inode may be gone now */

}

/*

* Do the cleanup once all the compressed pages hit the disk. This will clear

* writeback on the file pages and free the compressed pages.

*

* This also calls the writeback end hooks for the file pages so that metadata

* and checksums can be updated in the file.

*/

static void end_bbio_compressed_write(struct btrfs_bio *bbio)

{

struct compressed_bio *cb = to_compressed_bio(bbio);

struct folio_iter fi;

btrfs_finish_ordered_extent(cb->bbio.ordered, cb->start, cb->len,

cb->bbio.bio.bi_status == BLK_STS_OK);

if (cb->writeback)

end_compressed_writeback(cb);

/* Note, our inode could be gone now. */

bio_for_each_folio_all(fi, &bbio->bio)

btrfs_free_compr_folio(fi.folio);

bio_put(&cb->bbio.bio);

}

/*

* worker function to build and submit bios for previously compressed pages.

* The corresponding pages in the inode should be marked for writeback

* and the compressed pages should have a reference on them for dropping

* when the IO is complete.

*

* This also checksums the file bytes and gets things ready for

* the end io hooks.

*/

void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered,

struct compressed_bio *cb)

{

struct btrfs_inode *inode = ordered->inode;

struct btrfs_fs_info *fs_info = inode->root->fs_info;

ASSERT(IS_ALIGNED(ordered->file_offset, fs_info->sectorsize));

ASSERT(IS_ALIGNED(ordered->num_bytes, fs_info->sectorsize));

/*

* This flag determines if we should clear the writeback flag from the

* page cache. But this function is only utilized by encoded writes, it

* never goes through the page cache.

*/

ASSERT(!cb->writeback);

cb->start = ordered->file_offset;

cb->len = ordered->num_bytes;

ASSERT(cb->bbio.bio.bi_iter.bi_size == ordered->disk_num_bytes);

cb->bbio.bio.bi_iter.bi_sector = ordered->disk_bytenr >> SECTOR_SHIFT;

cb->bbio.ordered = ordered;

btrfs_submit_bbio(&cb->bbio, 0);

}

/*

* Allocate a compressed write bio for @inode file offset @start length @len.

*

* The caller still needs to properly queue all folios and populate involved

* members.

*/

struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode,

u64 start, u64 len)

{

struct compressed_bio *cb;

cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE, end_bbio_compressed_write);

cb->start = start;

cb->len = len;

cb->writeback = false;

return cb;

}

/*

* Add extra pages in the same compressed file extent so that we don't need to

* re-read the same extent again and again.

*

* NOTE: this won't work well for subpage, as for subpage read, we lock the

* full page then submit bio for each compressed/regular extents.

*

* This means, if we have several sectors in the same page points to the same

* on-disk compressed data, we will re-read the same extent many times and

* this function can only help for the next page.

*/

static noinline int add_ra_bio_pages(struct inode *inode,

u64 compressed_end,

struct compressed_bio *cb,

int *memstall, unsigned long *pflags,

bool direct_reclaim)

{

struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);

pgoff_t end_index;

struct bio *orig_bio = &cb->orig_bbio->bio;

u64 cur = cb->orig_bbio->file_offset + orig_bio->bi_iter.bi_size;

u64 isize = i_size_read(inode);

int ret;

gfp_t constraint_gfp, cache_gfp;

struct folio *folio;

struct extent_map *em;

struct address_space *mapping = inode->i_mapping;

struct extent_map_tree *em_tree;

struct extent_io_tree *tree;

int sectors_missed = 0;

em_tree = &BTRFS_I(inode)->extent_tree;

tree = &BTRFS_I(inode)->io_tree;

if (isize == 0)

return 0;

/*

* For current subpage support, we only support 64K page size,

* which means maximum compressed extent size (128K) is just 2x page

* size.

* This makes readahead less effective, so here disable readahead for

* subpage for now, until full compressed write is supported.

*/

if (fs_info->sectorsize < PAGE_SIZE)

return 0;

/* For bs > ps cases, we don't support readahead for compressed folios for now. */

if (fs_info->block_min_order)

return 0;

end_index = (i_size_read(inode) - 1) >> PAGE_SHIFT;

/*

* Avoid direct reclaim when the caller does not allow it. Since

* add_ra_bio_pages() is always speculative, suppress allocation warnings

* in either case.

*/

if (!direct_reclaim) {

constraint_gfp = ~(__GFP_FS | __GFP_DIRECT_RECLAIM) | __GFP_NOWARN;

cache_gfp = (GFP_NOFS & ~__GFP_DIRECT_RECLAIM) | __GFP_NOWARN;

} else {

constraint_gfp = (~__GFP_FS) | __GFP_NOWARN;

cache_gfp = GFP_NOFS | __GFP_NOWARN;

}

while (cur < compressed_end) {

pgoff_t page_end;

pgoff_t pg_index = cur >> PAGE_SHIFT;

u32 add_size;

if (pg_index > end_index)

break;

folio = filemap_get_folio(mapping, pg_index);

if (!IS_ERR(folio)) {

u64 folio_sz = folio_size(folio);

u64 offset = offset_in_folio(folio, cur);

folio_put(folio);

sectors_missed += (folio_sz - offset) >>

fs_info->sectorsize_bits;

/* Beyond threshold, no need to continue */

if (sectors_missed > 4)

break;

/*

* Jump to next page start as we already have page for

* current offset.

*/

cur += (folio_sz - offset);

continue;

}

folio = filemap_alloc_folio(mapping_gfp_constraint(mapping, constraint_gfp),

0, NULL);

if (!folio)

break;

if (filemap_add_folio(mapping, folio, pg_index, cache_gfp)) {

/* There is already a page, skip to page end */

cur += folio_size(folio);

folio_put(folio);

continue;

}

if (!*memstall && folio_test_workingset(folio)) {

psi_memstall_enter(pflags);

*memstall = 1;

}

ret = set_folio_extent_mapped(folio);

if (ret < 0) {

folio_unlock(folio);

folio_put(folio);

break;

}

page_end = (pg_index << PAGE_SHIFT) + folio_size(folio) - 1;

btrfs_lock_extent(tree, cur, page_end, NULL);

read_lock(&em_tree->lock);

em = btrfs_lookup_extent_mapping(em_tree, cur, page_end + 1 - cur);

read_unlock(&em_tree->lock);

/*

* At this point, we have a locked page in the page cache for

* these bytes in the file. But, we have to make sure they map

* to this compressed extent on disk.

*/

if (!em || cur < em->start ||

(cur + fs_info->sectorsize > btrfs_extent_map_end(em)) ||

(btrfs_extent_map_block_start(em) >> SECTOR_SHIFT) !=

orig_bio->bi_iter.bi_sector) {

btrfs_free_extent_map(em);

btrfs_unlock_extent(tree, cur, page_end, NULL);

folio_unlock(folio);

folio_put(folio);

break;

}

add_size = min(btrfs_extent_map_end(em), page_end + 1) - cur;

btrfs_free_extent_map(em);

btrfs_unlock_extent(tree, cur, page_end, NULL);

if (folio_contains(folio, end_index)) {

size_t zero_offset = offset_in_folio(folio, isize);

if (zero_offset) {

int zeros;

zeros = folio_size(folio) - zero_offset;

folio_zero_range(folio, zero_offset, zeros);

}

}

if (!bio_add_folio(orig_bio, folio, add_size,

offset_in_folio(folio, cur))) {

folio_unlock(folio);

folio_put(folio);

break;

}

/*

* If it's subpage, we also need to increase its

* subpage::readers number, as at endio we will decrease

* subpage::readers and to unlock the page.

*/

if (fs_info->sectorsize < PAGE_SIZE)

btrfs_folio_set_lock(fs_info, folio, cur, add_size);

folio_put(folio);

cur += add_size;

}

return 0;

}

/*

* for a compressed read, the bio we get passed has all the inode pages

* in it. We don't actually do IO on those pages but allocate new ones

* to hold the compressed pages on disk.

*

* bio->bi_iter.bi_sector points to the compressed extent on disk

* bio->bi_io_vec points to all of the inode pages

*

* After the compressed pages are read, we copy the bytes into the

* bio we were passed and then call the bio end_io calls

*/

void btrfs_submit_compressed_read(struct btrfs_bio *bbio)

{

struct btrfs_inode *inode = bbio->inode;

struct btrfs_fs_info *fs_info = inode->root->fs_info;

struct extent_map_tree *em_tree = &inode->extent_tree;

struct compressed_bio *cb;

unsigned int compressed_len;

const u32 min_folio_size = btrfs_min_folio_size(fs_info);

u64 file_offset = bbio->file_offset;

gfp_t gfp;

u64 em_len;

u64 em_start;

struct extent_map *em;

unsigned long pflags;

int memstall = 0;

int ret;

/*

* If this is a readahead bio, prevent direct reclaim. This is done to

* avoid stalling on speculative allocations when memory pressure is

* high. The demand fault will retry with GFP_NOFS and enter direct

* reclaim if needed.

*/

if (bbio->bio.bi_opf & REQ_RAHEAD)

gfp = (GFP_NOFS & ~__GFP_DIRECT_RECLAIM) | __GFP_NOWARN;

else

gfp = GFP_NOFS;

/* we need the actual starting offset of this extent in the file */

read_lock(&em_tree->lock);

em = btrfs_lookup_extent_mapping(em_tree, file_offset, fs_info->sectorsize);

read_unlock(&em_tree->lock);

if (!em) {

ret = -EIO;

goto out;

}

ASSERT(btrfs_extent_map_is_compressed(em));

compressed_len = em->disk_num_bytes;

cb = alloc_compressed_bio(inode, file_offset, REQ_OP_READ,

end_bbio_compressed_read);

cb->start = em->start - em->offset;

em_len = em->len;

em_start = em->start;

cb->len = bbio->bio.bi_iter.bi_size;

cb->compress_type = btrfs_extent_map_compression(em);

cb->orig_bbio = bbio;

cb->bbio.csum_search_commit_root = bbio->csum_search_commit_root;

btrfs_free_extent_map(em);

for (int i = 0; i * min_folio_size < compressed_len; i++) {

struct folio *folio;

u32 cur_len = min(compressed_len - i * min_folio_size, min_folio_size);

folio = btrfs_alloc_compr_folio(fs_info, gfp);

if (!folio) {

ret = -ENOMEM;

goto out_free_bio;

}

ret = bio_add_folio(&cb->bbio.bio, folio, cur_len, 0);

if (unlikely(!ret)) {

folio_put(folio);

ret = -EINVAL;

goto out_free_bio;

}

}

ASSERT(cb->bbio.bio.bi_iter.bi_size == compressed_len);

add_ra_bio_pages(&inode->vfs_inode, em_start + em_len, cb, &memstall,

&pflags, !(bbio->bio.bi_opf & REQ_RAHEAD));

cb->len = bbio->bio.bi_iter.bi_size;

cb->bbio.bio.bi_iter.bi_sector = bbio->bio.bi_iter.bi_sector;

if (memstall)

psi_memstall_leave(&pflags);

btrfs_submit_bbio(&cb->bbio, 0);

return;

out_free_bio:

cleanup_compressed_bio(cb);

out:

btrfs_bio_end_io(bbio, errno_to_blk_status(ret));

}

/*

* Heuristic uses systematic sampling to collect data from the input data

* range, the logic can be tuned by the following constants:

*

* @SAMPLING_READ_SIZE - how many bytes will be copied from for each sample

* @SAMPLING_INTERVAL - range from which the sampled data can be collected

*/

#define SAMPLING_READ_SIZE (16)

#define SAMPLING_INTERVAL (256)

/*

* For statistical analysis of the input data we consider bytes that form a

* Galois Field of 256 objects. Each object has an attribute count, ie. how

* many times the object appeared in the sample.

*/

#define BUCKET_SIZE (256)

/*

* The size of the sample is based on a statistical sampling rule of thumb.

* The common way is to perform sampling tests as long as the number of

* elements in each cell is at least 5.

*

* Instead of 5, we choose 32 to obtain more accurate results.

* If the data contain the maximum number of symbols, which is 256, we obtain a

* sample size bound by 8192.

*

* For a sample of at most 8KB of data per data range: 16 consecutive bytes

* from up to 512 locations.

*/

#define MAX_SAMPLE_SIZE (BTRFS_MAX_UNCOMPRESSED * \

SAMPLING_READ_SIZE / SAMPLING_INTERVAL)

struct bucket_item {

u32 count;

};

struct heuristic_ws {

/* Partial copy of input data */

u8 *sample;

u32 sample_size;

/* Buckets store counters for each byte value */

struct bucket_item *bucket;

/* Sorting buffer */

struct bucket_item *bucket_b;

struct list_head list;

};

static void free_heuristic_ws(struct list_head *ws)

{

struct heuristic_ws *workspace;

workspace = list_entry(ws, struct heuristic_ws, list);

kvfree(workspace->sample);

kfree(workspace->bucket);

kfree(workspace->bucket_b);

kfree(workspace);

}

static struct list_head *alloc_heuristic_ws(struct btrfs_fs_info *fs_info)

{

struct heuristic_ws *ws;

ws = kzalloc_obj(*ws);

if (!ws)

return ERR_PTR(-ENOMEM);

ws->sample = kvmalloc(MAX_SAMPLE_SIZE, GFP_KERNEL);

if (!ws->sample)

goto fail;

ws->bucket = kzalloc_objs(*ws->bucket, BUCKET_SIZE);

if (!ws->bucket)

goto fail;

ws->bucket_b = kzalloc_objs(*ws->bucket_b, BUCKET_SIZE);

if (!ws->bucket_b)

goto fail;

INIT_LIST_HEAD(&ws->list);

return &ws->list;

fail:

free_heuristic_ws(&ws->list);

return ERR_PTR(-ENOMEM);

}

const struct btrfs_compress_levels btrfs_heuristic_compress = { 0 };

static const struct btrfs_compress_levels * const btrfs_compress_levels[] = {

/* The heuristic is represented as compression type 0 */

&btrfs_heuristic_compress,

&btrfs_zlib_compress,

&btrfs_lzo_compress,

&btrfs_zstd_compress,

};

static struct list_head *alloc_workspace(struct btrfs_fs_info *fs_info, int type, int level)

{

switch (type) {

case BTRFS_COMPRESS_NONE: return alloc_heuristic_ws(fs_info);

case BTRFS_COMPRESS_ZLIB: return zlib_alloc_workspace(fs_info, level);

case BTRFS_COMPRESS_LZO: return lzo_alloc_workspace(fs_info);

case BTRFS_COMPRESS_ZSTD: return zstd_alloc_workspace(fs_info, level);

default:

/*

* This can't happen, the type is validated several times

* before we get here.

*/

BUG();

}

}

static void free_workspace(int type, struct list_head *ws)

{

switch (type) {

case BTRFS_COMPRESS_NONE: return free_heuristic_ws(ws);

case BTRFS_COMPRESS_ZLIB: return zlib_free_workspace(ws);

case BTRFS_COMPRESS_LZO: return lzo_free_workspace(ws);

case BTRFS_COMPRESS_ZSTD: return zstd_free_workspace(ws);

default:

/*

* This can't happen, the type is validated several times

* before we get here.

*/

BUG();

}

}

static int alloc_workspace_manager(struct btrfs_fs_info *fs_info,

enum btrfs_compression_type type)

{

struct workspace_manager *gwsm;

struct list_head *workspace;

ASSERT(fs_info->compr_wsm[type] == NULL);

gwsm = kzalloc_obj(*gwsm);

if (!gwsm)

return -ENOMEM;

INIT_LIST_HEAD(&gwsm->idle_ws);

spin_lock_init(&gwsm->ws_lock);

atomic_set(&gwsm->total_ws, 0);

init_waitqueue_head(&gwsm->ws_wait);

fs_info->compr_wsm[type] = gwsm;

/*

* Preallocate one workspace for each compression type so we can

* guarantee forward progress in the worst case

*/

workspace = alloc_workspace(fs_info, type, 0);

if (IS_ERR(workspace)) {

btrfs_warn(fs_info,

"cannot preallocate compression workspace for %s, will try later",

btrfs_compress_type2str(type));

} else {

atomic_set(&gwsm->total_ws, 1);

gwsm->free_ws = 1;

list_add(workspace, &gwsm->idle_ws);

}

return 0;

}

static void free_workspace_manager(struct btrfs_fs_info *fs_info,

enum btrfs_compression_type type)

{

struct list_head *ws;

struct workspace_manager *gwsm = fs_info->compr_wsm[type];

/* ZSTD uses its own workspace manager, should enter here. */

ASSERT(type != BTRFS_COMPRESS_ZSTD && type < BTRFS_NR_COMPRESS_TYPES);

if (!gwsm)

return;

fs_info->compr_wsm[type] = NULL;

while (!list_empty(&gwsm->idle_ws)) {

ws = gwsm->idle_ws.next;

list_del(ws);

free_workspace(type, ws);

atomic_dec(&gwsm->total_ws);

}

kfree(gwsm);

}

/*

* This finds an available workspace or allocates a new one.

* If it's not possible to allocate a new one, waits until there's one.

* Preallocation makes a forward progress guarantees and we do not return

* errors.

*/

struct list_head *btrfs_get_workspace(struct btrfs_fs_info *fs_info, int type, int level)

{

struct workspace_manager *wsm = fs_info->compr_wsm[type];

struct list_head *workspace;

int cpus = num_online_cpus();

unsigned nofs_flag;

struct list_head *idle_ws;

spinlock_t *ws_lock;

atomic_t *total_ws;

wait_queue_head_t *ws_wait;

int *free_ws;

ASSERT(wsm);

idle_ws = &wsm->idle_ws;

ws_lock = &wsm->ws_lock;

total_ws = &wsm->total_ws;

ws_wait = &wsm->ws_wait;

free_ws = &wsm->free_ws;

again:

spin_lock(ws_lock);

if (!list_empty(idle_ws)) {

workspace = idle_ws->next;

list_del(workspace);

(*free_ws)--;

spin_unlock(ws_lock);

return workspace;

}

if (atomic_read(total_ws) > cpus) {

DEFINE_WAIT(wait);

spin_unlock(ws_lock);

prepare_to_wait(ws_wait, &wait, TASK_UNINTERRUPTIBLE);

if (atomic_read(total_ws) > cpus && !*free_ws)

schedule();

finish_wait(ws_wait, &wait);

goto again;

}

atomic_inc(total_ws);

spin_unlock(ws_lock);

/*

* Allocation helpers call vmalloc that can't use GFP_NOFS, so we have

* to turn it off here because we might get called from the restricted

* context of btrfs_compress_bio/btrfs_compress_pages

*/

nofs_flag = memalloc_nofs_save();

workspace = alloc_workspace(fs_info, type, level);

memalloc_nofs_restore(nofs_flag);

if (IS_ERR(workspace)) {

atomic_dec(total_ws);

wake_up(ws_wait);

/*

* Do not return the error but go back to waiting. There's a

* workspace preallocated for each type and the compression

* time is bounded so we get to a workspace eventually. This

* makes our caller's life easier.

*

* To prevent silent and low-probability deadlocks (when the

* initial preallocation fails), check if there are any

* workspaces at all.

*/

if (atomic_read(total_ws) == 0) {

static DEFINE_RATELIMIT_STATE(_rs,

/* once per minute */ 60 * HZ,

/* no burst */ 1);

if (__ratelimit(&_rs))

btrfs_warn(fs_info,

"no compression workspaces, low memory, retrying");

}

goto again;

}

return workspace;

}

static struct list_head *get_workspace(struct btrfs_fs_info *fs_info, int type, int level)

{

switch (type) {

case BTRFS_COMPRESS_NONE: return btrfs_get_workspace(fs_info, type, level);

case BTRFS_COMPRESS_ZLIB: return zlib_get_workspace(fs_info, level);

case BTRFS_COMPRESS_LZO: return btrfs_get_workspace(fs_info, type, level);

case BTRFS_COMPRESS_ZSTD: return zstd_get_workspace(fs_info, level);

default:

/*

* This can't happen, the type is validated several times

* before we get here.

*/

BUG();

}

}

/*

* put a workspace struct back on the list or free it if we have enough

* idle ones sitting around

*/

void btrfs_put_workspace(struct btrfs_fs_info *fs_info, int type, struct list_head *ws)

{

struct workspace_manager *gwsm = fs_info->compr_wsm[type];

struct list_head *idle_ws;

spinlock_t *ws_lock;

atomic_t *total_ws;

wait_queue_head_t *ws_wait;

int *free_ws;

ASSERT(gwsm);

idle_ws = &gwsm->idle_ws;

ws_lock = &gwsm->ws_lock;

total_ws = &gwsm->total_ws;

ws_wait = &gwsm->ws_wait;

free_ws = &gwsm->free_ws;

spin_lock(ws_lock);

if (*free_ws <= num_online_cpus()) {

list_add(ws, idle_ws);

(*free_ws)++;

spin_unlock(ws_lock);

goto wake;

}

spin_unlock(ws_lock);

free_workspace(type, ws);

atomic_dec(total_ws);

wake:

cond_wake_up(ws_wait);

}

static void put_workspace(struct btrfs_fs_info *fs_info, int type, struct list_head *ws)

{

switch (type) {

case BTRFS_COMPRESS_NONE: return btrfs_put_workspace(fs_info, type, ws);

case BTRFS_COMPRESS_ZLIB: return btrfs_put_workspace(fs_info, type, ws);

case BTRFS_COMPRESS_LZO: return btrfs_put_workspace(fs_info, type, ws);

case BTRFS_COMPRESS_ZSTD: return zstd_put_workspace(fs_info, ws);

default:

/*

* This can't happen, the type is validated several times

* before we get here.

*/

BUG();

}

}

/*

* Adjust @level according to the limits of the compression algorithm or

* fallback to default

*/

static int btrfs_compress_set_level(unsigned int type, int level)

{

const struct btrfs_compress_levels *levels = btrfs_compress_levels[type];

if (level == 0)

level = levels->default_level;

else

level = clamp(level, levels->min_level, levels->max_level);

return level;

}

/*

* Check whether the @level is within the valid range for the given type.

*/

bool btrfs_compress_level_valid(unsigned int type, int level)

{

const struct btrfs_compress_levels *levels = btrfs_compress_levels[type];

return levels->min_level <= level && level <= levels->max_level;

}

/* Wrapper around find_get_page(), with extra error message. */

int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start,

struct folio **in_folio_ret)

{

struct folio *in_folio;

/*

* The compressed write path should have the folio locked already, thus

* we only need to grab one reference.