// SPDX-License-Identifier: GPL-2.0

/*

* fs/ext4/fast_commit.c

*

* Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>

*

* Ext4 fast commits routines.

*/

#include "ext4.h"

#include "ext4_jbd2.h"

#include "ext4_extents.h"

#include "mballoc.h"

/*

* Ext4 Fast Commits

* -----------------

*

* Ext4 fast commits implement fine grained journalling for Ext4.

*

* Fast commits are organized as a log of tag-length-value (TLV) structs. (See

* struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by

* TLV during the recovery phase. For the scenarios for which we currently

* don't have replay code, fast commit falls back to full commits.

* Fast commits record delta in one of the following three categories.

*

* (A) Directory entry updates:

*

* - EXT4_FC_TAG_UNLINK - records directory entry unlink

* - EXT4_FC_TAG_LINK - records directory entry link

* - EXT4_FC_TAG_CREAT - records inode and directory entry creation

*

* (B) File specific data range updates:

*

* - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode

* - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode

*

* (C) Inode metadata (mtime / ctime etc):

*

* - EXT4_FC_TAG_INODE - record the inode that should be replayed

* during recovery. Note that iblocks field is

* not replayed and instead derived during

* replay.

* Commit Operation

* ----------------

* With fast commits, we maintain all the directory entry operations in the

* order in which they are issued in an in-memory queue. This queue is flushed

* to disk during the commit operation. We also maintain a list of inodes

* that need to be committed during a fast commit in another in memory queue of

* inodes. During the commit operation, we commit in the following order:

*

* [1] Lock inodes for any further data updates by setting COMMITTING state

* [2] Submit data buffers of all the inodes

* [3] Wait for [2] to complete

* [4] Commit all the directory entry updates in the fast commit space

* [5] Commit all the changed inode structures

* [6] Write tail tag (this tag ensures the atomicity, please read the following

* section for more details).

* [7] Wait for [4], [5] and [6] to complete.

*

* All the inode updates must call ext4_fc_start_update() before starting an

* update. If such an ongoing update is present, fast commit waits for it to

* complete. The completion of such an update is marked by

* ext4_fc_stop_update().

*

* Fast Commit Ineligibility

* -------------------------

*

* Not all operations are supported by fast commits today (e.g extended

* attributes). Fast commit ineligibility is marked by calling

* ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back

* to full commit.

*

* Atomicity of commits

* --------------------

* In order to guarantee atomicity during the commit operation, fast commit

* uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail

* tag contains CRC of the contents and TID of the transaction after which

* this fast commit should be applied. Recovery code replays fast commit

* logs only if there's at least 1 valid tail present. For every fast commit

* operation, there is 1 tail. This means, we may end up with multiple tails

* in the fast commit space. Here's an example:

*

* - Create a new file A and remove existing file B

* - fsync()

* - Append contents to file A

* - Truncate file A

* - fsync()

*

* The fast commit space at the end of above operations would look like this:

* [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]

* |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|

*

* Replay code should thus check for all the valid tails in the FC area.

*

* Fast Commit Replay Idempotence

* ------------------------------

*

* Fast commits tags are idempotent in nature provided the recovery code follows

* certain rules. The guiding principle that the commit path follows while

* committing is that it stores the result of a particular operation instead of

* storing the procedure.

*

* Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'

* was associated with inode 10. During fast commit, instead of storing this

* operation as a procedure "rename a to b", we store the resulting file system

* state as a "series" of outcomes:

*

* - Link dirent b to inode 10

* - Unlink dirent a

* - Inode <10> with valid refcount

*

* Now when recovery code runs, it needs "enforce" this state on the file

* system. This is what guarantees idempotence of fast commit replay.

*

* Let's take an example of a procedure that is not idempotent and see how fast

* commits make it idempotent. Consider following sequence of operations:

*

* rm A; mv B A; read A

* (x) (y) (z)

*

* (x), (y) and (z) are the points at which we can crash. If we store this

* sequence of operations as is then the replay is not idempotent. Let's say

* while in replay, we crash at (z). During the second replay, file A (which was

* actually created as a result of "mv B A" operation) would get deleted. Thus,

* file named A would be absent when we try to read A. So, this sequence of

* operations is not idempotent. However, as mentioned above, instead of storing

* the procedure fast commits store the outcome of each procedure. Thus the fast

* commit log for above procedure would be as follows:

*

* (Let's assume dirent A was linked to inode 10 and dirent B was linked to

* inode 11 before the replay)

*

* [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11]

* (w) (x) (y) (z)

*

* If we crash at (z), we will have file A linked to inode 11. During the second

* replay, we will remove file A (inode 11). But we will create it back and make

* it point to inode 11. We won't find B, so we'll just skip that step. At this

* point, the refcount for inode 11 is not reliable, but that gets fixed by the

* replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled

* similarly. Thus, by converting a non-idempotent procedure into a series of

* idempotent outcomes, fast commits ensured idempotence during the replay.

*

* TODOs

* -----

*

* 0) Fast commit replay path hardening: Fast commit replay code should use

* journal handles to make sure all the updates it does during the replay

* path are atomic. With that if we crash during fast commit replay, after

* trying to do recovery again, we will find a file system where fast commit

* area is invalid (because new full commit would be found). In order to deal

* with that, fast commit replay code should ensure that the "FC_REPLAY"

* superblock state is persisted before starting the replay, so that after

* the crash, fast commit recovery code can look at that flag and perform

* fast commit recovery even if that area is invalidated by later full

* commits.

*

* 1) Fast commit's commit path locks the entire file system during fast

* commit. This has significant performance penalty. Instead of that, we

* should use ext4_fc_start/stop_update functions to start inode level

* updates from ext4_journal_start/stop. Once we do that we can drop file

* system locking during commit path.

*

* 2) Handle more ineligible cases.

*/

#include <trace/events/ext4.h>

static struct kmem_cache *ext4_fc_dentry_cachep;

static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)

{

BUFFER_TRACE(bh, "");

if (uptodate) {

ext4_debug("%s: Block %lld up-to-date",

__func__, bh->b_blocknr);

set_buffer_uptodate(bh);

} else {

ext4_debug("%s: Block %lld not up-to-date",

__func__, bh->b_blocknr);

clear_buffer_uptodate(bh);

}

unlock_buffer(bh);

}

static inline void ext4_fc_reset_inode(struct inode *inode)

{

struct ext4_inode_info *ei = EXT4_I(inode);

ei->i_fc_lblk_start = 0;

ei->i_fc_lblk_len = 0;

}

void ext4_fc_init_inode(struct inode *inode)

{

struct ext4_inode_info *ei = EXT4_I(inode);

ext4_fc_reset_inode(inode);

ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);

INIT_LIST_HEAD(&ei->i_fc_list);

INIT_LIST_HEAD(&ei->i_fc_dilist);

init_waitqueue_head(&ei->i_fc_wait);

atomic_set(&ei->i_fc_updates, 0);

}

/* This function must be called with sbi->s_fc_lock held. */

static void ext4_fc_wait_committing_inode(struct inode *inode)

__releases(&EXT4_SB(inode->i_sb)->s_fc_lock)

{

wait_queue_head_t *wq;

struct ext4_inode_info *ei = EXT4_I(inode);

#if (BITS_PER_LONG < 64)

DEFINE_WAIT_BIT(wait, &ei->i_state_flags,

EXT4_STATE_FC_COMMITTING);

wq = bit_waitqueue(&ei->i_state_flags,

EXT4_STATE_FC_COMMITTING);

#else

DEFINE_WAIT_BIT(wait, &ei->i_flags,

EXT4_STATE_FC_COMMITTING);

wq = bit_waitqueue(&ei->i_flags,

EXT4_STATE_FC_COMMITTING);

#endif

lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);

prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);

spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);

schedule();

finish_wait(wq, &wait.wq_entry);

}

static bool ext4_fc_disabled(struct super_block *sb)

{

return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||

(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));

}

/*

* Inform Ext4's fast about start of an inode update

*

* This function is called by the high level call VFS callbacks before

* performing any inode update. This function blocks if there's an ongoing

* fast commit on the inode in question.

*/

void ext4_fc_start_update(struct inode *inode)

{

struct ext4_inode_info *ei = EXT4_I(inode);

if (ext4_fc_disabled(inode->i_sb))

return;

restart:

spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);

if (list_empty(&ei->i_fc_list))

goto out;

if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {

ext4_fc_wait_committing_inode(inode);

goto restart;

}

out:

atomic_inc(&ei->i_fc_updates);

spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);

}

/*

* Stop inode update and wake up waiting fast commits if any.

*/

void ext4_fc_stop_update(struct inode *inode)

{

struct ext4_inode_info *ei = EXT4_I(inode);

if (ext4_fc_disabled(inode->i_sb))

return;

if (atomic_dec_and_test(&ei->i_fc_updates))

wake_up_all(&ei->i_fc_wait);

}

/*

* Remove inode from fast commit list. If the inode is being committed

* we wait until inode commit is done.

*/

void ext4_fc_del(struct inode *inode)

{

struct ext4_inode_info *ei = EXT4_I(inode);

struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);

struct ext4_fc_dentry_update *fc_dentry;

if (ext4_fc_disabled(inode->i_sb))

return;

restart:

spin_lock(&sbi->s_fc_lock);

if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) {

spin_unlock(&sbi->s_fc_lock);

return;

}

if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {

ext4_fc_wait_committing_inode(inode);

goto restart;

}

if (!list_empty(&ei->i_fc_list))

list_del_init(&ei->i_fc_list);

/*

* Since this inode is getting removed, let's also remove all FC

* dentry create references, since it is not needed to log it anyways.

*/

if (list_empty(&ei->i_fc_dilist)) {

spin_unlock(&sbi->s_fc_lock);

return;

}

fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist);

WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);

list_del_init(&fc_dentry->fcd_list);

list_del_init(&fc_dentry->fcd_dilist);

WARN_ON(!list_empty(&ei->i_fc_dilist));

spin_unlock(&sbi->s_fc_lock);

release_dentry_name_snapshot(&fc_dentry->fcd_name);

kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);

return;

}

/*

* Mark file system as fast commit ineligible, and record latest

* ineligible transaction tid. This means until the recorded

* transaction, commit operation would result in a full jbd2 commit.

*/

void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)

{

struct ext4_sb_info *sbi = EXT4_SB(sb);

tid_t tid;

bool has_transaction = true;

bool is_ineligible;

if (ext4_fc_disabled(sb))

return;

if (handle && !IS_ERR(handle))

tid = handle->h_transaction->t_tid;

else {

read_lock(&sbi->s_journal->j_state_lock);

if (sbi->s_journal->j_running_transaction)

tid = sbi->s_journal->j_running_transaction->t_tid;

else

has_transaction = false;

read_unlock(&sbi->s_journal->j_state_lock);

}

spin_lock(&sbi->s_fc_lock);

is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);

if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid)))

sbi->s_fc_ineligible_tid = tid;

ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);

spin_unlock(&sbi->s_fc_lock);

WARN_ON(reason >= EXT4_FC_REASON_MAX);

sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;

}

/*

* Generic fast commit tracking function. If this is the first time this we are

* called after a full commit, we initialize fast commit fields and then call

* __fc_track_fn() with update = 0. If we have already been called after a full

* commit, we pass update = 1. Based on that, the track function can determine

* if it needs to track a field for the first time or if it needs to just

* update the previously tracked value.

*

* If enqueue is set, this function enqueues the inode in fast commit list.

*/

static int ext4_fc_track_template(

handle_t *handle, struct inode *inode,

int (*__fc_track_fn)(handle_t *handle, struct inode *, void *, bool),

void *args, int enqueue)

{

bool update = false;

struct ext4_inode_info *ei = EXT4_I(inode);

struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);

tid_t tid = 0;

int ret;

tid = handle->h_transaction->t_tid;

mutex_lock(&ei->i_fc_lock);

if (tid == ei->i_sync_tid) {

update = true;

} else {

ext4_fc_reset_inode(inode);

ei->i_sync_tid = tid;

}

ret = __fc_track_fn(handle, inode, args, update);

mutex_unlock(&ei->i_fc_lock);

if (!enqueue)

return ret;

spin_lock(&sbi->s_fc_lock);

if (list_empty(&EXT4_I(inode)->i_fc_list))

list_add_tail(&EXT4_I(inode)->i_fc_list,

(sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||

sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?

&sbi->s_fc_q[FC_Q_STAGING] :

&sbi->s_fc_q[FC_Q_MAIN]);

spin_unlock(&sbi->s_fc_lock);

return ret;

}

struct __track_dentry_update_args {

struct dentry *dentry;

int op;

};

/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */

static int __track_dentry_update(handle_t *handle, struct inode *inode,

void *arg, bool update)

{

struct ext4_fc_dentry_update *node;

struct ext4_inode_info *ei = EXT4_I(inode);

struct __track_dentry_update_args *dentry_update =

(struct __track_dentry_update_args *)arg;

struct dentry *dentry = dentry_update->dentry;

struct inode *dir = dentry->d_parent->d_inode;

struct super_block *sb = inode->i_sb;

struct ext4_sb_info *sbi = EXT4_SB(sb);

mutex_unlock(&ei->i_fc_lock);

if (IS_ENCRYPTED(dir)) {

ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,

handle);

mutex_lock(&ei->i_fc_lock);

return -EOPNOTSUPP;

}

node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);

if (!node) {

ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, handle);

mutex_lock(&ei->i_fc_lock);

return -ENOMEM;

}

node->fcd_op = dentry_update->op;

node->fcd_parent = dir->i_ino;

node->fcd_ino = inode->i_ino;

take_dentry_name_snapshot(&node->fcd_name, dentry);

INIT_LIST_HEAD(&node->fcd_dilist);

spin_lock(&sbi->s_fc_lock);

if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||

sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)

list_add_tail(&node->fcd_list,

&sbi->s_fc_dentry_q[FC_Q_STAGING]);

else

list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);

/*

* This helps us keep a track of all fc_dentry updates which is part of

* this ext4 inode. So in case the inode is getting unlinked, before

* even we get a chance to fsync, we could remove all fc_dentry

* references while evicting the inode in ext4_fc_del().

* Also with this, we don't need to loop over all the inodes in

* sbi->s_fc_q to get the corresponding inode in

* ext4_fc_commit_dentry_updates().

*/

if (dentry_update->op == EXT4_FC_TAG_CREAT) {

WARN_ON(!list_empty(&ei->i_fc_dilist));

list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist);

}

spin_unlock(&sbi->s_fc_lock);

mutex_lock(&ei->i_fc_lock);

return 0;

}

void __ext4_fc_track_unlink(handle_t *handle,

struct inode *inode, struct dentry *dentry)

{

struct __track_dentry_update_args args;

int ret;

args.dentry = dentry;

args.op = EXT4_FC_TAG_UNLINK;

ret = ext4_fc_track_template(handle, inode, __track_dentry_update,

(void *)&args, 0);

trace_ext4_fc_track_unlink(handle, inode, dentry, ret);

}

void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)

{

struct inode *inode = d_inode(dentry);

if (ext4_fc_disabled(inode->i_sb))

return;

if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))

return;

__ext4_fc_track_unlink(handle, inode, dentry);

}

void __ext4_fc_track_link(handle_t *handle,

struct inode *inode, struct dentry *dentry)

{

struct __track_dentry_update_args args;

int ret;

args.dentry = dentry;

args.op = EXT4_FC_TAG_LINK;

ret = ext4_fc_track_template(handle, inode, __track_dentry_update,

(void *)&args, 0);

trace_ext4_fc_track_link(handle, inode, dentry, ret);

}

void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)

{

struct inode *inode = d_inode(dentry);

if (ext4_fc_disabled(inode->i_sb))

return;

if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))

return;

__ext4_fc_track_link(handle, inode, dentry);

}

void __ext4_fc_track_create(handle_t *handle, struct inode *inode,

struct dentry *dentry)

{

struct __track_dentry_update_args args;

int ret;

args.dentry = dentry;

args.op = EXT4_FC_TAG_CREAT;

ret = ext4_fc_track_template(handle, inode, __track_dentry_update,

(void *)&args, 0);

trace_ext4_fc_track_create(handle, inode, dentry, ret);

}

void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)

{

struct inode *inode = d_inode(dentry);

if (ext4_fc_disabled(inode->i_sb))

return;

if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))

return;

__ext4_fc_track_create(handle, inode, dentry);

}

/* __track_fn for inode tracking */

static int __track_inode(handle_t *handle, struct inode *inode, void *arg,

bool update)

{

if (update)

return -EEXIST;

EXT4_I(inode)->i_fc_lblk_len = 0;

return 0;

}

void ext4_fc_track_inode(handle_t *handle, struct inode *inode)

{

int ret;

if (S_ISDIR(inode->i_mode))

return;

if (ext4_fc_disabled(inode->i_sb))

return;

if (ext4_should_journal_data(inode)) {

ext4_fc_mark_ineligible(inode->i_sb,

EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);

return;

}

if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))

return;

ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);

trace_ext4_fc_track_inode(handle, inode, ret);

}

struct __track_range_args {

ext4_lblk_t start, end;

};

/* __track_fn for tracking data updates */

static int __track_range(handle_t *handle, struct inode *inode, void *arg,

bool update)

{

struct ext4_inode_info *ei = EXT4_I(inode);

ext4_lblk_t oldstart;

struct __track_range_args *__arg =

(struct __track_range_args *)arg;

if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {

ext4_debug("Special inode %ld being modified\n", inode->i_ino);

return -ECANCELED;

}

oldstart = ei->i_fc_lblk_start;

if (update && ei->i_fc_lblk_len > 0) {

ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);

ei->i_fc_lblk_len =

max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -

ei->i_fc_lblk_start + 1;

} else {

ei->i_fc_lblk_start = __arg->start;

ei->i_fc_lblk_len = __arg->end - __arg->start + 1;

}

return 0;

}

void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,

ext4_lblk_t end)

{

struct __track_range_args args;

int ret;

if (S_ISDIR(inode->i_mode))

return;

if (ext4_fc_disabled(inode->i_sb))

return;

if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))

return;

if (ext4_has_inline_data(inode)) {

ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR,

handle);

return;

}

args.start = start;

args.end = end;

ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1);

trace_ext4_fc_track_range(handle, inode, start, end, ret);

}

static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)

{

blk_opf_t write_flags = REQ_SYNC;

struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;

/* Add REQ_FUA | REQ_PREFLUSH only its tail */

if (test_opt(sb, BARRIER) && is_tail)

write_flags |= REQ_FUA | REQ_PREFLUSH;

lock_buffer(bh);

set_buffer_dirty(bh);

set_buffer_uptodate(bh);

bh->b_end_io = ext4_end_buffer_io_sync;

submit_bh(REQ_OP_WRITE | write_flags, bh);

EXT4_SB(sb)->s_fc_bh = NULL;

}

/* Ext4 commit path routines */

/*

* Allocate len bytes on a fast commit buffer.

*

* During the commit time this function is used to manage fast commit

* block space. We don't split a fast commit log onto different

* blocks. So this function makes sure that if there's not enough space

* on the current block, the remaining space in the current block is

* marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,

* new block is from jbd2 and CRC is updated to reflect the padding

* we added.

*/

static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)

{

struct ext4_fc_tl tl;

struct ext4_sb_info *sbi = EXT4_SB(sb);

struct buffer_head *bh;

int bsize = sbi->s_journal->j_blocksize;

int ret, off = sbi->s_fc_bytes % bsize;

int remaining;

u8 *dst;

/*

* If 'len' is too long to fit in any block alongside a PAD tlv, then we

* cannot fulfill the request.

*/

if (len > bsize - EXT4_FC_TAG_BASE_LEN)

return NULL;

if (!sbi->s_fc_bh) {

ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);

if (ret)

return NULL;

sbi->s_fc_bh = bh;

}

dst = sbi->s_fc_bh->b_data + off;

/*

* Allocate the bytes in the current block if we can do so while still

* leaving enough space for a PAD tlv.

*/

remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;

if (len <= remaining) {

sbi->s_fc_bytes += len;

return dst;

}

/*

* Else, terminate the current block with a PAD tlv, then allocate a new

* block and allocate the bytes at the start of that new block.

*/

tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);

tl.fc_len = cpu_to_le16(remaining);

memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);

memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining);

*crc = ext4_chksum(sbi, *crc, sbi->s_fc_bh->b_data, bsize);

ext4_fc_submit_bh(sb, false);

ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);

if (ret)

return NULL;

sbi->s_fc_bh = bh;

sbi->s_fc_bytes += bsize - off + len;

return sbi->s_fc_bh->b_data;

}

/*

* Complete a fast commit by writing tail tag.

*

* Writing tail tag marks the end of a fast commit. In order to guarantee

* atomicity, after writing tail tag, even if there's space remaining

* in the block, next commit shouldn't use it. That's why tail tag

* has the length as that of the remaining space on the block.

*/

static int ext4_fc_write_tail(struct super_block *sb, u32 crc)

{

struct ext4_sb_info *sbi = EXT4_SB(sb);

struct ext4_fc_tl tl;

struct ext4_fc_tail tail;

int off, bsize = sbi->s_journal->j_blocksize;

u8 *dst;

/*

* ext4_fc_reserve_space takes care of allocating an extra block if

* there's no enough space on this block for accommodating this tail.

*/

dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);

if (!dst)

return -ENOSPC;

off = sbi->s_fc_bytes % bsize;

tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);

tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));

sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);

memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);

dst += EXT4_FC_TAG_BASE_LEN;

tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);

memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid));

dst += sizeof(tail.fc_tid);

crc = ext4_chksum(sbi, crc, sbi->s_fc_bh->b_data,

dst - (u8 *)sbi->s_fc_bh->b_data);

tail.fc_crc = cpu_to_le32(crc);

memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc));

dst += sizeof(tail.fc_crc);

memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */

ext4_fc_submit_bh(sb, true);

return 0;

}

/*

* Adds tag, length, value and updates CRC. Returns true if tlv was added.

* Returns false if there's not enough space.

*/

static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,

u32 *crc)

{

struct ext4_fc_tl tl;

u8 *dst;

dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);

if (!dst)

return false;

tl.fc_tag = cpu_to_le16(tag);

tl.fc_len = cpu_to_le16(len);

memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);

memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len);

return true;

}

/* Same as above, but adds dentry tlv. */

static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,

struct ext4_fc_dentry_update *fc_dentry)

{

struct ext4_fc_dentry_info fcd;

struct ext4_fc_tl tl;

int dlen = fc_dentry->fcd_name.name.len;

u8 *dst = ext4_fc_reserve_space(sb,

EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);

if (!dst)

return false;

fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);

fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);

tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);

tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);

memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);

dst += EXT4_FC_TAG_BASE_LEN;

memcpy(dst, &fcd, sizeof(fcd));

dst += sizeof(fcd);

memcpy(dst, fc_dentry->fcd_name.name.name, dlen);

return true;

}

/*

* Writes inode in the fast commit space under TLV with tag @tag.

* Returns 0 on success, error on failure.

*/

static int ext4_fc_write_inode(struct inode *inode, u32 *crc)

{

struct ext4_inode_info *ei = EXT4_I(inode);

int inode_len = EXT4_GOOD_OLD_INODE_SIZE;

int ret;

struct ext4_iloc iloc;

struct ext4_fc_inode fc_inode;

struct ext4_fc_tl tl;

u8 *dst;

ret = ext4_get_inode_loc(inode, &iloc);

if (ret)

return ret;

if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))

inode_len = EXT4_INODE_SIZE(inode->i_sb);

else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)

inode_len += ei->i_extra_isize;

fc_inode.fc_ino = cpu_to_le32(inode->i_ino);

tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);

tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));

ret = -ECANCELED;

dst = ext4_fc_reserve_space(inode->i_sb,

EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);

if (!dst)

goto err;

memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);

dst += EXT4_FC_TAG_BASE_LEN;

memcpy(dst, &fc_inode, sizeof(fc_inode));

dst += sizeof(fc_inode);

memcpy(dst, (u8 *)ext4_raw_inode(&iloc), inode_len);

ret = 0;

err:

brelse(iloc.bh);

return ret;

}

/*

* Writes updated data ranges for the inode in question. Updates CRC.

* Returns 0 on success, error otherwise.

*/

static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)

{

ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;

struct ext4_inode_info *ei = EXT4_I(inode);

struct ext4_map_blocks map;

struct ext4_fc_add_range fc_ext;

struct ext4_fc_del_range lrange;

struct ext4_extent *ex;

int ret;

mutex_lock(&ei->i_fc_lock);

if (ei->i_fc_lblk_len == 0) {

mutex_unlock(&ei->i_fc_lock);

return 0;

}

old_blk_size = ei->i_fc_lblk_start;

new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;

ei->i_fc_lblk_len = 0;

mutex_unlock(&ei->i_fc_lock);

cur_lblk_off = old_blk_size;

ext4_debug("will try writing %d to %d for inode %ld\n",

cur_lblk_off, new_blk_size, inode->i_ino);

while (cur_lblk_off <= new_blk_size) {

map.m_lblk = cur_lblk_off;

map.m_len = new_blk_size - cur_lblk_off + 1;

ret = ext4_map_blocks(NULL, inode, &map, 0);

if (ret < 0)

return -ECANCELED;

if (map.m_len == 0) {

cur_lblk_off++;

continue;

}

if (ret == 0) {

lrange.fc_ino = cpu_to_le32(inode->i_ino);

lrange.fc_lblk = cpu_to_le32(map.m_lblk);

lrange.fc_len = cpu_to_le32(map.m_len);

if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,

sizeof(lrange), (u8 *)&lrange, crc))

return -ENOSPC;

} else {

unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?

EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;

/* Limit the number of blocks in one extent */

map.m_len = min(max, map.m_len);

fc_ext.fc_ino = cpu_to_le32(inode->i_ino);

ex = (struct ext4_extent *)&fc_ext.fc_ex;

ex->ee_block = cpu_to_le32(map.m_lblk);

ex->ee_len = cpu_to_le16(map.m_len);

ext4_ext_store_pblock(ex, map.m_pblk);

if (map.m_flags & EXT4_MAP_UNWRITTEN)

ext4_ext_mark_unwritten(ex);

else

ext4_ext_mark_initialized(ex);

if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,

sizeof(fc_ext), (u8 *)&fc_ext, crc))

return -ENOSPC;

}

cur_lblk_off += map.m_len;

}

return 0;

}

/* Submit data for all the fast commit inodes */

static int ext4_fc_submit_inode_data_all(journal_t *journal)

{

struct super_block *sb = journal->j_private;

struct ext4_sb_info *sbi = EXT4_SB(sb);

struct ext4_inode_info *ei;

int ret = 0;

spin_lock(&sbi->s_fc_lock);

list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {

ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);

while (atomic_read(&ei->i_fc_updates)) {

DEFINE_WAIT(wait);

prepare_to_wait(&ei->i_fc_wait, &wait,

TASK_UNINTERRUPTIBLE);

if (atomic_read(&ei->i_fc_updates)) {

spin_unlock(&sbi->s_fc_lock);

schedule();

spin_lock(&sbi->s_fc_lock);

}

finish_wait(&ei->i_fc_wait, &wait);

}

spin_unlock(&sbi->s_fc_lock);

ret = jbd2_submit_inode_data(journal, ei->jinode);

if (ret)

return ret;

spin_lock(&sbi->s_fc_lock);

}

spin_unlock(&sbi->s_fc_lock);

return ret;

}

/* Wait for completion of data for all the fast commit inodes */

static int ext4_fc_wait_inode_data_all(journal_t *journal)

{

struct super_block *sb = journal->j_private;

struct ext4_sb_info *sbi = EXT4_SB(sb);

struct ext4_inode_info *pos, *n;

int ret = 0;

spin_lock(&sbi->s_fc_lock);

list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {