/*

* fs/dax.c - Direct Access filesystem code

* Copyright (c) 2013-2014 Intel Corporation

* Author: Matthew Wilcox <matthew.r.wilcox@intel.com>

* Author: Ross Zwisler <ross.zwisler@linux.intel.com>

*

* This program is free software; you can redistribute it and/or modify it

* under the terms and conditions of the GNU General Public License,

* version 2, as published by the Free Software Foundation.

*

* This program is distributed in the hope it will be useful, but WITHOUT

* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for

* more details.

*/

#include <linux/atomic.h>

#include <linux/blkdev.h>

#include <linux/buffer_head.h>

#include <linux/dax.h>

#include <linux/fs.h>

#include <linux/genhd.h>

#include <linux/highmem.h>

#include <linux/memcontrol.h>

#include <linux/mm.h>

#include <linux/mutex.h>

#include <linux/pagevec.h>

#include <linux/sched.h>

#include <linux/sched/signal.h>

#include <linux/uio.h>

#include <linux/vmstat.h>

#include <linux/pfn_t.h>

#include <linux/sizes.h>

#include <linux/mmu_notifier.h>

#include <linux/iomap.h>

#include "internal.h"

#define CREATE_TRACE_POINTS

#include <trace/events/fs_dax.h>

/* We choose 4096 entries - same as per-zone page wait tables */

#define DAX_WAIT_TABLE_BITS 12

#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)

/* The 'colour' (ie low bits) within a PMD of a page offset. */

#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)

static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];

static int __init init_dax_wait_table(void)

{

int i;

for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)

init_waitqueue_head(wait_table + i);

return 0;

}

fs_initcall(init_dax_wait_table);

/*

* We use lowest available bit in exceptional entry for locking, one bit for

* the entry size (PMD) and two more to tell us if the entry is a zero page or

* an empty entry that is just used for locking. In total four special bits.

*

* If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE

* and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem

* block allocation.

*/

#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4)

#define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT)

#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))

#define RADIX_DAX_ZERO_PAGE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))

#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))

static unsigned long dax_radix_sector(void *entry)

{

return (unsigned long)entry >> RADIX_DAX_SHIFT;

}

static void *dax_radix_locked_entry(sector_t sector, unsigned long flags)

{

return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |

((unsigned long)sector << RADIX_DAX_SHIFT) |

RADIX_DAX_ENTRY_LOCK);

}

static unsigned int dax_radix_order(void *entry)

{

if ((unsigned long)entry & RADIX_DAX_PMD)

return PMD_SHIFT - PAGE_SHIFT;

return 0;

}

static int dax_is_pmd_entry(void *entry)

{

return (unsigned long)entry & RADIX_DAX_PMD;

}

static int dax_is_pte_entry(void *entry)

{

return !((unsigned long)entry & RADIX_DAX_PMD);

}

static int dax_is_zero_entry(void *entry)

{

return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;

}

static int dax_is_empty_entry(void *entry)

{

return (unsigned long)entry & RADIX_DAX_EMPTY;

}

/*

* DAX radix tree locking

*/

struct exceptional_entry_key {

struct address_space *mapping;

pgoff_t entry_start;

};

struct wait_exceptional_entry_queue {

wait_queue_entry_t wait;

struct exceptional_entry_key key;

};

static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,

pgoff_t index, void *entry, struct exceptional_entry_key *key)

{

unsigned long hash;

/*

* If 'entry' is a PMD, align the 'index' that we use for the wait

* queue to the start of that PMD. This ensures that all offsets in

* the range covered by the PMD map to the same bit lock.

*/

if (dax_is_pmd_entry(entry))

index &= ~PG_PMD_COLOUR;

key->mapping = mapping;

key->entry_start = index;

hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);

return wait_table + hash;

}

static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,

int sync, void *keyp)

{

struct exceptional_entry_key *key = keyp;

struct wait_exceptional_entry_queue *ewait =

container_of(wait, struct wait_exceptional_entry_queue, wait);

if (key->mapping != ewait->key.mapping ||

key->entry_start != ewait->key.entry_start)

return 0;

return autoremove_wake_function(wait, mode, sync, NULL);

}

/*

* We do not necessarily hold the mapping->tree_lock when we call this

* function so it is possible that 'entry' is no longer a valid item in the

* radix tree. This is okay because all we really need to do is to find the

* correct waitqueue where tasks might be waiting for that old 'entry' and

* wake them.

*/

static void dax_wake_mapping_entry_waiter(struct address_space *mapping,

pgoff_t index, void *entry, bool wake_all)

{

struct exceptional_entry_key key;

wait_queue_head_t *wq;

wq = dax_entry_waitqueue(mapping, index, entry, &key);

/*

* Checking for locked entry and prepare_to_wait_exclusive() happens

* under mapping->tree_lock, ditto for entry handling in our callers.

* So at this point all tasks that could have seen our entry locked

* must be in the waitqueue and the following check will see them.

*/

if (waitqueue_active(wq))

__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);

}

/*

* Check whether the given slot is locked. The function must be called with

* mapping->tree_lock held

*/

static inline int slot_locked(struct address_space *mapping, void **slot)

{

unsigned long entry = (unsigned long)

radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

return entry & RADIX_DAX_ENTRY_LOCK;

}

/*

* Mark the given slot is locked. The function must be called with

* mapping->tree_lock held

*/

static inline void *lock_slot(struct address_space *mapping, void **slot)

{

unsigned long entry = (unsigned long)

radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

entry |= RADIX_DAX_ENTRY_LOCK;

radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);

return (void *)entry;

}

/*

* Mark the given slot is unlocked. The function must be called with

* mapping->tree_lock held

*/

static inline void *unlock_slot(struct address_space *mapping, void **slot)

{

unsigned long entry = (unsigned long)

radix_tree_deref_slot_protected(slot, &mapping->tree_lock);

entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;

radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);

return (void *)entry;

}

/*

* Lookup entry in radix tree, wait for it to become unlocked if it is

* exceptional entry and return it. The caller must call

* put_unlocked_mapping_entry() when he decided not to lock the entry or

* put_locked_mapping_entry() when he locked the entry and now wants to

* unlock it.

*

* The function must be called with mapping->tree_lock held.

*/

static void *get_unlocked_mapping_entry(struct address_space *mapping,

pgoff_t index, void ***slotp)

{

void *entry, **slot;

struct wait_exceptional_entry_queue ewait;

wait_queue_head_t *wq;

init_wait(&ewait.wait);

ewait.wait.func = wake_exceptional_entry_func;

for (;;) {

entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,

&slot);

if (!entry ||

WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||

!slot_locked(mapping, slot)) {

if (slotp)

*slotp = slot;

return entry;

}

wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);

prepare_to_wait_exclusive(wq, &ewait.wait,

TASK_UNINTERRUPTIBLE);

spin_unlock_irq(&mapping->tree_lock);

schedule();

finish_wait(wq, &ewait.wait);

spin_lock_irq(&mapping->tree_lock);

}

}

static void dax_unlock_mapping_entry(struct address_space *mapping,

pgoff_t index)

{

void *entry, **slot;

spin_lock_irq(&mapping->tree_lock);

entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);

if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||

!slot_locked(mapping, slot))) {

spin_unlock_irq(&mapping->tree_lock);

return;

}

unlock_slot(mapping, slot);

spin_unlock_irq(&mapping->tree_lock);

dax_wake_mapping_entry_waiter(mapping, index, entry, false);

}

static void put_locked_mapping_entry(struct address_space *mapping,

pgoff_t index)

{

dax_unlock_mapping_entry(mapping, index);

}

/*

* Called when we are done with radix tree entry we looked up via

* get_unlocked_mapping_entry() and which we didn't lock in the end.

*/

static void put_unlocked_mapping_entry(struct address_space *mapping,

pgoff_t index, void *entry)

{

if (!entry)

return;

/* We have to wake up next waiter for the radix tree entry lock */

dax_wake_mapping_entry_waiter(mapping, index, entry, false);

}

/*

* Find radix tree entry at given index. If it points to an exceptional entry,

* return it with the radix tree entry locked. If the radix tree doesn't

* contain given index, create an empty exceptional entry for the index and

* return with it locked.

*

* When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will

* either return that locked entry or will return an error. This error will

* happen if there are any 4k entries within the 2MiB range that we are

* requesting.

*

* We always favor 4k entries over 2MiB entries. There isn't a flow where we

* evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB

* insertion will fail if it finds any 4k entries already in the tree, and a

* 4k insertion will cause an existing 2MiB entry to be unmapped and

* downgraded to 4k entries. This happens for both 2MiB huge zero pages as

* well as 2MiB empty entries.

*

* The exception to this downgrade path is for 2MiB DAX PMD entries that have

* real storage backing them. We will leave these real 2MiB DAX entries in

* the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.

*

* Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For

* persistent memory the benefit is doubtful. We can add that later if we can

* show it helps.

*/

static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,

unsigned long size_flag)

{

bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */

void *entry, **slot;

restart:

spin_lock_irq(&mapping->tree_lock);

entry = get_unlocked_mapping_entry(mapping, index, &slot);

if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {

entry = ERR_PTR(-EIO);

goto out_unlock;

}

if (entry) {

if (size_flag & RADIX_DAX_PMD) {

if (dax_is_pte_entry(entry)) {

put_unlocked_mapping_entry(mapping, index,

entry);

entry = ERR_PTR(-EEXIST);

goto out_unlock;

}

} else { /* trying to grab a PTE entry */

if (dax_is_pmd_entry(entry) &&

(dax_is_zero_entry(entry) ||

dax_is_empty_entry(entry))) {

pmd_downgrade = true;

}

}

}

/* No entry for given index? Make sure radix tree is big enough. */

if (!entry || pmd_downgrade) {

int err;

if (pmd_downgrade) {

/*

* Make sure 'entry' remains valid while we drop

* mapping->tree_lock.

*/

entry = lock_slot(mapping, slot);

}

spin_unlock_irq(&mapping->tree_lock);

/*

* Besides huge zero pages the only other thing that gets

* downgraded are empty entries which don't need to be

* unmapped.

*/

if (pmd_downgrade && dax_is_zero_entry(entry))

unmap_mapping_range(mapping,

(index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);

err = radix_tree_preload(

mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);

if (err) {

if (pmd_downgrade)

put_locked_mapping_entry(mapping, index);

return ERR_PTR(err);

}

spin_lock_irq(&mapping->tree_lock);

if (!entry) {

/*

* We needed to drop the page_tree lock while calling

* radix_tree_preload() and we didn't have an entry to

* lock. See if another thread inserted an entry at

* our index during this time.

*/

entry = __radix_tree_lookup(&mapping->page_tree, index,

NULL, &slot);

if (entry) {

radix_tree_preload_end();

spin_unlock_irq(&mapping->tree_lock);

goto restart;

}

}

if (pmd_downgrade) {

radix_tree_delete(&mapping->page_tree, index);

mapping->nrexceptional--;

dax_wake_mapping_entry_waiter(mapping, index, entry,

true);

}

entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);

err = __radix_tree_insert(&mapping->page_tree, index,

dax_radix_order(entry), entry);

radix_tree_preload_end();

if (err) {

spin_unlock_irq(&mapping->tree_lock);

/*

* Our insertion of a DAX entry failed, most likely

* because we were inserting a PMD entry and it

* collided with a PTE sized entry at a different

* index in the PMD range. We haven't inserted

* anything into the radix tree and have no waiters to

* wake.

*/

return ERR_PTR(err);

}

/* Good, we have inserted empty locked entry into the tree. */

mapping->nrexceptional++;

spin_unlock_irq(&mapping->tree_lock);

return entry;

}

entry = lock_slot(mapping, slot);

out_unlock:

spin_unlock_irq(&mapping->tree_lock);

return entry;

}

static int __dax_invalidate_mapping_entry(struct address_space *mapping,

pgoff_t index, bool trunc)

{

int ret = 0;

void *entry;

struct radix_tree_root *page_tree = &mapping->page_tree;

spin_lock_irq(&mapping->tree_lock);

entry = get_unlocked_mapping_entry(mapping, index, NULL);

if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))

goto out;

if (!trunc &&

(radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||

radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))

goto out;

radix_tree_delete(page_tree, index);

mapping->nrexceptional--;

ret = 1;

out:

put_unlocked_mapping_entry(mapping, index, entry);

spin_unlock_irq(&mapping->tree_lock);

return ret;

}

/*

* Delete exceptional DAX entry at @index from @mapping. Wait for radix tree

* entry to get unlocked before deleting it.

*/

int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)

{

int ret = __dax_invalidate_mapping_entry(mapping, index, true);

/*

* This gets called from truncate / punch_hole path. As such, the caller

* must hold locks protecting against concurrent modifications of the

* radix tree (usually fs-private i_mmap_sem for writing). Since the

* caller has seen exceptional entry for this index, we better find it

* at that index as well...

*/

WARN_ON_ONCE(!ret);

return ret;

}

/*

* Invalidate exceptional DAX entry if it is clean.

*/

int dax_invalidate_mapping_entry_sync(struct address_space *mapping,

pgoff_t index)

{

return __dax_invalidate_mapping_entry(mapping, index, false);

}

static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,

sector_t sector, size_t size, struct page *to,

unsigned long vaddr)

{

void *vto, *kaddr;

pgoff_t pgoff;

pfn_t pfn;

long rc;

int id;

rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);

if (rc)

return rc;

id = dax_read_lock();

rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);

if (rc < 0) {

dax_read_unlock(id);

return rc;

}

vto = kmap_atomic(to);

copy_user_page(vto, (void __force *)kaddr, vaddr, to);

kunmap_atomic(vto);

dax_read_unlock(id);

return 0;

}

/*

* By this point grab_mapping_entry() has ensured that we have a locked entry

* of the appropriate size so we don't have to worry about downgrading PMDs to

* PTEs. If we happen to be trying to insert a PTE and there is a PMD

* already in the tree, we will skip the insertion and just dirty the PMD as

* appropriate.

*/

static void *dax_insert_mapping_entry(struct address_space *mapping,

struct vm_fault *vmf,

void *entry, sector_t sector,

unsigned long flags)

{

struct radix_tree_root *page_tree = &mapping->page_tree;

void *new_entry;

pgoff_t index = vmf->pgoff;

if (vmf->flags & FAULT_FLAG_WRITE)

__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);

if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {

/* we are replacing a zero page with block mapping */

if (dax_is_pmd_entry(entry))

unmap_mapping_range(mapping,

(vmf->pgoff << PAGE_SHIFT) & PMD_MASK,

PMD_SIZE, 0);

else /* pte entry */

unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,

PAGE_SIZE, 0);

}

spin_lock_irq(&mapping->tree_lock);

new_entry = dax_radix_locked_entry(sector, flags);

if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {

/*

* Only swap our new entry into the radix tree if the current

* entry is a zero page or an empty entry. If a normal PTE or

* PMD entry is already in the tree, we leave it alone. This

* means that if we are trying to insert a PTE and the

* existing entry is a PMD, we will just leave the PMD in the

* tree and dirty it if necessary.

*/

struct radix_tree_node *node;

void **slot;

void *ret;

ret = __radix_tree_lookup(page_tree, index, &node, &slot);

WARN_ON_ONCE(ret != entry);

__radix_tree_replace(page_tree, node, slot,

new_entry, NULL, NULL);

entry = new_entry;

}

if (vmf->flags & FAULT_FLAG_WRITE)

radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);

spin_unlock_irq(&mapping->tree_lock);

return entry;

}

static inline unsigned long

pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)

{

unsigned long address;

address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);

VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);

return address;

}

/* Walk all mappings of a given index of a file and writeprotect them */

static void dax_mapping_entry_mkclean(struct address_space *mapping,

pgoff_t index, unsigned long pfn)

{

struct vm_area_struct *vma;

pte_t pte, *ptep = NULL;

pmd_t *pmdp = NULL;

spinlock_t *ptl;

i_mmap_lock_read(mapping);

vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {

unsigned long address, start, end;

cond_resched();

if (!(vma->vm_flags & VM_SHARED))

continue;

address = pgoff_address(index, vma);

/*

* Note because we provide start/end to follow_pte_pmd it will

* call mmu_notifier_invalidate_range_start() on our behalf

* before taking any lock.

*/

if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))

continue;

if (pmdp) {

#ifdef CONFIG_FS_DAX_PMD

pmd_t pmd;

if (pfn != pmd_pfn(*pmdp))

goto unlock_pmd;

if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))

goto unlock_pmd;

flush_cache_page(vma, address, pfn);

pmd = pmdp_huge_clear_flush(vma, address, pmdp);

pmd = pmd_wrprotect(pmd);

pmd = pmd_mkclean(pmd);

set_pmd_at(vma->vm_mm, address, pmdp, pmd);

mmu_notifier_invalidate_range(vma->vm_mm, start, end);

unlock_pmd:

spin_unlock(ptl);

#endif

} else {

if (pfn != pte_pfn(*ptep))

goto unlock_pte;

if (!pte_dirty(*ptep) && !pte_write(*ptep))

goto unlock_pte;

flush_cache_page(vma, address, pfn);

pte = ptep_clear_flush(vma, address, ptep);

pte = pte_wrprotect(pte);

pte = pte_mkclean(pte);

set_pte_at(vma->vm_mm, address, ptep, pte);

mmu_notifier_invalidate_range(vma->vm_mm, start, end);

unlock_pte:

pte_unmap_unlock(ptep, ptl);

}

mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);

}

i_mmap_unlock_read(mapping);

}

static int dax_writeback_one(struct block_device *bdev,

struct dax_device *dax_dev, struct address_space *mapping,

pgoff_t index, void *entry)

{

struct radix_tree_root *page_tree = &mapping->page_tree;

void *entry2, **slot, *kaddr;

long ret = 0, id;

sector_t sector;

pgoff_t pgoff;

size_t size;

pfn_t pfn;

/*

* A page got tagged dirty in DAX mapping? Something is seriously

* wrong.

*/

if (WARN_ON(!radix_tree_exceptional_entry(entry)))

return -EIO;

spin_lock_irq(&mapping->tree_lock);

entry2 = get_unlocked_mapping_entry(mapping, index, &slot);

/* Entry got punched out / reallocated? */

if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))

goto put_unlocked;

/*

* Entry got reallocated elsewhere? No need to writeback. We have to

* compare sectors as we must not bail out due to difference in lockbit

* or entry type.

*/

if (dax_radix_sector(entry2) != dax_radix_sector(entry))

goto put_unlocked;

if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||

dax_is_zero_entry(entry))) {

ret = -EIO;

goto put_unlocked;

}

/* Another fsync thread may have already written back this entry */

if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))

goto put_unlocked;

/* Lock the entry to serialize with page faults */

entry = lock_slot(mapping, slot);

/*

* We can clear the tag now but we have to be careful so that concurrent

* dax_writeback_one() calls for the same index cannot finish before we

* actually flush the caches. This is achieved as the calls will look

* at the entry only under tree_lock and once they do that they will

* see the entry locked and wait for it to unlock.

*/

radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);

spin_unlock_irq(&mapping->tree_lock);

/*

* Even if dax_writeback_mapping_range() was given a wbc->range_start

* in the middle of a PMD, the 'index' we are given will be aligned to

* the start index of the PMD, as will the sector we pull from

* 'entry'. This allows us to flush for PMD_SIZE and not have to

* worry about partial PMD writebacks.

*/

sector = dax_radix_sector(entry);

size = PAGE_SIZE << dax_radix_order(entry);

id = dax_read_lock();

ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);

if (ret)

goto dax_unlock;

/*

* dax_direct_access() may sleep, so cannot hold tree_lock over

* its invocation.

*/

ret = dax_direct_access(dax_dev, pgoff, size / PAGE_SIZE, &kaddr, &pfn);

if (ret < 0)

goto dax_unlock;

if (WARN_ON_ONCE(ret < size / PAGE_SIZE)) {

ret = -EIO;

goto dax_unlock;

}

dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn));

dax_flush(dax_dev, kaddr, size);

/*

* After we have flushed the cache, we can clear the dirty tag. There

* cannot be new dirty data in the pfn after the flush has completed as

* the pfn mappings are writeprotected and fault waits for mapping

* entry lock.

*/

spin_lock_irq(&mapping->tree_lock);

radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);

spin_unlock_irq(&mapping->tree_lock);

trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);

dax_unlock:

dax_read_unlock(id);

put_locked_mapping_entry(mapping, index);

return ret;

put_unlocked:

put_unlocked_mapping_entry(mapping, index, entry2);

spin_unlock_irq(&mapping->tree_lock);

return ret;

}

/*

* Flush the mapping to the persistent domain within the byte range of [start,

* end]. This is required by data integrity operations to ensure file data is

* on persistent storage prior to completion of the operation.

*/

int dax_writeback_mapping_range(struct address_space *mapping,

struct block_device *bdev, struct writeback_control *wbc)

{

struct inode *inode = mapping->host;

pgoff_t start_index, end_index;

pgoff_t indices[PAGEVEC_SIZE];

struct dax_device *dax_dev;

struct pagevec pvec;

bool done = false;

int i, ret = 0;

if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))

return -EIO;

if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)

return 0;

dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);

if (!dax_dev)

return -EIO;

start_index = wbc->range_start >> PAGE_SHIFT;

end_index = wbc->range_end >> PAGE_SHIFT;

trace_dax_writeback_range(inode, start_index, end_index);

tag_pages_for_writeback(mapping, start_index, end_index);

pagevec_init(&pvec, 0);

while (!done) {

pvec.nr = find_get_entries_tag(mapping, start_index,

PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,

pvec.pages, indices);

if (pvec.nr == 0)

break;

for (i = 0; i < pvec.nr; i++) {

if (indices[i] > end_index) {

done = true;

break;

}

ret = dax_writeback_one(bdev, dax_dev, mapping,

indices[i], pvec.pages[i]);

if (ret < 0) {

mapping_set_error(mapping, ret);

goto out;

}

}

start_index = indices[pvec.nr - 1] + 1;

}

out:

put_dax(dax_dev);

trace_dax_writeback_range_done(inode, start_index, end_index);

return (ret < 0 ? ret : 0);

}

EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);

static int dax_insert_mapping(struct address_space *mapping,

struct block_device *bdev, struct dax_device *dax_dev,

sector_t sector, size_t size, void *entry,

struct vm_area_struct *vma, struct vm_fault *vmf)

{

unsigned long vaddr = vmf->address;

void *ret, *kaddr;

pgoff_t pgoff;

int id, rc;

pfn_t pfn;

rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);

if (rc)

return rc;

id = dax_read_lock();

rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);

if (rc < 0) {

dax_read_unlock(id);

return rc;

}

dax_read_unlock(id);

ret = dax_insert_mapping_entry(mapping, vmf, entry, sector, 0);

if (IS_ERR(ret))

return PTR_ERR(ret);

trace_dax_insert_mapping(mapping->host, vmf, ret);

if (vmf->flags & FAULT_FLAG_WRITE)

return vm_insert_mixed_mkwrite(vma, vaddr, pfn);

else

return vm_insert_mixed(vma, vaddr, pfn);

}

/*

* The user has performed a load from a hole in the file. Allocating a new

* page in the file would cause excessive storage usage for workloads with

* sparse files. Instead we insert a read-only mapping of the 4k zero page.

* If this page is ever written to we will re-fault and change the mapping to

* point to real DAX storage instead.

*/

static int dax_load_hole(struct address_space *mapping, void *entry,

struct vm_fault *vmf)

{

struct inode *inode = mapping->host;

unsigned long vaddr = vmf->address;

int ret = VM_FAULT_NOPAGE;

struct page *zero_page;

void *entry2;

zero_page = ZERO_PAGE(0);

if (unlikely(!zero_page)) {

ret = VM_FAULT_OOM;

goto out;

}

entry2 = dax_insert_mapping_entry(mapping, vmf, entry, 0,

RADIX_DAX_ZERO_PAGE);

if (IS_ERR(entry2)) {

ret = VM_FAULT_SIGBUS;

goto out;

}

vm_insert_mixed(vmf->vma, vaddr, page_to_pfn_t(zero_page));

out:

trace_dax_load_hole(inode, vmf, ret);

return ret;

}

static bool dax_range_is_aligned(struct block_device *bdev,

unsigned int offset, unsigned int length)

{

unsigned short sector_size = bdev_logical_block_size(bdev);

if (!IS_ALIGNED(offset, sector_size))

return false;

if (!IS_ALIGNED(length, sector_size))

return false;

return true;

}

int __dax_zero_page_range(struct block_device *bdev,

struct dax_device *dax_dev, sector_t sector,

unsigned int offset, unsigned int size)

{

if (dax_range_is_aligned(bdev, offset, size)) {

sector_t start_sector = sector + (offset >> 9);

return blkdev_issue_zeroout(bdev, start_sector,

size >> 9, GFP_NOFS, 0);

} else {

pgoff_t pgoff;

long rc, id;

void *kaddr;

pfn_t pfn;

rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);

if (rc)

return rc;

id = dax_read_lock();

rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr,

&pfn);

if (rc < 0) {

dax_read_unlock(id);

return rc;

}

memset(kaddr + offset, 0, size);

dax_flush(dax_dev, kaddr + offset, size);

dax_read_unlock(id);

}

return 0;

}

EXPORT_SYMBOL_GPL(__dax_zero_page_range);

static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)

{

return iomap->blkno + (((pos & PAGE_MASK) - iomap->offset) >> 9);

}

static loff_t

dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,

struct iomap *iomap)

{

struct block_device *bdev = iomap->bdev;

struct dax_device *dax_dev = iomap->dax_dev;

struct iov_iter *iter = data;

loff_t end = pos + length, done = 0;

ssize_t ret = 0;

int id;

if (iov_iter_rw(iter) == READ) {

end = min(end, i_size_read(inode));

if (pos >= end)

return 0;

if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)

return iov_iter_zero(min(length, end - pos), iter);

}

if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))

return -EIO;

/*

* Write can allocate block for an area which has a hole page mapped

* into page tables. We have to tear down these mappings so that data

* written by write(2) is visible in mmap.

*/

if (iomap->flags & IOMAP_F_NEW) {

invalidate_inode_pages2_range(inode->i_mapping,

pos >> PAGE_SHIFT,

(end - 1) >> PAGE_SHIFT);

}

id = dax_read_lock();

while (pos < end) {

unsigned offset = pos & (PAGE_SIZE - 1);

const size_t size = ALIGN(length + offset, PAGE_SIZE);

const sector_t sector = dax_iomap_sector(iomap, pos);

ssize_t map_len;

pgoff_t pgoff;

void *kaddr;

pfn_t pfn;

if (fatal_signal_pending(current)) {

ret = -EINTR;

break;

}

ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);

if (ret)

break;

map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),

&kaddr, &pfn);

if (map_len < 0) {

ret = map_len;