// SPDX-License-Identifier: GPL-2.0

/*

* VMware Balloon driver.

*

* Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved.

*

* This is VMware physical memory management driver for Linux. The driver

* acts like a "balloon" that can be inflated to reclaim physical pages by

* reserving them in the guest and invalidating them in the monitor,

* freeing up the underlying machine pages so they can be allocated to

* other guests. The balloon can also be deflated to allow the guest to

* use more physical memory. Higher level policies can control the sizes

* of balloons in VMs in order to manage physical memory resources.

*/

//#define DEBUG

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/types.h>

#include <linux/io.h>

#include <linux/kernel.h>

#include <linux/mm.h>

#include <linux/vmalloc.h>

#include <linux/sched.h>

#include <linux/module.h>

#include <linux/workqueue.h>

#include <linux/debugfs.h>

#include <linux/seq_file.h>

#include <linux/rwsem.h>

#include <linux/slab.h>

#include <linux/spinlock.h>

#include <linux/balloon.h>

#include <linux/vmw_vmci_defs.h>

#include <linux/vmw_vmci_api.h>

#include <asm/hypervisor.h>

MODULE_AUTHOR("VMware, Inc.");

MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");

MODULE_ALIAS("dmi:*:svnVMware*:*");

MODULE_ALIAS("vmware_vmmemctl");

MODULE_LICENSE("GPL");

static bool __read_mostly vmwballoon_shrinker_enable;

module_param(vmwballoon_shrinker_enable, bool, 0444);

MODULE_PARM_DESC(vmwballoon_shrinker_enable,

"Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance.");

/* Delay in seconds after shrink before inflation. */

#define VMBALLOON_SHRINK_DELAY (5)

/* Maximum number of refused pages we accumulate during inflation cycle */

#define VMW_BALLOON_MAX_REFUSED 16

/* Magic number for the balloon mount-point */

#define BALLOON_VMW_MAGIC 0x0ba11007

/*

* Hypervisor communication port definitions.

*/

#define VMW_BALLOON_HV_PORT 0x5670

#define VMW_BALLOON_HV_MAGIC 0x456c6d6f

#define VMW_BALLOON_GUEST_ID 1 /* Linux */

enum vmwballoon_capabilities {

/*

* Bit 0 is reserved and not associated to any capability.

*/

VMW_BALLOON_BASIC_CMDS = (1 << 1),

VMW_BALLOON_BATCHED_CMDS = (1 << 2),

VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3),

VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4),

VMW_BALLOON_64_BIT_TARGET = (1 << 5)

};

#define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \

| VMW_BALLOON_BATCHED_CMDS \

| VMW_BALLOON_BATCHED_2M_CMDS \

| VMW_BALLOON_SIGNALLED_WAKEUP_CMD)

#define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)

/*

* 64-bit targets are only supported in 64-bit

*/

#ifdef CONFIG_64BIT

#define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \

| VMW_BALLOON_64_BIT_TARGET)

#else

#define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON

#endif

enum vmballoon_page_size_type {

VMW_BALLOON_4K_PAGE,

VMW_BALLOON_2M_PAGE,

VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE

};

#define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)

static const char * const vmballoon_page_size_names[] = {

[VMW_BALLOON_4K_PAGE] = "4k",

[VMW_BALLOON_2M_PAGE] = "2M"

};

enum vmballoon_op {

VMW_BALLOON_INFLATE,

VMW_BALLOON_DEFLATE

};

enum vmballoon_op_stat_type {

VMW_BALLOON_OP_STAT,

VMW_BALLOON_OP_FAIL_STAT

};

#define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1)

/**

* enum vmballoon_cmd_type - backdoor commands.

*

* Availability of the commands is as followed:

*

* %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and

* %VMW_BALLOON_CMD_GUEST_ID are always available.

*

* If the host reports %VMW_BALLOON_BASIC_CMDS are supported then

* %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available.

*

* If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then

* %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands

* are available.

*

* If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then

* %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK

* are supported.

*

* If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then

* VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported.

*

* @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor.

* @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.

* @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.

* @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about

* to be deflated from the balloon.

* @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that

* runs in the VM.

* @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of

* ballooned pages (up to 512).

* @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of

* pages that are about to be deflated from the

* balloon (up to 512).

* @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK

* for 2MB pages.

* @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to

* @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB

* pages.

* @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification

* that would be invoked when the balloon

* size changes.

* @VMW_BALLOON_CMD_LAST: Value of the last command.

*/

enum vmballoon_cmd_type {

VMW_BALLOON_CMD_START,

VMW_BALLOON_CMD_GET_TARGET,

VMW_BALLOON_CMD_LOCK,

VMW_BALLOON_CMD_UNLOCK,

VMW_BALLOON_CMD_GUEST_ID,

/* No command 5 */

VMW_BALLOON_CMD_BATCHED_LOCK = 6,

VMW_BALLOON_CMD_BATCHED_UNLOCK,

VMW_BALLOON_CMD_BATCHED_2M_LOCK,

VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,

VMW_BALLOON_CMD_VMCI_DOORBELL_SET,

VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,

};

#define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1)

enum vmballoon_error_codes {

VMW_BALLOON_SUCCESS,

VMW_BALLOON_ERROR_CMD_INVALID,

VMW_BALLOON_ERROR_PPN_INVALID,

VMW_BALLOON_ERROR_PPN_LOCKED,

VMW_BALLOON_ERROR_PPN_UNLOCKED,

VMW_BALLOON_ERROR_PPN_PINNED,

VMW_BALLOON_ERROR_PPN_NOTNEEDED,

VMW_BALLOON_ERROR_RESET,

VMW_BALLOON_ERROR_BUSY

};

#define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000)

#define VMW_BALLOON_CMD_WITH_TARGET_MASK \

((1UL << VMW_BALLOON_CMD_GET_TARGET) | \

(1UL << VMW_BALLOON_CMD_LOCK) | \

(1UL << VMW_BALLOON_CMD_UNLOCK) | \

(1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \

(1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \

(1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \

(1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))

static const char * const vmballoon_cmd_names[] = {

[VMW_BALLOON_CMD_START] = "start",

[VMW_BALLOON_CMD_GET_TARGET] = "target",

[VMW_BALLOON_CMD_LOCK] = "lock",

[VMW_BALLOON_CMD_UNLOCK] = "unlock",

[VMW_BALLOON_CMD_GUEST_ID] = "guestType",

[VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock",

[VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock",

[VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock",

[VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock",

[VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"

};

enum vmballoon_stat_page {

VMW_BALLOON_PAGE_STAT_ALLOC,

VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,

VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,

VMW_BALLOON_PAGE_STAT_REFUSED_FREE,

VMW_BALLOON_PAGE_STAT_FREE,

VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE

};

#define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1)

enum vmballoon_stat_general {

VMW_BALLOON_STAT_TIMER,

VMW_BALLOON_STAT_DOORBELL,

VMW_BALLOON_STAT_RESET,

VMW_BALLOON_STAT_SHRINK,

VMW_BALLOON_STAT_SHRINK_FREE,

VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE

};

#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)

static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);

static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);

struct vmballoon_ctl {

struct list_head pages;

struct list_head refused_pages;

struct list_head prealloc_pages;

unsigned int n_refused_pages;

unsigned int n_pages;

enum vmballoon_page_size_type page_size;

enum vmballoon_op op;

};

/**

* struct vmballoon_batch_entry - a batch entry for lock or unlock.

*

* @status: the status of the operation, which is written by the hypervisor.

* @reserved: reserved for future use. Must be set to zero.

* @pfn: the physical frame number of the page to be locked or unlocked.

*/

struct vmballoon_batch_entry {

u64 status : 5;

u64 reserved : PAGE_SHIFT - 5;

u64 pfn : 52;

} __packed;

struct vmballoon {

/**

* @max_page_size: maximum supported page size for ballooning.

*

* Protected by @conf_sem

*/

enum vmballoon_page_size_type max_page_size;

/**

* @size: balloon actual size in basic page size (frames).

*

* While we currently do not support size which is bigger than 32-bit,

* in preparation for future support, use 64-bits.

*/

atomic64_t size;

/**

* @target: balloon target size in basic page size (frames).

*

* We do not protect the target under the assumption that setting the

* value is always done through a single write. If this assumption ever

* breaks, we would have to use X_ONCE for accesses, and suffer the less

* optimized code. Although we may read stale target value if multiple

* accesses happen at once, the performance impact should be minor.

*/

unsigned long target;

/**

* @reset_required: reset flag

*

* Setting this flag may introduce races, but the code is expected to

* handle them gracefully. In the worst case, another operation will

* fail as reset did not take place. Clearing the flag is done while

* holding @conf_sem for write.

*/

bool reset_required;

/**

* @capabilities: hypervisor balloon capabilities.

*

* Protected by @conf_sem.

*/

unsigned long capabilities;

/**

* @batch_page: pointer to communication batch page.

*

* When batching is used, batch_page points to a page, which holds up to

* %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.

*/

struct vmballoon_batch_entry *batch_page;

/**

* @batch_max_pages: maximum pages that can be locked/unlocked.

*

* Indicates the number of pages that the hypervisor can lock or unlock

* at once, according to whether batching is enabled. If batching is

* disabled, only a single page can be locked/unlock on each operation.

*

* Protected by @conf_sem.

*/

unsigned int batch_max_pages;

/**

* @page: page to be locked/unlocked by the hypervisor

*

* @page is only used when batching is disabled and a single page is

* reclaimed on each iteration.

*

* Protected by @comm_lock.

*/

struct page *page;

/**

* @shrink_timeout: timeout until the next inflation.

*

* After an shrink event, indicates the time in jiffies after which

* inflation is allowed again. Can be written concurrently with reads,

* so must use READ_ONCE/WRITE_ONCE when accessing.

*/

unsigned long shrink_timeout;

/* statistics */

struct vmballoon_stats *stats;

/**

* @b_dev_info: balloon device information descriptor.

*/

struct balloon_dev_info b_dev_info;

struct delayed_work dwork;

/**

* @huge_pages - list of the inflated 2MB pages.

*

* Protected by @huge_pages_lock.

*/

struct list_head huge_pages;

/**

* @huge_pages_lock: lock for the list of inflated 2MB pages.

*/

spinlock_t huge_pages_lock;

/**

* @vmci_doorbell.

*

* Protected by @conf_sem.

*/

struct vmci_handle vmci_doorbell;

/**

* @conf_sem: semaphore to protect the configuration and the statistics.

*/

struct rw_semaphore conf_sem;

/**

* @comm_lock: lock to protect the communication with the host.

*

* Lock ordering: @conf_sem -> @comm_lock .

*/

spinlock_t comm_lock;

/**

* @shrinker: shrinker interface that is used to avoid over-inflation.

*/

struct shrinker *shrinker;

};

static struct vmballoon balloon;

struct vmballoon_stats {

/* timer / doorbell operations */

atomic64_t general_stat[VMW_BALLOON_STAT_NUM];

/* allocation statistics for huge and small pages */

atomic64_t

page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES];

/* Monitor operations: total operations, and failures */

atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];

};

static inline bool is_vmballoon_stats_on(void)

{

return IS_ENABLED(CONFIG_DEBUG_FS) &&

static_branch_unlikely(&balloon_stat_enabled);

}

static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,

enum vmballoon_op_stat_type type)

{

if (is_vmballoon_stats_on())

atomic64_inc(&b->stats->ops[op][type]);

}

static inline void vmballoon_stats_gen_inc(struct vmballoon *b,

enum vmballoon_stat_general stat)

{

if (is_vmballoon_stats_on())

atomic64_inc(&b->stats->general_stat[stat]);

}

static inline void vmballoon_stats_gen_add(struct vmballoon *b,

enum vmballoon_stat_general stat,

unsigned int val)

{

if (is_vmballoon_stats_on())

atomic64_add(val, &b->stats->general_stat[stat]);

}

static inline void vmballoon_stats_page_inc(struct vmballoon *b,

enum vmballoon_stat_page stat,

enum vmballoon_page_size_type size)

{

if (is_vmballoon_stats_on())

atomic64_inc(&b->stats->page_stat[stat][size]);

}

static inline void vmballoon_stats_page_add(struct vmballoon *b,

enum vmballoon_stat_page stat,

enum vmballoon_page_size_type size,

unsigned int val)

{

if (is_vmballoon_stats_on())

atomic64_add(val, &b->stats->page_stat[stat][size]);

}

static inline unsigned long

__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,

unsigned long arg2, unsigned long *result)

{

unsigned long status, dummy1, dummy2, dummy3, local_result;

vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);

asm volatile ("inl %%dx" :

"=a"(status),

"=c"(dummy1),

"=d"(dummy2),

"=b"(local_result),

"=S"(dummy3) :

"0"(VMW_BALLOON_HV_MAGIC),

"1"(cmd),

"2"(VMW_BALLOON_HV_PORT),

"3"(arg1),

"4"(arg2) :

"memory");

/* update the result if needed */

if (result)

*result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :

local_result;

/* update target when applicable */

if (status == VMW_BALLOON_SUCCESS &&

((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))

WRITE_ONCE(b->target, local_result);

if (status != VMW_BALLOON_SUCCESS &&

status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {

vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);

pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",

__func__, vmballoon_cmd_names[cmd], arg1, arg2,

status);

}

/* mark reset required accordingly */

if (status == VMW_BALLOON_ERROR_RESET)

b->reset_required = true;

return status;

}

static __always_inline unsigned long

vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,

unsigned long arg2)

{

unsigned long dummy;

return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);

}

/*

* Send "start" command to the host, communicating supported version

* of the protocol.

*/

static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)

{

unsigned long status, capabilities;

status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,

&capabilities);

switch (status) {

case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:

b->capabilities = capabilities;

break;

case VMW_BALLOON_SUCCESS:

b->capabilities = VMW_BALLOON_BASIC_CMDS;

break;

default:

return -EIO;

}

/*

* 2MB pages are only supported with batching. If batching is for some

* reason disabled, do not use 2MB pages, since otherwise the legacy

* mechanism is used with 2MB pages, causing a failure.

*/

b->max_page_size = VMW_BALLOON_4K_PAGE;

if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&

(b->capabilities & VMW_BALLOON_BATCHED_CMDS))

b->max_page_size = VMW_BALLOON_2M_PAGE;

return 0;

}

/**

* vmballoon_send_guest_id - communicate guest type to the host.

*

* @b: pointer to the balloon.

*

* Communicate guest type to the host so that it can adjust ballooning

* algorithm to the one most appropriate for the guest. This command

* is normally issued after sending "start" command and is part of

* standard reset sequence.

*

* Return: zero on success or appropriate error code.

*/

static int vmballoon_send_guest_id(struct vmballoon *b)

{

unsigned long status;

status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,

VMW_BALLOON_GUEST_ID, 0);

return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;

}

/**

* vmballoon_page_order() - return the order of the page

* @page_size: the size of the page.

*

* Return: the allocation order.

*/

static inline

unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)

{

return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;

}

/**

* vmballoon_page_in_frames() - returns the number of frames in a page.

* @page_size: the size of the page.

*

* Return: the number of 4k frames.

*/

static inline unsigned int

vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)

{

return 1 << vmballoon_page_order(page_size);

}

/**

* vmballoon_mark_page_offline() - mark a page as offline

* @page: pointer for the page.

* @page_size: the size of the page.

*/

static void

vmballoon_mark_page_offline(struct page *page,

enum vmballoon_page_size_type page_size)

{

int i;

for (i = 0; i < vmballoon_page_in_frames(page_size); i++)

__SetPageOffline(page + i);

}

/**

* vmballoon_mark_page_online() - mark a page as online

* @page: pointer for the page.

* @page_size: the size of the page.

*/

static void

vmballoon_mark_page_online(struct page *page,

enum vmballoon_page_size_type page_size)

{

int i;

for (i = 0; i < vmballoon_page_in_frames(page_size); i++)

__ClearPageOffline(page + i);

}

/**

* vmballoon_send_get_target() - Retrieve desired balloon size from the host.

*

* @b: pointer to the balloon.

*

* Return: zero on success, EINVAL if limit does not fit in 32-bit, as required

* by the host-guest protocol and EIO if an error occurred in communicating with

* the host.

*/

static int vmballoon_send_get_target(struct vmballoon *b)

{

unsigned long status;

unsigned long limit;

limit = totalram_pages();

/* Ensure limit fits in 32-bits if 64-bit targets are not supported */

if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&

limit != (u32)limit)

return -EINVAL;

status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);

return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;

}

/**

* vmballoon_alloc_page_list - allocates a list of pages.

*

* @b: pointer to the balloon.

* @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.

* @req_n_pages: the number of requested pages.

*

* Tries to allocate @req_n_pages. Add them to the list of balloon pages in

* @ctl.pages and updates @ctl.n_pages to reflect the number of pages.

*

* Return: zero on success or error code otherwise.

*/

static int vmballoon_alloc_page_list(struct vmballoon *b,

struct vmballoon_ctl *ctl,

unsigned int req_n_pages)

{

struct page *page;

unsigned int i;

for (i = 0; i < req_n_pages; i++) {

/*

* First check if we happen to have pages that were allocated

* before. This happens when 2MB page rejected during inflation

* by the hypervisor, and then split into 4KB pages.

*/

if (!list_empty(&ctl->prealloc_pages)) {

page = list_first_entry(&ctl->prealloc_pages,

struct page, lru);

list_del(&page->lru);

} else {

if (ctl->page_size == VMW_BALLOON_2M_PAGE)

page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|

__GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);

else

page = balloon_page_alloc();

vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,

ctl->page_size);

}

if (page) {

/* Success. Add the page to the list and continue. */

list_add(&page->lru, &ctl->pages);

continue;

}

/* Allocation failed. Update statistics and stop. */

vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,

ctl->page_size);

break;

}

ctl->n_pages = i;

return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;

}

/**

* vmballoon_handle_one_result - Handle lock/unlock result for a single page.

*

* @b: pointer for %struct vmballoon.

* @page: pointer for the page whose result should be handled.

* @page_size: size of the page.

* @status: status of the operation as provided by the hypervisor.

*/

static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,

enum vmballoon_page_size_type page_size,

unsigned long status)

{

/* On success do nothing. The page is already on the balloon list. */

if (likely(status == VMW_BALLOON_SUCCESS))

return 0;

pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,

page_to_pfn(page), status,

vmballoon_page_size_names[page_size]);

/* Error occurred */

vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,

page_size);

return -EIO;

}

/**

* vmballoon_status_page - returns the status of (un)lock operation

*

* @b: pointer to the balloon.

* @idx: index for the page for which the operation is performed.

* @p: pointer to where the page struct is returned.

*

* Following a lock or unlock operation, returns the status of the operation for

* an individual page. Provides the page that the operation was performed on on

* the @page argument.

*

* Returns: The status of a lock or unlock operation for an individual page.

*/

static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,

struct page **p)

{

if (static_branch_likely(&vmw_balloon_batching)) {

/* batching mode */

*p = pfn_to_page(b->batch_page[idx].pfn);

return b->batch_page[idx].status;

}

/* non-batching mode */

*p = b->page;

/*

* If a failure occurs, the indication will be provided in the status

* of the entire operation, which is considered before the individual

* page status. So for non-batching mode, the indication is always of

* success.

*/

return VMW_BALLOON_SUCCESS;

}

/**

* vmballoon_lock_op - notifies the host about inflated/deflated pages.

* @b: pointer to the balloon.

* @num_pages: number of inflated/deflated pages.

* @page_size: size of the page.

* @op: the type of operation (lock or unlock).

*

* Notify the host about page(s) that were ballooned (or removed from the

* balloon) so that host can use it without fear that guest will need it (or

* stop using them since the VM does). Host may reject some pages, we need to

* check the return value and maybe submit a different page. The pages that are

* inflated/deflated are pointed by @b->page.

*

* Return: result as provided by the hypervisor.

*/

static unsigned long vmballoon_lock_op(struct vmballoon *b,

unsigned int num_pages,

enum vmballoon_page_size_type page_size,

enum vmballoon_op op)

{

unsigned long cmd, pfn;

lockdep_assert_held(&b->comm_lock);

if (static_branch_likely(&vmw_balloon_batching)) {

if (op == VMW_BALLOON_INFLATE)

cmd = page_size == VMW_BALLOON_2M_PAGE ?

VMW_BALLOON_CMD_BATCHED_2M_LOCK :

VMW_BALLOON_CMD_BATCHED_LOCK;

else

cmd = page_size == VMW_BALLOON_2M_PAGE ?

VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :

VMW_BALLOON_CMD_BATCHED_UNLOCK;

pfn = PHYS_PFN(virt_to_phys(b->batch_page));

} else {

cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :

VMW_BALLOON_CMD_UNLOCK;

pfn = page_to_pfn(b->page);

/* In non-batching mode, PFNs must fit in 32-bit */

if (unlikely(pfn != (u32)pfn))

return VMW_BALLOON_ERROR_PPN_INVALID;

}

return vmballoon_cmd(b, cmd, pfn, num_pages);

}

/**

* vmballoon_add_page - adds a page towards lock/unlock operation.

*

* @b: pointer to the balloon.

* @idx: index of the page to be ballooned in this batch.

* @p: pointer to the page that is about to be ballooned.

*

* Adds the page to be ballooned. Must be called while holding @comm_lock.

*/

static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,

struct page *p)

{

lockdep_assert_held(&b->comm_lock);

if (static_branch_likely(&vmw_balloon_batching))

b->batch_page[idx] = (struct vmballoon_batch_entry)

{ .pfn = page_to_pfn(p) };

else

b->page = p;

}

/**

* vmballoon_lock - lock or unlock a batch of pages.

*

* @b: pointer to the balloon.

* @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.

*

* Notifies the host of about ballooned pages (after inflation or deflation,

* according to @ctl). If the host rejects the page put it on the

* @ctl refuse list. These refused page are then released when moving to the

* next size of pages.

*

* Note that we neither free any @page here nor put them back on the ballooned

* pages list. Instead we queue it for later processing. We do that for several

* reasons. First, we do not want to free the page under the lock. Second, it

* allows us to unify the handling of lock and unlock. In the inflate case, the

* caller will check if there are too many refused pages and release them.

* Although it is not identical to the past behavior, it should not affect

* performance.

*/

static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)

{

unsigned long batch_status;

struct page *page;

unsigned int i, num_pages;

num_pages = ctl->n_pages;

if (num_pages == 0)

return 0;

/* communication with the host is done under the communication lock */

spin_lock(&b->comm_lock);

i = 0;

list_for_each_entry(page, &ctl->pages, lru)

vmballoon_add_page(b, i++, page);

batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,

ctl->op);

/*

* Iterate over the pages in the provided list. Since we are changing

* @ctl->n_pages we are saving the original value in @num_pages and

* use this value to bound the loop.

*/

for (i = 0; i < num_pages; i++) {

unsigned long status;

status = vmballoon_status_page(b, i, &page);

/*

* Failure of the whole batch overrides a single operation

* results.

*/

if (batch_status != VMW_BALLOON_SUCCESS)

status = batch_status;

/* Continue if no error happened */

if (!vmballoon_handle_one_result(b, page, ctl->page_size,

status))

continue;

/*

* Error happened. Move the pages to the refused list and update

* the pages number.

*/

list_move(&page->lru, &ctl->refused_pages);

ctl->n_pages--;

ctl->n_refused_pages++;

}

spin_unlock(&b->comm_lock);

return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;

}

/**

* vmballoon_release_page_list() - Releases a page list

*

* @page_list: list of pages to release.

* @n_pages: pointer to the number of pages.

* @page_size: whether the pages in the list are 2MB (or else 4KB).

*

* Releases the list of pages and zeros the number of pages.

*/

static void vmballoon_release_page_list(struct list_head *page_list,

int *n_pages,

enum vmballoon_page_size_type page_size)

{

struct page *page, *tmp;

list_for_each_entry_safe(page, tmp, page_list, lru) {

list_del(&page->lru);

__free_pages(page, vmballoon_page_order(page_size));

}

if (n_pages)

*n_pages = 0;

}

/*

* Release pages that were allocated while attempting to inflate the

* balloon but were refused by the host for one reason or another.

*/

static void vmballoon_release_refused_pages(struct vmballoon *b,

struct vmballoon_ctl *ctl)

{

vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,

ctl->page_size);

vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,

ctl->page_size);

}

/**

* vmballoon_change - retrieve the required balloon change

*

* @b: pointer for the balloon.

*

* Return: the required change for the balloon size. A positive number

* indicates inflation, a negative number indicates a deflation.

*/

static int64_t vmballoon_change(struct vmballoon *b)

{

int64_t size, target;

size = atomic64_read(&b->size);

target = READ_ONCE(b->target);

/*

* We must cast first because of int sizes

* Otherwise we might get huge positives instead of negatives

*/

if (b->reset_required)

return 0;

/* consider a 2MB slack on deflate, unless the balloon is emptied */

if (target < size && target != 0 &&

size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))

return 0;

/* If an out-of-memory recently occurred, inflation is disallowed. */

if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))

return 0;

return target - size;

}

/**

* vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.

*

* @b: pointer to balloon.

* @pages: list of pages to enqueue.

* @n_pages: pointer to number of pages in list. The value is zeroed.

* @page_size: whether the pages are 2MB or 4KB pages.

*

* Enqueues the provides list of pages in the ballooned page list, clears the

* list and zeroes the number of pages that was provided.

*/

static void vmballoon_enqueue_page_list(struct vmballoon *b,

struct list_head *pages,

unsigned int *n_pages,

enum vmballoon_page_size_type page_size)

{

struct page *page;

if (page_size == VMW_BALLOON_4K_PAGE) {

balloon_page_list_enqueue(&b->b_dev_info, pages);

} else {

/*