#include <linux/seq_file.h>
#include <linux/rwsem.h>
#include <linux/slab.h>
+#include <linux/spinlock.h>
#include <linux/vmw_vmci_defs.h>
#include <linux/vmw_vmci_api.h>
#include <asm/hypervisor.h>
#define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)
+static const char * const vmballoon_page_size_names[] = {
+ [VMW_BALLOON_4K_PAGE] = "4k",
+ [VMW_BALLOON_2M_PAGE] = "2M"
+};
+
+enum vmballoon_op {
+ VMW_BALLOON_INFLATE,
+ VMW_BALLOON_DEFLATE
+};
+
enum vmballoon_op_stat_type {
VMW_BALLOON_OP_STAT,
VMW_BALLOON_OP_FAIL_STAT
static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
-struct vmballoon_page_size {
- /* list of reserved physical pages */
+struct vmballoon_ctl {
struct list_head pages;
-
- /* transient list of non-balloonable pages */
struct list_head refused_pages;
unsigned int n_refused_pages;
+ unsigned int n_pages;
+ enum vmballoon_page_size_type page_size;
+ enum vmballoon_op op;
+};
+
+struct vmballoon_page_size {
+ /* list of reserved physical pages */
+ struct list_head pages;
};
/**
struct vmballoon {
struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES];
- /* supported page sizes. 1 == 4k pages only, 2 == 4k and 2m pages */
- unsigned supported_page_sizes;
+ /**
+ * @max_page_size: maximum supported page size for ballooning.
+ *
+ * Protected by @conf_sem
+ */
+ enum vmballoon_page_size_type max_page_size;
+
+ /**
+ * @size: balloon actual size in basic page size (frames).
+ *
+ * While we currently do not support size which is bigger than 32-bit,
+ * in preparation for future support, use 64-bits.
+ */
+ atomic64_t size;
- /* balloon size in pages */
- unsigned int size;
- unsigned int target;
+ /**
+ * @target: balloon target size in basic page size (frames).
+ *
+ * We do not protect the target under the assumption that setting the
+ * value is always done through a single write. If this assumption ever
+ * breaks, we would have to use X_ONCE for accesses, and suffer the less
+ * optimized code. Although we may read stale target value if multiple
+ * accesses happen at once, the performance impact should be minor.
+ */
+ unsigned long target;
- /* reset flag */
+ /**
+ * @reset_required: reset flag
+ *
+ * Setting this flag may introduce races, but the code is expected to
+ * handle them gracefully. In the worst case, another operation will
+ * fail as reset did not take place. Clearing the flag is done while
+ * holding @conf_sem for write.
+ */
bool reset_required;
+ /**
+ * @capabilities: hypervisor balloon capabilities.
+ *
+ * Protected by @conf_sem.
+ */
unsigned long capabilities;
/**
*/
struct vmballoon_batch_entry *batch_page;
+ /**
+ * @batch_max_pages: maximum pages that can be locked/unlocked.
+ *
+ * Indicates the number of pages that the hypervisor can lock or unlock
+ * at once, according to whether batching is enabled. If batching is
+ * disabled, only a single page can be locked/unlock on each operation.
+ *
+ * Protected by @conf_sem.
+ */
unsigned int batch_max_pages;
+
+ /**
+ * @page: page to be locked/unlocked by the hypervisor
+ *
+ * @page is only used when batching is disabled and a single page is
+ * reclaimed on each iteration.
+ *
+ * Protected by @comm_lock.
+ */
struct page *page;
/* statistics */
struct delayed_work dwork;
+ /**
+ * @vmci_doorbell.
+ *
+ * Protected by @conf_sem.
+ */
struct vmci_handle vmci_doorbell;
/**
* @conf_sem: semaphore to protect the configuration and the statistics.
*/
struct rw_semaphore conf_sem;
+
+ /**
+ * @comm_lock: lock to protect the communication with the host.
+ *
+ * Lock ordering: @conf_sem -> @comm_lock .
+ */
+ spinlock_t comm_lock;
};
static struct vmballoon balloon;
static inline void vmballoon_stats_page_inc(struct vmballoon *b,
enum vmballoon_stat_page stat,
- bool is_2m_page)
+ enum vmballoon_page_size_type size)
{
if (is_vmballoon_stats_on())
- atomic64_inc(&b->stats->page_stat[stat][is_2m_page]);
+ atomic64_inc(&b->stats->page_stat[stat][size]);
+}
+
+static inline void vmballoon_stats_page_add(struct vmballoon *b,
+ enum vmballoon_stat_page stat,
+ enum vmballoon_page_size_type size,
+ unsigned int val)
+{
+ if (is_vmballoon_stats_on())
+ atomic64_add(val, &b->stats->page_stat[stat][size]);
}
static inline unsigned long
/* update target when applicable */
if (status == VMW_BALLOON_SUCCESS &&
((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
- b->target = local_result;
+ WRITE_ONCE(b->target, local_result);
if (status != VMW_BALLOON_SUCCESS &&
status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
* reason disabled, do not use 2MB pages, since otherwise the legacy
* mechanism is used with 2MB pages, causing a failure.
*/
+ b->max_page_size = VMW_BALLOON_4K_PAGE;
if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&
(b->capabilities & VMW_BALLOON_BATCHED_CMDS))
- b->supported_page_sizes = 2;
- else
- b->supported_page_sizes = 1;
+ b->max_page_size = VMW_BALLOON_2M_PAGE;
+
return success;
}
return false;
}
-static u16 vmballoon_page_size(bool is_2m_page)
+/**
+ * vmballoon_page_order() - return the order of the page
+ * @page_size: the size of the page.
+ *
+ * Return: the allocation order.
+ */
+static inline
+unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)
{
- if (is_2m_page)
- return 1 << VMW_BALLOON_2M_ORDER;
+ return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;
+}
- return 1;
+/**
+ * vmballoon_page_in_frames() - returns the number of frames in a page.
+ * @page_size: the size of the page.
+ *
+ * Return: the number of 4k frames.
+ */
+static inline unsigned int
+vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)
+{
+ return 1 << vmballoon_page_order(page_size);
}
/**
return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
-static struct page *vmballoon_alloc_page(bool is_2m_page)
+/**
+ * vmballoon_alloc_page_list - allocates a list of pages.
+ *
+ * @b: pointer to the balloon.
+ * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
+ * @req_n_pages: the number of requested pages.
+ *
+ * Tries to allocate @req_n_pages. Add them to the list of balloon pages in
+ * @ctl.pages and updates @ctl.n_pages to reflect the number of pages.
+ *
+ * Return: zero on success or error code otherwise.
+ */
+static int vmballoon_alloc_page_list(struct vmballoon *b,
+ struct vmballoon_ctl *ctl,
+ unsigned int req_n_pages)
{
- if (is_2m_page)
- return alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
- VMW_BALLOON_2M_ORDER);
+ struct page *page;
+ unsigned int i;
- return alloc_page(VMW_PAGE_ALLOC_FLAGS);
-}
+ for (i = 0; i < req_n_pages; i++) {
+ if (ctl->page_size == VMW_BALLOON_2M_PAGE)
+ page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
+ VMW_BALLOON_2M_ORDER);
+ else
+ page = alloc_page(VMW_PAGE_ALLOC_FLAGS);
-static void vmballoon_free_page(struct page *page, bool is_2m_page)
-{
- if (is_2m_page)
- __free_pages(page, VMW_BALLOON_2M_ORDER);
- else
- __free_page(page);
+ /* Update statistics */
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
+ ctl->page_size);
+
+ if (page) {
+ /* Success. Add the page to the list and continue. */
+ list_add(&page->lru, &ctl->pages);
+ continue;
+ }
+
+ /* Allocation failed. Update statistics and stop. */
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
+ ctl->page_size);
+ break;
+ }
+
+ ctl->n_pages = i;
+
+ return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;
}
-/*
- * Quickly release all pages allocated for the balloon. This function is
- * called when host decides to "reset" balloon for one reason or another.
- * Unlike normal "deflate" we do not (shall not) notify host of the pages
- * being released.
+/**
+ * vmballoon_handle_one_result - Handle lock/unlock result for a single page.
+ *
+ * @b: pointer for %struct vmballoon.
+ * @page: pointer for the page whose result should be handled.
+ * @page_size: size of the page.
+ * @status: status of the operation as provided by the hypervisor.
*/
-static void vmballoon_pop(struct vmballoon *b)
+static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,
+ enum vmballoon_page_size_type page_size,
+ unsigned long status)
{
- struct page *page, *next;
- unsigned is_2m_pages;
-
- for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES;
- is_2m_pages++) {
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
- u16 size_per_page = vmballoon_page_size(is_2m_pages);
-
- list_for_each_entry_safe(page, next, &page_size->pages, lru) {
- list_del(&page->lru);
- vmballoon_free_page(page, is_2m_pages);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_FREE,
- is_2m_pages);
- b->size -= size_per_page;
- cond_resched();
- }
- }
+ /* On success do nothing. The page is already on the balloon list. */
+ if (likely(status == VMW_BALLOON_SUCCESS))
+ return 0;
- /* Clearing the batch_page unconditionally has no adverse effect */
- free_page((unsigned long)b->batch_page);
- b->batch_page = NULL;
+ pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,
+ page_to_pfn(page), status,
+ vmballoon_page_size_names[page_size]);
+
+ /* Error occurred */
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
+ page_size);
+
+ return -EIO;
}
/**
* vmballoon_lock_op - notifies the host about inflated/deflated pages.
* @b: pointer to the balloon.
* @num_pages: number of inflated/deflated pages.
- * @is_2m_pages: whether the page(s) are 2M (or 4k).
- * @lock: whether the operation is lock (or unlock).
+ * @page_size: size of the page.
+ * @op: the type of operation (lock or unlock).
*
* Notify the host about page(s) that were ballooned (or removed from the
* balloon) so that host can use it without fear that guest will need it (or
*/
static unsigned long vmballoon_lock_op(struct vmballoon *b,
unsigned int num_pages,
- bool is_2m_pages, bool lock)
+ enum vmballoon_page_size_type page_size,
+ enum vmballoon_op op)
{
unsigned long cmd, pfn;
+ lockdep_assert_held(&b->comm_lock);
+
if (static_branch_likely(&vmw_balloon_batching)) {
- if (lock)
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_LOCK :
- VMW_BALLOON_CMD_BATCHED_LOCK;
+ if (op == VMW_BALLOON_INFLATE)
+ cmd = page_size == VMW_BALLOON_2M_PAGE ?
+ VMW_BALLOON_CMD_BATCHED_2M_LOCK :
+ VMW_BALLOON_CMD_BATCHED_LOCK;
else
- cmd = is_2m_pages ? VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
- VMW_BALLOON_CMD_BATCHED_UNLOCK;
+ cmd = page_size == VMW_BALLOON_2M_PAGE ?
+ VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
+ VMW_BALLOON_CMD_BATCHED_UNLOCK;
pfn = PHYS_PFN(virt_to_phys(b->batch_page));
} else {
- cmd = lock ? VMW_BALLOON_CMD_LOCK : VMW_BALLOON_CMD_UNLOCK;
+ cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :
+ VMW_BALLOON_CMD_UNLOCK;
pfn = page_to_pfn(b->page);
/* In non-batching mode, PFNs must fit in 32-bit */
return vmballoon_cmd(b, cmd, pfn, num_pages);
}
-static int vmballoon_lock(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages)
+/**
+ * vmballoon_add_page - adds a page towards lock/unlock operation.
+ *
+ * @b: pointer to the balloon.
+ * @idx: index of the page to be ballooned in this batch.
+ * @p: pointer to the page that is about to be ballooned.
+ *
+ * Adds the page to be ballooned. Must be called while holding @comm_lock.
+ */
+static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,
+ struct page *p)
+{
+ lockdep_assert_held(&b->comm_lock);
+
+ if (static_branch_likely(&vmw_balloon_batching))
+ b->batch_page[idx] = (struct vmballoon_batch_entry)
+ { .pfn = page_to_pfn(p) };
+ else
+ b->page = p;
+}
+
+/**
+ * vmballoon_lock - lock or unlock a batch of pages.
+ *
+ * @b: pointer to the balloon.
+ * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
+ *
+ * Notifies the host of about ballooned pages (after inflation or deflation,
+ * according to @ctl). If the host rejects the page put it on the
+ * @ctl refuse list. These refused page are then released when moving to the
+ * next size of pages.
+ *
+ * Note that we neither free any @page here nor put them back on the ballooned
+ * pages list. Instead we queue it for later processing. We do that for several
+ * reasons. First, we do not want to free the page under the lock. Second, it
+ * allows us to unify the handling of lock and unlock. In the inflate case, the
+ * caller will check if there are too many refused pages and release them.
+ * Although it is not identical to the past behavior, it should not affect
+ * performance.
+ */
+static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)
{
unsigned long batch_status;
- int i;
- u16 size_per_page = vmballoon_page_size(is_2m_pages);
+ struct page *page;
+ unsigned int i, num_pages;
+
+ num_pages = ctl->n_pages;
+ if (num_pages == 0)
+ return 0;
- batch_status = vmballoon_lock_op(b, num_pages, is_2m_pages, true);
+ /* communication with the host is done under the communication lock */
+ spin_lock(&b->comm_lock);
+ i = 0;
+ list_for_each_entry(page, &ctl->pages, lru)
+ vmballoon_add_page(b, i++, page);
+
+ batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,
+ ctl->op);
+
+ /*
+ * Iterate over the pages in the provided list. Since we are changing
+ * @ctl->n_pages we are saving the original value in @num_pages and
+ * use this value to bound the loop.
+ */
for (i = 0; i < num_pages; i++) {
unsigned long status;
- struct page *p;
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
- status = vmballoon_status_page(b, i, &p);
+ status = vmballoon_status_page(b, i, &page);
/*
* Failure of the whole batch overrides a single operation
if (batch_status != VMW_BALLOON_SUCCESS)
status = batch_status;
- if (status == VMW_BALLOON_SUCCESS) {
- /* track allocated page */
- list_add(&p->lru, &page_size->pages);
-
- /* update balloon size */
- b->size += size_per_page;
+ /* Continue if no error happened */
+ if (!vmballoon_handle_one_result(b, page, ctl->page_size,
+ status))
continue;
- }
-
- /* Error occurred */
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
- is_2m_pages);
/*
- * Place page on the list of non-balloonable pages
- * and retry allocation, unless we already accumulated
- * too many of them, in which case take a breather.
+ * Error happened. Move the pages to the refused list and update
+ * the pages number.
*/
- list_add(&p->lru, &page_size->refused_pages);
- page_size->n_refused_pages++;
+ list_move(&page->lru, &ctl->refused_pages);
+ ctl->n_pages--;
+ ctl->n_refused_pages++;
}
+ spin_unlock(&b->comm_lock);
+
return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
}
-/*
- * Release the page allocated for the balloon. Note that we first notify
- * the host so it can make sure the page will be available for the guest
- * to use, if needed.
+/**
+ * vmballoon_release_page_list() - Releases a page list
+ *
+ * @page_list: list of pages to release.
+ * @n_pages: pointer to the number of pages.
+ * @page_size: whether the pages in the list are 2MB (or else 4KB).
+ *
+ * Releases the list of pages and zeros the number of pages.
*/
-static int vmballoon_unlock(struct vmballoon *b, unsigned int num_pages,
- bool is_2m_pages)
+static void vmballoon_release_page_list(struct list_head *page_list,
+ int *n_pages,
+ enum vmballoon_page_size_type page_size)
{
- int i;
- unsigned long batch_status;
- u16 size_per_page = vmballoon_page_size(is_2m_pages);
-
- batch_status = vmballoon_lock_op(b, num_pages, is_2m_pages, false);
-
- for (i = 0; i < num_pages; i++) {
- struct vmballoon_page_size *page_size;
- unsigned long status;
- struct page *p;
+ struct page *page, *tmp;
- status = vmballoon_status_page(b, i, &p);
- page_size = &b->page_sizes[is_2m_pages];
-
- /*
- * Failure of the whole batch overrides a single operation
- * results.
- */
- if (batch_status != VMW_BALLOON_SUCCESS)
- status = batch_status;
-
- if (status != VMW_BALLOON_SUCCESS) {
- /*
- * That page wasn't successfully unlocked by the
- * hypervisor, re-add it to the list of pages owned by
- * the balloon driver.
- */
- list_add(&p->lru, &page_size->pages);
- } else {
- /* deallocate page */
- vmballoon_free_page(p, is_2m_pages);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_FREE,
- is_2m_pages);
-
- /* update balloon size */
- b->size -= size_per_page;
- }
+ list_for_each_entry_safe(page, tmp, page_list, lru) {
+ list_del(&page->lru);
+ __free_pages(page, vmballoon_page_order(page_size));
}
- return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
+ *n_pages = 0;
}
+
/*
* Release pages that were allocated while attempting to inflate the
* balloon but were refused by the host for one reason or another.
*/
static void vmballoon_release_refused_pages(struct vmballoon *b,
- bool is_2m_pages)
+ struct vmballoon_ctl *ctl)
{
- struct page *page, *next;
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
+ vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
+ ctl->page_size);
- list_for_each_entry_safe(page, next, &page_size->refused_pages, lru) {
- list_del(&page->lru);
- vmballoon_free_page(page, is_2m_pages);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
- is_2m_pages);
- }
-
- page_size->n_refused_pages = 0;
-}
-
-static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p)
-{
- if (static_branch_likely(&vmw_balloon_batching))
- b->batch_page[idx] = (struct vmballoon_batch_entry)
- { .pfn = page_to_pfn(p) };
- else
- b->page = p;
+ vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,
+ ctl->page_size);
}
/**
{
int64_t size, target;
- size = b->size;
- target = b->target;
+ size = atomic64_read(&b->size);
+ target = READ_ONCE(b->target);
/*
* We must cast first because of int sizes
return 0;
/* consider a 2MB slack on deflate, unless the balloon is emptied */
- if (target < size && size - target < vmballoon_page_size(true) &&
- target != 0)
+ if (target < size && target != 0 &&
+ size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))
return 0;
return target - size;
}
-/*
- * Inflate the balloon towards its target size. Note that we try to limit
- * the rate of allocation to make sure we are not choking the rest of the
- * system.
+/**
+ * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.
+ *
+ * @b: pointer to balloon.
+ * @pages: list of pages to enqueue.
+ * @n_pages: pointer to number of pages in list. The value is zeroed.
+ * @page_size: whether the pages are 2MB or 4KB pages.
+ *
+ * Enqueues the provides list of pages in the ballooned page list, clears the
+ * list and zeroes the number of pages that was provided.
+ */
+static void vmballoon_enqueue_page_list(struct vmballoon *b,
+ struct list_head *pages,
+ unsigned int *n_pages,
+ enum vmballoon_page_size_type page_size)
+{
+ struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
+
+ list_splice_init(pages, &page_size_info->pages);
+ *n_pages = 0;
+}
+
+/**
+ * vmballoon_dequeue_page_list() - Dequeues page lists for deflation.
+ *
+ * @b: pointer to balloon.
+ * @pages: list of pages to enqueue.
+ * @n_pages: pointer to number of pages in list. The value is zeroed.
+ * @page_size: whether the pages are 2MB or 4KB pages.
+ * @n_req_pages: the number of requested pages.
+ *
+ * Dequeues the number of requested pages from the balloon for deflation. The
+ * number of dequeued pages may be lower, if not enough pages in the requested
+ * size are available.
+ */
+static void vmballoon_dequeue_page_list(struct vmballoon *b,
+ struct list_head *pages,
+ unsigned int *n_pages,
+ enum vmballoon_page_size_type page_size,
+ unsigned int n_req_pages)
+{
+ struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
+ struct page *page, *tmp;
+ unsigned int i = 0;
+
+ list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) {
+ list_move(&page->lru, pages);
+ if (++i == n_req_pages)
+ break;
+ }
+ *n_pages = i;
+}
+
+/**
+ * vmballoon_inflate() - Inflate the balloon towards its target size.
+ *
+ * @b: pointer to the balloon.
*/
static void vmballoon_inflate(struct vmballoon *b)
{
- unsigned int num_pages = 0;
- int error = 0;
- bool is_2m_pages;
+ int64_t to_inflate_frames;
+ struct vmballoon_ctl ctl = {
+ .pages = LIST_HEAD_INIT(ctl.pages),
+ .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
+ .page_size = b->max_page_size,
+ .op = VMW_BALLOON_INFLATE
+ };
- /*
- * First try NOSLEEP page allocations to inflate balloon.
- *
- * If we do not throttle nosleep allocations, we can drain all
- * free pages in the guest quickly (if the balloon target is high).
- * As a side-effect, draining free pages helps to inform (force)
- * the guest to start swapping if balloon target is not met yet,
- * which is a desired behavior. However, balloon driver can consume
- * all available CPU cycles if too many pages are allocated in a
- * second. Therefore, we throttle nosleep allocations even when
- * the guest is not under memory pressure. OTOH, if we have already
- * predicted that the guest is under memory pressure, then we
- * slowdown page allocations considerably.
- */
+ while ((to_inflate_frames = vmballoon_change(b)) > 0) {
+ unsigned int to_inflate_pages, page_in_frames;
+ int alloc_error, lock_error = 0;
- /*
- * Start with no sleep allocation rate which may be higher
- * than sleeping allocation rate.
- */
- is_2m_pages = b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES;
+ VM_BUG_ON(!list_empty(&ctl.pages));
+ VM_BUG_ON(ctl.n_pages != 0);
- while ((int64_t)(num_pages * vmballoon_page_size(is_2m_pages)) <
- vmballoon_change(b)) {
- struct page *page;
+ page_in_frames = vmballoon_page_in_frames(ctl.page_size);
- vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
- is_2m_pages);
-
- page = vmballoon_alloc_page(is_2m_pages);
- if (!page) {
- vmballoon_stats_page_inc(b,
- VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, is_2m_pages);
-
- if (is_2m_pages) {
- vmballoon_lock(b, num_pages, true);
-
- /*
- * ignore errors from locking as we now switch
- * to 4k pages and we might get different
- * errors.
- */
-
- num_pages = 0;
- is_2m_pages = false;
- continue;
- }
+ to_inflate_pages = min_t(unsigned long, b->batch_max_pages,
+ DIV_ROUND_UP_ULL(to_inflate_frames,
+ page_in_frames));
+
+ /* Start by allocating */
+ alloc_error = vmballoon_alloc_page_list(b, &ctl,
+ to_inflate_pages);
+
+ /* Actually lock the pages by telling the hypervisor */
+ lock_error = vmballoon_lock(b, &ctl);
+
+ /*
+ * If an error indicates that something serious went wrong,
+ * stop the inflation.
+ */
+ if (lock_error)
break;
- }
- vmballoon_add_page(b, num_pages++, page);
- if (num_pages == b->batch_max_pages) {
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
+ /* Update the balloon size */
+ atomic64_add(ctl.n_pages * page_in_frames, &b->size);
- error = vmballoon_lock(b, num_pages, is_2m_pages);
+ vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,
+ ctl.page_size);
- num_pages = 0;
+ /*
+ * If allocation failed or the number of refused pages exceeds
+ * the maximum allowed, move to the next page size.
+ */
+ if (alloc_error ||
+ ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) {
+ if (ctl.page_size == VMW_BALLOON_4K_PAGE)
+ break;
/*
- * Stop allocating this page size if we already
- * accumulated too many pages that the hypervisor
- * refused.
+ * Ignore errors from locking as we now switch to 4k
+ * pages and we might get different errors.
*/
- if (page_size->n_refused_pages >=
- VMW_BALLOON_MAX_REFUSED) {
- if (!is_2m_pages)
- break;
-
- /*
- * Release the refused pages as we move to 4k
- * pages.
- */
- vmballoon_release_refused_pages(b, true);
- is_2m_pages = true;
- }
-
- if (error)
- break;
+ vmballoon_release_refused_pages(b, &ctl);
+ ctl.page_size--;
}
cond_resched();
}
- if (num_pages > 0)
- vmballoon_lock(b, num_pages, is_2m_pages);
-
- vmballoon_release_refused_pages(b, true);
- vmballoon_release_refused_pages(b, false);
+ /*
+ * Release pages that were allocated while attempting to inflate the
+ * balloon but were refused by the host for one reason or another,
+ * and update the statistics.
+ */
+ if (ctl.n_refused_pages != 0)
+ vmballoon_release_refused_pages(b, &ctl);
}
-/*
+/**
+ * vmballoon_deflate() - Decrease the size of the balloon.
+ *
+ * @b: pointer to the balloon
+ * @n_frames: the number of frames to deflate. If zero, automatically
+ * calculated according to the target size.
+ * @coordinated: whether to coordinate with the host
+ *
* Decrease the size of the balloon allowing guest to use more memory.
+ *
+ * Return: The number of deflated frames (i.e., basic page size units)
*/
-static void vmballoon_deflate(struct vmballoon *b)
+static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,
+ bool coordinated)
{
- unsigned is_2m_pages;
+ unsigned long deflated_frames = 0;
+ unsigned long tried_frames = 0;
+ struct vmballoon_ctl ctl = {
+ .pages = LIST_HEAD_INIT(ctl.pages),
+ .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
+ .page_size = VMW_BALLOON_4K_PAGE,
+ .op = VMW_BALLOON_DEFLATE
+ };
/* free pages to reach target */
- for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes;
- is_2m_pages++) {
- struct page *page, *next;
- unsigned int num_pages = 0;
- struct vmballoon_page_size *page_size =
- &b->page_sizes[is_2m_pages];
-
- list_for_each_entry_safe(page, next, &page_size->pages, lru) {
- if ((int64_t)(num_pages *
- vmballoon_page_size(is_2m_pages)) >=
- -vmballoon_change(b))
- break;
+ while (true) {
+ unsigned int to_deflate_pages, n_unlocked_frames;
+ unsigned int page_in_frames;
+ int64_t to_deflate_frames;
+ bool deflated_all;
+
+ page_in_frames = vmballoon_page_in_frames(ctl.page_size);
+
+ VM_BUG_ON(!list_empty(&ctl.pages));
+ VM_BUG_ON(ctl.n_pages);
+ VM_BUG_ON(!list_empty(&ctl.refused_pages));
+ VM_BUG_ON(ctl.n_refused_pages);
+
+ /*
+ * If we were requested a specific number of frames, we try to
+ * deflate this number of frames. Otherwise, deflation is
+ * performed according to the target and balloon size.
+ */
+ to_deflate_frames = n_frames ? n_frames - tried_frames :
+ -vmballoon_change(b);
+
+ /* break if no work to do */
+ if (to_deflate_frames <= 0)
+ break;
+
+ /*
+ * Calculate the number of frames based on current page size,
+ * but limit the deflated frames to a single chunk
+ */
+ to_deflate_pages = min_t(unsigned long, b->batch_max_pages,
+ DIV_ROUND_UP_ULL(to_deflate_frames,
+ page_in_frames));
+
+ /* First take the pages from the balloon pages. */
+ vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,
+ ctl.page_size, to_deflate_pages);
- list_del(&page->lru);
- vmballoon_add_page(b, num_pages++, page);
+ /*
+ * Before pages are moving to the refused list, count their
+ * frames as frames that we tried to deflate.
+ */
+ tried_frames += ctl.n_pages * page_in_frames;
+
+ /*
+ * Unlock the pages by communicating with the hypervisor if the
+ * communication is coordinated (i.e., not pop). We ignore the
+ * return code. Instead we check if all the pages we manage to
+ * unlock all the pages. If we failed, we will move to the next
+ * page size, and would eventually try again later.
+ */
+ if (coordinated)
+ vmballoon_lock(b, &ctl);
+
+ /*
+ * Check if we deflated enough. We will move to the next page
+ * size if we did not manage to do so. This calculation takes
+ * place now, as once the pages are released, the number of
+ * pages is zeroed.
+ */
+ deflated_all = (ctl.n_pages == to_deflate_pages);
- if (num_pages == b->batch_max_pages) {
- int error;
+ /* Update local and global counters */
+ n_unlocked_frames = ctl.n_pages * page_in_frames;
+ atomic64_sub(n_unlocked_frames, &b->size);
+ deflated_frames += n_unlocked_frames;
- error = vmballoon_unlock(b, num_pages,
- is_2m_pages);
- num_pages = 0;
- if (error)
- return;
- }
+ vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,
+ ctl.page_size, ctl.n_pages);
- cond_resched();
+ /* free the ballooned pages */
+ vmballoon_release_page_list(&ctl.pages, &ctl.n_pages,
+ ctl.page_size);
+
+ /* Return the refused pages to the ballooned list. */
+ vmballoon_enqueue_page_list(b, &ctl.refused_pages,
+ &ctl.n_refused_pages,
+ ctl.page_size);
+
+ /* If we failed to unlock all the pages, move to next size. */
+ if (!deflated_all) {
+ if (ctl.page_size == b->max_page_size)
+ break;
+ ctl.page_size++;
}
- if (num_pages > 0)
- vmballoon_unlock(b, num_pages, is_2m_pages);
+ cond_resched();
}
+
+ return deflated_frames;
}
/**
return -EIO;
}
+/**
+ * vmballoon_pop - Quickly release all pages allocate for the balloon.
+ *
+ * @b: pointer to the balloon.
+ *
+ * This function is called when host decides to "reset" balloon for one reason
+ * or another. Unlike normal "deflate" we do not (shall not) notify host of the
+ * pages being released.
+ */
+static void vmballoon_pop(struct vmballoon *b)
+{
+ unsigned long size;
+
+ while ((size = atomic64_read(&b->size)))
+ vmballoon_deflate(b, size, false);
+}
+
/*
* Perform standard reset sequence by popping the balloon (in case it
* is not empty) and then restarting protocol. This operation normally
change = vmballoon_change(b);
if (change != 0) {
- pr_debug("%s - size: %u, target %u", __func__,
- b->size, b->target);
+ pr_debug("%s - size: %llu, target %lu\n", __func__,
+ atomic64_read(&b->size), READ_ONCE(b->target));
if (change > 0)
vmballoon_inflate(b);
else /* (change < 0) */
- vmballoon_deflate(b);
+ vmballoon_deflate(b, 0, true);
}
up_read(&b->conf_sem);
[VMW_BALLOON_STAT_DOORBELL] = "doorbell"
};
-static const char * const vmballoon_page_size_names[] = {
- [VMW_BALLOON_4K_PAGE] = "4k",
- [VMW_BALLOON_2M_PAGE] = "2M"
-};
-
static int vmballoon_enable_stats(struct vmballoon *b)
{
int r = 0;
}
/* format capabilities info */
- seq_printf(f, "%-22s: %#4x\n", "balloon capabilities",
+ seq_printf(f, "%-22s: %#16x\n", "balloon capabilities",
VMW_BALLOON_CAPABILITIES);
- seq_printf(f, "%-22s: %#4lx\n", "used capabilities",
- b->capabilities);
+ seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);
seq_printf(f, "%-22s: %16s\n", "is resetting",
b->reset_required ? "y" : "n");
/* format size info */
- seq_printf(f, "%-22s: %16u\n", "target", b->target);
- seq_printf(f, "%-22s: %16u\n", "current", b->size);
+ seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));
+ seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));
for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
if (vmballoon_cmd_names[i] == NULL)
static int __init vmballoon_init(void)
{
+ enum vmballoon_page_size_type page_size;
int error;
- unsigned is_2m_pages;
+
/*
* Check if we are running on VMware's hypervisor and bail out
* if we are not.
if (x86_hyper_type != X86_HYPER_VMWARE)
return -ENODEV;
- for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES;
- is_2m_pages++) {
- INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].pages);
- INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].refused_pages);
- }
+ for (page_size = VMW_BALLOON_4K_PAGE;
+ page_size <= VMW_BALLOON_LAST_SIZE; page_size++)
+ INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages);
+
INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
if (error)
return error;
+ spin_lock_init(&balloon.comm_lock);
init_rwsem(&balloon.conf_sem);
balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
balloon.batch_page = NULL;
projects / openwrt / staging / blogic.git / commitdiff