@@ -693,34 +693,17 @@ static struct page_info *alloc_heap_pages(
unsigned int order, unsigned int memflags,
struct domain *d)
{
- unsigned int i, j, zone = 0, nodemask_retry = 0;
- nodeid_t first_node, node = MEMF_get_node(memflags), req_node = node;
+ unsigned int i, j, zone, nodemask_retry;
+ nodeid_t first_node, node, req_node;
unsigned long request = 1UL << order;
struct page_info *pg;
- nodemask_t nodemask = (d != NULL ) ? d->node_affinity : node_online_map;
- bool_t need_tlbflush = 0;
+ nodemask_t nodemask;
+ bool_t need_tlbflush = 0, use_unscrubbed = 0;
uint32_t tlbflush_timestamp = 0;
/* Make sure there are enough bits in memflags for nodeID. */
BUILD_BUG_ON((_MEMF_bits - _MEMF_node) < (8 * sizeof(nodeid_t)));
- if ( node == NUMA_NO_NODE )
- {
- if ( d != NULL )
- {
- node = next_node(d->last_alloc_node, nodemask);
- if ( node >= MAX_NUMNODES )
- node = first_node(nodemask);
- }
- if ( node >= MAX_NUMNODES )
- node = cpu_to_node(smp_processor_id());
- }
- first_node = node;
-
- ASSERT(node < MAX_NUMNODES);
- ASSERT(zone_lo <= zone_hi);
- ASSERT(zone_hi < NR_ZONES);
-
if ( unlikely(order > MAX_ORDER) )
return NULL;
@@ -734,7 +717,10 @@ static struct page_info *alloc_heap_pages(
total_avail_pages + tmem_freeable_pages()) &&
((memflags & MEMF_no_refcount) ||
!d || d->outstanding_pages < request) )
- goto not_found;
+ {
+ spin_unlock(&heap_lock);
+ return NULL;
+ }
/*
* TMEM: When available memory is scarce due to tmem absorbing it, allow
@@ -747,6 +733,28 @@ static struct page_info *alloc_heap_pages(
tmem_freeable_pages() )
goto try_tmem;
+ again:
+
+ nodemask_retry = 0;
+ nodemask = (d != NULL ) ? d->node_affinity : node_online_map;
+ node = req_node = MEMF_get_node(memflags);
+ if ( node == NUMA_NO_NODE )
+ {
+ if ( d != NULL )
+ {
+ node = next_node(d->last_alloc_node, nodemask);
+ if ( node >= MAX_NUMNODES )
+ node = first_node(nodemask);
+ }
+ if ( node >= MAX_NUMNODES )
+ node = cpu_to_node(smp_processor_id());
+ }
+ first_node = node;
+
+ ASSERT(node < MAX_NUMNODES);
+ ASSERT(zone_lo <= zone_hi);
+ ASSERT(zone_hi < NR_ZONES);
+
/*
* Start with requested node, but exhaust all node memory in requested
* zone before failing, only calc new node value if we fail to find memory
@@ -762,8 +770,16 @@ static struct page_info *alloc_heap_pages(
/* Find smallest order which can satisfy the request. */
for ( j = order; j <= MAX_ORDER; j++ )
+ {
if ( (pg = page_list_remove_head(&heap(node, zone, j))) )
- goto found;
+ {
+ if ( (order == 0) || use_unscrubbed ||
+ !test_bit(_PGC_need_scrub, &pg[0].count_info) )
+ goto found;
+
+ page_list_add_tail(pg, &heap(node, zone, j));
+ }
+ }
} while ( zone-- > zone_lo ); /* careful: unsigned zone may wrap */
if ( (memflags & MEMF_exact_node) && req_node != NUMA_NO_NODE )
@@ -802,6 +818,16 @@ static struct page_info *alloc_heap_pages(
}
not_found:
+ /*
+ * If we couldn't find clean page let's search again and this time
+ * take unscrubbed pages if available.
+ */
+ if ( !use_unscrubbed )
+ {
+ use_unscrubbed = 1;
+ goto again;
+ }
+
/* No suitable memory blocks. Fail the request. */
spin_unlock(&heap_lock);
return NULL;
@@ -811,7 +837,10 @@ static struct page_info *alloc_heap_pages(
while ( j != order )
{
PFN_ORDER(pg) = --j;
- page_list_add(pg, &heap(node, zone, j));
+ if ( test_bit(_PGC_need_scrub, &pg[0].count_info) )
+ page_list_add_tail(pg, &heap(node, zone, j));
+ else
+ page_list_add(pg, &heap(node, zone, j));
pg += 1 << j;
}
@@ -825,6 +854,21 @@ static struct page_info *alloc_heap_pages(
if ( d != NULL )
d->last_alloc_node = node;
+ /*
+ * Whole chunk is either clean or dirty so we can
+ * test only the head.
+ */
+ if ( test_bit(_PGC_need_scrub, &pg[0].count_info) )
+ {
+ for ( i = 0; i < (1 << order); i++ )
+ {
+ scrub_one_page(&pg[i]);
+ pg[i].count_info &= ~PGC_need_scrub;
+ }
+ node_need_scrub[node] -= (1 << order);
+ }
+
+
for ( i = 0; i < (1 << order); i++ )
{
/* Reference count must continuously be zero for free pages. */
When allocating pages in alloc_heap_pages() first look for clean pages. If none is found then retry, take pages marked as unscrubbed and scrub them. Note that we shouldn't find unscrubbed pages in alloc_heap_pages() yet. However, this will become possible when we stop scrubbing from free_heap_pages() and instead do it from idle loop. Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com> --- xen/common/page_alloc.c | 92 ++++++++++++++++++++++++++++++++++------------ 1 files changed, 68 insertions(+), 24 deletions(-)