@@ -257,9 +257,12 @@ struct swap_cluster_info {
unsigned int data:24;
unsigned int flags:4;
unsigned int order:4;
+ unsigned int nr_scanners;
};
#define CLUSTER_FLAG_FREE 1 /* This cluster is free */
#define CLUSTER_FLAG_NEXT_NULL 2 /* This cluster has no next cluster */
+#define CLUSTER_FLAG_SKIP_SCAN 4 /* Skip cluster for per-order scan */
+#define CLUSTER_FLAG_DECREMENT 8 /* A swap entry was freed from cluster */
/*
* swap_info_struct::max is an unsigned int, so the maximum number of pages in
@@ -574,6 +574,9 @@ static void add_cluster_info_page(struct swap_info_struct *p,
VM_BUG_ON(cluster_count(&cluster_info[idx]) + count > SWAPFILE_CLUSTER);
cluster_set_count(&cluster_info[idx],
cluster_count(&cluster_info[idx]) + count);
+
+ if (SWAPFILE_CLUSTER - cluster_count(&cluster_info[idx]) < count)
+ cluster_info[idx].flags |= CLUSTER_FLAG_SKIP_SCAN;
}
/*
@@ -595,6 +598,7 @@ static void dec_cluster_info_page(struct swap_info_struct *p,
struct swap_cluster_info *cluster_info, unsigned long page_nr)
{
unsigned long idx = page_nr / SWAPFILE_CLUSTER;
+ unsigned long count = 1 << cluster_info[idx].order;
if (!cluster_info)
return;
@@ -603,6 +607,10 @@ static void dec_cluster_info_page(struct swap_info_struct *p,
cluster_set_count(&cluster_info[idx],
cluster_count(&cluster_info[idx]) - 1);
+ cluster_info[idx].flags |= CLUSTER_FLAG_DECREMENT;
+ if (SWAPFILE_CLUSTER - cluster_count(&cluster_info[idx]) >= count)
+ cluster_info[idx].flags &= ~CLUSTER_FLAG_SKIP_SCAN;
+
if (cluster_count(&cluster_info[idx]) == 0)
free_cluster(p, idx);
}
@@ -708,7 +716,8 @@ static unsigned int next_cluster_for_scan(struct swap_info_struct *si,
end = offset_to_cluster(si, *stop);
while (ci != end) {
- if ((ci->flags & CLUSTER_FLAG_FREE) == 0 && ci->order == order)
+ if ((ci->flags & (CLUSTER_FLAG_SKIP_SCAN | CLUSTER_FLAG_FREE)) == 0
+ && ci->order == order)
break;
ci = next_cluster_circular(si, ci);
}
@@ -722,6 +731,21 @@ static unsigned int next_cluster_for_scan(struct swap_info_struct *si,
return cluster_to_offset(si, ci);
}
+static inline void cluster_inc_scanners(struct swap_cluster_info *ci)
+{
+ /* Protected by si lock. */
+ ci->nr_scanners++;
+ ci->flags &= ~CLUSTER_FLAG_DECREMENT;
+}
+
+static inline void cluster_dec_scanners(struct swap_cluster_info *ci)
+{
+ /* Protected by si lock. */
+ ci->nr_scanners--;
+ if (ci->nr_scanners == 0 && (ci->flags & CLUSTER_FLAG_DECREMENT) == 0)
+ ci->flags |= CLUSTER_FLAG_SKIP_SCAN;
+}
+
/*
* Try to get swap entries with specified order from current cpu's swap entry
* pool (a cluster). This might involve allocating a new cluster for current CPU
@@ -764,6 +788,8 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
return false;
} else
return false;
+
+ cluster_inc_scanners(offset_to_cluster(si, tmp));
}
/*
@@ -780,13 +806,19 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
}
unlock_cluster(ci);
if (tmp >= max) {
+ cluster_dec_scanners(ci);
cluster->next[order] = SWAP_NEXT_INVALID;
goto new_cluster;
}
*offset = tmp;
*scan_base = tmp;
tmp += nr_pages;
- cluster->next[order] = tmp < max ? tmp : SWAP_NEXT_INVALID;
+ if (tmp >= max) {
+ cluster_dec_scanners(ci);
+ cluster->next[order] = SWAP_NEXT_INVALID;
+ } else {
+ cluster->next[order] = tmp;
+ }
return true;
}
Add CLUSTER_FLAG_SKIP_SCAN cluster flag, which is applied to a cluster under 1 of 2 conditions. When present, the cluster will be skipped during a scan. - When the number of free entries is less than the number of entries that would be required for a new allocation of the order that the cluster serves. - When scanning completes for the cluster, and no further scanners are active for the cluster and no swap entries were freed for the cluster since the last scan began. In this case, it has been proven that there are no contiguous free entries of sufficient size to allcoate the order that the cluster serves. In this case the cluster is made eligible for scanning again when the next entry is freed. The latter is implemented to permit multiple CPUs to scan the same cluster, which in turn garrantees that if there is a free block available in a cluster allocated for the desired order then it will be allocated on a first come, first served basis. As a result, the number of active scanners for a cluster must be tracked, costing 4 bytes per cluster. Signed-off-by: Ryan Roberts <ryan.roberts@arm.com> --- include/linux/swap.h | 3 +++ mm/swapfile.c | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) -- 2.43.0