@@ -333,7 +333,7 @@ static inline struct page *grab_cache_page_nowait(struct address_space *mapping,
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset);
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset);
-unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
+unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
unsigned int nr_entries, struct page **entries,
pgoff_t *indices);
unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
@@ -24,7 +24,7 @@ void __pagevec_release(struct pagevec *pvec);
void __pagevec_lru_add(struct pagevec *pvec);
unsigned pagevec_lookup_entries(struct pagevec *pvec,
struct address_space *mapping,
- pgoff_t start, unsigned nr_entries,
+ pgoff_t *start, unsigned nr_entries,
pgoff_t *indices);
void pagevec_remove_exceptionals(struct pagevec *pvec);
unsigned pagevec_lookup_range(struct pagevec *pvec,
@@ -1373,11 +1373,11 @@ EXPORT_SYMBOL(pagecache_get_page);
* Any shadow entries of evicted pages, or swap entries from
* shmem/tmpfs, are included in the returned array.
*
- * find_get_entries() returns the number of pages and shadow entries
- * which were found.
+ * find_get_entries() returns the number of pages and shadow entries which were
+ * found. It also updates @start to index the next page for the traversal.
*/
unsigned find_get_entries(struct address_space *mapping,
- pgoff_t start, unsigned int nr_entries,
+ pgoff_t *start, unsigned int nr_entries,
struct page **entries, pgoff_t *indices)
{
void **slot;
@@ -1388,7 +1388,7 @@ unsigned find_get_entries(struct address_space *mapping,
return 0;
rcu_read_lock();
- radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
+ radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) {
struct page *head, *page;
repeat:
page = radix_tree_deref_slot(slot);
@@ -1429,6 +1429,9 @@ unsigned find_get_entries(struct address_space *mapping,
break;
}
rcu_read_unlock();
+
+ if (ret)
+ *start = indices[ret - 1] + 1;
return ret;
}
@@ -768,26 +768,25 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
pagevec_init(&pvec, 0);
index = start;
while (index < end) {
- if (!pagevec_lookup_entries(&pvec, mapping, index,
+ if (!pagevec_lookup_entries(&pvec, mapping, &index,
min(end - index, (pgoff_t)PAGEVEC_SIZE),
indices))
break;
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
- index = indices[i];
- if (index >= end)
+ if (indices[i] >= end)
break;
if (radix_tree_exceptional_entry(page)) {
if (unfalloc)
continue;
nr_swaps_freed += !shmem_free_swap(mapping,
- index, page);
+ indices[i], page);
continue;
}
- VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
+ VM_BUG_ON_PAGE(page_to_pgoff(page) != indices[i], page);
if (!trylock_page(page))
continue;
@@ -798,7 +797,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
unlock_page(page);
continue;
} else if (PageTransHuge(page)) {
- if (index == round_down(end, HPAGE_PMD_NR)) {
+ if (indices[i] ==
+ round_down(end, HPAGE_PMD_NR)) {
/*
* Range ends in the middle of THP:
* zero out the page
@@ -807,7 +807,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
unlock_page(page);
continue;
}
- index += HPAGE_PMD_NR - 1;
+ if (indices[i] + HPAGE_PMD_NR > index)
+ index = indices[i] + HPAGE_PMD_NR;
i += HPAGE_PMD_NR - 1;
}
@@ -823,7 +824,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
- index++;
}
if (partial_start) {
@@ -856,13 +856,15 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
index = start;
while (index < end) {
+ pgoff_t lookup_start = index;
+
cond_resched();
- if (!pagevec_lookup_entries(&pvec, mapping, index,
+ if (!pagevec_lookup_entries(&pvec, mapping, &index,
min(end - index, (pgoff_t)PAGEVEC_SIZE),
indices)) {
/* If all gone or hole-punch or unfalloc, we're done */
- if (index == start || end != -1)
+ if (lookup_start == start || end != -1)
break;
/* But if truncating, restart to make sure all gone */
index = start;
@@ -871,16 +873,16 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
- index = indices[i];
- if (index >= end)
+ if (indices[i] >= end)
break;
if (radix_tree_exceptional_entry(page)) {
if (unfalloc)
continue;
- if (shmem_free_swap(mapping, index, page)) {
+ if (shmem_free_swap(mapping, indices[i],
+ page)) {
/* Swap was replaced by page: retry */
- index--;
+ index = indices[i];
break;
}
nr_swaps_freed++;
@@ -898,11 +900,12 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
* of THP: don't need to look on these pages
* again on !pvec.nr restart.
*/
- if (index != round_down(end, HPAGE_PMD_NR))
+ if (indices[i] != round_down(end, HPAGE_PMD_NR))
start++;
continue;
} else if (PageTransHuge(page)) {
- if (index == round_down(end, HPAGE_PMD_NR)) {
+ if (indices[i] ==
+ round_down(end, HPAGE_PMD_NR)) {
/*
* Range ends in the middle of THP:
* zero out the page
@@ -911,7 +914,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
unlock_page(page);
continue;
}
- index += HPAGE_PMD_NR - 1;
+ if (indices[i] + HPAGE_PMD_NR > index)
+ index = indices[i] + HPAGE_PMD_NR;
i += HPAGE_PMD_NR - 1;
}
@@ -923,7 +927,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
} else {
/* Page was replaced by swap: retry */
unlock_page(page);
- index--;
+ index = indices[i];
break;
}
}
@@ -931,7 +935,6 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
- index++;
}
spin_lock_irq(&info->lock);
@@ -2487,31 +2490,33 @@ static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
pgoff_t indices[PAGEVEC_SIZE];
bool done = false;
int i;
+ pgoff_t last;
pagevec_init(&pvec, 0);
pvec.nr = 1; /* start small: we may be there already */
while (!done) {
- pvec.nr = find_get_entries(mapping, index,
+ last = index;
+ pvec.nr = find_get_entries(mapping, &index,
pvec.nr, pvec.pages, indices);
if (!pvec.nr) {
if (whence == SEEK_DATA)
- index = end;
+ last = end;
break;
}
- for (i = 0; i < pvec.nr; i++, index++) {
- if (index < indices[i]) {
+ for (i = 0; i < pvec.nr; i++, last++) {
+ if (last < indices[i]) {
if (whence == SEEK_HOLE) {
done = true;
break;
}
- index = indices[i];
+ last = indices[i];
}
page = pvec.pages[i];
if (page && !radix_tree_exceptional_entry(page)) {
if (!PageUptodate(page))
page = NULL;
}
- if (index >= end ||
+ if (last >= end ||
(page && whence == SEEK_DATA) ||
(!page && whence == SEEK_HOLE)) {
done = true;
@@ -2523,7 +2528,7 @@ static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
pvec.nr = PAGEVEC_SIZE;
cond_resched();
}
- return index;
+ return last;
}
static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
@@ -906,11 +906,11 @@ EXPORT_SYMBOL(__pagevec_lru_add);
* not-present entries.
*
* pagevec_lookup_entries() returns the number of entries which were
- * found.
+ * found. It also updates @start to index the next page for the traversal.
*/
unsigned pagevec_lookup_entries(struct pagevec *pvec,
struct address_space *mapping,
- pgoff_t start, unsigned nr_pages,
+ pgoff_t *start, unsigned nr_pages,
pgoff_t *indices)
{
pvec->nr = find_get_entries(mapping, start, nr_pages,
@@ -289,26 +289,25 @@ void truncate_inode_pages_range(struct address_space *mapping,
pagevec_init(&pvec, 0);
index = start;
- while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
+ while (index < end && pagevec_lookup_entries(&pvec, mapping, &index,
min(end - index, (pgoff_t)PAGEVEC_SIZE),
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
- index = indices[i];
- if (index >= end)
+ if (indices[i] >= end)
break;
if (radix_tree_exceptional_entry(page)) {
- truncate_exceptional_entry(mapping, index,
+ truncate_exceptional_entry(mapping, indices[i],
page);
continue;
}
if (!trylock_page(page))
continue;
- WARN_ON(page_to_index(page) != index);
+ WARN_ON(page_to_index(page) != indices[i]);
if (PageWriteback(page)) {
unlock_page(page);
continue;
@@ -319,7 +318,6 @@ void truncate_inode_pages_range(struct address_space *mapping,
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
- index++;
}
if (partial_start) {
@@ -363,17 +361,19 @@ void truncate_inode_pages_range(struct address_space *mapping,
index = start;
for ( ; ; ) {
+ pgoff_t lookup_start = index;
+
cond_resched();
- if (!pagevec_lookup_entries(&pvec, mapping, index,
+ if (!pagevec_lookup_entries(&pvec, mapping, &index,
min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
/* If all gone from start onwards, we're done */
- if (index == start)
+ if (lookup_start == start)
break;
/* Otherwise restart to make sure all gone */
index = start;
continue;
}
- if (index == start && indices[0] >= end) {
+ if (lookup_start == start && indices[0] >= end) {
/* All gone out of hole to be punched, we're done */
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
@@ -383,28 +383,26 @@ void truncate_inode_pages_range(struct address_space *mapping,
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
- index = indices[i];
- if (index >= end) {
+ if (indices[i] >= end) {
/* Restart punch to make sure all gone */
- index = start - 1;
+ index = start;
break;
}
if (radix_tree_exceptional_entry(page)) {
- truncate_exceptional_entry(mapping, index,
+ truncate_exceptional_entry(mapping, indices[i],
page);
continue;
}
lock_page(page);
- WARN_ON(page_to_index(page) != index);
+ WARN_ON(page_to_index(page) != indices[i]);
wait_on_page_writeback(page);
truncate_inode_page(mapping, page);
unlock_page(page);
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
- index++;
}
out:
@@ -501,44 +499,44 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
int i;
pagevec_init(&pvec, 0);
- while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
+ while (index <= end && pagevec_lookup_entries(&pvec, mapping, &index,
min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
- index = indices[i];
- if (index > end)
+ if (indices[i] > end)
break;
if (radix_tree_exceptional_entry(page)) {
- invalidate_exceptional_entry(mapping, index,
- page);
+ invalidate_exceptional_entry(mapping,
+ indices[i], page);
continue;
}
if (!trylock_page(page))
continue;
- WARN_ON(page_to_index(page) != index);
+ WARN_ON(page_to_index(page) != indices[i]);
/* Middle of THP: skip */
if (PageTransTail(page)) {
unlock_page(page);
continue;
} else if (PageTransHuge(page)) {
- index += HPAGE_PMD_NR - 1;
- i += HPAGE_PMD_NR - 1;
+ if (index < indices[i] + HPAGE_PMD_NR)
+ index = indices[i] + HPAGE_PMD_NR;
/*
* 'end' is in the middle of THP. Don't
* invalidate the page as the part outside of
* 'end' could be still useful.
*/
- if (index > end) {
+ if (indices[i] + HPAGE_PMD_NR - 1 > end) {
unlock_page(page);
- continue;
+ break;
}
+ i += HPAGE_PMD_NR - 1;
}
ret = invalidate_inode_page(page);
@@ -554,7 +552,6 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
- index++;
}
return count;
}
@@ -632,26 +629,25 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
pagevec_init(&pvec, 0);
index = start;
- while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
+ while (index <= end && pagevec_lookup_entries(&pvec, mapping, &index,
min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
- index = indices[i];
- if (index > end)
+ if (indices[i] > end)
break;
if (radix_tree_exceptional_entry(page)) {
if (!invalidate_exceptional_entry2(mapping,
- index, page))
+ indices[i], page))
ret = -EBUSY;
continue;
}
lock_page(page);
- WARN_ON(page_to_index(page) != index);
+ WARN_ON(page_to_index(page) != indices[i]);
if (page->mapping != mapping) {
unlock_page(page);
continue;
@@ -663,8 +659,8 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
* Zap the rest of the file in one hit.
*/
unmap_mapping_range(mapping,
- (loff_t)index << PAGE_SHIFT,
- (loff_t)(1 + end - index)
+ (loff_t)indices[i] << PAGE_SHIFT,
+ (loff_t)(1 + end - indices[i])
<< PAGE_SHIFT,
0);
did_range_unmap = 1;
@@ -673,7 +669,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
* Just zap this page
*/
unmap_mapping_range(mapping,
- (loff_t)index << PAGE_SHIFT,
+ (loff_t)indices[i] << PAGE_SHIFT,
PAGE_SIZE, 0);
}
}
@@ -690,7 +686,6 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
- index++;
}
/*
* For DAX we invalidate page tables after invalidating radix tree. We
Make pagevec_lookup_entries() (and underlying find_get_entries()) update index to the next page where iteration should continue. This is mostly for consistency with pagevec_lookup() and future pagevec_lookup_entries_range(). Signed-off-by: Jan Kara <jack@suse.cz> --- include/linux/pagemap.h | 2 +- include/linux/pagevec.h | 2 +- mm/filemap.c | 11 ++++++--- mm/shmem.c | 57 +++++++++++++++++++++++-------------------- mm/swap.c | 4 +-- mm/truncate.c | 65 +++++++++++++++++++++++-------------------------- 6 files changed, 72 insertions(+), 69 deletions(-)