@@ -834,41 +834,58 @@ static int __add_to_page_cache_locked(struct page *page,
XA_STATE(xas, &mapping->i_pages, offset);
int huge = PageHuge(page);
int error;
+ unsigned int nr = 1;
void *old;
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(PageSwapBacked(page), page);
mapping_set_update(&xas, mapping);
- get_page(page);
- page->mapping = mapping;
- page->index = offset;
-
if (!huge) {
error = mem_cgroup_charge(page, current->mm, gfp_mask);
if (error)
- goto error;
+ return error;
+ xas_set_order(&xas, offset, thp_order(page));
+ nr = thp_nr_pages(page);
}
+ page_ref_add(page, nr);
+ page->mapping = mapping;
+ page->index = offset;
+
do {
+ unsigned long exceptional = 0;
+ unsigned int i = 0;
+
xas_lock_irq(&xas);
- old = xas_load(&xas);
- if (old && !xa_is_value(old))
- xas_set_err(&xas, -EEXIST);
- xas_store(&xas, page);
+ xas_for_each_conflict(&xas, old) {
+ if (!xa_is_value(old)) {
+ xas_set_err(&xas, -EEXIST);
+ break;
+ }
+ exceptional++;
+ if (shadowp)
+ *shadowp = old;
+ }
+ xas_create_range(&xas);
if (xas_error(&xas))
goto unlock;
- if (xa_is_value(old)) {
- mapping->nrexceptional--;
- if (shadowp)
- *shadowp = old;
+next:
+ xas_store(&xas, page);
+ if (++i < nr) {
+ xas_next(&xas);
+ goto next;
}
- mapping->nrpages++;
+ mapping->nrexceptional -= exceptional;
+ mapping->nrpages += nr;
/* hugetlb pages do not participate in page cache accounting */
- if (!huge)
- __inc_lruvec_page_state(page, NR_FILE_PAGES);
+ if (!huge) {
+ __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
+ if (nr > 1)
+ __inc_lruvec_page_state(page, NR_FILE_THPS);
+ }
unlock:
xas_unlock_irq(&xas);
} while (xas_nomem(&xas, gfp_mask & GFP_RECLAIM_MASK));
@@ -883,7 +900,8 @@ static int __add_to_page_cache_locked(struct page *page,
error:
page->mapping = NULL;
/* Leave page->index set: truncation relies upon it */
- put_page(page);
+ page_ref_sub(page, nr);
+ VM_BUG_ON_PAGE(page_count(page) <= 0, page);
return error;
}
ALLOW_ERROR_INJECTION(__add_to_page_cache_locked, ERRNO);