| Message ID | 1507625625-8665-4-git-send-email-deathsimple@vodafone.de (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Tue, Oct 10, 2017 at 4:53 AM, Christian König <ckoenig.leichtzumerken@gmail.com> wrote: > From: Christian König <christian.koenig@amd.com> > > Add a new huge page pool and try to allocate from it when it makes sense. > > v2: avoid compound pages for now > > Signed-off-by: Christian König <christian.koenig@amd.com> > Acked-by: Alex Deucher <alexander.deucher@amd.com> Series is: Acked-by: Alex Deucher <alexander.deucher@amd.com> > --- > drivers/gpu/drm/ttm/ttm_page_alloc.c | 136 ++++++++++++++++++++++++++++------- > 1 file changed, 109 insertions(+), 27 deletions(-) > > diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c > index 3974732..b6f16e7ff 100644 > --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c > +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c > @@ -95,7 +95,7 @@ struct ttm_pool_opts { > unsigned small; > }; > > -#define NUM_POOLS 4 > +#define NUM_POOLS 6 > > /** > * struct ttm_pool_manager - Holds memory pools for fst allocation > @@ -122,6 +122,8 @@ struct ttm_pool_manager { > struct ttm_page_pool uc_pool; > struct ttm_page_pool wc_pool_dma32; > struct ttm_page_pool uc_pool_dma32; > + struct ttm_page_pool wc_pool_huge; > + struct ttm_page_pool uc_pool_huge; > } ; > }; > }; > @@ -256,8 +258,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray) > > /** > * Select the right pool or requested caching state and ttm flags. */ > -static struct ttm_page_pool *ttm_get_pool(int flags, > - enum ttm_caching_state cstate) > +static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, > + enum ttm_caching_state cstate) > { > int pool_index; > > @@ -269,9 +271,15 @@ static struct ttm_page_pool *ttm_get_pool(int flags, > else > pool_index = 0x1; > > - if (flags & TTM_PAGE_FLAG_DMA32) > + if (flags & TTM_PAGE_FLAG_DMA32) { > + if (huge) > + return NULL; > pool_index |= 0x2; > > + } else if (huge) { > + pool_index |= 0x4; > + } > + > return &_manager->pools[pool_index]; > } > > @@ -494,12 +502,14 @@ static void ttm_handle_caching_state_failure(struct list_head *pages, > * pages returned in pages array. > */ > static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, > - int ttm_flags, enum ttm_caching_state cstate, unsigned count) > + int ttm_flags, enum ttm_caching_state cstate, > + unsigned count, unsigned order) > { > struct page **caching_array; > struct page *p; > int r = 0; > - unsigned i, cpages; > + unsigned i, j, cpages; > + unsigned npages = 1 << order; > unsigned max_cpages = min(count, > (unsigned)(PAGE_SIZE/sizeof(struct page *))); > > @@ -512,7 +522,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, > } > > for (i = 0, cpages = 0; i < count; ++i) { > - p = alloc_page(gfp_flags); > + p = alloc_pages(gfp_flags, order); > > if (!p) { > pr_err("Unable to get page %u\n", i); > @@ -531,14 +541,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, > goto out; > } > > + list_add(&p->lru, pages); > + > #ifdef CONFIG_HIGHMEM > /* gfp flags of highmem page should never be dma32 so we > * we should be fine in such case > */ > - if (!PageHighMem(p)) > + if (PageHighMem(p)) > + continue; > + > #endif > - { > - caching_array[cpages++] = p; > + for (j = 0; j < npages; ++j) { > + caching_array[cpages++] = p++; > if (cpages == max_cpages) { > > r = ttm_set_pages_caching(caching_array, > @@ -552,8 +566,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, > cpages = 0; > } > } > - > - list_add(&p->lru, pages); > } > > if (cpages) { > @@ -573,9 +585,9 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, > * Fill the given pool if there aren't enough pages and the requested number of > * pages is small. > */ > -static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, > - int ttm_flags, enum ttm_caching_state cstate, unsigned count, > - unsigned long *irq_flags) > +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, > + enum ttm_caching_state cstate, > + unsigned count, unsigned long *irq_flags) > { > struct page *p; > int r; > @@ -605,7 +617,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, > > INIT_LIST_HEAD(&new_pages); > r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, > - cstate, alloc_size); > + cstate, alloc_size, 0); > spin_lock_irqsave(&pool->lock, *irq_flags); > > if (!r) { > @@ -635,7 +647,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, > struct list_head *pages, > int ttm_flags, > enum ttm_caching_state cstate, > - unsigned count) > + unsigned count, unsigned order) > { > unsigned long irq_flags; > struct list_head *p; > @@ -643,7 +655,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, > int r = 0; > > spin_lock_irqsave(&pool->lock, irq_flags); > - ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); > + if (!order) > + ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, > + &irq_flags); > > if (count >= pool->npages) { > /* take all pages from the pool */ > @@ -698,7 +712,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, > * multiple requests in parallel. > **/ > r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, > - count); > + count, order); > } > > return r; > @@ -708,8 +722,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, > static void ttm_put_pages(struct page **pages, unsigned npages, int flags, > enum ttm_caching_state cstate) > { > + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); > + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); > unsigned long irq_flags; > - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); > unsigned i; > > if (pool == NULL) { > @@ -737,8 +752,48 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, > return; > } > > + i = 0; > +#ifdef CONFIG_TRANSPARENT_HUGEPAGE > + if (huge) { > + unsigned max_size, n2free; > + > + spin_lock_irqsave(&huge->lock, irq_flags); > + while (i < npages) { > + struct page *p = pages[i]; > + unsigned j; > + > + if (!p) > + break; > + > + for (j = 0; j < HPAGE_PMD_NR; ++j) > + if (p++ != pages[i + j]) > + break; > + > + if (j != HPAGE_PMD_NR) > + break; > + > + list_add_tail(&pages[i]->lru, &huge->list); > + > + for (j = 0; j < HPAGE_PMD_NR; ++j) > + pages[i++] = NULL; > + huge->npages++; > + } > + > + /* Check that we don't go over the pool limit */ > + max_size = _manager->options.max_size; > + max_size /= HPAGE_PMD_NR; > + if (huge->npages > max_size) > + n2free = huge->npages - max_size; > + else > + n2free = 0; > + spin_unlock_irqrestore(&huge->lock, irq_flags); > + if (n2free) > + ttm_page_pool_free(huge, n2free, false); > + } > +#endif > + > spin_lock_irqsave(&pool->lock, irq_flags); > - for (i = 0; i < npages; i++) { > + while (i < npages) { > if (pages[i]) { > if (page_count(pages[i]) != 1) > pr_err("Erroneous page count. Leaking pages.\n"); > @@ -746,6 +801,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, > pages[i] = NULL; > pool->npages++; > } > + ++i; > } > /* Check that we don't go over the pool limit */ > npages = 0; > @@ -768,7 +824,8 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, > static int ttm_get_pages(struct page **pages, unsigned npages, int flags, > enum ttm_caching_state cstate) > { > - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); > + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); > + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); > struct list_head plist; > struct page *p = NULL; > unsigned count; > @@ -821,11 +878,28 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, > return 0; > } > > - /* First we take pages from the pool */ > + count = 0; > + > +#ifdef CONFIG_TRANSPARENT_HUGEPAGE > + if (huge && npages >= HPAGE_PMD_NR) { > + INIT_LIST_HEAD(&plist); > + ttm_page_pool_get_pages(huge, &plist, flags, cstate, > + npages / HPAGE_PMD_NR, > + HPAGE_PMD_ORDER); > + > + list_for_each_entry(p, &plist, lru) { > + unsigned j; > + > + for (j = 0; j < HPAGE_PMD_NR; ++j) > + pages[count++] = &p[j]; > + } > + } > +#endif > + > INIT_LIST_HEAD(&plist); > - r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); > + r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, > + npages - count, 0); > > - count = 0; > list_for_each_entry(p, &plist, lru) > pages[count++] = p; > > @@ -872,6 +946,14 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) > ttm_page_pool_init_locked(&_manager->uc_pool_dma32, > GFP_USER | GFP_DMA32, "uc dma"); > > + ttm_page_pool_init_locked(&_manager->wc_pool_huge, > + GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP), > + "wc huge"); > + > + ttm_page_pool_init_locked(&_manager->uc_pool_huge, > + GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP) > + , "uc huge"); > + > _manager->options.max_size = max_pages; > _manager->options.small = SMALL_ALLOCATION; > _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; > @@ -1041,12 +1123,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data) > seq_printf(m, "No pool allocator running.\n"); > return 0; > } > - seq_printf(m, "%6s %12s %13s %8s\n", > + seq_printf(m, "%7s %12s %13s %8s\n", > h[0], h[1], h[2], h[3]); > for (i = 0; i < NUM_POOLS; ++i) { > p = &_manager->pools[i]; > > - seq_printf(m, "%6s %12ld %13ld %8d\n", > + seq_printf(m, "%7s %12ld %13ld %8d\n", > p->name, p->nrefills, > p->nfrees, p->npages); > } > -- > 2.7.4 > > _______________________________________________ > amd-gfx mailing list > amd-gfx@lists.freedesktop.org > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 3974732..b6f16e7ff 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -95,7 +95,7 @@ struct ttm_pool_opts { unsigned small; }; -#define NUM_POOLS 4 +#define NUM_POOLS 6 /** * struct ttm_pool_manager - Holds memory pools for fst allocation @@ -122,6 +122,8 @@ struct ttm_pool_manager { struct ttm_page_pool uc_pool; struct ttm_page_pool wc_pool_dma32; struct ttm_page_pool uc_pool_dma32; + struct ttm_page_pool wc_pool_huge; + struct ttm_page_pool uc_pool_huge; } ; }; }; @@ -256,8 +258,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray) /** * Select the right pool or requested caching state and ttm flags. */ -static struct ttm_page_pool *ttm_get_pool(int flags, - enum ttm_caching_state cstate) +static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, + enum ttm_caching_state cstate) { int pool_index; @@ -269,9 +271,15 @@ static struct ttm_page_pool *ttm_get_pool(int flags, else pool_index = 0x1; - if (flags & TTM_PAGE_FLAG_DMA32) + if (flags & TTM_PAGE_FLAG_DMA32) { + if (huge) + return NULL; pool_index |= 0x2; + } else if (huge) { + pool_index |= 0x4; + } + return &_manager->pools[pool_index]; } @@ -494,12 +502,14 @@ static void ttm_handle_caching_state_failure(struct list_head *pages, * pages returned in pages array. */ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, - int ttm_flags, enum ttm_caching_state cstate, unsigned count) + int ttm_flags, enum ttm_caching_state cstate, + unsigned count, unsigned order) { struct page **caching_array; struct page *p; int r = 0; - unsigned i, cpages; + unsigned i, j, cpages; + unsigned npages = 1 << order; unsigned max_cpages = min(count, (unsigned)(PAGE_SIZE/sizeof(struct page *))); @@ -512,7 +522,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, } for (i = 0, cpages = 0; i < count; ++i) { - p = alloc_page(gfp_flags); + p = alloc_pages(gfp_flags, order); if (!p) { pr_err("Unable to get page %u\n", i); @@ -531,14 +541,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, goto out; } + list_add(&p->lru, pages); + #ifdef CONFIG_HIGHMEM /* gfp flags of highmem page should never be dma32 so we * we should be fine in such case */ - if (!PageHighMem(p)) + if (PageHighMem(p)) + continue; + #endif - { - caching_array[cpages++] = p; + for (j = 0; j < npages; ++j) { + caching_array[cpages++] = p++; if (cpages == max_cpages) { r = ttm_set_pages_caching(caching_array, @@ -552,8 +566,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, cpages = 0; } } - - list_add(&p->lru, pages); } if (cpages) { @@ -573,9 +585,9 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, * Fill the given pool if there aren't enough pages and the requested number of * pages is small. */ -static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, - int ttm_flags, enum ttm_caching_state cstate, unsigned count, - unsigned long *irq_flags) +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, + enum ttm_caching_state cstate, + unsigned count, unsigned long *irq_flags) { struct page *p; int r; @@ -605,7 +617,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, INIT_LIST_HEAD(&new_pages); r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, - cstate, alloc_size); + cstate, alloc_size, 0); spin_lock_irqsave(&pool->lock, *irq_flags); if (!r) { @@ -635,7 +647,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, struct list_head *pages, int ttm_flags, enum ttm_caching_state cstate, - unsigned count) + unsigned count, unsigned order) { unsigned long irq_flags; struct list_head *p; @@ -643,7 +655,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, int r = 0; spin_lock_irqsave(&pool->lock, irq_flags); - ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); + if (!order) + ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, + &irq_flags); if (count >= pool->npages) { /* take all pages from the pool */ @@ -698,7 +712,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, * multiple requests in parallel. **/ r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, - count); + count, order); } return r; @@ -708,8 +722,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, static void ttm_put_pages(struct page **pages, unsigned npages, int flags, enum ttm_caching_state cstate) { + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); unsigned long irq_flags; - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); unsigned i; if (pool == NULL) { @@ -737,8 +752,48 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, return; } + i = 0; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (huge) { + unsigned max_size, n2free; + + spin_lock_irqsave(&huge->lock, irq_flags); + while (i < npages) { + struct page *p = pages[i]; + unsigned j; + + if (!p) + break; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + if (p++ != pages[i + j]) + break; + + if (j != HPAGE_PMD_NR) + break; + + list_add_tail(&pages[i]->lru, &huge->list); + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[i++] = NULL; + huge->npages++; + } + + /* Check that we don't go over the pool limit */ + max_size = _manager->options.max_size; + max_size /= HPAGE_PMD_NR; + if (huge->npages > max_size) + n2free = huge->npages - max_size; + else + n2free = 0; + spin_unlock_irqrestore(&huge->lock, irq_flags); + if (n2free) + ttm_page_pool_free(huge, n2free, false); + } +#endif + spin_lock_irqsave(&pool->lock, irq_flags); - for (i = 0; i < npages; i++) { + while (i < npages) { if (pages[i]) { if (page_count(pages[i]) != 1) pr_err("Erroneous page count. Leaking pages.\n"); @@ -746,6 +801,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, pages[i] = NULL; pool->npages++; } + ++i; } /* Check that we don't go over the pool limit */ npages = 0; @@ -768,7 +824,8 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, static int ttm_get_pages(struct page **pages, unsigned npages, int flags, enum ttm_caching_state cstate) { - struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); struct list_head plist; struct page *p = NULL; unsigned count; @@ -821,11 +878,28 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, return 0; } - /* First we take pages from the pool */ + count = 0; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (huge && npages >= HPAGE_PMD_NR) { + INIT_LIST_HEAD(&plist); + ttm_page_pool_get_pages(huge, &plist, flags, cstate, + npages / HPAGE_PMD_NR, + HPAGE_PMD_ORDER); + + list_for_each_entry(p, &plist, lru) { + unsigned j; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[count++] = &p[j]; + } + } +#endif + INIT_LIST_HEAD(&plist); - r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); + r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, + npages - count, 0); - count = 0; list_for_each_entry(p, &plist, lru) pages[count++] = p; @@ -872,6 +946,14 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) ttm_page_pool_init_locked(&_manager->uc_pool_dma32, GFP_USER | GFP_DMA32, "uc dma"); + ttm_page_pool_init_locked(&_manager->wc_pool_huge, + GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP), + "wc huge"); + + ttm_page_pool_init_locked(&_manager->uc_pool_huge, + GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP) + , "uc huge"); + _manager->options.max_size = max_pages; _manager->options.small = SMALL_ALLOCATION; _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; @@ -1041,12 +1123,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data) seq_printf(m, "No pool allocator running.\n"); return 0; } - seq_printf(m, "%6s %12s %13s %8s\n", + seq_printf(m, "%7s %12s %13s %8s\n", h[0], h[1], h[2], h[3]); for (i = 0; i < NUM_POOLS; ++i) { p = &_manager->pools[i]; - seq_printf(m, "%6s %12ld %13ld %8d\n", + seq_printf(m, "%7s %12ld %13ld %8d\n", p->name, p->nrefills, p->nfrees, p->npages); }