From patchwork Fri Jul 5 21:40:18 2013 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ben Widawsky X-Patchwork-Id: 2824355 Return-Path: X-Original-To: patchwork-dri-devel@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.19.201]) by patchwork1.web.kernel.org (Postfix) with ESMTP id 450859F3C3 for ; Fri, 5 Jul 2013 21:39:27 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id B9415200F0 for ; Fri, 5 Jul 2013 21:39:25 +0000 (UTC) Received: from gabe.freedesktop.org (gabe.freedesktop.org [131.252.210.177]) by mail.kernel.org (Postfix) with ESMTP id 5C52A200ED for ; Fri, 5 Jul 2013 21:39:24 +0000 (UTC) Received: from gabe.freedesktop.org (localhost [127.0.0.1]) by gabe.freedesktop.org (Postfix) with ESMTP id 22B95E602C for ; Fri, 5 Jul 2013 14:39:24 -0700 (PDT) X-Original-To: dri-devel@lists.freedesktop.org Delivered-To: dri-devel@lists.freedesktop.org Received: from shiva.localdomain (unknown [209.20.75.48]) by gabe.freedesktop.org (Postfix) with ESMTP id 16A52E5DC6 for ; Fri, 5 Jul 2013 14:37:25 -0700 (PDT) Received: by shiva.localdomain (Postfix, from userid 99) id CE441886AF; Fri, 5 Jul 2013 21:37:24 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Spam-Level: X-Spam-Status: No, score=-4.4 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_MED, RP_MATCHES_RCVD,UNPARSEABLE_RELAY autolearn=ham version=3.3.1 Received: from lundgren.kumite (c-24-21-100-90.hsd1.or.comcast.net [24.21.100.90]) by shiva.localdomain (Postfix) with ESMTPSA id 275C0886B0; Fri, 5 Jul 2013 21:37:09 +0000 (UTC) From: Ben Widawsky To: "Intel GFX^Cintel-gfx"@lists.freedesktop.org> Subject: [PATCH 9/9] drm: Optionally create mm blocks from top-to-bottom Date: Fri, 5 Jul 2013 14:40:18 -0700 Message-Id: <1373060418-815-9-git-send-email-ben@bwidawsk.net> X-Mailer: git-send-email 1.8.3.2 In-Reply-To: <1373060418-815-1-git-send-email-ben@bwidawsk.net> References: <1373060418-815-1-git-send-email-ben@bwidawsk.net> Cc: Ben Widawsky , dri-devel@lists.freedesktop.org X-BeenThere: dri-devel@lists.freedesktop.org X-Mailman-Version: 2.1.13 Precedence: list List-Id: Direct Rendering Infrastructure - Development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , MIME-Version: 1.0 Sender: dri-devel-bounces+patchwork-dri-devel=patchwork.kernel.org@lists.freedesktop.org Errors-To: dri-devel-bounces+patchwork-dri-devel=patchwork.kernel.org@lists.freedesktop.org X-Virus-Scanned: ClamAV using ClamSMTP From: Chris Wilson Clients like i915 needs to segregate cache domains within the GTT which can lead to small amounts of fragmentation. By allocating the uncached buffers from the bottom and the cacheable buffers from the top, we can reduce the amount of wasted space and also optimize allocation of the mappable portion of the GTT to only those buffers that require CPU access through the GTT. v2 by Ben: Update callers in i915_gem_object_bind_to_gtt() Turn search flags and allocation flags into separate enums Make checkpatch happy where logical/easy v3: by Ben: Rebased on create_block removal v4: by Ben: Rebased on reserve_node addition. Signed-off-by: Chris Wilson CC: Signed-off-by: Ben Widawsky Conflicts: drivers/gpu/drm/drm_mm.c include/drm/drm_mm.h Conflicts: drivers/gpu/drm/drm_mm.c drivers/gpu/drm/i915/i915_gem_gtt.c drivers/gpu/drm/i915/i915_gem_stolen.c include/drm/drm_mm.h --- drivers/gpu/drm/drm_mm.c | 118 +++++++++++++++++-------------- drivers/gpu/drm/i915/i915_gem.c | 4 +- include/drm/drm_mm.h | 149 +++++++++++++++++++++++++++------------- 3 files changed, 169 insertions(+), 102 deletions(-) diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index fe304f9..5037b1a 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -49,7 +49,7 @@ #define MM_UNUSED_TARGET 4 -static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, int atomic) +static struct drm_mm_node *drm_mm_kmalloc(struct drm_mm *mm, bool atomic) { struct drm_mm_node *child; @@ -105,7 +105,8 @@ EXPORT_SYMBOL(drm_mm_pre_get); static void drm_mm_insert_helper(struct drm_mm_node *hole_node, struct drm_mm_node *node, unsigned long size, unsigned alignment, - unsigned long color) + unsigned long color, + enum drm_mm_allocator_flags flags) { struct drm_mm *mm = hole_node->mm; unsigned long hole_start = drm_mm_hole_node_start(hole_node); @@ -118,12 +119,22 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node, if (mm->color_adjust) mm->color_adjust(hole_node, color, &adj_start, &adj_end); + if (flags & DRM_MM_CREATE_TOP) + adj_start = adj_end - size; + if (alignment) { unsigned tmp = adj_start % alignment; - if (tmp) - adj_start += alignment - tmp; + if (tmp) { + if (flags & DRM_MM_CREATE_TOP) + adj_start -= tmp; + else + adj_start += alignment - tmp; + } } + BUG_ON(adj_start < hole_start); + BUG_ON(adj_end > hole_end); + if (adj_start == hole_start) { hole_node->hole_follows = 0; list_del(&hole_node->hole_stack); @@ -191,15 +202,15 @@ struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *hole_node, unsigned long size, unsigned alignment, unsigned long color, - int atomic) + enum drm_mm_allocator_flags flags) { struct drm_mm_node *node; - node = drm_mm_kmalloc(hole_node->mm, atomic); + node = drm_mm_kmalloc(hole_node->mm, flags & DRM_MM_CREATE_ATOMIC); if (unlikely(node == NULL)) return NULL; - drm_mm_insert_helper(hole_node, node, size, alignment, color); + drm_mm_insert_helper(hole_node, node, size, alignment, color, flags); return node; } @@ -212,32 +223,28 @@ EXPORT_SYMBOL(drm_mm_get_block_generic); */ int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node, unsigned long size, unsigned alignment, - unsigned long color) + unsigned long color, + enum drm_mm_allocator_flags aflags, + enum drm_mm_search_flags sflags) { struct drm_mm_node *hole_node; hole_node = drm_mm_search_free_generic(mm, size, alignment, - color, 0); + color, sflags); if (!hole_node) return -ENOSPC; - drm_mm_insert_helper(hole_node, node, size, alignment, color); + drm_mm_insert_helper(hole_node, node, size, alignment, color, aflags); return 0; } EXPORT_SYMBOL(drm_mm_insert_node_generic); -int drm_mm_insert_node(struct drm_mm *mm, struct drm_mm_node *node, - unsigned long size, unsigned alignment) -{ - return drm_mm_insert_node_generic(mm, node, size, alignment, 0); -} -EXPORT_SYMBOL(drm_mm_insert_node); - static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node, struct drm_mm_node *node, unsigned long size, unsigned alignment, unsigned long color, - unsigned long start, unsigned long end) + unsigned long start, unsigned long end, + enum drm_mm_search_flags flags) { struct drm_mm *mm = hole_node->mm; unsigned long hole_start = drm_mm_hole_node_start(hole_node); @@ -252,13 +259,20 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node, if (adj_end > end) adj_end = end; + if (flags & DRM_MM_CREATE_TOP) + adj_start = adj_end - size; + if (mm->color_adjust) mm->color_adjust(hole_node, color, &adj_start, &adj_end); if (alignment) { unsigned tmp = adj_start % alignment; - if (tmp) - adj_start += alignment - tmp; + if (tmp) { + if (flags & DRM_MM_CREATE_TOP) + adj_start -= tmp; + else + adj_start += alignment - tmp; + } } if (adj_start == hole_start) { @@ -275,6 +289,8 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node, INIT_LIST_HEAD(&node->hole_stack); list_add(&node->node_list, &hole_node->node_list); + BUG_ON(node->start < start); + BUG_ON(node->start < adj_start); BUG_ON(node->start + node->size > adj_end); BUG_ON(node->start + node->size > end); @@ -285,22 +301,23 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node, } } -struct drm_mm_node *drm_mm_get_block_range_generic(struct drm_mm_node *hole_node, - unsigned long size, - unsigned alignment, - unsigned long color, - unsigned long start, - unsigned long end, - int atomic) +struct drm_mm_node * +drm_mm_get_block_range_generic(struct drm_mm_node *hole_node, + unsigned long size, + unsigned alignment, + unsigned long color, + unsigned long start, + unsigned long end, + enum drm_mm_allocator_flags flags) { struct drm_mm_node *node; - node = drm_mm_kmalloc(hole_node->mm, atomic); + node = drm_mm_kmalloc(hole_node->mm, flags & DRM_MM_CREATE_ATOMIC); if (unlikely(node == NULL)) return NULL; drm_mm_insert_helper_range(hole_node, node, size, alignment, color, - start, end); + start, end, flags); return node; } @@ -313,31 +330,25 @@ EXPORT_SYMBOL(drm_mm_get_block_range_generic); */ int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, struct drm_mm_node *node, unsigned long size, unsigned alignment, unsigned long color, - unsigned long start, unsigned long end) + unsigned long start, unsigned long end, + enum drm_mm_allocator_flags aflags, + enum drm_mm_search_flags sflags) { struct drm_mm_node *hole_node; hole_node = drm_mm_search_free_in_range_generic(mm, size, alignment, color, - start, end, 0); + start, end, sflags); if (!hole_node) return -ENOSPC; drm_mm_insert_helper_range(hole_node, node, size, alignment, color, - start, end); + start, end, aflags); return 0; } EXPORT_SYMBOL(drm_mm_insert_node_in_range_generic); -int drm_mm_insert_node_in_range(struct drm_mm *mm, struct drm_mm_node *node, - unsigned long size, unsigned alignment, - unsigned long start, unsigned long end) -{ - return drm_mm_insert_node_in_range_generic(mm, node, size, alignment, 0, start, end); -} -EXPORT_SYMBOL(drm_mm_insert_node_in_range); - /** * Remove a memory node from the allocator. */ @@ -413,7 +424,7 @@ struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, unsigned long size, unsigned alignment, unsigned long color, - bool best_match) + enum drm_mm_search_flags flags) { struct drm_mm_node *entry; struct drm_mm_node *best; @@ -426,7 +437,8 @@ struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, best = NULL; best_size = ~0UL; - drm_mm_for_each_hole(entry, mm, adj_start, adj_end) { + __drm_mm_for_each_hole(entry, mm, adj_start, adj_end, + flags & DRM_MM_SEARCH_BELOW) { if (mm->color_adjust) { mm->color_adjust(entry, color, &adj_start, &adj_end); if (adj_end <= adj_start) @@ -436,7 +448,7 @@ struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, if (!check_free_hole(adj_start, adj_end, size, alignment)) continue; - if (!best_match) + if ((flags & DRM_MM_SEARCH_BEST) == 0) return entry; if (entry->size < best_size) { @@ -449,13 +461,14 @@ struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, } EXPORT_SYMBOL(drm_mm_search_free_generic); -struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_mm *mm, - unsigned long size, - unsigned alignment, - unsigned long color, - unsigned long start, - unsigned long end, - bool best_match) +struct drm_mm_node * +drm_mm_search_free_in_range_generic(const struct drm_mm *mm, + unsigned long size, + unsigned alignment, + unsigned long color, + unsigned long start, + unsigned long end, + enum drm_mm_search_flags flags) { struct drm_mm_node *entry; struct drm_mm_node *best; @@ -468,7 +481,8 @@ struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_mm *mm, best = NULL; best_size = ~0UL; - drm_mm_for_each_hole(entry, mm, adj_start, adj_end) { + __drm_mm_for_each_hole(entry, mm, adj_start, adj_end, + flags & DRM_MM_SEARCH_BELOW) { if (adj_start < start) adj_start = start; if (adj_end > end) @@ -483,7 +497,7 @@ struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_mm *mm, if (!check_free_hole(adj_start, adj_end, size, alignment)) continue; - if (!best_match) + if ((flags & DRM_MM_SEARCH_BEST) == 0) return entry; if (entry->size < best_size) { diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index af61be8..9275f90 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -3115,7 +3115,9 @@ search_free: ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, &obj->gtt_space, size, alignment, - obj->cache_level, 0, gtt_max); + obj->cache_level, 0, gtt_max, + DRM_MM_CREATE_DEFAULT, + DRM_MM_SEARCH_DEFAULT); if (ret) { ret = i915_gem_evict_something(dev, size, alignment, obj->cache_level, diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h index b87d05e..5a69db2 100644 --- a/include/drm/drm_mm.h +++ b/include/drm/drm_mm.h @@ -41,6 +41,21 @@ #include #endif +enum drm_mm_allocator_flags { + DRM_MM_CREATE_DEFAULT = 0, + DRM_MM_CREATE_ATOMIC = 1<<0, + DRM_MM_CREATE_TOP = 1<<1, +}; + +enum drm_mm_search_flags { + DRM_MM_SEARCH_DEFAULT = 0, + DRM_MM_SEARCH_BEST = 1<<0, + DRM_MM_SEARCH_BELOW = 1<<1, +}; + +#define DRM_MM_BOTTOMUP DRM_MM_CREATE_DEFAULT, DRM_MM_SEARCH_DEFAULT +#define DRM_MM_TOPDOWN DRM_MM_CREATE_TOP, DRM_MM_SEARCH_BELOW + struct drm_mm_node { struct list_head node_list; struct list_head hole_stack; @@ -135,27 +150,37 @@ static inline unsigned long drm_mm_hole_node_end(struct drm_mm_node *hole_node) 1 : 0; \ entry = list_entry(entry->hole_stack.next, struct drm_mm_node, hole_stack)) +#define __drm_mm_for_each_hole(entry, mm, hole_start, hole_end, backwards) \ + for (entry = list_entry((backwards) ? (mm)->hole_stack.prev : (mm)->hole_stack.next, struct drm_mm_node, hole_stack); \ + &entry->hole_stack != &(mm)->hole_stack ? \ + hole_start = drm_mm_hole_node_start(entry), \ + hole_end = drm_mm_hole_node_end(entry), \ + 1 : 0; \ + entry = list_entry((backwards) ? entry->hole_stack.prev : entry->hole_stack.next, struct drm_mm_node, hole_stack)) + /* * Basic range manager support (drm_mm.c) */ extern int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node); -extern struct drm_mm_node *drm_mm_get_block_generic(struct drm_mm_node *node, - unsigned long size, - unsigned alignment, - unsigned long color, - int atomic); -extern struct drm_mm_node *drm_mm_get_block_range_generic( - struct drm_mm_node *node, - unsigned long size, - unsigned alignment, - unsigned long color, - unsigned long start, - unsigned long end, - int atomic); +extern struct drm_mm_node * +drm_mm_get_block_generic(struct drm_mm_node *node, + unsigned long size, + unsigned alignment, + unsigned long color, + enum drm_mm_allocator_flags flags); +extern struct drm_mm_node * +drm_mm_get_block_range_generic(struct drm_mm_node *node, + unsigned long size, + unsigned alignment, + unsigned long color, + unsigned long start, + unsigned long end, + enum drm_mm_allocator_flags flags); -static inline struct drm_mm_node *drm_mm_get_block(struct drm_mm_node *parent, - unsigned long size, - unsigned alignment) +static inline struct drm_mm_node * +drm_mm_get_block(struct drm_mm_node *parent, + unsigned long size, + unsigned alignment) { return drm_mm_get_block_generic(parent, size, alignment, 0, 0); } @@ -163,7 +188,8 @@ static inline struct drm_mm_node *drm_mm_get_block_atomic(struct drm_mm_node *pa unsigned long size, unsigned alignment) { - return drm_mm_get_block_generic(parent, size, alignment, 0, 1); + return drm_mm_get_block_generic(parent, size, alignment, 0, + DRM_MM_CREATE_ATOMIC); } static inline struct drm_mm_node *drm_mm_get_block_range( struct drm_mm_node *parent, @@ -183,39 +209,41 @@ static inline struct drm_mm_node *drm_mm_get_block_atomic_range( unsigned long end) { return drm_mm_get_block_range_generic(parent, size, alignment, 0, - start, end, 1); + start, end, + DRM_MM_CREATE_ATOMIC); } -extern int drm_mm_insert_node(struct drm_mm *mm, - struct drm_mm_node *node, - unsigned long size, - unsigned alignment); -extern int drm_mm_insert_node_in_range(struct drm_mm *mm, - struct drm_mm_node *node, - unsigned long size, - unsigned alignment, - unsigned long start, - unsigned long end); extern int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node, unsigned long size, unsigned alignment, - unsigned long color); -extern int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, - struct drm_mm_node *node, - unsigned long size, - unsigned alignment, - unsigned long color, - unsigned long start, - unsigned long end); + unsigned long color, + enum drm_mm_allocator_flags aflags, + enum drm_mm_search_flags sflags); +#define drm_mm_insert_node(mm, node, size, alignment) \ + drm_mm_insert_node_generic(mm, node, size, alignment, 0, 0) +extern int +drm_mm_insert_node_in_range_generic(struct drm_mm *mm, + struct drm_mm_node *node, + unsigned long size, + unsigned alignment, + unsigned long color, + unsigned long start, + unsigned long end, + enum drm_mm_allocator_flags aflags, + enum drm_mm_search_flags sflags); +#define drm_mm_insert_node_in_range(mm, node, size, alignment, start, end) \ + drm_mm_insert_node_in_range_generic(mm, node, size, alignment, 0, start, end, 0) extern void drm_mm_put_block(struct drm_mm_node *cur); extern void drm_mm_remove_node(struct drm_mm_node *node); extern void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new); -extern struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, - unsigned long size, - unsigned alignment, - unsigned long color, - bool best_match); + +extern struct drm_mm_node * +drm_mm_search_free_generic(const struct drm_mm *mm, + unsigned long size, + unsigned alignment, + unsigned long color, + enum drm_mm_search_flags flags); extern struct drm_mm_node *drm_mm_search_free_in_range_generic( const struct drm_mm *mm, unsigned long size, @@ -223,13 +251,15 @@ extern struct drm_mm_node *drm_mm_search_free_in_range_generic( unsigned long color, unsigned long start, unsigned long end, - bool best_match); -static inline struct drm_mm_node *drm_mm_search_free(const struct drm_mm *mm, - unsigned long size, - unsigned alignment, - bool best_match) + enum drm_mm_search_flags flags); + +static inline struct drm_mm_node * +drm_mm_search_free(const struct drm_mm *mm, + unsigned long size, + unsigned alignment, + enum drm_mm_search_flags flags) { - return drm_mm_search_free_generic(mm,size, alignment, 0, best_match); + return drm_mm_search_free_generic(mm, size, alignment, 0, flags); } static inline struct drm_mm_node *drm_mm_search_free_in_range( const struct drm_mm *mm, @@ -237,12 +267,33 @@ static inline struct drm_mm_node *drm_mm_search_free_in_range( unsigned alignment, unsigned long start, unsigned long end, - bool best_match) + enum drm_mm_search_flags flags) { return drm_mm_search_free_in_range_generic(mm, size, alignment, 0, - start, end, best_match); + start, end, flags); } +static inline struct drm_mm_node * +drm_mm_search_free_color(const struct drm_mm *mm, + unsigned long size, + unsigned alignment, + unsigned long color, + enum drm_mm_search_flags flags) +{ + return drm_mm_search_free_generic(mm, size, alignment, color, flags); +} +static inline struct drm_mm_node *drm_mm_search_free_in_range_color( + const struct drm_mm *mm, + unsigned long size, + unsigned alignment, + unsigned long color, + unsigned long start, + unsigned long end, + enum drm_mm_search_flags flags) +{ + return drm_mm_search_free_in_range_generic(mm, size, alignment, color, + start, end, flags); +} extern void drm_mm_init(struct drm_mm *mm, unsigned long start, unsigned long size);