@@ -512,6 +512,7 @@ struct device_physical_location {
* @dma_io_tlb_mem: Software IO TLB allocator. Not for driver use.
* @dma_io_tlb_pools: List of transient swiotlb memory pools.
* @dma_io_tlb_lock: Protects changes to the list of active pools.
+ * @dma_uses_io_tlb: %true if device has used the software IO TLB.
* @archdata: For arch-specific additions.
* @of_node: Associated device tree node.
* @fwnode: Associated device node supplied by platform firmware.
@@ -619,6 +620,7 @@ struct device {
struct io_tlb_mem *dma_io_tlb_mem;
struct list_head dma_io_tlb_pools;
spinlock_t dma_io_tlb_lock;
+ bool dma_uses_io_tlb;
#endif
/* arch specific additions */
struct dev_archdata archdata;
@@ -148,7 +148,11 @@ struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
*/
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
{
- return dev->dma_io_tlb_mem &&
+ /* Pairs with smp_wmb() in swiotlb_find_slots() and
+ * swiotlb_dyn_alloc(), which modify the RCU lists.
+ */
+ smp_rmb();
+ return dev->dma_uses_io_tlb &&
!!swiotlb_find_pool(dev, paddr);
}
@@ -701,7 +701,7 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
add_mem_pool(mem, pool);
- /* Pairs with smp_rmb() in swiotlb_find_pool(). */
+ /* Pairs with smp_rmb() in is_swiotlb_buffer(). */
smp_wmb();
}
@@ -729,6 +729,7 @@ void swiotlb_dev_init(struct device *dev)
dev->dma_io_tlb_mem = &io_tlb_default_mem;
INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
spin_lock_init(&dev->dma_io_tlb_lock);
+ dev->dma_uses_io_tlb = false;
}
/**
@@ -746,11 +747,6 @@ struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
- /* Pairs with smp_wmb() in swiotlb_find_slots() and
- * swiotlb_dyn_alloc(), which modify the RCU lists.
- */
- smp_rmb();
-
rcu_read_lock();
list_for_each_entry_rcu(pool, &mem->pools, node) {
if (paddr >= pool->start && paddr < pool->end)
@@ -1125,9 +1121,11 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
- /* Pairs with smp_rmb() in swiotlb_find_pool(). */
- smp_wmb();
found:
+ dev->dma_uses_io_tlb = true;
+ /* Pairs with smp_rmb() in is_swiotlb_buffer() */
+ smp_wmb();
+
*retpool = pool;
return index;
}