@@ -1375,7 +1375,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
u64 *cmds, int n, bool sync)
{
u64 cmd_sync[CMDQ_ENT_DWORDS];
- u32 prod;
+ u32 prod, prodx;
unsigned long flags;
bool owner;
struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
@@ -1383,33 +1383,21 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
.max_n_shift = cmdq->q.llq.max_n_shift,
}, head = llq;
int ret = 0;
+ u32 owner_val = 1 << cmdq->q.llq.owner_count_shift;
+ u32 prod_mask = GENMASK(cmdq->q.llq.max_n_shift, 0);
+ u32 owner_mask = GENMASK(30, cmdq->q.llq.owner_count_shift);
+
+ /* always issue a CMD_SYNC TODO: fixup callers for this */
+ sync = true;
/* 1. Allocate some space in the queue */
local_irq_save(flags);
- llq.val = READ_ONCE(cmdq->q.llq.val);
- do {
- u64 old;
-
- while (!queue_has_space(&llq, n + sync)) {
- local_irq_restore(flags);
- if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
- dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
- local_irq_save(flags);
- }
- head.cons = llq.cons;
- head.prod = queue_inc_prod_n(&llq, n + sync) |
- CMDQ_PROD_OWNED_FLAG;
+ prodx = atomic_fetch_add(n + sync + owner_val, &cmdq->q.llq.atomic.prod);
- old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
- if (old == llq.val)
- break;
-
- llq.val = old;
- } while (1);
- owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
- head.prod &= ~CMDQ_PROD_OWNED_FLAG;
- llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
+ owner = !(prodx & owner_mask);
+ llq.prod = prod_mask & prodx;
+ head.prod = queue_inc_prod_n(&llq, n + sync);
/*
* 2. Write our commands into the queue As with simple binary counting, once the prod+wrap fields overflow, they
It has been shown that the cmpxchg() for finding space in the cmdq can be a bottleneck: - For more CPUs contenting the cmdq, cmpxchg() will fail more often. - Since the software-maintained cons pointer is updated on the same 64b memory region, the chance of cmpxchg() failure increases again. The cmpxchg() is removed as part of 2 related changes: - If a CMD_SYNC is always issued for a batch, the cmdq can - in theory - never fill, since we always wait for a CMD_SYNC to be consumed. We must issue the CMD_SYNC so that a CPU will be always limited to issuing max_cmd_per_batch commands. Generally for DMA unmap ops, a CMD_SYNC is always issued anyway. As such, the cmdq locking is removed, and we only longer maintain cons in software (this needs to be revised for !MSI support). - Update prod and cmdq owner in a single operation. For this, we count the prod and owner in separate regions in arm_smmu_ll_queue.prod. As with simple binary counting, once the prod+wrap fields overflow, they will zero. They will overflow into "owner" region, but this is ok as we have accounted for the extra value. As for the "owner", we now count this value, instead of setting a flag. Similar to before, once the owner has finished gathering, it will clear this mask. As such, a CPU declares itself as the "owner" when it reads zero for this field. This zeroing will also clear possible overflow in wrap+prod region. Signed-off-by: John Garry <john.garry@huawei.com> --- drivers/iommu/arm-smmu-v3.c | 58 +++++++++++---------------------------------- 1 file changed, 14 insertions(+), 44 deletions(-) will zero. They will overflow into "owner" region, but this is ok as we have accounted for the extra value. @@ -1420,14 +1408,6 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, prod = queue_inc_prod_n(&llq, n); arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod); queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS); - - /* - * In order to determine completion of our CMD_SYNC, we must - * ensure that the queue can't wrap twice without us noticing. - * We achieve that by taking the cmdq lock as shared before - * marking our slot as valid. - */ - arm_smmu_cmdq_shared_lock(cmdq); } /* 3. Mark our slots as valid, ensuring commands are visible first */ @@ -1439,11 +1419,10 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, /* a. Wait for previous owner to finish */ atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod); - /* b. Stop gathering work by clearing the owned flag */ - prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG, + /* b. Stop gathering work by clearing the owned mask */ + prod = atomic_fetch_andnot_relaxed(owner_mask, &cmdq->q.llq.atomic.prod); - prod &= ~CMDQ_PROD_OWNED_FLAG; - + prod &= prod_mask; /* * c. Wait for any gathered work to be written to the queue. * Note that we read our own entries so that we have the control @@ -1476,15 +1455,6 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, readl_relaxed(cmdq->q.prod_reg), readl_relaxed(cmdq->q.cons_reg)); } - - /* - * Try to unlock the cmq lock. This will fail if we're the last - * reader, in which case we can safely update cmdq->q.llq.cons - */ - if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) { - WRITE_ONCE(cmdq->q.llq.cons, llq.cons); - arm_smmu_cmdq_shared_unlock(cmdq); - } } local_irq_restore(flags);