@@ -932,6 +932,16 @@ static void queue_inc_cons(struct arm_smmu_queue *q)
writel(q->cons, q->cons_reg);
}
+static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
+{
+ /* Acknowledge overflow condition if any */
+ if (Q_OVF(q, q->prod) == Q_OVF(q, q->cons))
+ return;
+
+ q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
+ writel(q->cons, q->cons_reg);
+}
+
static int queue_sync_prod(struct arm_smmu_queue *q)
{
int ret = 0;
@@ -1782,7 +1792,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
} while (!queue_empty(q));
/* Sync our overflow flag, as we believe we're up to speed */
- q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
+ queue_sync_cons_ovf(q);
return IRQ_HANDLED;
}
@@ -1846,7 +1856,7 @@ static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
} while (!queue_empty(q));
/* Sync our overflow flag, as we believe we're up to speed */
- q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
+ queue_sync_cons_ovf(q);
smmu->priq.batch++;
wake_up_locked(&smmu->priq.wq);
When an overflow occurs in the PRI queue, the SMMU toggles the overflow flag in the PROD register. To exit the overflow condition, the PRI thread is supposed to acknowledge it by toggling this flag in the CONS register. Currently with an overflow condition, the flag is toggled in q->cons after clearing the PRI queue, but is never published to the hardware. It would be done next time we execute the thread. However, we never get a chance because the SMMU doesn't append anything to the queue while in overflow condition, and the thread is not scheduled unless the queue transitions from empty to non-empty. To fix it, synchronize the hardware CONS register before leaving the PRIQ thread. This bug doesn't affect the event queue, since the SMMU still adds elements to that queue when the overflow condition is active. Even missing an overflow condition because one is already active doesn't matter. We won't miss fault records for stalled transactions. But it feels nicer to keep the SMMU in sync when possible, so do it there as well. Signed-off-by: Jean-Philippe Brucker <jean-philippe.brucker@arm.com> --- drivers/iommu/arm-smmu-v3.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)