From patchwork Tue Aug 4 23:27:47 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Dave Jiang X-Patchwork-Id: 6945021 Return-Path: X-Original-To: patchwork-dmaengine@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork2.web.kernel.org (Postfix) with ESMTP id 7D6EEC05AC for ; Tue, 4 Aug 2015 23:27:58 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 9E8EE20553 for ; Tue, 4 Aug 2015 23:27:54 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 605B72041A for ; Tue, 4 Aug 2015 23:27:50 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754490AbbHDX1u (ORCPT ); Tue, 4 Aug 2015 19:27:50 -0400 Received: from mga03.intel.com ([134.134.136.65]:34776 "EHLO mga03.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754501AbbHDX1t (ORCPT ); Tue, 4 Aug 2015 19:27:49 -0400 Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga103.jf.intel.com with ESMTP; 04 Aug 2015 16:27:47 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.15,612,1432623600"; d="scan'208";a="536110785" Received: from djiang5-desk3.ch.intel.com ([143.182.51.113]) by FMSMGA003.fm.intel.com with ESMTP; 04 Aug 2015 16:27:48 -0700 Subject: [PATCH 04/12] ioatdma: clean up local dma channel data structure From: Dave Jiang To: vinod.koul@intel.com Cc: dmaengine@vger.kernel.org, dan.j.williams@intel.com, daniel.verkamp@intel.com Date: Tue, 04 Aug 2015 16:27:47 -0700 Message-ID: <20150804232747.40926.26637.stgit@djiang5-desk3.ch.intel.com> In-Reply-To: <20150804231837.40926.14487.stgit@djiang5-desk3.ch.intel.com> References: <20150804231837.40926.14487.stgit@djiang5-desk3.ch.intel.com> User-Agent: StGit/0.17.1-dirty MIME-Version: 1.0 Sender: dmaengine-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: dmaengine@vger.kernel.org X-Spam-Status: No, score=-7.0 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=unavailable version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Kill the common ioatdma channel structure and everything that is not dma_chan to be ioat_dma_chan. Since we don't have to worry about v1 and v2 ioatdma anymore this makes it much cleaner and obvious for maintenance. Signed-off-by: Dave Jiang --- drivers/dma/ioat/dma.c | 116 ++++++++------- drivers/dma/ioat/dma.h | 89 +++++++---- drivers/dma/ioat/dma_v2.c | 351 +++++++++++++++++++++++---------------------- drivers/dma/ioat/dma_v2.h | 82 +++-------- drivers/dma/ioat/dma_v3.c | 307 +++++++++++++++++++-------------------- 5 files changed, 470 insertions(+), 475 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe dmaengine" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index fe63ff8..60aa04d 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -50,7 +50,7 @@ MODULE_PARM_DESC(ioat_pending_level, static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) { struct ioatdma_device *instance = data; - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; unsigned long attnstatus; int bit; u8 intrctrl; @@ -67,9 +67,9 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { - chan = ioat_chan_by_index(instance, bit); - if (test_bit(IOAT_RUN, &chan->state)) - tasklet_schedule(&chan->cleanup_task); + ioat_chan = ioat_chan_by_index(instance, bit); + if (test_bit(IOAT_RUN, &ioat_chan->state)) + tasklet_schedule(&ioat_chan->cleanup_task); } writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); @@ -83,45 +83,47 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) */ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) { - struct ioat_chan_common *chan = data; + struct ioatdma_chan *ioat_chan = data; - if (test_bit(IOAT_RUN, &chan->state)) - tasklet_schedule(&chan->cleanup_task); + if (test_bit(IOAT_RUN, &ioat_chan->state)) + tasklet_schedule(&ioat_chan->cleanup_task); return IRQ_HANDLED; } /* common channel initialization */ -void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) +void +ioat_init_channel(struct ioatdma_device *device, struct ioatdma_chan *ioat_chan, + int idx) { struct dma_device *dma = &device->common; - struct dma_chan *c = &chan->common; + struct dma_chan *c = &ioat_chan->dma_chan; unsigned long data = (unsigned long) c; - chan->device = device; - chan->reg_base = device->reg_base + (0x80 * (idx + 1)); - spin_lock_init(&chan->cleanup_lock); - chan->common.device = dma; - dma_cookie_init(&chan->common); - list_add_tail(&chan->common.device_node, &dma->channels); - device->idx[idx] = chan; - init_timer(&chan->timer); - chan->timer.function = device->timer_fn; - chan->timer.data = data; - tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); + ioat_chan->device = device; + ioat_chan->reg_base = device->reg_base + (0x80 * (idx + 1)); + spin_lock_init(&ioat_chan->cleanup_lock); + ioat_chan->dma_chan.device = dma; + dma_cookie_init(&ioat_chan->dma_chan); + list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels); + device->idx[idx] = ioat_chan; + init_timer(&ioat_chan->timer); + ioat_chan->timer.function = device->timer_fn; + ioat_chan->timer.data = data; + tasklet_init(&ioat_chan->cleanup_task, device->cleanup_fn, data); } -void ioat_stop(struct ioat_chan_common *chan) +void ioat_stop(struct ioatdma_chan *ioat_chan) { - struct ioatdma_device *device = chan->device; + struct ioatdma_device *device = ioat_chan->device; struct pci_dev *pdev = device->pdev; - int chan_id = chan_num(chan); + int chan_id = chan_num(ioat_chan); struct msix_entry *msix; /* 1/ stop irq from firing tasklets * 2/ stop the tasklet from re-arming irqs */ - clear_bit(IOAT_RUN, &chan->state); + clear_bit(IOAT_RUN, &ioat_chan->state); /* flush inflight interrupts */ switch (device->irq_mode) { @@ -138,29 +140,30 @@ void ioat_stop(struct ioat_chan_common *chan) } /* flush inflight timers */ - del_timer_sync(&chan->timer); + del_timer_sync(&ioat_chan->timer); /* flush inflight tasklet runs */ - tasklet_kill(&chan->cleanup_task); + tasklet_kill(&ioat_chan->cleanup_task); /* final cleanup now that everything is quiesced and can't re-arm */ - device->cleanup_fn((unsigned long) &chan->common); + device->cleanup_fn((unsigned long)&ioat_chan->dma_chan); } -dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) +dma_addr_t ioat_get_current_completion(struct ioatdma_chan *ioat_chan) { dma_addr_t phys_complete; u64 completion; - completion = *chan->completion; + completion = *ioat_chan->completion; phys_complete = ioat_chansts_to_addr(completion); - dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, + dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__, (unsigned long long) phys_complete); if (is_ioat_halted(completion)) { - u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - dev_err(to_dev(chan), "Channel halted, chanerr = %x\n", + u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + + dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n", chanerr); /* TODO do something to salvage the situation */ @@ -169,14 +172,14 @@ dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) return phys_complete; } -bool ioat_cleanup_preamble(struct ioat_chan_common *chan, +bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, dma_addr_t *phys_complete) { - *phys_complete = ioat_get_current_completion(chan); - if (*phys_complete == chan->last_completion) + *phys_complete = ioat_get_current_completion(ioat_chan); + if (*phys_complete == ioat_chan->last_completion) return false; - clear_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); return true; } @@ -185,8 +188,8 @@ enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate) { - struct ioat_chan_common *chan = to_chan_common(c); - struct ioatdma_device *device = chan->device; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *device = ioat_chan->device; enum dma_status ret; ret = dma_cookie_status(c, cookie, txstate); @@ -322,7 +325,7 @@ MODULE_PARM_DESC(ioat_interrupt_style, */ int ioat_dma_setup_interrupts(struct ioatdma_device *device) { - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; struct pci_dev *pdev = device->pdev; struct device *dev = &pdev->dev; struct msix_entry *msix; @@ -351,15 +354,15 @@ msix: for (i = 0; i < msixcnt; i++) { msix = &device->msix_entries[i]; - chan = ioat_chan_by_index(device, i); + ioat_chan = ioat_chan_by_index(device, i); err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt_msix, 0, - "ioat-msix", chan); + "ioat-msix", ioat_chan); if (err) { for (j = 0; j < i; j++) { msix = &device->msix_entries[j]; - chan = ioat_chan_by_index(device, j); - devm_free_irq(dev, msix->vector, chan); + ioat_chan = ioat_chan_by_index(device, j); + devm_free_irq(dev, msix->vector, ioat_chan); } goto msi; } @@ -507,14 +510,14 @@ static ssize_t ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) { struct ioat_sysfs_entry *entry; - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; entry = container_of(attr, struct ioat_sysfs_entry, attr); - chan = container_of(kobj, struct ioat_chan_common, kobj); + ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); if (!entry->show) return -EIO; - return entry->show(&chan->common, page); + return entry->show(&ioat_chan->dma_chan, page); } const struct sysfs_ops ioat_sysfs_ops = { @@ -527,16 +530,17 @@ void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type) struct dma_chan *c; list_for_each_entry(c, &dma->channels, device_node) { - struct ioat_chan_common *chan = to_chan_common(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); struct kobject *parent = &c->dev->device.kobj; int err; - err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); + err = kobject_init_and_add(&ioat_chan->kobj, type, + parent, "quickdata"); if (err) { - dev_warn(to_dev(chan), + dev_warn(to_dev(ioat_chan), "sysfs init error (%d), continuing...\n", err); - kobject_put(&chan->kobj); - set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state); + kobject_put(&ioat_chan->kobj); + set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state); } } } @@ -547,11 +551,11 @@ void ioat_kobject_del(struct ioatdma_device *device) struct dma_chan *c; list_for_each_entry(c, &dma->channels, device_node) { - struct ioat_chan_common *chan = to_chan_common(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); - if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) { - kobject_del(&chan->kobj); - kobject_put(&chan->kobj); + if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) { + kobject_del(&ioat_chan->kobj); + kobject_put(&ioat_chan->kobj); } } } diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 1e96eaa..43290d1 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -78,20 +78,20 @@ struct ioatdma_device { struct dma_device common; u8 version; struct msix_entry msix_entries[4]; - struct ioat_chan_common *idx[4]; + struct ioatdma_chan *idx[4]; struct dca_provider *dca; enum ioat_irq_mode irq_mode; u32 cap; void (*intr_quirk)(struct ioatdma_device *device); int (*enumerate_channels)(struct ioatdma_device *device); - int (*reset_hw)(struct ioat_chan_common *chan); + int (*reset_hw)(struct ioatdma_chan *ioat_chan); void (*cleanup_fn)(unsigned long data); void (*timer_fn)(unsigned long data); int (*self_test)(struct ioatdma_device *device); }; -struct ioat_chan_common { - struct dma_chan common; +struct ioatdma_chan { + struct dma_chan dma_chan; void __iomem *reg_base; dma_addr_t last_completion; spinlock_t cleanup_lock; @@ -112,6 +112,27 @@ struct ioat_chan_common { u64 *completion; struct tasklet_struct cleanup_task; struct kobject kobj; + +/* ioat v2 / v3 channel attributes + * @xfercap_log; log2 of channel max transfer length (for fast division) + * @head: allocated index + * @issued: hardware notification point + * @tail: cleanup index + * @dmacount: identical to 'head' except for occasionally resetting to zero + * @alloc_order: log2 of the number of allocated descriptors + * @produce: number of descriptors to produce at submit time + * @ring: software ring buffer implementation of hardware ring + * @prep_lock: serializes descriptor preparation (producers) + */ + size_t xfercap_log; + u16 head; + u16 issued; + u16 tail; + u16 dmacount; + u16 alloc_order; + u16 produce; + struct ioat_ring_ent **ring; + spinlock_t prep_lock; }; struct ioat_sysfs_entry { @@ -133,11 +154,13 @@ struct ioat_sed_ent { unsigned int hw_pool; }; -static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) +static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c) { - return container_of(c, struct ioat_chan_common, common); + return container_of(c, struct ioatdma_chan, dma_chan); } + + /* wrapper around hardware descriptor format + additional software fields */ #ifdef DEBUG @@ -149,10 +172,10 @@ static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) #endif static inline void -__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, +__dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw, struct dma_async_tx_descriptor *tx, int id) { - struct device *dev = to_dev(chan); + struct device *dev = to_dev(ioat_chan); dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x" " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id, @@ -162,25 +185,25 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, } #define dump_desc_dbg(c, d) \ - ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) + ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; }) -static inline struct ioat_chan_common * +static inline struct ioatdma_chan * ioat_chan_by_index(struct ioatdma_device *device, int index) { return device->idx[index]; } -static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) +static inline u64 ioat_chansts_32(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->device->version; u64 status; u32 status_lo; /* We need to read the low address first as this causes the * chipset to latch the upper bits for the subsequent read */ - status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); - status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); + status_lo = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); + status = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); status <<= 32; status |= status_lo; @@ -189,16 +212,16 @@ static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) #if BITS_PER_LONG == 64 -static inline u64 ioat_chansts(struct ioat_chan_common *chan) +static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->device->version; u64 status; /* With IOAT v3.3 the status register is 64bit. */ if (ver >= IOAT_VER_3_3) - status = readq(chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); + status = readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); else - status = ioat_chansts_32(chan); + status = ioat_chansts_32(ioat_chan); return status; } @@ -212,31 +235,33 @@ static inline u64 ioat_chansts_to_addr(u64 status) return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; } -static inline u32 ioat_chanerr(struct ioat_chan_common *chan) +static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan) { - return readl(chan->reg_base + IOAT_CHANERR_OFFSET); + return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); } -static inline void ioat_suspend(struct ioat_chan_common *chan) +static inline void ioat_suspend(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->device->version; - writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + writeb(IOAT_CHANCMD_SUSPEND, + ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); } -static inline void ioat_reset(struct ioat_chan_common *chan) +static inline void ioat_reset(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->device->version; - writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + writeb(IOAT_CHANCMD_RESET, + ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); } -static inline bool ioat_reset_pending(struct ioat_chan_common *chan) +static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->device->version; u8 cmd; - cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; } @@ -272,15 +297,15 @@ int ioat_dma_self_test(struct ioatdma_device *device); void ioat_dma_remove(struct ioatdma_device *device); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); void ioat_init_channel(struct ioatdma_device *device, - struct ioat_chan_common *chan, int idx); + struct ioatdma_chan *ioat_chan, int idx); enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate); -bool ioat_cleanup_preamble(struct ioat_chan_common *chan, +bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, dma_addr_t *phys_complete); void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); void ioat_kobject_del(struct ioatdma_device *device); int ioat_dma_setup_interrupts(struct ioatdma_device *device); -void ioat_stop(struct ioat_chan_common *chan); +void ioat_stop(struct ioatdma_chan *ioat_chan); extern const struct sysfs_ops ioat_sysfs_ops; extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_cap_attr; diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index f5e1521..6b7322a 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -48,26 +48,26 @@ module_param(ioat_ring_max_alloc_order, int, 0644); MODULE_PARM_DESC(ioat_ring_max_alloc_order, "ioat2+: upper limit for ring size (default: 16)"); -void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) +void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan) { - struct ioat_chan_common *chan = &ioat->base; - - ioat->dmacount += ioat2_ring_pending(ioat); - ioat->issued = ioat->head; - writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); - dev_dbg(to_dev(chan), + ioat_chan->dmacount += ioat2_ring_pending(ioat_chan); + ioat_chan->issued = ioat_chan->head; + writew(ioat_chan->dmacount, + ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); + dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x count: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); + __func__, ioat_chan->head, ioat_chan->tail, + ioat_chan->issued, ioat_chan->dmacount); } void ioat2_issue_pending(struct dma_chan *c) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); - if (ioat2_ring_pending(ioat)) { - spin_lock_bh(&ioat->prep_lock); - __ioat2_issue_pending(ioat); - spin_unlock_bh(&ioat->prep_lock); + if (ioat2_ring_pending(ioat_chan)) { + spin_lock_bh(&ioat_chan->prep_lock); + __ioat2_issue_pending(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); } } @@ -78,26 +78,27 @@ void ioat2_issue_pending(struct dma_chan *c) * Check if the number of unsubmitted descriptors has exceeded the * watermark. Called with prep_lock held */ -static void ioat2_update_pending(struct ioat2_dma_chan *ioat) +static void ioat2_update_pending(struct ioatdma_chan *ioat_chan) { - if (ioat2_ring_pending(ioat) > ioat_pending_level) - __ioat2_issue_pending(ioat); + if (ioat2_ring_pending(ioat_chan) > ioat_pending_level) + __ioat2_issue_pending(ioat_chan); } -static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) +static void __ioat2_start_null_desc(struct ioatdma_chan *ioat_chan) { struct ioat_ring_ent *desc; struct ioat_dma_descriptor *hw; - if (ioat2_ring_space(ioat) < 1) { - dev_err(to_dev(&ioat->base), + if (ioat2_ring_space(ioat_chan) < 1) { + dev_err(to_dev(ioat_chan), "Unable to start null desc - ring full\n"); return; } - dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued); - desc = ioat2_get_ring_ent(ioat, ioat->head); + dev_dbg(to_dev(ioat_chan), + "%s: head: %#x tail: %#x issued: %#x\n", + __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); + desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head); hw = desc->hw; hw->ctl = 0; @@ -109,72 +110,71 @@ static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) hw->src_addr = 0; hw->dst_addr = 0; async_tx_ack(&desc->txd); - ioat2_set_chainaddr(ioat, desc->txd.phys); - dump_desc_dbg(ioat, desc); + ioat2_set_chainaddr(ioat_chan, desc->txd.phys); + dump_desc_dbg(ioat_chan, desc); wmb(); - ioat->head += 1; - __ioat2_issue_pending(ioat); + ioat_chan->head += 1; + __ioat2_issue_pending(ioat_chan); } -static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat) +static void ioat2_start_null_desc(struct ioatdma_chan *ioat_chan) { - spin_lock_bh(&ioat->prep_lock); - __ioat2_start_null_desc(ioat); - spin_unlock_bh(&ioat->prep_lock); + spin_lock_bh(&ioat_chan->prep_lock); + __ioat2_start_null_desc(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); } -void __ioat2_restart_chan(struct ioat2_dma_chan *ioat) +void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan) { - struct ioat_chan_common *chan = &ioat->base; - /* set the tail to be re-issued */ - ioat->issued = ioat->tail; - ioat->dmacount = 0; - set_bit(IOAT_COMPLETION_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + ioat_chan->issued = ioat_chan->tail; + ioat_chan->dmacount = 0; + set_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); - dev_dbg(to_dev(chan), + dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x count: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); + __func__, ioat_chan->head, ioat_chan->tail, + ioat_chan->issued, ioat_chan->dmacount); - if (ioat2_ring_pending(ioat)) { + if (ioat2_ring_pending(ioat_chan)) { struct ioat_ring_ent *desc; - desc = ioat2_get_ring_ent(ioat, ioat->tail); - ioat2_set_chainaddr(ioat, desc->txd.phys); - __ioat2_issue_pending(ioat); + desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail); + ioat2_set_chainaddr(ioat_chan, desc->txd.phys); + __ioat2_issue_pending(ioat_chan); } else - __ioat2_start_null_desc(ioat); + __ioat2_start_null_desc(ioat_chan); } -int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo) +int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo) { unsigned long end = jiffies + tmo; int err = 0; u32 status; - status = ioat_chansts(chan); + status = ioat_chansts(ioat_chan); if (is_ioat_active(status) || is_ioat_idle(status)) - ioat_suspend(chan); + ioat_suspend(ioat_chan); while (is_ioat_active(status) || is_ioat_idle(status)) { if (tmo && time_after(jiffies, end)) { err = -ETIMEDOUT; break; } - status = ioat_chansts(chan); + status = ioat_chansts(ioat_chan); cpu_relax(); } return err; } -int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo) +int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo) { unsigned long end = jiffies + tmo; int err = 0; - ioat_reset(chan); - while (ioat_reset_pending(chan)) { + ioat_reset(ioat_chan); + while (ioat_reset_pending(ioat_chan)) { if (end && time_after(jiffies, end)) { err = -ETIMEDOUT; break; @@ -191,7 +191,7 @@ int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo) */ int ioat2_enumerate_channels(struct ioatdma_device *device) { - struct ioat2_dma_chan *ioat; + struct ioatdma_chan *ioat_chan; struct device *dev = &device->pdev->dev; struct dma_device *dma = &device->common; u8 xfercap_log; @@ -212,14 +212,14 @@ int ioat2_enumerate_channels(struct ioatdma_device *device) dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); for (i = 0; i < dma->chancnt; i++) { - ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); - if (!ioat) + ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL); + if (!ioat_chan) break; - ioat_init_channel(device, &ioat->base, i); - ioat->xfercap_log = xfercap_log; - spin_lock_init(&ioat->prep_lock); - if (device->reset_hw(&ioat->base)) { + ioat_init_channel(device, ioat_chan, i); + ioat_chan->xfercap_log = xfercap_log; + spin_lock_init(&ioat_chan->prep_lock); + if (device->reset_hw(ioat_chan)) { i = 0; break; } @@ -231,15 +231,14 @@ int ioat2_enumerate_channels(struct ioatdma_device *device) static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx) { struct dma_chan *c = tx->chan; - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); dma_cookie_t cookie; cookie = dma_cookie_assign(tx); - dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); + dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie); - if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &chan->state)) - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); /* make descriptor updates visible before advancing ioat->head, * this is purposefully not smp_wmb() since we are also @@ -247,10 +246,10 @@ static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx) */ wmb(); - ioat->head += ioat->produce; + ioat_chan->head += ioat_chan->produce; - ioat2_update_pending(ioat); - spin_unlock_bh(&ioat->prep_lock); + ioat2_update_pending(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); return cookie; } @@ -333,79 +332,78 @@ void ioat2_free_chan_resources(struct dma_chan *c); */ int ioat2_alloc_chan_resources(struct dma_chan *c) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); struct ioat_ring_ent **ring; u64 status; int order; int i = 0; /* have we already been set up? */ - if (ioat->ring) - return 1 << ioat->alloc_order; + if (ioat_chan->ring) + return 1 << ioat_chan->alloc_order; /* Setup register to interrupt and write completion status on error */ - writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); + writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); /* allocate a completion writeback area */ /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ - chan->completion = pci_pool_alloc(chan->device->completion_pool, - GFP_KERNEL, &chan->completion_dma); - if (!chan->completion) + ioat_chan->completion = + pci_pool_alloc(ioat_chan->device->completion_pool, + GFP_KERNEL, &ioat_chan->completion_dma); + if (!ioat_chan->completion) return -ENOMEM; - memset(chan->completion, 0, sizeof(*chan->completion)); - writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, - chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); - writel(((u64) chan->completion_dma) >> 32, - chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); + memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion)); + writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF, + ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); + writel(((u64)ioat_chan->completion_dma) >> 32, + ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); order = ioat_get_alloc_order(); ring = ioat2_alloc_ring(c, order, GFP_KERNEL); if (!ring) return -ENOMEM; - spin_lock_bh(&chan->cleanup_lock); - spin_lock_bh(&ioat->prep_lock); - ioat->ring = ring; - ioat->head = 0; - ioat->issued = 0; - ioat->tail = 0; - ioat->alloc_order = order; - set_bit(IOAT_RUN, &chan->state); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + ioat_chan->ring = ring; + ioat_chan->head = 0; + ioat_chan->issued = 0; + ioat_chan->tail = 0; + ioat_chan->alloc_order = order; + set_bit(IOAT_RUN, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); - ioat2_start_null_desc(ioat); + ioat2_start_null_desc(ioat_chan); /* check that we got off the ground */ do { udelay(1); - status = ioat_chansts(chan); + status = ioat_chansts(ioat_chan); } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); if (is_ioat_active(status) || is_ioat_idle(status)) { - return 1 << ioat->alloc_order; + return 1 << ioat_chan->alloc_order; } else { - u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); - dev_WARN(to_dev(chan), + dev_WARN(to_dev(ioat_chan), "failed to start channel chanerr: %#x\n", chanerr); ioat2_free_chan_resources(c); return -EFAULT; } } -bool reshape_ring(struct ioat2_dma_chan *ioat, int order) +bool reshape_ring(struct ioatdma_chan *ioat_chan, int order) { /* reshape differs from normal ring allocation in that we want * to allocate a new software ring while only * extending/truncating the hardware ring */ - struct ioat_chan_common *chan = &ioat->base; - struct dma_chan *c = &chan->common; - const u32 curr_size = ioat2_ring_size(ioat); - const u16 active = ioat2_ring_active(ioat); + struct dma_chan *c = &ioat_chan->dma_chan; + const u32 curr_size = ioat2_ring_size(ioat_chan); + const u16 active = ioat2_ring_active(ioat_chan); const u32 new_size = 1 << order; struct ioat_ring_ent **ring; u16 i; @@ -432,21 +430,22 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) if (new_size > curr_size) { /* copy current descriptors to the new ring */ for (i = 0; i < curr_size; i++) { - u16 curr_idx = (ioat->tail+i) & (curr_size-1); - u16 new_idx = (ioat->tail+i) & (new_size-1); + u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); - ring[new_idx] = ioat->ring[curr_idx]; + ring[new_idx] = ioat_chan->ring[curr_idx]; set_desc_id(ring[new_idx], new_idx); } /* add new descriptors to the ring */ for (i = curr_size; i < new_size; i++) { - u16 new_idx = (ioat->tail+i) & (new_size-1); + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT); if (!ring[new_idx]) { while (i--) { - u16 new_idx = (ioat->tail+i) & (new_size-1); + u16 new_idx = (ioat_chan->tail+i) & + (new_size-1); ioat2_free_ring_ent(ring[new_idx], c); } @@ -458,7 +457,7 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) /* hw link new descriptors */ for (i = curr_size-1; i < new_size; i++) { - u16 new_idx = (ioat->tail+i) & (new_size-1); + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)]; struct ioat_dma_descriptor *hw = ring[new_idx]->hw; @@ -472,10 +471,10 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) * removed descriptors */ for (i = 0; i < new_size; i++) { - u16 curr_idx = (ioat->tail+i) & (curr_size-1); - u16 new_idx = (ioat->tail+i) & (new_size-1); + u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); - ring[new_idx] = ioat->ring[curr_idx]; + ring[new_idx] = ioat_chan->ring[curr_idx]; set_desc_id(ring[new_idx], new_idx); } @@ -483,22 +482,22 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) for (i = new_size; i < curr_size; i++) { struct ioat_ring_ent *ent; - ent = ioat2_get_ring_ent(ioat, ioat->tail+i); + ent = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail+i); ioat2_free_ring_ent(ent, c); } /* fix up hardware ring */ - hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw; - next = ring[(ioat->tail+new_size) & (new_size-1)]; + hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw; + next = ring[(ioat_chan->tail+new_size) & (new_size-1)]; hw->next = next->txd.phys; } - dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", + dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n", __func__, new_size); - kfree(ioat->ring); - ioat->ring = ring; - ioat->alloc_order = order; + kfree(ioat_chan->ring); + ioat_chan->ring = ring; + ioat_chan->alloc_order = order; return true; } @@ -508,55 +507,57 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order) * @ioat: ioat2,3 channel (ring) to operate on * @num_descs: allocation length */ -int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs) +int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs) { - struct ioat_chan_common *chan = &ioat->base; bool retry; retry: - spin_lock_bh(&ioat->prep_lock); + spin_lock_bh(&ioat_chan->prep_lock); /* never allow the last descriptor to be consumed, we need at * least one free at all times to allow for on-the-fly ring * resizing. */ - if (likely(ioat2_ring_space(ioat) > num_descs)) { - dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n", - __func__, num_descs, ioat->head, ioat->tail, ioat->issued); - ioat->produce = num_descs; + if (likely(ioat2_ring_space(ioat_chan) > num_descs)) { + dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", + __func__, num_descs, ioat_chan->head, + ioat_chan->tail, ioat_chan->issued); + ioat_chan->produce = num_descs; return 0; /* with ioat->prep_lock held */ } - retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state); - spin_unlock_bh(&ioat->prep_lock); + retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); /* is another cpu already trying to expand the ring? */ if (retry) goto retry; - spin_lock_bh(&chan->cleanup_lock); - spin_lock_bh(&ioat->prep_lock); - retry = reshape_ring(ioat, ioat->alloc_order + 1); - clear_bit(IOAT_RESHAPE_PENDING, &chan->state); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1); + clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); /* if we were able to expand the ring retry the allocation */ if (retry) goto retry; if (printk_ratelimit()) - dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n", - __func__, num_descs, ioat->head, ioat->tail, ioat->issued); + dev_dbg(to_dev(ioat_chan), + "%s: ring full! num_descs: %d (%x:%x:%x)\n", + __func__, num_descs, ioat_chan->head, + ioat_chan->tail, ioat_chan->issued); /* progress reclaim in the allocation failure case we may be * called under bh_disabled so we need to trigger the timer * event directly */ - if (time_is_before_jiffies(chan->timer.expires) - && timer_pending(&chan->timer)) { - struct ioatdma_device *device = chan->device; + if (time_is_before_jiffies(ioat_chan->timer.expires) + && timer_pending(&ioat_chan->timer)) { + struct ioatdma_device *device = ioat_chan->device; - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - device->timer_fn((unsigned long) &chan->common); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + device->timer_fn((unsigned long)ioat_chan); } return -ENOMEM; @@ -566,7 +567,7 @@ struct dma_async_tx_descriptor * ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, dma_addr_t dma_src, size_t len, unsigned long flags) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); struct ioat_dma_descriptor *hw; struct ioat_ring_ent *desc; dma_addr_t dst = dma_dest; @@ -574,16 +575,17 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, size_t total_len = len; int num_descs, idx, i; - num_descs = ioat2_xferlen_to_descs(ioat, len); - if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0) - idx = ioat->head; + num_descs = ioat2_xferlen_to_descs(ioat_chan, len); + if (likely(num_descs) && + ioat2_check_space_lock(ioat_chan, num_descs) == 0) + idx = ioat_chan->head; else return NULL; i = 0; do { - size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log); + size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log); - desc = ioat2_get_ring_ent(ioat, idx + i); + desc = ioat2_get_ring_ent(ioat_chan, idx + i); hw = desc->hw; hw->size = copy; @@ -594,7 +596,7 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, len -= copy; dst += copy; src += copy; - dump_desc_dbg(ioat, desc); + dump_desc_dbg(ioat_chan, desc); } while (++i < num_descs); desc->txd.flags = flags; @@ -602,7 +604,7 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); hw->ctl_f.compl_write = 1; - dump_desc_dbg(ioat, desc); + dump_desc_dbg(ioat_chan, desc); /* we leave the channel locked to ensure in order submission */ return &desc->txd; @@ -614,69 +616,68 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, */ void ioat2_free_chan_resources(struct dma_chan *c) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *device = ioat_chan->device; struct ioat_ring_ent *desc; - const u16 total_descs = 1 << ioat->alloc_order; + const u16 total_descs = 1 << ioat_chan->alloc_order; int descs; int i; /* Before freeing channel resources first check * if they have been previously allocated for this channel. */ - if (!ioat->ring) + if (!ioat_chan->ring) return; - ioat_stop(chan); - device->reset_hw(chan); + ioat_stop(ioat_chan); + device->reset_hw(ioat_chan); - spin_lock_bh(&chan->cleanup_lock); - spin_lock_bh(&ioat->prep_lock); - descs = ioat2_ring_space(ioat); - dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs); + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + descs = ioat2_ring_space(ioat_chan); + dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs); for (i = 0; i < descs; i++) { - desc = ioat2_get_ring_ent(ioat, ioat->head + i); + desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head + i); ioat2_free_ring_ent(desc, c); } if (descs < total_descs) - dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", + dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n", total_descs - descs); for (i = 0; i < total_descs - descs; i++) { - desc = ioat2_get_ring_ent(ioat, ioat->tail + i); - dump_desc_dbg(ioat, desc); + desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail + i); + dump_desc_dbg(ioat_chan, desc); ioat2_free_ring_ent(desc, c); } - kfree(ioat->ring); - ioat->ring = NULL; - ioat->alloc_order = 0; - pci_pool_free(device->completion_pool, chan->completion, - chan->completion_dma); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); - - chan->last_completion = 0; - chan->completion_dma = 0; - ioat->dmacount = 0; + kfree(ioat_chan->ring); + ioat_chan->ring = NULL; + ioat_chan->alloc_order = 0; + pci_pool_free(device->completion_pool, ioat_chan->completion, + ioat_chan->completion_dma); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); + + ioat_chan->last_completion = 0; + ioat_chan->completion_dma = 0; + ioat_chan->dmacount = 0; } static ssize_t ring_size_show(struct dma_chan *c, char *page) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); - return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1); + return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1); } static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); static ssize_t ring_active_show(struct dma_chan *c, char *page) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); /* ...taken outside the lock, no need to be precise */ - return sprintf(page, "%d\n", ioat2_ring_active(ioat)); + return sprintf(page, "%d\n", ioat2_ring_active(ioat_chan)); } static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index b7d3583..d3b73c8 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -38,65 +38,36 @@ extern int ioat_ring_alloc_order; #define ioat_get_max_alloc_order() \ (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) -/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes - * @base: common ioat channel parameters - * @xfercap_log; log2 of channel max transfer length (for fast division) - * @head: allocated index - * @issued: hardware notification point - * @tail: cleanup index - * @dmacount: identical to 'head' except for occasionally resetting to zero - * @alloc_order: log2 of the number of allocated descriptors - * @produce: number of descriptors to produce at submit time - * @ring: software ring buffer implementation of hardware ring - * @prep_lock: serializes descriptor preparation (producers) - */ -struct ioat2_dma_chan { - struct ioat_chan_common base; - size_t xfercap_log; - u16 head; - u16 issued; - u16 tail; - u16 dmacount; - u16 alloc_order; - u16 produce; - struct ioat_ring_ent **ring; - spinlock_t prep_lock; -}; - -static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c) -{ - struct ioat_chan_common *chan = to_chan_common(c); - - return container_of(chan, struct ioat2_dma_chan, base); -} - -static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat) +static inline u32 ioat2_ring_size(struct ioatdma_chan *ioat_chan) { - return 1 << ioat->alloc_order; + return 1 << ioat_chan->alloc_order; } /* count of descriptors in flight with the engine */ -static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat) +static inline u16 ioat2_ring_active(struct ioatdma_chan *ioat_chan) { - return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat)); + return CIRC_CNT(ioat_chan->head, ioat_chan->tail, + ioat2_ring_size(ioat_chan)); } /* count of descriptors pending submission to hardware */ -static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat) +static inline u16 ioat2_ring_pending(struct ioatdma_chan *ioat_chan) { - return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat)); + return CIRC_CNT(ioat_chan->head, ioat_chan->issued, + ioat2_ring_size(ioat_chan)); } -static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat) +static inline u32 ioat2_ring_space(struct ioatdma_chan *ioat_chan) { - return ioat2_ring_size(ioat) - ioat2_ring_active(ioat); + return ioat2_ring_size(ioat_chan) - ioat2_ring_active(ioat_chan); } -static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len) +static inline u16 +ioat2_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len) { - u16 num_descs = len >> ioat->xfercap_log; + u16 num_descs = len >> ioat_chan->xfercap_log; - num_descs += !!(len & ((1 << ioat->xfercap_log) - 1)); + num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1)); return num_descs; } @@ -136,25 +107,24 @@ struct ioat_ring_ent { }; static inline struct ioat_ring_ent * -ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx) +ioat2_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx) { - return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)]; + return ioat_chan->ring[idx & (ioat2_ring_size(ioat_chan) - 1)]; } -static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) +static inline void +ioat2_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr) { - struct ioat_chan_common *chan = &ioat->base; - writel(addr & 0x00000000FFFFFFFF, - chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); + ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); writel(addr >> 32, - chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); + ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); } int ioat2_dma_probe(struct ioatdma_device *dev, int dca); int ioat3_dma_probe(struct ioatdma_device *dev, int dca); struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); -int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); +int ioat2_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs); int ioat2_enumerate_channels(struct ioatdma_device *device); struct dma_async_tx_descriptor * ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, @@ -162,12 +132,12 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, void ioat2_issue_pending(struct dma_chan *chan); int ioat2_alloc_chan_resources(struct dma_chan *c); void ioat2_free_chan_resources(struct dma_chan *c); -void __ioat2_restart_chan(struct ioat2_dma_chan *ioat); -bool reshape_ring(struct ioat2_dma_chan *ioat, int order); -void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); +void __ioat2_restart_chan(struct ioatdma_chan *ioat_chan); +bool reshape_ring(struct ioatdma_chan *ioat, int order); +void __ioat2_issue_pending(struct ioatdma_chan *ioat_chan); void ioat2_timer_event(unsigned long data); -int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); -int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); +int ioat2_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo); +int ioat2_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo); extern struct kobj_type ioat2_ktype; extern struct kmem_cache *ioat2_cache; #endif /* IOATDMA_V2_H */ diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 64790a4..2d598fa 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -85,7 +85,7 @@ static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 }; static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6 }; -static void ioat3_eh(struct ioat2_dma_chan *ioat); +static void ioat3_eh(struct ioatdma_chan *ioat_chan); static void xor_set_src(struct ioat_raw_descriptor *descs[2], dma_addr_t addr, u32 offset, int idx) @@ -304,35 +304,35 @@ static bool desc_has_ext(struct ioat_ring_ent *desc) return false; } -static u64 ioat3_get_current_completion(struct ioat_chan_common *chan) +static u64 ioat3_get_current_completion(struct ioatdma_chan *ioat_chan) { u64 phys_complete; u64 completion; - completion = *chan->completion; + completion = *ioat_chan->completion; phys_complete = ioat_chansts_to_addr(completion); - dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, + dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__, (unsigned long long) phys_complete); return phys_complete; } -static bool ioat3_cleanup_preamble(struct ioat_chan_common *chan, +static bool ioat3_cleanup_preamble(struct ioatdma_chan *ioat_chan, u64 *phys_complete) { - *phys_complete = ioat3_get_current_completion(chan); - if (*phys_complete == chan->last_completion) + *phys_complete = ioat3_get_current_completion(ioat_chan); + if (*phys_complete == ioat_chan->last_completion) return false; - clear_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); return true; } static void -desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc) +desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) { struct ioat_dma_descriptor *hw = desc->hw; @@ -368,17 +368,16 @@ desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc) * The difference from the dma_v2.c __cleanup() is that this routine * handles extended descriptors and dma-unmapping raid operations. */ -static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) +static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) { - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; + struct ioatdma_device *device = ioat_chan->device; struct ioat_ring_ent *desc; bool seen_current = false; - int idx = ioat->tail, i; + int idx = ioat_chan->tail, i; u16 active; - dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued); + dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n", + __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); /* * At restart of the channel, the completion address and the @@ -390,18 +389,18 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) if (!phys_complete) return; - active = ioat2_ring_active(ioat); + active = ioat2_ring_active(ioat_chan); for (i = 0; i < active && !seen_current; i++) { struct dma_async_tx_descriptor *tx; smp_read_barrier_depends(); - prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); - desc = ioat2_get_ring_ent(ioat, idx + i); - dump_desc_dbg(ioat, desc); + prefetch(ioat2_get_ring_ent(ioat_chan, idx + i + 1)); + desc = ioat2_get_ring_ent(ioat_chan, idx + i); + dump_desc_dbg(ioat_chan, desc); /* set err stat if we are using dwbes */ if (device->cap & IOAT_CAP_DWBES) - desc_get_errstat(ioat, desc); + desc_get_errstat(ioat_chan, desc); tx = &desc->txd; if (tx->cookie) { @@ -429,70 +428,66 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) } } smp_mb(); /* finish all descriptor reads before incrementing tail */ - ioat->tail = idx + i; + ioat_chan->tail = idx + i; BUG_ON(active && !seen_current); /* no active descs have written a completion? */ - chan->last_completion = phys_complete; + ioat_chan->last_completion = phys_complete; if (active - i == 0) { - dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", + dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n", __func__); - clear_bit(IOAT_COMPLETION_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); + clear_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); } /* 5 microsecond delay per pending descriptor */ writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK), - chan->device->reg_base + IOAT_INTRDELAY_OFFSET); + ioat_chan->device->reg_base + IOAT_INTRDELAY_OFFSET); } -static void ioat3_cleanup(struct ioat2_dma_chan *ioat) +static void ioat3_cleanup(struct ioatdma_chan *ioat_chan) { - struct ioat_chan_common *chan = &ioat->base; u64 phys_complete; - spin_lock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat_chan->cleanup_lock); - if (ioat3_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); + if (ioat3_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); - if (is_ioat_halted(*chan->completion)) { - u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + if (is_ioat_halted(*ioat_chan->completion)) { + u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); if (chanerr & IOAT_CHANERR_HANDLE_MASK) { - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - ioat3_eh(ioat); + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + ioat3_eh(ioat_chan); } } - spin_unlock_bh(&chan->cleanup_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); } static void ioat3_cleanup_event(unsigned long data) { - struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; + struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data); - ioat3_cleanup(ioat); - if (!test_bit(IOAT_RUN, &chan->state)) + ioat3_cleanup(ioat_chan); + if (!test_bit(IOAT_RUN, &ioat_chan->state)) return; - writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); + writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); } -static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) +static void ioat3_restart_channel(struct ioatdma_chan *ioat_chan) { - struct ioat_chan_common *chan = &ioat->base; u64 phys_complete; - ioat2_quiesce(chan, 0); - if (ioat3_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); + ioat2_quiesce(ioat_chan, 0); + if (ioat3_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); - __ioat2_restart_chan(ioat); + __ioat2_restart_chan(ioat_chan); } -static void ioat3_eh(struct ioat2_dma_chan *ioat) +static void ioat3_eh(struct ioatdma_chan *ioat_chan) { - struct ioat_chan_common *chan = &ioat->base; - struct pci_dev *pdev = to_pdev(chan); + struct pci_dev *pdev = to_pdev(ioat_chan); struct ioat_dma_descriptor *hw; struct dma_async_tx_descriptor *tx; u64 phys_complete; @@ -502,18 +497,18 @@ static void ioat3_eh(struct ioat2_dma_chan *ioat) u32 chanerr; /* cleanup so tail points to descriptor that caused the error */ - if (ioat3_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); + if (ioat3_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int); - dev_dbg(to_dev(chan), "%s: error = %x:%x\n", + dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n", __func__, chanerr, chanerr_int); - desc = ioat2_get_ring_ent(ioat, ioat->tail); + desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->tail); hw = desc->hw; - dump_desc_dbg(ioat, desc); + dump_desc_dbg(ioat_chan, desc); switch (hw->ctl_f.op) { case IOAT_OP_XOR_VAL: @@ -537,7 +532,7 @@ static void ioat3_eh(struct ioat2_dma_chan *ioat) /* fault on unhandled error or spurious halt */ if (chanerr ^ err_handled || chanerr == 0) { - dev_err(to_dev(chan), "%s: fatal error (%x:%x)\n", + dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n", __func__, chanerr, err_handled); BUG(); } else { /* cleanup the faulty descriptor */ @@ -552,51 +547,48 @@ static void ioat3_eh(struct ioat2_dma_chan *ioat) } } - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); + writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); /* mark faulting descriptor as complete */ - *chan->completion = desc->txd.phys; + *ioat_chan->completion = desc->txd.phys; - spin_lock_bh(&ioat->prep_lock); - ioat3_restart_channel(ioat); - spin_unlock_bh(&ioat->prep_lock); + spin_lock_bh(&ioat_chan->prep_lock); + ioat3_restart_channel(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); } -static void check_active(struct ioat2_dma_chan *ioat) +static void check_active(struct ioatdma_chan *ioat_chan) { - struct ioat_chan_common *chan = &ioat->base; - - if (ioat2_ring_active(ioat)) { - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + if (ioat2_ring_active(ioat_chan)) { + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); return; } - if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state)) - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - else if (ioat->alloc_order > ioat_get_alloc_order()) { + if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + else if (ioat_chan->alloc_order > ioat_get_alloc_order()) { /* if the ring is idle, empty, and oversized try to step * down the size */ - reshape_ring(ioat, ioat->alloc_order - 1); + reshape_ring(ioat_chan, ioat_chan->alloc_order - 1); /* keep shrinking until we get back to our minimum * default size */ - if (ioat->alloc_order > ioat_get_alloc_order()) - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); + if (ioat_chan->alloc_order > ioat_get_alloc_order()) + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); } } static void ioat3_timer_event(unsigned long data) { - struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; + struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data); dma_addr_t phys_complete; u64 status; - status = ioat_chansts(chan); + status = ioat_chansts(ioat_chan); /* when halted due to errors check for channel * programming errors before advancing the completion state @@ -604,10 +596,10 @@ static void ioat3_timer_event(unsigned long data) if (is_ioat_halted(status)) { u32 chanerr; - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - dev_err(to_dev(chan), "%s: Channel halted (%x)\n", + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n", __func__, chanerr); - if (test_bit(IOAT_RUN, &chan->state)) + if (test_bit(IOAT_RUN, &ioat_chan->state)) BUG_ON(is_ioat_bug(chanerr)); else /* we never got off the ground */ return; @@ -617,43 +609,43 @@ static void ioat3_timer_event(unsigned long data) * acknowledged a pending completion once, then be more * forceful with a restart */ - spin_lock_bh(&chan->cleanup_lock); - if (ioat_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { - spin_lock_bh(&ioat->prep_lock); - ioat3_restart_channel(ioat); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); + spin_lock_bh(&ioat_chan->cleanup_lock); + if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); + else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) { + spin_lock_bh(&ioat_chan->prep_lock); + ioat3_restart_channel(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); return; } else { - set_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); } - if (ioat2_ring_active(ioat)) - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); + if (ioat2_ring_active(ioat_chan)) + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); else { - spin_lock_bh(&ioat->prep_lock); - check_active(ioat); - spin_unlock_bh(&ioat->prep_lock); + spin_lock_bh(&ioat_chan->prep_lock); + check_active(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); } - spin_unlock_bh(&chan->cleanup_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); } static enum dma_status ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); enum dma_status ret; ret = dma_cookie_status(c, cookie, txstate); if (ret == DMA_COMPLETE) return ret; - ioat3_cleanup(ioat); + ioat3_cleanup(ioat_chan); return dma_cookie_status(c, cookie, txstate); } @@ -663,7 +655,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, size_t len, unsigned long flags) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); struct ioat_ring_ent *compl_desc; struct ioat_ring_ent *desc; struct ioat_ring_ent *ext; @@ -677,7 +669,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, BUG_ON(src_cnt < 2); - num_descs = ioat2_xferlen_to_descs(ioat, len); + num_descs = ioat2_xferlen_to_descs(ioat_chan, len); /* we need 2x the number of descriptors to cover greater than 5 * sources */ @@ -692,24 +684,26 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, * (legacy) descriptor to ensure all completion writes arrive in * order. */ - if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0) - idx = ioat->head; + if (likely(num_descs) && + ioat2_check_space_lock(ioat_chan, num_descs+1) == 0) + idx = ioat_chan->head; else return NULL; i = 0; do { struct ioat_raw_descriptor *descs[2]; - size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); + size_t xfer_size = min_t(size_t, + len, 1 << ioat_chan->xfercap_log); int s; - desc = ioat2_get_ring_ent(ioat, idx + i); + desc = ioat2_get_ring_ent(ioat_chan, idx + i); xor = desc->xor; /* save a branch by unconditionally retrieving the * extended descriptor xor_set_src() knows to not write * to it in the single descriptor case */ - ext = ioat2_get_ring_ent(ioat, idx + i + 1); + ext = ioat2_get_ring_ent(ioat_chan, idx + i + 1); xor_ex = ext->xor_ex; descs[0] = (struct ioat_raw_descriptor *) xor; @@ -724,7 +718,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, len -= xfer_size; offset += xfer_size; - dump_desc_dbg(ioat, desc); + dump_desc_dbg(ioat_chan, desc); } while ((i += 1 + with_ext) < num_descs); /* last xor descriptor carries the unmap parameters and fence bit */ @@ -735,7 +729,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); /* completion descriptor carries interrupt bit */ - compl_desc = ioat2_get_ring_ent(ioat, idx + i); + compl_desc = ioat2_get_ring_ent(ioat_chan, idx + i); compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; hw = compl_desc->hw; hw->ctl = 0; @@ -743,7 +737,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.compl_write = 1; hw->size = NULL_DESC_BUFFER_SIZE; - dump_desc_dbg(ioat, compl_desc); + dump_desc_dbg(ioat_chan, compl_desc); /* we leave the channel locked to ensure in order submission */ return &compl_desc->txd; @@ -771,9 +765,10 @@ ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, } static void -dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext) +dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc, + struct ioat_ring_ent *ext) { - struct device *dev = to_dev(&ioat->base); + struct device *dev = to_dev(ioat_chan); struct ioat_pq_descriptor *pq = desc->pq; struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL; struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex }; @@ -797,10 +792,10 @@ dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct dev_dbg(dev, "\tNEXT: %#llx\n", pq->next); } -static void dump_pq16_desc_dbg(struct ioat2_dma_chan *ioat, +static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) { - struct device *dev = to_dev(&ioat->base); + struct device *dev = to_dev(ioat_chan); struct ioat_pq_descriptor *pq = desc->pq; struct ioat_raw_descriptor *descs[] = { (void *)pq, (void *)pq, @@ -838,9 +833,8 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *device = ioat_chan->device; struct ioat_ring_ent *compl_desc; struct ioat_ring_ent *desc; struct ioat_ring_ent *ext; @@ -853,13 +847,13 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, int i, s, idx, with_ext, num_descs; int cb32 = (device->version < IOAT_VER_3_3) ? 1 : 0; - dev_dbg(to_dev(chan), "%s\n", __func__); + dev_dbg(to_dev(ioat_chan), "%s\n", __func__); /* the engine requires at least two sources (we provide * at least 1 implied source in the DMA_PREP_CONTINUE case) */ BUG_ON(src_cnt + dmaf_continue(flags) < 2); - num_descs = ioat2_xferlen_to_descs(ioat, len); + num_descs = ioat2_xferlen_to_descs(ioat_chan, len); /* we need 2x the number of descriptors to cover greater than 3 * sources (we need 1 extra source in the q-only continuation * case and 3 extra sources in the p+q continuation case. @@ -877,23 +871,24 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, * order. */ if (likely(num_descs) && - ioat2_check_space_lock(ioat, num_descs + cb32) == 0) - idx = ioat->head; + ioat2_check_space_lock(ioat_chan, num_descs + cb32) == 0) + idx = ioat_chan->head; else return NULL; i = 0; do { struct ioat_raw_descriptor *descs[2]; - size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); + size_t xfer_size = min_t(size_t, len, + 1 << ioat_chan->xfercap_log); - desc = ioat2_get_ring_ent(ioat, idx + i); + desc = ioat2_get_ring_ent(ioat_chan, idx + i); pq = desc->pq; /* save a branch by unconditionally retrieving the * extended descriptor pq_set_src() knows to not write * to it in the single descriptor case */ - ext = ioat2_get_ring_ent(ioat, idx + i + with_ext); + ext = ioat2_get_ring_ent(ioat_chan, idx + i + with_ext); pq_ex = ext->pq_ex; descs[0] = (struct ioat_raw_descriptor *) pq; @@ -932,7 +927,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, if (result) desc->result = result; pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); - dump_pq_desc_dbg(ioat, desc, ext); + dump_pq_desc_dbg(ioat_chan, desc, ext); if (!cb32) { pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); @@ -940,7 +935,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, compl_desc = desc; } else { /* completion descriptor carries interrupt bit */ - compl_desc = ioat2_get_ring_ent(ioat, idx + i); + compl_desc = ioat2_get_ring_ent(ioat_chan, idx + i); compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; hw = compl_desc->hw; hw->ctl = 0; @@ -948,7 +943,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); hw->ctl_f.compl_write = 1; hw->size = NULL_DESC_BUFFER_SIZE; - dump_desc_dbg(ioat, compl_desc); + dump_desc_dbg(ioat_chan, compl_desc); } @@ -962,9 +957,8 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *device = ioat_chan->device; struct ioat_ring_ent *desc; size_t total_len = len; struct ioat_pq_descriptor *pq; @@ -975,16 +969,16 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, /* this function is only called with 9-16 sources */ op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; - dev_dbg(to_dev(chan), "%s\n", __func__); + dev_dbg(to_dev(ioat_chan), "%s\n", __func__); - num_descs = ioat2_xferlen_to_descs(ioat, len); + num_descs = ioat2_xferlen_to_descs(ioat_chan, len); /* * 16 source pq is only available on cb3.3 and has no completion * write hw bug. */ - if (num_descs && ioat2_check_space_lock(ioat, num_descs) == 0) - idx = ioat->head; + if (num_descs && ioat2_check_space_lock(ioat_chan, num_descs) == 0) + idx = ioat_chan->head; else return NULL; @@ -992,16 +986,17 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, do { struct ioat_raw_descriptor *descs[4]; - size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); + size_t xfer_size = min_t(size_t, len, + 1 << ioat_chan->xfercap_log); - desc = ioat2_get_ring_ent(ioat, idx + i); + desc = ioat2_get_ring_ent(ioat_chan, idx + i); pq = desc->pq; descs[0] = (struct ioat_raw_descriptor *) pq; desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3); if (!desc->sed) { - dev_err(to_dev(chan), + dev_err(to_dev(ioat_chan), "%s: no free sed entries\n", __func__); return NULL; } @@ -1051,7 +1046,7 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); pq->ctl_f.compl_write = 1; - dump_pq16_desc_dbg(ioat, desc); + dump_pq16_desc_dbg(ioat_chan, desc); /* we leave the channel locked to ensure in order submission */ return &desc->txd; @@ -1177,12 +1172,12 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, static struct dma_async_tx_descriptor * ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) { - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); struct ioat_ring_ent *desc; struct ioat_dma_descriptor *hw; - if (ioat2_check_space_lock(ioat, 1) == 0) - desc = ioat2_get_ring_ent(ioat, ioat->head); + if (ioat2_check_space_lock(ioat_chan, 1) == 0) + desc = ioat2_get_ring_ent(ioat_chan, ioat_chan->head); else return NULL; @@ -1199,7 +1194,7 @@ ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) desc->txd.flags = flags; desc->len = 1; - dump_desc_dbg(ioat, desc); + dump_desc_dbg(ioat_chan, desc); /* we leave the channel locked to ensure in order submission */ return &desc->txd; @@ -1504,10 +1499,10 @@ static int ioat3_irq_reinit(struct ioatdma_device *device) case IOAT_MSIX: for (i = 0; i < device->common.chancnt; i++) { struct msix_entry *msix = &device->msix_entries[i]; - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; - chan = ioat_chan_by_index(device, i); - devm_free_irq(&pdev->dev, msix->vector, chan); + ioat_chan = ioat_chan_by_index(device, i); + devm_free_irq(&pdev->dev, msix->vector, ioat_chan); } pci_disable_msix(pdev); @@ -1526,21 +1521,21 @@ static int ioat3_irq_reinit(struct ioatdma_device *device) return ioat_dma_setup_interrupts(device); } -static int ioat3_reset_hw(struct ioat_chan_common *chan) +static int ioat3_reset_hw(struct ioatdma_chan *ioat_chan) { /* throw away whatever the channel was doing and get it * initialized, with ioat3 specific workarounds */ - struct ioatdma_device *device = chan->device; + struct ioatdma_device *device = ioat_chan->device; struct pci_dev *pdev = device->pdev; u32 chanerr; u16 dev_id; int err; - ioat2_quiesce(chan, msecs_to_jiffies(100)); + ioat2_quiesce(ioat_chan, msecs_to_jiffies(100)); - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); if (device->version < IOAT_VER_3_3) { /* clear any pending errors */ @@ -1565,7 +1560,7 @@ static int ioat3_reset_hw(struct ioat_chan_common *chan) } } - err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); + err = ioat2_reset_sync(ioat_chan, msecs_to_jiffies(200)); if (!err) err = ioat3_irq_reinit(device); @@ -1579,7 +1574,7 @@ static void ioat3_intr_quirk(struct ioatdma_device *device) { struct dma_device *dma; struct dma_chan *c; - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; u32 errmask; dma = &device->common; @@ -1590,12 +1585,12 @@ static void ioat3_intr_quirk(struct ioatdma_device *device) */ if (device->cap & IOAT_CAP_DWBES) { list_for_each_entry(c, &dma->channels, device_node) { - chan = to_chan_common(c); - errmask = readl(chan->reg_base + + ioat_chan = to_ioat_chan(c); + errmask = readl(ioat_chan->reg_base + IOAT_CHANERR_MASK_OFFSET); errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR | IOAT_CHANERR_XOR_Q_ERR; - writel(errmask, chan->reg_base + + writel(errmask, ioat_chan->reg_base + IOAT_CHANERR_MASK_OFFSET); } } @@ -1607,7 +1602,7 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) int dca_en = system_has_dca_enabled(pdev); struct dma_device *dma; struct dma_chan *c; - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; bool is_raid_device = false; int err; @@ -1699,9 +1694,9 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) return err; list_for_each_entry(c, &dma->channels, device_node) { - chan = to_chan_common(c); + ioat_chan = to_ioat_chan(c); writel(IOAT_DMA_DCA_ANY_CPU, - chan->reg_base + IOAT_DCACTRL_OFFSET); + ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); } err = ioat_register(device);