[v5,05/14] cxl: Calculate and store PCI link latency for the downstream ports

Message ID	168357884387.2756219.5365448705913389325.stgit@djiang5-mobl3
State	Superseded
Headers	show Return-Path: <linux-cxl-owner@vger.kernel.org> Subject: [PATCH v5 05/14] cxl: Calculate and store PCI link latency for the downstream ports From: Dave Jiang <dave.jiang@intel.com> To: linux-cxl@vger.kernel.org Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com>, dan.j.williams@intel.com, ira.weiny@intel.com, vishal.l.verma@intel.com, alison.schofield@intel.com, Jonathan.Cameron@huawei.com Date: Mon, 08 May 2023 13:47:23 -0700 Message-ID: <168357884387.2756219.5365448705913389325.stgit@djiang5-mobl3> In-Reply-To: <168357873843.2756219.5839806150467356492.stgit@djiang5-mobl3> References: <168357873843.2756219.5839806150467356492.stgit@djiang5-mobl3> User-Agent: StGit/1.5 MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Precedence: bulk
Series	cxl: Add support for QTG ID retrieval for CXL subsystem \| expand [v5,00/14] cxl: Add support for QTG ID retrieval for CXL subsystem [v5,01/14] cxl: Add callback to parse the DSMAS subtables from CDAT [v5,02/14] cxl: Add callback to parse the DSLBIS subtable from CDAT [v5,03/14] cxl: Add callback to parse the SSLBIS subtable from CDAT [v5,04/14] cxl: Add support for _DSM Function for retrieving QTG ID [v5,05/14] cxl: Calculate and store PCI link latency for the downstream ports [v5,06/14] cxl: Store the access coordinates for the generic ports [v5,07/14] cxl: Add helper function that calculate performance data for downstream ports [v5,08/14] cxl: Compute the entire CXL path latency and bandwidth data [v5,09/14] cxl: Wait Memory_Info_Valid before access memory related info [v5,10/14] cxl: Move identify and partition query from pci probe to port probe [v5,11/14] cxl: Move read_cdat_data() to after media is ready [v5,12/14] cxl: Store QTG IDs and related info to the CXL memory device context [v5,13/14] cxl: Export sysfs attributes for memory device QoS class [v5,14/14] cxl/mem: Add debugfs output for QTG related data

diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 58051154ab1a..6c99a964eb54 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ +#include <linux/units.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/device.h> #include <linux/delay.h> @@ -718,3 +719,75 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, return PCI_ERS_RESULT_NEED_RESET; } EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL); + +extern const unsigned char pcie_link_speed[]; + +static enum pci_bus_speed get_link_speed(struct pci_dev *pdev) +{ + u16 linkstat; + int err; + + err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &linkstat); + if (err) + return -EINVAL; + + return pcie_link_speed[linkstat & PCI_EXP_LNKSTA_CLS]; +} + +static int pci_bus_speed_to_mbps(enum pci_bus_speed speed) +{ + switch (speed) { + case PCIE_SPEED_2_5GT: + return 2500; + case PCIE_SPEED_5_0GT: + return 5000; + case PCIE_SPEED_8_0GT: + return 8000; + case PCIE_SPEED_16_0GT: + return 16000; + case PCIE_SPEED_32_0GT: + return 32000; + case PCIE_SPEED_64_0GT: + return 64000; + default: + break; + } + + return -EINVAL; +} + +static int cxl_flit_size(struct pci_dev *pdev) +{ + if (cxl_pci_flit_256(pdev)) + return 256; + + return 68; +} + +/** + * cxl_pci_get_latency - calculate the link latency for the PCIe link + * @pdev - PCI device + * + * return: calculated latency or 0 for no latency + * + * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation + * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency + * LinkProgationLatency is negligible, so 0 will be used + * RetimerLatency is assumed to be negligible and 0 will be used + * FlitLatency = FlitSize / LinkBandwidth + * FlitSize is defined by spec. CXL rev3.0 4.2.1. + * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used. + * The FlitLatency is converted to picoseconds. + */ +long cxl_pci_get_latency(struct pci_dev *pdev) +{ + long bw; + + bw = pci_bus_speed_to_mbps(get_link_speed(pdev)); + if (bw < 0) + return 0; + bw /= BITS_PER_BYTE; + + return cxl_flit_size(pdev) * MEGA / bw; +} +EXPORT_SYMBOL_NS_GPL(cxl_pci_get_latency, CXL); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 60693c3f85f6..e4f75847b851 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -751,6 +751,9 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, if (rc) return ERR_PTR(rc); + if (parent_dport && dev_is_pci(uport)) + port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport)); + return port; err: @@ -1008,6 +1011,9 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (rc) return ERR_PTR(rc); + if (dev_is_pci(dport_dev)) + dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev)); + return dport; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 16fc14d43aa4..c1e2c3703a63 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -562,6 +562,7 @@ struct cxl_dax_region { * @depth: How deep this port is relative to the root. depth 0 is the root. * @cdat: Cached CDAT data * @cdat_available: Should a CDAT attribute be available in sysfs + * @pci_latency: Upstream latency in picoseconds */ struct cxl_port { struct device dev; @@ -584,6 +585,7 @@ struct cxl_port { size_t length; } cdat; bool cdat_available; + long pci_latency; }; struct qtg_dsm_input; @@ -625,6 +627,7 @@ cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev) * @rch: Indicate whether this dport was enumerated in RCH or VH mode * @port: reference to cxl_port that contains this downstream port * @coord: access coordinates (performance) for switch from CDAT + * @link_latency: calculated PCIe downstream latency */ struct cxl_dport { struct device *dport; @@ -634,6 +637,7 @@ struct cxl_dport { bool rch; struct cxl_port *port; struct access_coordinate coord; + long link_latency; }; /** diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 0465ef963cd6..84c9f73c7d92 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -82,6 +82,19 @@ struct cdat_entry_header { __le16 length; } __packed; +/* + * CXL v3.0 6.2.3 Table 6-4 + * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits + * mode, otherwise it's 68B flits mode. + */ +static inline bool cxl_pci_flit_256(struct pci_dev *pdev) +{ + u16 lnksta2; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2); + return lnksta2 & PCI_EXP_LNKSTA2_FLIT; +} + int devm_cxl_port_enumerate_dports(struct cxl_port *port); struct cxl_dev_state; int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, @@ -90,4 +103,6 @@ void read_cdat_data(struct cxl_port *port); void cxl_cor_error_detected(struct pci_dev *pdev); pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, pci_channel_state_t state); +long cxl_pci_get_latency(struct pci_dev *pdev); + #endif /* __CXL_PCI_H__ */ diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index ea38bd49b0cf..ed39d133b70d 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -365,19 +365,6 @@ static bool is_cxl_restricted(struct pci_dev *pdev) return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END; } -/* - * CXL v3.0 6.2.3 Table 6-4 - * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits - * mode, otherwise it's 68B flits mode. - */ -static bool cxl_pci_flit_256(struct pci_dev *pdev) -{ - u16 lnksta2; - - pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2); - return lnksta2 & PCI_EXP_LNKSTA2_FLIT; -} - static int cxl_pci_ras_unmask(struct pci_dev *pdev) { struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus);

[v5,05/14] cxl: Calculate and store PCI link latency for the downstream ports

Commit Message

Patch