diff mbox series

[V2,4/9] PCI/TPH: Implement a command line option to force No ST Mode

Message ID 20240531213841.3246055-5-wei.huang2@amd.com (mailing list archive)
State Not Applicable
Headers show
Series PCIe TPH and cache direct injection support | expand

Checks

Context Check Description
netdev/series_format success Posting correctly formatted
netdev/tree_selection success Guessed tree name to be net-next
netdev/ynl success Generated files up to date; no warnings/errors; no diff in generated;
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit success Errors and warnings before: 7366 this patch: 7366
netdev/build_tools success Errors and warnings before: 0 this patch: 0
netdev/cc_maintainers warning 1 maintainers not CCed: paulmck@kernel.org
netdev/build_clang success Errors and warnings before: 1037 this patch: 1037
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn success Errors and warnings before: 8072 this patch: 8072
netdev/checkpatch success total: 0 errors, 0 warnings, 0 checks, 97 lines checked
netdev/build_clang_rust success No Rust files in patch. Skipping build
netdev/kdoc success Errors and warnings before: 123 this patch: 123
netdev/source_inline success Was 0 now: 0

Commit Message

Wei Huang May 31, 2024, 9:38 p.m. UTC
When "No ST mode" is enabled, end-point devices can generate TPH headers
but with all steering tags treated as zero. A steering tag of zero is
interpreted as "using the default policy" by the root complex. This is
essential to quantify the benefit of steering tags for some given
workloads.

Co-developed-by: Eric Van Tassell <Eric.VanTassell@amd.com>
Signed-off-by: Eric Van Tassell <Eric.VanTassell@amd.com>
Signed-off-by: Wei Huang <wei.huang2@amd.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
Reviewed-by: Somnath Kotur <somnath.kotur@broadcom.com> 
Reviewed-by: Andy Gospodarek <andrew.gospodarek@broadcom.com>
---
 .../admin-guide/kernel-parameters.txt         |  1 +
 drivers/pci/pci-driver.c                      |  7 ++++++-
 drivers/pci/pci.c                             | 12 +++++++++++
 drivers/pci/pcie/tph.c                        | 21 +++++++++++++++++++
 include/linux/pci-tph.h                       |  3 +++
 include/linux/pci.h                           |  1 +
 6 files changed, 44 insertions(+), 1 deletion(-)

Comments

Jonathan Cameron June 7, 2024, 4:32 p.m. UTC | #1
On Fri, 31 May 2024 16:38:36 -0500
Wei Huang <wei.huang2@amd.com> wrote:

> When "No ST mode" is enabled, end-point devices can generate TPH headers
> but with all steering tags treated as zero. A steering tag of zero is
> interpreted as "using the default policy" by the root complex. This is
> essential to quantify the benefit of steering tags for some given
> workloads.

This is a good explanation. Need similar in the previous patch to
justify the disable TPH entirely.

> diff --git a/drivers/pci/pcie/tph.c b/drivers/pci/pcie/tph.c
> index 5dc533b89a33..d5f7309fdf52 100644
> --- a/drivers/pci/pcie/tph.c
> +++ b/drivers/pci/pcie/tph.c
> @@ -43,6 +43,27 @@ static int tph_set_reg_field_u32(struct pci_dev *dev, u8 offset, u32 mask,
>  	return ret;
>  }
>  
> +int tph_set_dev_nostmode(struct pci_dev *dev)
> +{
> +	int ret;
> +
> +	/* set ST Mode Select to "No ST Mode" */
> +	ret = tph_set_reg_field_u32(dev, PCI_TPH_CTRL,
> +				    PCI_TPH_CTRL_MODE_SEL_MASK,
> +				    PCI_TPH_CTRL_MODE_SEL_SHIFT,
> +				    PCI_TPH_NO_ST_MODE);
> +	if (ret)
> +		return ret;
> +
> +	/* set "TPH Requester Enable" to "TPH only" */
> +	ret = tph_set_reg_field_u32(dev, PCI_TPH_CTRL,
> +				    PCI_TPH_CTRL_REQ_EN_MASK,
> +				    PCI_TPH_CTRL_REQ_EN_SHIFT,
> +				    PCI_TPH_REQ_TPH_ONLY);

Unless these have to be two RMW operations. (if they do add a spec reference)
then this is a good example of why a field update function may not be
the right option.  We probably want to RMW once.

return tph_set_reg_field_u32()

> +
> +	return ret;
> +}
> +
>  int pcie_tph_disable(struct pci_dev *dev)
>  {
>  	return  tph_set_reg_field_u32(dev, PCI_TPH_CTRL,
Bjorn Helgaas June 7, 2024, 5:42 p.m. UTC | #2
On Fri, May 31, 2024 at 04:38:36PM -0500, Wei Huang wrote:
> When "No ST mode" is enabled, end-point devices can generate TPH headers
> but with all steering tags treated as zero. A steering tag of zero is
> interpreted as "using the default policy" by the root complex. This is
> essential to quantify the benefit of steering tags for some given
> workloads.

>  #ifdef CONFIG_PCIE_TPH
>  int pcie_tph_disable(struct pci_dev *dev);
> +int tph_set_dev_nostmode(struct pci_dev *dev);

Exported functions need a "pci" or "pcie_" prefix.  We haven't been
completely consistent on this; we have "pci_acs_*", which is obviously
PCIe-specific, but we do also have pcie_find_root_port() etc.
diff mbox series

Patch

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index fedcc69e35c1..e97a4a239563 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -4624,6 +4624,7 @@ 
 		norid		[S390] ignore the RID field and force use of
 				one PCI domain per PCI function
 		notph		[PCIE] Do not use PCIe TPH
+		nostmode	[PCIE] Force TPH to use No ST Mode
 
 	pcie_aspm=	[PCIE] Forcibly enable or ignore PCIe Active State Power
 			Management.
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index 9722d070c0ca..aa98843d9884 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -324,8 +324,13 @@  static long local_pci_probe(void *_ddi)
 	pci_dev->driver = pci_drv;
 	rc = pci_drv->probe(pci_dev, ddi->id);
 	if (!rc) {
-		if (pci_tph_disabled())
+		if (pci_tph_disabled()) {
 			pcie_tph_disable(pci_dev);
+			return rc;
+		}
+
+		if (pci_tph_nostmode())
+			tph_set_dev_nostmode(pci_dev);
 
 		return rc;
 	}
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 31c443504ce9..f3558a551bf2 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -160,6 +160,9 @@  static bool pcie_ats_disabled;
 /* If set, the PCIe TPH capability will not be used. */
 static bool pcie_tph_disabled;
 
+/* If TPH is enabled, "No ST Mode" will be enforced. */
+static bool pcie_tph_nostmode;
+
 /* If set, the PCI config space of each device is printed during boot. */
 bool pci_early_dump;
 
@@ -175,6 +178,12 @@  bool pci_tph_disabled(void)
 }
 EXPORT_SYMBOL_GPL(pci_tph_disabled);
 
+bool pci_tph_nostmode(void)
+{
+	return pcie_tph_nostmode;
+}
+EXPORT_SYMBOL_GPL(pci_tph_nostmode);
+
 /* Disable bridge_d3 for all PCIe ports */
 static bool pci_bridge_d3_disable;
 /* Force bridge_d3 for all PCIe ports */
@@ -6818,6 +6827,9 @@  static int __init pci_setup(char *str)
 			} else if (!strcmp(str, "notph")) {
 				pr_info("PCIe: TPH is disabled\n");
 				pcie_tph_disabled = true;
+			} else if (!strcmp(str, "nostmode")) {
+				pr_info("PCIe: TPH No ST Mode is enabled\n");
+				pcie_tph_nostmode = true;
 			} else if (!strncmp(str, "cbiosize=", 9)) {
 				pci_cardbus_io_size = memparse(str + 9, &str);
 			} else if (!strncmp(str, "cbmemsize=", 10)) {
diff --git a/drivers/pci/pcie/tph.c b/drivers/pci/pcie/tph.c
index 5dc533b89a33..d5f7309fdf52 100644
--- a/drivers/pci/pcie/tph.c
+++ b/drivers/pci/pcie/tph.c
@@ -43,6 +43,27 @@  static int tph_set_reg_field_u32(struct pci_dev *dev, u8 offset, u32 mask,
 	return ret;
 }
 
+int tph_set_dev_nostmode(struct pci_dev *dev)
+{
+	int ret;
+
+	/* set ST Mode Select to "No ST Mode" */
+	ret = tph_set_reg_field_u32(dev, PCI_TPH_CTRL,
+				    PCI_TPH_CTRL_MODE_SEL_MASK,
+				    PCI_TPH_CTRL_MODE_SEL_SHIFT,
+				    PCI_TPH_NO_ST_MODE);
+	if (ret)
+		return ret;
+
+	/* set "TPH Requester Enable" to "TPH only" */
+	ret = tph_set_reg_field_u32(dev, PCI_TPH_CTRL,
+				    PCI_TPH_CTRL_REQ_EN_MASK,
+				    PCI_TPH_CTRL_REQ_EN_SHIFT,
+				    PCI_TPH_REQ_TPH_ONLY);
+
+	return ret;
+}
+
 int pcie_tph_disable(struct pci_dev *dev)
 {
 	return  tph_set_reg_field_u32(dev, PCI_TPH_CTRL,
diff --git a/include/linux/pci-tph.h b/include/linux/pci-tph.h
index e187d7e89e8c..95269afc8b7d 100644
--- a/include/linux/pci-tph.h
+++ b/include/linux/pci-tph.h
@@ -11,9 +11,12 @@ 
 
 #ifdef CONFIG_PCIE_TPH
 int pcie_tph_disable(struct pci_dev *dev);
+int tph_set_dev_nostmode(struct pci_dev *dev);
 #else
 static inline int pcie_tph_disable(struct pci_dev *dev)
 { return -EOPNOTSUPP; }
+static inline int tph_set_dev_nostmode(struct pci_dev *dev)
+{ return -EOPNOTSUPP; }
 #endif
 
 #endif /* LINUX_PCI_TPH_H */
diff --git a/include/linux/pci.h b/include/linux/pci.h
index d88ebe87815a..5f520624d133 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1842,6 +1842,7 @@  static inline bool pci_aer_available(void) { return false; }
 
 bool pci_ats_disabled(void);
 bool pci_tph_disabled(void);
+bool pci_tph_nostmode(void);
 
 #ifdef CONFIG_PCIE_PTM
 int pci_enable_ptm(struct pci_dev *dev, u8 *granularity);