diff mbox series

[v3,20/20] efx: support pio mapping based on cxl

Message ID 20240907081836.5801-21-alejandro.lucero-palau@amd.com (mailing list archive)
State Changes Requested
Delegated to: Netdev Maintainers
Headers show
Series cxl: add Type2 device support | expand

Checks

Context Check Description
netdev/series_format fail Series longer than 15 patches
netdev/tree_selection success Guessed tree name to be net-next
netdev/ynl success Generated files up to date; no warnings/errors; no diff in generated;
netdev/fixes_present success Fixes tag not required for -next series
netdev/header_inline success No static functions without inline keyword in header files
netdev/build_32bit fail Errors and warnings before: 24 this patch: 25
netdev/build_tools success No tools touched, skip
netdev/cc_maintainers warning 3 maintainers not CCed: habetsm.xilinx@gmail.com linux-net-drivers@amd.com ecree.xilinx@gmail.com
netdev/build_clang success Errors and warnings before: 17 this patch: 17
netdev/verify_signedoff success Signed-off-by tag matches author and committer
netdev/deprecated_api success None detected
netdev/check_selftest success No net selftest shell script
netdev/verify_fixes success No Fixes tag
netdev/build_allmodconfig_warn fail Errors and warnings before: 22 this patch: 23
netdev/checkpatch warning WARNING: line length of 86 exceeds 80 columns
netdev/build_clang_rust success No Rust files in patch. Skipping build
netdev/kdoc success Errors and warnings before: 67 this patch: 67
netdev/source_inline success Was 0 now: 0

Commit Message

Lucero Palau, Alejandro Sept. 7, 2024, 8:18 a.m. UTC
From: Alejandro Lucero <alucerop@amd.com>

With a device supporting CXL and successfully initialised, use the cxl
region to map the memory range and use this mapping for PIO buffers.

Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
 drivers/net/ethernet/sfc/ef10.c       | 32 +++++++++++++++++++++------
 drivers/net/ethernet/sfc/efx_cxl.c    | 20 ++++++++++++++++-
 drivers/net/ethernet/sfc/mcdi_pcol.h  | 12 ++++++++++
 drivers/net/ethernet/sfc/net_driver.h |  2 ++
 drivers/net/ethernet/sfc/nic.h        |  2 ++
 5 files changed, 60 insertions(+), 8 deletions(-)

Comments

Edward Cree Sept. 13, 2024, 5:45 p.m. UTC | #1
On 07/09/2024 09:18, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
> 
> With a device supporting CXL and successfully initialised, use the cxl
> region to map the memory range and use this mapping for PIO buffers.
> 
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
<snip>> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
> index cd297e19cddc..c158a1e8d01b 100644
> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
> @@ -16799,6 +16799,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  
>  /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
>  #define    MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
> @@ -17303,6 +17306,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  /* These bits are reserved for communicating test-specific capabilities to
>   * host-side test software. All production drivers should treat this field as
>   * opaque.
> @@ -17821,6 +17827,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  /* These bits are reserved for communicating test-specific capabilities to
>   * host-side test software. All production drivers should treat this field as
>   * opaque.
> @@ -18374,6 +18383,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  /* These bits are reserved for communicating test-specific capabilities to
>   * host-side test software. All production drivers should treat this field as
>   * opaque.

Please, do not make targeted edits to mcdi_pcol.h.  Our standard
 process for this file is to regenerate the whole thing from
 smartnic_registry whenever we want to pick up new additions;
 this helps us have certainty and traceability that the driver
 and firmware-side definitions of the MCDI protocol are consistent
 and limit the risk of copy/paste errors etc. introducing
 differences that could cause backwards-compatibility headaches
 later.  Ideally the MCDI update should also be a separate commit.
See previous commits to this file, such as
    1f17708b47a9 ("sfc: update MCDI protocol headers")
 for examples of how this should look.

-ed
Dave Jiang Sept. 13, 2024, 5:52 p.m. UTC | #2
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
> 
> With a device supporting CXL and successfully initialised, use the cxl
> region to map the memory range and use this mapping for PIO buffers.
> 
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
>  drivers/net/ethernet/sfc/ef10.c       | 32 +++++++++++++++++++++------
>  drivers/net/ethernet/sfc/efx_cxl.c    | 20 ++++++++++++++++-
>  drivers/net/ethernet/sfc/mcdi_pcol.h  | 12 ++++++++++
>  drivers/net/ethernet/sfc/net_driver.h |  2 ++
>  drivers/net/ethernet/sfc/nic.h        |  2 ++
>  5 files changed, 60 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
> index 7d69302ffa0a..d4e64cd0f7a4 100644
> --- a/drivers/net/ethernet/sfc/ef10.c
> +++ b/drivers/net/ethernet/sfc/ef10.c
> @@ -24,6 +24,7 @@
>  #include <linux/wait.h>
>  #include <linux/workqueue.h>
>  #include <net/udp_tunnel.h>
> +#include "efx_cxl.h"
>  
>  /* Hardware control for EF10 architecture including 'Huntington'. */
>  
> @@ -177,6 +178,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
>  			  efx->num_mac_stats);
>  	}
>  
> +	if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
> +		nic_data->datapath_caps3 = 0;
> +	else
> +		nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
> +						      GET_CAPABILITIES_V7_OUT_FLAGS3);
> +
>  	return 0;
>  }
>  
> @@ -949,7 +956,7 @@ static void efx_ef10_remove(struct efx_nic *efx)
>  
>  	efx_mcdi_rx_free_indir_table(efx);
>  
> -	if (nic_data->wc_membase)
> +	if (nic_data->wc_membase && !efx->efx_cxl_pio_in_use)
>  		iounmap(nic_data->wc_membase);
>  
>  	rc = efx_mcdi_free_vis(efx);
> @@ -1263,8 +1270,19 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
>  	iounmap(efx->membase);
>  	efx->membase = membase;
>  
> -	/* Set up the WC mapping if needed */
> -	if (wc_mem_map_size) {
> +	if (!wc_mem_map_size)
> +		return 0;
> +
> +	/* Using PIO through CXL mapping? */
> +	if ((nic_data->datapath_caps3 &
> +	    (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&

Maybe a FIELD_GET() call would make this cleaner

DJ


> +	    efx->efx_cxl_pio_initialised) {
> +		nic_data->pio_write_base = efx->cxl->ctpio_cxl +
> +					   (pio_write_vi_base * efx->vi_stride +
> +					    ER_DZ_TX_PIOBUF - uc_mem_map_size);
> +		efx->efx_cxl_pio_in_use = true;
> +	} else {
> +		/* Using legacy PIO BAR mapping */
>  		nic_data->wc_membase = ioremap_wc(efx->membase_phys +
>  						  uc_mem_map_size,
>  						  wc_mem_map_size);
> @@ -1279,12 +1297,12 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
>  			nic_data->wc_membase +
>  			(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
>  			 uc_mem_map_size);
> -
> -		rc = efx_ef10_link_piobufs(efx);
> -		if (rc)
> -			efx_ef10_free_piobufs(efx);
>  	}
>  
> +	rc = efx_ef10_link_piobufs(efx);
> +	if (rc)
> +		efx_ef10_free_piobufs(efx);
> +
>  	netif_dbg(efx, probe, efx->net_dev,
>  		  "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
>  		  &efx->membase_phys, efx->membase, uc_mem_map_size,
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index dd2dbfb8ba15..ef57f833b8a7 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -21,9 +21,9 @@
>  int efx_cxl_init(struct efx_nic *efx)
>  {
>  	struct pci_dev *pci_dev = efx->pci_dev;
> +	resource_size_t start, end, max = 0;
>  	struct efx_cxl *cxl;
>  	struct resource res;
> -	resource_size_t max;
>  	u16 dvsec;
>  	int rc;
>  
> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
>  		goto err_region;
>  	}
>  
> +	rc = cxl_get_region_params(cxl->efx_region, &start, &end);
> +	if (rc) {
> +		pci_err(pci_dev, "CXL getting regions params failed");
> +		goto err_map;
> +	}
> +
> +	cxl->ctpio_cxl = ioremap(start, end - start);
> +	if (!cxl->ctpio_cxl) {
> +		pci_err(pci_dev, "CXL ioremap region failed");
> +		rc = -EIO;
> +		goto err_map;
> +	}
> +
> +	efx->efx_cxl_pio_initialised = true;
> +
>  	cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>  
>  	return 0;
>  
> +err_map:
> +		cxl_region_detach(cxl->cxled);
>  err_region:
>  	cxl_dpa_free(efx->cxl->cxled);
>  err_release:
> @@ -151,6 +168,7 @@ int efx_cxl_init(struct efx_nic *efx)
>  void efx_cxl_exit(struct efx_nic *efx)
>  {
>  	if (efx->cxl) {
> +		iounmap(efx->cxl->ctpio_cxl);
>  		cxl_region_detach(efx->cxl->cxled);
>  		cxl_dpa_free(efx->cxl->cxled);
>  		cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
> index cd297e19cddc..c158a1e8d01b 100644
> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
> @@ -16799,6 +16799,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  
>  /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
>  #define    MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
> @@ -17303,6 +17306,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  /* These bits are reserved for communicating test-specific capabilities to
>   * host-side test software. All production drivers should treat this field as
>   * opaque.
> @@ -17821,6 +17827,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  /* These bits are reserved for communicating test-specific capabilities to
>   * host-side test software. All production drivers should treat this field as
>   * opaque.
> @@ -18374,6 +18383,9 @@
>  #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>  #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>  #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>  /* These bits are reserved for communicating test-specific capabilities to
>   * host-side test software. All production drivers should treat this field as
>   * opaque.
> diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
> index 77261de65e63..893e7841ffb4 100644
> --- a/drivers/net/ethernet/sfc/net_driver.h
> +++ b/drivers/net/ethernet/sfc/net_driver.h
> @@ -967,6 +967,7 @@ struct efx_cxl;
>   * @dl_port: devlink port associated with the PF
>   * @cxl: details of related cxl objects
>   * @efx_cxl_pio_initialised: clx initialization outcome.
> + * @efx_cxl_pio_in_use: PIO using CXL mapping
>   * @mem_bar: The BAR that is mapped into membase.
>   * @reg_base: Offset from the start of the bar to the function control window.
>   * @monitor_work: Hardware monitor workitem
> @@ -1154,6 +1155,7 @@ struct efx_nic {
>  	struct devlink_port *dl_port;
>  	struct efx_cxl *cxl;
>  	bool efx_cxl_pio_initialised;
> +	bool efx_cxl_pio_in_use;
>  	unsigned int mem_bar;
>  	u32 reg_base;
>  
> diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
> index 1db64fc6e909..b7148810acdb 100644
> --- a/drivers/net/ethernet/sfc/nic.h
> +++ b/drivers/net/ethernet/sfc/nic.h
> @@ -151,6 +151,7 @@ enum {
>   * @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
>   *	%MC_CMD_GET_CAPABILITIES response)
>   * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
> + * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
>   * %MC_CMD_GET_CAPABILITIES response)
>   * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
>   * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
> @@ -186,6 +187,7 @@ struct efx_ef10_nic_data {
>  	bool must_check_datapath_caps;
>  	u32 datapath_caps;
>  	u32 datapath_caps2;
> +	u32 datapath_caps3;
>  	unsigned int rx_dpcpu_fw_id;
>  	unsigned int tx_dpcpu_fw_id;
>  	bool must_probe_vswitching;
Jonathan Cameron Sept. 13, 2024, 6:10 p.m. UTC | #3
On Sat, 7 Sep 2024 09:18:36 +0100
alejandro.lucero-palau@amd.com wrote:

> From: Alejandro Lucero <alucerop@amd.com>
> 
> With a device supporting CXL and successfully initialised, use the cxl
> region to map the memory range and use this mapping for PIO buffers.
> 
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
One trivial thing.

> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index dd2dbfb8ba15..ef57f833b8a7 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -21,9 +21,9 @@
>  int efx_cxl_init(struct efx_nic *efx)
>  {
>  	struct pci_dev *pci_dev = efx->pci_dev;
> +	resource_size_t start, end, max = 0;
>  	struct efx_cxl *cxl;
>  	struct resource res;
> -	resource_size_t max;
>  	u16 dvsec;
>  	int rc;
>  
> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
>  		goto err_region;
>  	}
>  
> +	rc = cxl_get_region_params(cxl->efx_region, &start, &end);
> +	if (rc) {
> +		pci_err(pci_dev, "CXL getting regions params failed");
> +		goto err_map;
> +	}
> +
> +	cxl->ctpio_cxl = ioremap(start, end - start);
> +	if (!cxl->ctpio_cxl) {
> +		pci_err(pci_dev, "CXL ioremap region failed");
> +		rc = -EIO;
> +		goto err_map;
> +	}
> +
> +	efx->efx_cxl_pio_initialised = true;
> +
>  	cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>  
>  	return 0;
>  
> +err_map:
> +		cxl_region_detach(cxl->cxled);

Odd looking indent.

>  err_region:
>  	cxl_dpa_free(efx->cxl->cxled);
>  err_release:
Alejandro Lucero Palau Sept. 16, 2024, 4:12 p.m. UTC | #4
On 9/13/24 18:45, Edward Cree wrote:
> On 07/09/2024 09:18, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> With a device supporting CXL and successfully initialised, use the cxl
>> region to map the memory range and use this mapping for PIO buffers.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> <snip>> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> index cd297e19cddc..c158a1e8d01b 100644
>> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
>> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> @@ -16799,6 +16799,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   
>>   /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
>>   #define    MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
>> @@ -17303,6 +17306,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   /* These bits are reserved for communicating test-specific capabilities to
>>    * host-side test software. All production drivers should treat this field as
>>    * opaque.
>> @@ -17821,6 +17827,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   /* These bits are reserved for communicating test-specific capabilities to
>>    * host-side test software. All production drivers should treat this field as
>>    * opaque.
>> @@ -18374,6 +18383,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   /* These bits are reserved for communicating test-specific capabilities to
>>    * host-side test software. All production drivers should treat this field as
>>    * opaque.


Hi Ed,

It is really kind of you to comment here, specifically this kind of comment.


> Please, do not make targeted edits to mcdi_pcol.h.  Our standard
>   process for this file is to regenerate the whole thing from
>   smartnic_registry whenever we want to pick up new additions;
>   this helps us have certainty and traceability that the driver
>   and firmware-side definitions of the MCDI protocol are consistent
>   and limit the risk of copy/paste errors etc. introducing
>   differences that could cause backwards-compatibility headaches
>   later.  Ideally the MCDI update should also be a separate commit.


Thank you for telling me ... I already knew it.

FWIW, I have followed the process. I guess you did not see it in our 
internal driver repo, but it is in the firmware repo which is the one 
that matters here.

You know there are some mcdi_pcol.h definitions not upstreamed, and 
therefore it is not possible to just submit the new generated file 
upstream but only those bits needed in a patch. I wonder how copy & 
paste can be avoided here. Is there a tool we have unbeknownst to me?


BTW, it is almost an standard process those daily team meetings you have 
been missing "lately" ... we could have avoided this public exchange ...


> See previous commits to this file, such as
>      1f17708b47a9 ("sfc: update MCDI protocol headers")
>   for examples of how this should look.
>
> -ed
Alejandro Lucero Palau Sept. 16, 2024, 4:23 p.m. UTC | #5
On 9/13/24 18:52, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> With a device supporting CXL and successfully initialised, use the cxl
>> region to map the memory range and use this mapping for PIO buffers.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>>   drivers/net/ethernet/sfc/ef10.c       | 32 +++++++++++++++++++++------
>>   drivers/net/ethernet/sfc/efx_cxl.c    | 20 ++++++++++++++++-
>>   drivers/net/ethernet/sfc/mcdi_pcol.h  | 12 ++++++++++
>>   drivers/net/ethernet/sfc/net_driver.h |  2 ++
>>   drivers/net/ethernet/sfc/nic.h        |  2 ++
>>   5 files changed, 60 insertions(+), 8 deletions(-)
>>
>> diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
>> index 7d69302ffa0a..d4e64cd0f7a4 100644
>> --- a/drivers/net/ethernet/sfc/ef10.c
>> +++ b/drivers/net/ethernet/sfc/ef10.c
>> @@ -24,6 +24,7 @@
>>   #include <linux/wait.h>
>>   #include <linux/workqueue.h>
>>   #include <net/udp_tunnel.h>
>> +#include "efx_cxl.h"
>>   
>>   /* Hardware control for EF10 architecture including 'Huntington'. */
>>   
>> @@ -177,6 +178,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
>>   			  efx->num_mac_stats);
>>   	}
>>   
>> +	if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
>> +		nic_data->datapath_caps3 = 0;
>> +	else
>> +		nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
>> +						      GET_CAPABILITIES_V7_OUT_FLAGS3);
>> +
>>   	return 0;
>>   }
>>   
>> @@ -949,7 +956,7 @@ static void efx_ef10_remove(struct efx_nic *efx)
>>   
>>   	efx_mcdi_rx_free_indir_table(efx);
>>   
>> -	if (nic_data->wc_membase)
>> +	if (nic_data->wc_membase && !efx->efx_cxl_pio_in_use)
>>   		iounmap(nic_data->wc_membase);
>>   
>>   	rc = efx_mcdi_free_vis(efx);
>> @@ -1263,8 +1270,19 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
>>   	iounmap(efx->membase);
>>   	efx->membase = membase;
>>   
>> -	/* Set up the WC mapping if needed */
>> -	if (wc_mem_map_size) {
>> +	if (!wc_mem_map_size)
>> +		return 0;
>> +
>> +	/* Using PIO through CXL mapping? */
>> +	if ((nic_data->datapath_caps3 &
>> +	    (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&
> Maybe a FIELD_GET() call would make this cleaner
>
> DJ


I agree.

Thanks


>
>> +	    efx->efx_cxl_pio_initialised) {
>> +		nic_data->pio_write_base = efx->cxl->ctpio_cxl +
>> +					   (pio_write_vi_base * efx->vi_stride +
>> +					    ER_DZ_TX_PIOBUF - uc_mem_map_size);
>> +		efx->efx_cxl_pio_in_use = true;
>> +	} else {
>> +		/* Using legacy PIO BAR mapping */
>>   		nic_data->wc_membase = ioremap_wc(efx->membase_phys +
>>   						  uc_mem_map_size,
>>   						  wc_mem_map_size);
>> @@ -1279,12 +1297,12 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
>>   			nic_data->wc_membase +
>>   			(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
>>   			 uc_mem_map_size);
>> -
>> -		rc = efx_ef10_link_piobufs(efx);
>> -		if (rc)
>> -			efx_ef10_free_piobufs(efx);
>>   	}
>>   
>> +	rc = efx_ef10_link_piobufs(efx);
>> +	if (rc)
>> +		efx_ef10_free_piobufs(efx);
>> +
>>   	netif_dbg(efx, probe, efx->net_dev,
>>   		  "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
>>   		  &efx->membase_phys, efx->membase, uc_mem_map_size,
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index dd2dbfb8ba15..ef57f833b8a7 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -21,9 +21,9 @@
>>   int efx_cxl_init(struct efx_nic *efx)
>>   {
>>   	struct pci_dev *pci_dev = efx->pci_dev;
>> +	resource_size_t start, end, max = 0;
>>   	struct efx_cxl *cxl;
>>   	struct resource res;
>> -	resource_size_t max;
>>   	u16 dvsec;
>>   	int rc;
>>   
>> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
>>   		goto err_region;
>>   	}
>>   
>> +	rc = cxl_get_region_params(cxl->efx_region, &start, &end);
>> +	if (rc) {
>> +		pci_err(pci_dev, "CXL getting regions params failed");
>> +		goto err_map;
>> +	}
>> +
>> +	cxl->ctpio_cxl = ioremap(start, end - start);
>> +	if (!cxl->ctpio_cxl) {
>> +		pci_err(pci_dev, "CXL ioremap region failed");
>> +		rc = -EIO;
>> +		goto err_map;
>> +	}
>> +
>> +	efx->efx_cxl_pio_initialised = true;
>> +
>>   	cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>>   
>>   	return 0;
>>   
>> +err_map:
>> +		cxl_region_detach(cxl->cxled);
>>   err_region:
>>   	cxl_dpa_free(efx->cxl->cxled);
>>   err_release:
>> @@ -151,6 +168,7 @@ int efx_cxl_init(struct efx_nic *efx)
>>   void efx_cxl_exit(struct efx_nic *efx)
>>   {
>>   	if (efx->cxl) {
>> +		iounmap(efx->cxl->ctpio_cxl);
>>   		cxl_region_detach(efx->cxl->cxled);
>>   		cxl_dpa_free(efx->cxl->cxled);
>>   		cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
>> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> index cd297e19cddc..c158a1e8d01b 100644
>> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
>> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> @@ -16799,6 +16799,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   
>>   /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
>>   #define    MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
>> @@ -17303,6 +17306,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   /* These bits are reserved for communicating test-specific capabilities to
>>    * host-side test software. All production drivers should treat this field as
>>    * opaque.
>> @@ -17821,6 +17827,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   /* These bits are reserved for communicating test-specific capabilities to
>>    * host-side test software. All production drivers should treat this field as
>>    * opaque.
>> @@ -18374,6 +18383,9 @@
>>   #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>>   #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>>   #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>   /* These bits are reserved for communicating test-specific capabilities to
>>    * host-side test software. All production drivers should treat this field as
>>    * opaque.
>> diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
>> index 77261de65e63..893e7841ffb4 100644
>> --- a/drivers/net/ethernet/sfc/net_driver.h
>> +++ b/drivers/net/ethernet/sfc/net_driver.h
>> @@ -967,6 +967,7 @@ struct efx_cxl;
>>    * @dl_port: devlink port associated with the PF
>>    * @cxl: details of related cxl objects
>>    * @efx_cxl_pio_initialised: clx initialization outcome.
>> + * @efx_cxl_pio_in_use: PIO using CXL mapping
>>    * @mem_bar: The BAR that is mapped into membase.
>>    * @reg_base: Offset from the start of the bar to the function control window.
>>    * @monitor_work: Hardware monitor workitem
>> @@ -1154,6 +1155,7 @@ struct efx_nic {
>>   	struct devlink_port *dl_port;
>>   	struct efx_cxl *cxl;
>>   	bool efx_cxl_pio_initialised;
>> +	bool efx_cxl_pio_in_use;
>>   	unsigned int mem_bar;
>>   	u32 reg_base;
>>   
>> diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
>> index 1db64fc6e909..b7148810acdb 100644
>> --- a/drivers/net/ethernet/sfc/nic.h
>> +++ b/drivers/net/ethernet/sfc/nic.h
>> @@ -151,6 +151,7 @@ enum {
>>    * @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
>>    *	%MC_CMD_GET_CAPABILITIES response)
>>    * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
>> + * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
>>    * %MC_CMD_GET_CAPABILITIES response)
>>    * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
>>    * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
>> @@ -186,6 +187,7 @@ struct efx_ef10_nic_data {
>>   	bool must_check_datapath_caps;
>>   	u32 datapath_caps;
>>   	u32 datapath_caps2;
>> +	u32 datapath_caps3;
>>   	unsigned int rx_dpcpu_fw_id;
>>   	unsigned int tx_dpcpu_fw_id;
>>   	bool must_probe_vswitching;
Alejandro Lucero Palau Sept. 16, 2024, 4:23 p.m. UTC | #6
On 9/13/24 19:10, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:36 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> With a device supporting CXL and successfully initialised, use the cxl
>> region to map the memory range and use this mapping for PIO buffers.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> One trivial thing.
>
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index dd2dbfb8ba15..ef57f833b8a7 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -21,9 +21,9 @@
>>   int efx_cxl_init(struct efx_nic *efx)
>>   {
>>   	struct pci_dev *pci_dev = efx->pci_dev;
>> +	resource_size_t start, end, max = 0;
>>   	struct efx_cxl *cxl;
>>   	struct resource res;
>> -	resource_size_t max;
>>   	u16 dvsec;
>>   	int rc;
>>   
>> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
>>   		goto err_region;
>>   	}
>>   
>> +	rc = cxl_get_region_params(cxl->efx_region, &start, &end);
>> +	if (rc) {
>> +		pci_err(pci_dev, "CXL getting regions params failed");
>> +		goto err_map;
>> +	}
>> +
>> +	cxl->ctpio_cxl = ioremap(start, end - start);
>> +	if (!cxl->ctpio_cxl) {
>> +		pci_err(pci_dev, "CXL ioremap region failed");
>> +		rc = -EIO;
>> +		goto err_map;
>> +	}
>> +
>> +	efx->efx_cxl_pio_initialised = true;
>> +
>>   	cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>>   
>>   	return 0;
>>   
>> +err_map:
>> +		cxl_region_detach(cxl->cxled);
> Odd looking indent.


I'll fix it.

Thanks!


>>   err_region:
>>   	cxl_dpa_free(efx->cxl->cxled);
>>   err_release:
diff mbox series

Patch

diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index 7d69302ffa0a..d4e64cd0f7a4 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -24,6 +24,7 @@ 
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 #include <net/udp_tunnel.h>
+#include "efx_cxl.h"
 
 /* Hardware control for EF10 architecture including 'Huntington'. */
 
@@ -177,6 +178,12 @@  static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 			  efx->num_mac_stats);
 	}
 
+	if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
+		nic_data->datapath_caps3 = 0;
+	else
+		nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
+						      GET_CAPABILITIES_V7_OUT_FLAGS3);
+
 	return 0;
 }
 
@@ -949,7 +956,7 @@  static void efx_ef10_remove(struct efx_nic *efx)
 
 	efx_mcdi_rx_free_indir_table(efx);
 
-	if (nic_data->wc_membase)
+	if (nic_data->wc_membase && !efx->efx_cxl_pio_in_use)
 		iounmap(nic_data->wc_membase);
 
 	rc = efx_mcdi_free_vis(efx);
@@ -1263,8 +1270,19 @@  static int efx_ef10_dimension_resources(struct efx_nic *efx)
 	iounmap(efx->membase);
 	efx->membase = membase;
 
-	/* Set up the WC mapping if needed */
-	if (wc_mem_map_size) {
+	if (!wc_mem_map_size)
+		return 0;
+
+	/* Using PIO through CXL mapping? */
+	if ((nic_data->datapath_caps3 &
+	    (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&
+	    efx->efx_cxl_pio_initialised) {
+		nic_data->pio_write_base = efx->cxl->ctpio_cxl +
+					   (pio_write_vi_base * efx->vi_stride +
+					    ER_DZ_TX_PIOBUF - uc_mem_map_size);
+		efx->efx_cxl_pio_in_use = true;
+	} else {
+		/* Using legacy PIO BAR mapping */
 		nic_data->wc_membase = ioremap_wc(efx->membase_phys +
 						  uc_mem_map_size,
 						  wc_mem_map_size);
@@ -1279,12 +1297,12 @@  static int efx_ef10_dimension_resources(struct efx_nic *efx)
 			nic_data->wc_membase +
 			(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
 			 uc_mem_map_size);
-
-		rc = efx_ef10_link_piobufs(efx);
-		if (rc)
-			efx_ef10_free_piobufs(efx);
 	}
 
+	rc = efx_ef10_link_piobufs(efx);
+	if (rc)
+		efx_ef10_free_piobufs(efx);
+
 	netif_dbg(efx, probe, efx->net_dev,
 		  "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
 		  &efx->membase_phys, efx->membase, uc_mem_map_size,
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index dd2dbfb8ba15..ef57f833b8a7 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -21,9 +21,9 @@ 
 int efx_cxl_init(struct efx_nic *efx)
 {
 	struct pci_dev *pci_dev = efx->pci_dev;
+	resource_size_t start, end, max = 0;
 	struct efx_cxl *cxl;
 	struct resource res;
-	resource_size_t max;
 	u16 dvsec;
 	int rc;
 
@@ -132,10 +132,27 @@  int efx_cxl_init(struct efx_nic *efx)
 		goto err_region;
 	}
 
+	rc = cxl_get_region_params(cxl->efx_region, &start, &end);
+	if (rc) {
+		pci_err(pci_dev, "CXL getting regions params failed");
+		goto err_map;
+	}
+
+	cxl->ctpio_cxl = ioremap(start, end - start);
+	if (!cxl->ctpio_cxl) {
+		pci_err(pci_dev, "CXL ioremap region failed");
+		rc = -EIO;
+		goto err_map;
+	}
+
+	efx->efx_cxl_pio_initialised = true;
+
 	cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
 
 	return 0;
 
+err_map:
+		cxl_region_detach(cxl->cxled);
 err_region:
 	cxl_dpa_free(efx->cxl->cxled);
 err_release:
@@ -151,6 +168,7 @@  int efx_cxl_init(struct efx_nic *efx)
 void efx_cxl_exit(struct efx_nic *efx)
 {
 	if (efx->cxl) {
+		iounmap(efx->cxl->ctpio_cxl);
 		cxl_region_detach(efx->cxl->cxled);
 		cxl_dpa_free(efx->cxl->cxled);
 		cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
index cd297e19cddc..c158a1e8d01b 100644
--- a/drivers/net/ethernet/sfc/mcdi_pcol.h
+++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
@@ -16799,6 +16799,9 @@ 
 #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
 #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
 #define        MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
 
 /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
 #define    MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
@@ -17303,6 +17306,9 @@ 
 #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
 #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
 #define        MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
 /* These bits are reserved for communicating test-specific capabilities to
  * host-side test software. All production drivers should treat this field as
  * opaque.
@@ -17821,6 +17827,9 @@ 
 #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
 #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
 #define        MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
 /* These bits are reserved for communicating test-specific capabilities to
  * host-side test software. All production drivers should treat this field as
  * opaque.
@@ -18374,6 +18383,9 @@ 
 #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
 #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
 #define        MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
 /* These bits are reserved for communicating test-specific capabilities to
  * host-side test software. All production drivers should treat this field as
  * opaque.
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 77261de65e63..893e7841ffb4 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -967,6 +967,7 @@  struct efx_cxl;
  * @dl_port: devlink port associated with the PF
  * @cxl: details of related cxl objects
  * @efx_cxl_pio_initialised: clx initialization outcome.
+ * @efx_cxl_pio_in_use: PIO using CXL mapping
  * @mem_bar: The BAR that is mapped into membase.
  * @reg_base: Offset from the start of the bar to the function control window.
  * @monitor_work: Hardware monitor workitem
@@ -1154,6 +1155,7 @@  struct efx_nic {
 	struct devlink_port *dl_port;
 	struct efx_cxl *cxl;
 	bool efx_cxl_pio_initialised;
+	bool efx_cxl_pio_in_use;
 	unsigned int mem_bar;
 	u32 reg_base;
 
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index 1db64fc6e909..b7148810acdb 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -151,6 +151,7 @@  enum {
  * @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
  *	%MC_CMD_GET_CAPABILITIES response)
  * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
+ * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
  * %MC_CMD_GET_CAPABILITIES response)
  * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
  * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
@@ -186,6 +187,7 @@  struct efx_ef10_nic_data {
 	bool must_check_datapath_caps;
 	u32 datapath_caps;
 	u32 datapath_caps2;
+	u32 datapath_caps3;
 	unsigned int rx_dpcpu_fw_id;
 	unsigned int tx_dpcpu_fw_id;
 	bool must_probe_vswitching;