diff mbox series

[v13,13/22] cxl: define a driver interface for DPA allocation

Message ID 20250414151336.3852990-14-alejandro.lucero-palau@amd.com
State New
Headers show
Series Type2 device basic support | expand

Commit Message

Lucero Palau, Alejandro April 14, 2025, 3:13 p.m. UTC 
From: Alejandro Lucero <alucerop@amd.com>

Region creation involves finding available DPA (device-physical-address)
capacity to map into HPA (host-physical-address) space. Define an API,
cxl_request_dpa(), that tries to allocate the DPA memory the driver
requires to operate. The memory requested should not be bigger than the
max available HPA obtained previously with cxl_get_hpa_freespace.

Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/

Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Reviewed-by: Ben Cheatham <benjamin.cheatham@amd.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
---
 drivers/cxl/core/hdm.c | 77 ++++++++++++++++++++++++++++++++++++++++++
 include/cxl/cxl.h      |  4 +++
 2 files changed, 81 insertions(+)

Comments

kernel test robot April 15, 2025, 5:19 p.m. UTC | #1 
Hi,

kernel test robot noticed the following build errors:

[auto build test ERROR on 73c117c17b562213242f432db2ddf1bcc22f39dd]

url:    https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20250414-233241
base:   73c117c17b562213242f432db2ddf1bcc22f39dd
patch link:    https://lore.kernel.org/r/20250414151336.3852990-14-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v13 13/22] cxl: define a driver interface for DPA allocation
config: csky-randconfig-002-20250415
compiler: csky-linux-gcc (GCC) 14.2.0
reproduce (this is a W=1 build):

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202504160058.DWk5ApNJ-lkp@intel.com/

All errors (new ones prefixed by >>):

   In file included from drivers/cxl/core/hdm.c:6:
>> include/cxl/cxl.h:153:22: error: field 'dpa_range' has incomplete type
     153 |         struct range dpa_range;
         |                      ^~~~~~~~~
>> include/cxl/cxl.h:227:30: error: field 'range' has incomplete type
     227 |                 struct range range;
         |                              ^~~~~


vim +/dpa_range +153 include/cxl/cxl.h

04ccf61aaf26953 Alejandro Lucero 2025-04-14  144  
04ccf61aaf26953 Alejandro Lucero 2025-04-14  145  /**
04ccf61aaf26953 Alejandro Lucero 2025-04-14  146   * struct cxl_dpa_perf - DPA performance property entry
04ccf61aaf26953 Alejandro Lucero 2025-04-14  147   * @dpa_range: range for DPA address
04ccf61aaf26953 Alejandro Lucero 2025-04-14  148   * @coord: QoS performance data (i.e. latency, bandwidth)
04ccf61aaf26953 Alejandro Lucero 2025-04-14  149   * @cdat_coord: raw QoS performance data from CDAT
04ccf61aaf26953 Alejandro Lucero 2025-04-14  150   * @qos_class: QoS Class cookies
04ccf61aaf26953 Alejandro Lucero 2025-04-14  151   */
04ccf61aaf26953 Alejandro Lucero 2025-04-14  152  struct cxl_dpa_perf {
04ccf61aaf26953 Alejandro Lucero 2025-04-14 @153  	struct range dpa_range;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  154  	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
04ccf61aaf26953 Alejandro Lucero 2025-04-14  155  	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
04ccf61aaf26953 Alejandro Lucero 2025-04-14  156  	int qos_class;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  157  };
04ccf61aaf26953 Alejandro Lucero 2025-04-14  158  
04ccf61aaf26953 Alejandro Lucero 2025-04-14  159  enum cxl_partition_mode {
04ccf61aaf26953 Alejandro Lucero 2025-04-14  160  	CXL_PARTMODE_RAM,
04ccf61aaf26953 Alejandro Lucero 2025-04-14  161  	CXL_PARTMODE_PMEM,
04ccf61aaf26953 Alejandro Lucero 2025-04-14  162  };
04ccf61aaf26953 Alejandro Lucero 2025-04-14  163  
04ccf61aaf26953 Alejandro Lucero 2025-04-14  164  /**
04ccf61aaf26953 Alejandro Lucero 2025-04-14  165   * struct cxl_dpa_partition - DPA partition descriptor
04ccf61aaf26953 Alejandro Lucero 2025-04-14  166   * @res: shortcut to the partition in the DPA resource tree (cxlds->dpa_res)
04ccf61aaf26953 Alejandro Lucero 2025-04-14  167   * @perf: performance attributes of the partition from CDAT
04ccf61aaf26953 Alejandro Lucero 2025-04-14  168   * @mode: operation mode for the DPA capacity, e.g. ram, pmem, dynamic...
04ccf61aaf26953 Alejandro Lucero 2025-04-14  169   */
04ccf61aaf26953 Alejandro Lucero 2025-04-14  170  struct cxl_dpa_partition {
04ccf61aaf26953 Alejandro Lucero 2025-04-14  171  	struct resource res;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  172  	struct cxl_dpa_perf perf;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  173  	enum cxl_partition_mode mode;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  174  };
04ccf61aaf26953 Alejandro Lucero 2025-04-14  175  
04ccf61aaf26953 Alejandro Lucero 2025-04-14  176  #define CXL_NR_PARTITIONS_MAX 2
04ccf61aaf26953 Alejandro Lucero 2025-04-14  177  
04ccf61aaf26953 Alejandro Lucero 2025-04-14  178  /**
04ccf61aaf26953 Alejandro Lucero 2025-04-14  179   * struct cxl_dev_state - The driver device state
04ccf61aaf26953 Alejandro Lucero 2025-04-14  180   *
04ccf61aaf26953 Alejandro Lucero 2025-04-14  181   * cxl_dev_state represents the CXL driver/device state.  It provides an
04ccf61aaf26953 Alejandro Lucero 2025-04-14  182   * interface to mailbox commands as well as some cached data about the device.
04ccf61aaf26953 Alejandro Lucero 2025-04-14  183   * Currently only memory devices are represented.
04ccf61aaf26953 Alejandro Lucero 2025-04-14  184   *
04ccf61aaf26953 Alejandro Lucero 2025-04-14  185   * @dev: The device associated with this CXL state
04ccf61aaf26953 Alejandro Lucero 2025-04-14  186   * @cxlmd: The device representing the CXL.mem capabilities of @dev
04ccf61aaf26953 Alejandro Lucero 2025-04-14  187   * @reg_map: component and ras register mapping parameters
04ccf61aaf26953 Alejandro Lucero 2025-04-14  188   * @regs: Parsed register blocks
04ccf61aaf26953 Alejandro Lucero 2025-04-14  189   * @cxl_dvsec: Offset to the PCIe device DVSEC
04ccf61aaf26953 Alejandro Lucero 2025-04-14  190   * @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH)
04ccf61aaf26953 Alejandro Lucero 2025-04-14  191   * @media_ready: Indicate whether the device media is usable
04ccf61aaf26953 Alejandro Lucero 2025-04-14  192   * @dpa_res: Overall DPA resource tree for the device
04ccf61aaf26953 Alejandro Lucero 2025-04-14  193   * @part: DPA partition array
04ccf61aaf26953 Alejandro Lucero 2025-04-14  194   * @nr_partitions: Number of DPA partitions
04ccf61aaf26953 Alejandro Lucero 2025-04-14  195   * @serial: PCIe Device Serial Number
04ccf61aaf26953 Alejandro Lucero 2025-04-14  196   * @type: Generic Memory Class device or Vendor Specific Memory device
04ccf61aaf26953 Alejandro Lucero 2025-04-14  197   * @cxl_mbox: CXL mailbox context
04ccf61aaf26953 Alejandro Lucero 2025-04-14  198   * @cxlfs: CXL features context
04ccf61aaf26953 Alejandro Lucero 2025-04-14  199   */
04ccf61aaf26953 Alejandro Lucero 2025-04-14  200  struct cxl_dev_state {
04ccf61aaf26953 Alejandro Lucero 2025-04-14  201  	/* public for Type2 drivers */
04ccf61aaf26953 Alejandro Lucero 2025-04-14  202  	struct device *dev;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  203  	struct cxl_memdev *cxlmd;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  204  
04ccf61aaf26953 Alejandro Lucero 2025-04-14  205  	/* private for Type2 drivers */
04ccf61aaf26953 Alejandro Lucero 2025-04-14  206  	struct cxl_register_map reg_map;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  207  	struct cxl_regs regs;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  208  	int cxl_dvsec;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  209  	bool rcd;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  210  	bool media_ready;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  211  	struct resource dpa_res;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  212  	struct cxl_dpa_partition part[CXL_NR_PARTITIONS_MAX];
04ccf61aaf26953 Alejandro Lucero 2025-04-14  213  	unsigned int nr_partitions;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  214  	u64 serial;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  215  	enum cxl_devtype type;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  216  	struct cxl_mailbox cxl_mbox;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  217  #ifdef CONFIG_CXL_FEATURES
04ccf61aaf26953 Alejandro Lucero 2025-04-14  218  	struct cxl_features_state *cxlfs;
04ccf61aaf26953 Alejandro Lucero 2025-04-14  219  #endif
04ccf61aaf26953 Alejandro Lucero 2025-04-14  220  };
04ccf61aaf26953 Alejandro Lucero 2025-04-14  221  
dd50535762e809d Alejandro Lucero 2025-04-14  222  #define CXL_NR_PARTITIONS_MAX 2
dd50535762e809d Alejandro Lucero 2025-04-14  223  
dd50535762e809d Alejandro Lucero 2025-04-14  224  struct cxl_dpa_info {
dd50535762e809d Alejandro Lucero 2025-04-14  225  	u64 size;
dd50535762e809d Alejandro Lucero 2025-04-14  226  	struct cxl_dpa_part_info {
dd50535762e809d Alejandro Lucero 2025-04-14 @227  		struct range range;
dd50535762e809d Alejandro Lucero 2025-04-14  228  		enum cxl_partition_mode mode;
dd50535762e809d Alejandro Lucero 2025-04-14  229  	} part[CXL_NR_PARTITIONS_MAX];
dd50535762e809d Alejandro Lucero 2025-04-14  230  	int nr_partitions;
dd50535762e809d Alejandro Lucero 2025-04-14  231  };
dd50535762e809d Alejandro Lucero 2025-04-14  232
kernel test robot April 15, 2025, 5:40 p.m. UTC | #2 
Hi,

kernel test robot noticed the following build warnings:

[auto build test WARNING on 73c117c17b562213242f432db2ddf1bcc22f39dd]

url:    https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20250414-233241
base:   73c117c17b562213242f432db2ddf1bcc22f39dd
patch link:    https://lore.kernel.org/r/20250414151336.3852990-14-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v13 13/22] cxl: define a driver interface for DPA allocation
config: loongarch-randconfig-002-20250415
compiler: loongarch64-linux-gcc (GCC) 14.2.0
reproduce (this is a W=1 build):

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202504160127.9vvEhASi-lkp@intel.com/

All warnings (new ones prefixed by >>):

>> drivers/cxl/core/hdm.c:728: warning: Function parameter or struct member 'mode' not described in 'cxl_request_dpa'
>> drivers/cxl/core/hdm.c:728: warning: Excess function parameter 'is_ram' description in 'cxl_request_dpa'


vim +728 drivers/cxl/core/hdm.c

   707	
   708	/**
   709	 * cxl_request_dpa - search and reserve DPA given input constraints
   710	 * @cxlmd: memdev with an endpoint port with available decoders
   711	 * @is_ram: DPA operation mode (ram vs pmem)
   712	 * @alloc: dpa size required
   713	 *
   714	 * Given that a region needs to allocate from limited HPA capacity it
   715	 * may be the case that a device has more mappable DPA capacity than
   716	 * available HPA. The expectation is that @alloc is a driver known
   717	 * value based on the device capacity but it could not be available
   718	 * due to HPA constraints.
   719	 *
   720	 * Returns a pinned cxl_decoder with at least @alloc bytes of capacity
   721	 * reserved, or an error pointer. The caller is also expected to own the
   722	 * lifetime of the memdev registration associated with the endpoint to
   723	 * pin the decoder registered as well.
   724	 */
   725	struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd,
   726						     enum cxl_partition_mode mode,
   727						     resource_size_t alloc)
 > 728	{
   729		struct cxl_port *endpoint = cxlmd->endpoint;
   730		struct cxl_endpoint_decoder *cxled;
   731		struct device *cxled_dev;
   732		int rc;
   733	
   734		if (!IS_ALIGNED(alloc, SZ_256M))
   735			return ERR_PTR(-EINVAL);
   736	
   737		down_read(&cxl_dpa_rwsem);
   738		cxled_dev = device_find_child(&endpoint->dev, NULL, find_free_decoder);
   739		up_read(&cxl_dpa_rwsem);
   740	
   741		if (!cxled_dev)
   742			return ERR_PTR(-ENXIO);
   743	
   744		cxled = to_cxl_endpoint_decoder(cxled_dev);
   745	
   746		if (!cxled) {
   747			rc = -ENODEV;
   748			goto err;
   749		}
   750	
   751		rc = cxl_dpa_set_part(cxled, mode);
   752		if (rc)
   753			goto err;
   754	
   755		rc = cxl_dpa_alloc(cxled, alloc);
   756		if (rc)
   757			goto err;
   758	
   759		return cxled;
   760	err:
   761		put_device(cxled_dev);
   762		return ERR_PTR(rc);
   763	}
   764	EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, "CXL");
   765
kernel test robot April 15, 2025, 7:02 p.m. UTC | #3 
Hi,

kernel test robot noticed the following build errors:

[auto build test ERROR on 73c117c17b562213242f432db2ddf1bcc22f39dd]

url:    https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20250414-233241
base:   73c117c17b562213242f432db2ddf1bcc22f39dd
patch link:    https://lore.kernel.org/r/20250414151336.3852990-14-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v13 13/22] cxl: define a driver interface for DPA allocation
config: powerpc-randconfig-002-20250415
compiler: clang version 17.0.6 (https://github.com/llvm/llvm-project 6009708b4367171ccdbf4b5905cb6a803753fe18)
reproduce (this is a W=1 build):

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202504160231.EoEr710d-lkp@intel.com/

All errors (new ones prefixed by >>):

   In file included from drivers/cxl/core/hdm.c:6:
>> include/cxl/cxl.h:153:15: error: field has incomplete type 'struct range'
     153 |         struct range dpa_range;
         |                      ^
   include/linux/memory_hotplug.h:247:8: note: forward declaration of 'struct range'
     247 | struct range arch_get_mappable_range(void);
         |        ^
   In file included from drivers/cxl/core/hdm.c:6:
   include/cxl/cxl.h:227:16: error: field has incomplete type 'struct range'
     227 |                 struct range range;
         |                              ^
   include/linux/memory_hotplug.h:247:8: note: forward declaration of 'struct range'
     247 | struct range arch_get_mappable_range(void);
         |        ^
   2 errors generated.


vim +153 include/cxl/cxl.h

04ccf61aaf2695 Alejandro Lucero 2025-04-14  144  
04ccf61aaf2695 Alejandro Lucero 2025-04-14  145  /**
04ccf61aaf2695 Alejandro Lucero 2025-04-14  146   * struct cxl_dpa_perf - DPA performance property entry
04ccf61aaf2695 Alejandro Lucero 2025-04-14  147   * @dpa_range: range for DPA address
04ccf61aaf2695 Alejandro Lucero 2025-04-14  148   * @coord: QoS performance data (i.e. latency, bandwidth)
04ccf61aaf2695 Alejandro Lucero 2025-04-14  149   * @cdat_coord: raw QoS performance data from CDAT
04ccf61aaf2695 Alejandro Lucero 2025-04-14  150   * @qos_class: QoS Class cookies
04ccf61aaf2695 Alejandro Lucero 2025-04-14  151   */
04ccf61aaf2695 Alejandro Lucero 2025-04-14  152  struct cxl_dpa_perf {
04ccf61aaf2695 Alejandro Lucero 2025-04-14 @153  	struct range dpa_range;
04ccf61aaf2695 Alejandro Lucero 2025-04-14  154  	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
04ccf61aaf2695 Alejandro Lucero 2025-04-14  155  	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
04ccf61aaf2695 Alejandro Lucero 2025-04-14  156  	int qos_class;
04ccf61aaf2695 Alejandro Lucero 2025-04-14  157  };
04ccf61aaf2695 Alejandro Lucero 2025-04-14  158
diff mbox series

Patch

diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 70cae4ebf8a4..18a6ee74f600 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -3,6 +3,7 @@ 
 #include <linux/seq_file.h>
 #include <linux/device.h>
 #include <linux/delay.h>
+#include <cxl/cxl.h>
 
 #include "cxlmem.h"
 #include "core.h"
@@ -572,6 +573,7 @@  int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
 	devm_cxl_dpa_release(cxled);
 	return 0;
 }
+EXPORT_SYMBOL_NS_GPL(cxl_dpa_free, "CXL");
 
 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
 		     enum cxl_partition_mode mode)
@@ -686,6 +688,81 @@  int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
 	return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
 }
 
+static int find_free_decoder(struct device *dev, const void *data)
+{
+	struct cxl_endpoint_decoder *cxled;
+	struct cxl_port *port;
+
+	if (!is_endpoint_decoder(dev))
+		return 0;
+
+	cxled = to_cxl_endpoint_decoder(dev);
+	port = cxled_to_port(cxled);
+
+	if (cxled->cxld.id != port->hdm_end + 1)
+		return 0;
+
+	return 1;
+}
+
+/**
+ * cxl_request_dpa - search and reserve DPA given input constraints
+ * @cxlmd: memdev with an endpoint port with available decoders
+ * @is_ram: DPA operation mode (ram vs pmem)
+ * @alloc: dpa size required
+ *
+ * Given that a region needs to allocate from limited HPA capacity it
+ * may be the case that a device has more mappable DPA capacity than
+ * available HPA. The expectation is that @alloc is a driver known
+ * value based on the device capacity but it could not be available
+ * due to HPA constraints.
+ *
+ * Returns a pinned cxl_decoder with at least @alloc bytes of capacity
+ * reserved, or an error pointer. The caller is also expected to own the
+ * lifetime of the memdev registration associated with the endpoint to
+ * pin the decoder registered as well.
+ */
+struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd,
+					     enum cxl_partition_mode mode,
+					     resource_size_t alloc)
+{
+	struct cxl_port *endpoint = cxlmd->endpoint;
+	struct cxl_endpoint_decoder *cxled;
+	struct device *cxled_dev;
+	int rc;
+
+	if (!IS_ALIGNED(alloc, SZ_256M))
+		return ERR_PTR(-EINVAL);
+
+	down_read(&cxl_dpa_rwsem);
+	cxled_dev = device_find_child(&endpoint->dev, NULL, find_free_decoder);
+	up_read(&cxl_dpa_rwsem);
+
+	if (!cxled_dev)
+		return ERR_PTR(-ENXIO);
+
+	cxled = to_cxl_endpoint_decoder(cxled_dev);
+
+	if (!cxled) {
+		rc = -ENODEV;
+		goto err;
+	}
+
+	rc = cxl_dpa_set_part(cxled, mode);
+	if (rc)
+		goto err;
+
+	rc = cxl_dpa_alloc(cxled, alloc);
+	if (rc)
+		goto err;
+
+	return cxled;
+err:
+	put_device(cxled_dev);
+	return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, "CXL");
+
 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
 {
 	u16 eig;
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
index 0334c8cc9a01..5d07a75aaab5 100644
--- a/include/cxl/cxl.h
+++ b/include/cxl/cxl.h
@@ -262,4 +262,8 @@  struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_memdev *cxlmd,
 					       unsigned long flags,
 					       resource_size_t *max);
 void cxl_put_root_decoder(struct cxl_root_decoder *cxlrd);
+struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd,
+					     enum cxl_partition_mode mode,
+					     resource_size_t alloc);
+int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
 #endif /* __CXL_CXL_H__ */