diff mbox series

[v11,13/23] cxl: define a driver interface for DPA allocation

Message ID 20250310210340.3234884-14-alejandro.lucero-palau@amd.com
State New
Headers show
Series add type2 device basic support | expand

Commit Message

Lucero Palau, Alejandro March 10, 2025, 9:03 p.m. UTC 
From: Alejandro Lucero <alucerop@amd.com>

Region creation involves finding available DPA (device-physical-address)
capacity to map into HPA (host-physical-address) space. Define an API,
cxl_request_dpa(), that tries to allocate the DPA memory the driver
requires to operate. The memory requested should not be bigger than the
max available HPA obtained previously with cxl_get_hpa_freespace.

Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/

Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
 drivers/cxl/core/hdm.c | 83 ++++++++++++++++++++++++++++++++++++++++++
 include/cxl/cxl.h      |  4 ++
 2 files changed, 87 insertions(+)

Comments

kernel test robot March 11, 2025, 7:12 p.m. UTC | #1 
Hi,

kernel test robot noticed the following build errors:

[auto build test ERROR on 0a14566be090ca51a32ebdd8a8e21678062dac08]

url:    https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20250311-050914
base:   0a14566be090ca51a32ebdd8a8e21678062dac08
patch link:    https://lore.kernel.org/r/20250310210340.3234884-14-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v11 13/23] cxl: define a driver interface for DPA allocation
config: csky-randconfig-002-20250312 (https://download.01.org/0day-ci/archive/20250312/202503120207.vNlP2uB3-lkp@intel.com/config)
compiler: csky-linux-gcc (GCC) 14.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250312/202503120207.vNlP2uB3-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202503120207.vNlP2uB3-lkp@intel.com/

All errors (new ones prefixed by >>):

   In file included from drivers/cxl/core/hdm.c:6:
>> include/cxl/cxl.h:150:22: error: field 'dpa_range' has incomplete type
     150 |         struct range dpa_range;
         |                      ^~~~~~~~~
>> include/cxl/cxl.h:221:30: error: field 'range' has incomplete type
     221 |                 struct range range;
         |                              ^~~~~


vim +/dpa_range +150 include/cxl/cxl.h

98e0e4ae7d20491 Alejandro Lucero 2025-03-10  141  
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  142  /**
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  143   * struct cxl_dpa_perf - DPA performance property entry
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  144   * @dpa_range: range for DPA address
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  145   * @coord: QoS performance data (i.e. latency, bandwidth)
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  146   * @cdat_coord: raw QoS performance data from CDAT
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  147   * @qos_class: QoS Class cookies
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  148   */
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  149  struct cxl_dpa_perf {
98e0e4ae7d20491 Alejandro Lucero 2025-03-10 @150  	struct range dpa_range;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  151  	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  152  	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  153  	int qos_class;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  154  };
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  155  
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  156  enum cxl_partition_mode {
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  157  	CXL_PARTMODE_RAM,
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  158  	CXL_PARTMODE_PMEM,
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  159  };
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  160  
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  161  /**
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  162   * struct cxl_dpa_partition - DPA partition descriptor
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  163   * @res: shortcut to the partition in the DPA resource tree (cxlds->dpa_res)
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  164   * @perf: performance attributes of the partition from CDAT
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  165   * @mode: operation mode for the DPA capacity, e.g. ram, pmem, dynamic...
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  166   */
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  167  struct cxl_dpa_partition {
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  168  	struct resource res;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  169  	struct cxl_dpa_perf perf;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  170  	enum cxl_partition_mode mode;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  171  };
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  172  
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  173  #define CXL_NR_PARTITIONS_MAX 2
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  174  
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  175  /**
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  176   * struct cxl_dev_state - The driver device state
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  177   *
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  178   * cxl_dev_state represents the CXL driver/device state.  It provides an
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  179   * interface to mailbox commands as well as some cached data about the device.
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  180   * Currently only memory devices are represented.
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  181   *
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  182   * @dev: The device associated with this CXL state
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  183   * @cxlmd: The device representing the CXL.mem capabilities of @dev
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  184   * @reg_map: component and ras register mapping parameters
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  185   * @regs: Parsed register blocks
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  186   * @cxl_dvsec: Offset to the PCIe device DVSEC
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  187   * @rcd: operating in RCD mode (CXL 3.0 9.11.8 CXL Devices Attached to an RCH)
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  188   * @media_ready: Indicate whether the device media is usable
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  189   * @dpa_res: Overall DPA resource tree for the device
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  190   * @part: DPA partition array
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  191   * @nr_partitions: Number of DPA partitions
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  192   * @serial: PCIe Device Serial Number
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  193   * @type: Generic Memory Class device or Vendor Specific Memory device
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  194   * @cxl_mbox: CXL mailbox context
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  195   * @cxlfs: CXL features context
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  196   */
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  197  struct cxl_dev_state {
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  198  	struct device *dev;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  199  	struct cxl_memdev *cxlmd;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  200  	struct cxl_register_map reg_map;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  201  	struct cxl_regs regs;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  202  	int cxl_dvsec;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  203  	bool rcd;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  204  	bool media_ready;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  205  	struct resource dpa_res;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  206  	struct cxl_dpa_partition part[CXL_NR_PARTITIONS_MAX];
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  207  	unsigned int nr_partitions;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  208  	u64 serial;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  209  	enum cxl_devtype type;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  210  	struct cxl_mailbox cxl_mbox;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  211  #ifdef CONFIG_CXL_FEATURES
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  212  	struct cxl_features_state *cxlfs;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  213  #endif
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  214  };
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  215  
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  216  #define CXL_NR_PARTITIONS_MAX 2
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  217  
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  218  struct cxl_dpa_info {
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  219  	u64 size;
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  220  	struct cxl_dpa_part_info {
fe6f26dd4c64059 Alejandro Lucero 2025-03-10 @221  		struct range range;
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  222  		enum cxl_partition_mode mode;
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  223  	} part[CXL_NR_PARTITIONS_MAX];
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  224  	int nr_partitions;
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  225  };
fe6f26dd4c64059 Alejandro Lucero 2025-03-10  226
Ben Cheatham March 11, 2025, 8:06 p.m. UTC | #2 
On 3/10/25 4:03 PM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
> 
> Region creation involves finding available DPA (device-physical-address)
> capacity to map into HPA (host-physical-address) space. Define an API,
> cxl_request_dpa(), that tries to allocate the DPA memory the driver
> requires to operate. The memory requested should not be bigger than the
> max available HPA obtained previously with cxl_get_hpa_freespace.
> 
> Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
> 
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---

Reviewed-by: Ben Cheatham <benjamin.cheatham@amd.com>
kernel test robot March 11, 2025, 8:17 p.m. UTC | #3 
Hi,

kernel test robot noticed the following build errors:

[auto build test ERROR on 0a14566be090ca51a32ebdd8a8e21678062dac08]

url:    https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20250311-050914
base:   0a14566be090ca51a32ebdd8a8e21678062dac08
patch link:    https://lore.kernel.org/r/20250310210340.3234884-14-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v11 13/23] cxl: define a driver interface for DPA allocation
config: arm64-randconfig-001-20250312 (https://download.01.org/0day-ci/archive/20250312/202503120331.TSbIvphi-lkp@intel.com/config)
compiler: clang version 21.0.0git (https://github.com/llvm/llvm-project e15545cad8297ec7555f26e5ae74a9f0511203e7)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250312/202503120331.TSbIvphi-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202503120331.TSbIvphi-lkp@intel.com/

All errors (new ones prefixed by >>):

   In file included from drivers/cxl/core/hdm.c:6:
>> include/cxl/cxl.h:150:15: error: field has incomplete type 'struct range'
     150 |         struct range dpa_range;
         |                      ^
   include/linux/memory_hotplug.h:247:8: note: forward declaration of 'struct range'
     247 | struct range arch_get_mappable_range(void);
         |        ^
   In file included from drivers/cxl/core/hdm.c:6:
   include/cxl/cxl.h:221:16: error: field has incomplete type 'struct range'
     221 |                 struct range range;
         |                              ^
   include/linux/memory_hotplug.h:247:8: note: forward declaration of 'struct range'
     247 | struct range arch_get_mappable_range(void);
         |        ^
   In file included from drivers/cxl/core/hdm.c:8:
   In file included from drivers/cxl/cxlmem.h:6:
   In file included from include/linux/pci.h:1660:
   In file included from include/linux/dmapool.h:14:
   In file included from include/linux/scatterlist.h:8:
   In file included from include/linux/mm.h:2224:
   include/linux/vmstat.h:504:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
     504 |         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
         |                            ~~~~~~~~~~~~~~~~~~~~~ ^
     505 |                            item];
         |                            ~~~~
   include/linux/vmstat.h:511:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
     511 |         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
         |                            ~~~~~~~~~~~~~~~~~~~~~ ^
     512 |                            NR_VM_NUMA_EVENT_ITEMS +
         |                            ~~~~~~~~~~~~~~~~~~~~~~
   include/linux/vmstat.h:524:43: warning: arithmetic between different enumeration types ('enum zone_stat_item' and 'enum numa_stat_item') [-Wenum-enum-conversion]
     524 |         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
         |                            ~~~~~~~~~~~~~~~~~~~~~ ^
     525 |                            NR_VM_NUMA_EVENT_ITEMS +
         |                            ~~~~~~~~~~~~~~~~~~~~~~
   3 warnings and 2 errors generated.


vim +150 include/cxl/cxl.h

98e0e4ae7d20491 Alejandro Lucero 2025-03-10  141  
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  142  /**
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  143   * struct cxl_dpa_perf - DPA performance property entry
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  144   * @dpa_range: range for DPA address
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  145   * @coord: QoS performance data (i.e. latency, bandwidth)
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  146   * @cdat_coord: raw QoS performance data from CDAT
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  147   * @qos_class: QoS Class cookies
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  148   */
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  149  struct cxl_dpa_perf {
98e0e4ae7d20491 Alejandro Lucero 2025-03-10 @150  	struct range dpa_range;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  151  	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  152  	struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  153  	int qos_class;
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  154  };
98e0e4ae7d20491 Alejandro Lucero 2025-03-10  155
Simon Horman March 20, 2025, 4:18 p.m. UTC | #4 
On Mon, Mar 10, 2025 at 09:03:30PM +0000, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
> 
> Region creation involves finding available DPA (device-physical-address)
> capacity to map into HPA (host-physical-address) space. Define an API,
> cxl_request_dpa(), that tries to allocate the DPA memory the driver
> requires to operate. The memory requested should not be bigger than the
> max available HPA obtained previously with cxl_get_hpa_freespace.
> 
> Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
> 
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>

Hi Alejandro,

As reported by the Kernel Test Robot, in some circumstances this
patch fails to build.

I did not see this with x86_64 or arm64 allmodconfig.
But I did see the problem on ARM and was able to reproduce it (quickly)
like this using the toolchain here [*].

$ PATH=.../gcc-12.3.0-nolibc/arm-linux-gnueabi/bin:$PATH

$ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make allmodconfig
$ echo CONFIG_GCC_PLUGINS=n >> .config
$ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make oldconfig

$ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make drivers/cxl/core/hdm.o
...
  CC [M]  drivers/cxl/core/hdm.o
In file included from drivers/cxl/core/hdm.c:6:
./include/cxl/cxl.h:150:22: error: field 'dpa_range' has incomplete type
  150 |         struct range dpa_range;
      |                      ^~~~~~~~~
./include/cxl/cxl.h:221:30: error: field 'range' has incomplete type
  221 |                 struct range range;
      | 

[*] https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/14.2.0/

...

> diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c

...

> +/**
> + * cxl_request_dpa - search and reserve DPA given input constraints
> + * @cxlmd: memdev with an endpoint port with available decoders
> + * @is_ram: DPA operation mode (ram vs pmem)
> + * @min: the minimum amount of capacity the call needs

nit: @alloc should be documented instead of @min

> + *
> + * Given that a region needs to allocate from limited HPA capacity it
> + * may be the case that a device has more mappable DPA capacity than
> + * available HPA. So, the expectation is that @min is a driver known
> + * value for how much capacity is needed, and @max is the limit of
> + * how much HPA space is available for a new region.
> + *
> + * Returns a pinned cxl_decoder with at least @min bytes of capacity
> + * reserved, or an error pointer. The caller is also expected to own the
> + * lifetime of the memdev registration associated with the endpoint to
> + * pin the decoder registered as well.
> + */
> +struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd,
> +					     bool is_ram,
> +					     resource_size_t alloc)

...
Alejandro Lucero Palau March 24, 2025, 4:16 p.m. UTC | #5 
On 3/20/25 16:18, Simon Horman wrote:
> On Mon, Mar 10, 2025 at 09:03:30PM +0000, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Region creation involves finding available DPA (device-physical-address)
>> capacity to map into HPA (host-physical-address) space. Define an API,
>> cxl_request_dpa(), that tries to allocate the DPA memory the driver
>> requires to operate. The memory requested should not be bigger than the
>> max available HPA obtained previously with cxl_get_hpa_freespace.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Hi Alejandro,


Hi Simon,


>
> As reported by the Kernel Test Robot, in some circumstances this
> patch fails to build.
>
> I did not see this with x86_64 or arm64 allmodconfig.
> But I did see the problem on ARM and was able to reproduce it (quickly)
> like this using the toolchain here [*].
>
> $ PATH=.../gcc-12.3.0-nolibc/arm-linux-gnueabi/bin:$PATH
>
> $ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make allmodconfig
> $ echo CONFIG_GCC_PLUGINS=n >> .config
> $ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make oldconfig
>
> $ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make drivers/cxl/core/hdm.o
> ...
>    CC [M]  drivers/cxl/core/hdm.o
> In file included from drivers/cxl/core/hdm.c:6:
> ./include/cxl/cxl.h:150:22: error: field 'dpa_range' has incomplete type
>    150 |         struct range dpa_range;
>        |                      ^~~~~~~~~
> ./include/cxl/cxl.h:221:30: error: field 'range' has incomplete type
>    221 |                 struct range range;
>        |
>
> [*] https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/14.2.0/
>
> ...


Thanks for the references. I'll try it and figure out what is required.


>> diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
> ...
>
>> +/**
>> + * cxl_request_dpa - search and reserve DPA given input constraints
>> + * @cxlmd: memdev with an endpoint port with available decoders
>> + * @is_ram: DPA operation mode (ram vs pmem)
>> + * @min: the minimum amount of capacity the call needs
> nit: @alloc should be documented instead of @min
>

I'll fix it.

Thanks!
Simon Horman March 25, 2025, 3:23 p.m. UTC | #6 
On Mon, Mar 24, 2025 at 04:16:05PM +0000, Alejandro Lucero Palau wrote:
> 
> On 3/20/25 16:18, Simon Horman wrote:
> > On Mon, Mar 10, 2025 at 09:03:30PM +0000, alejandro.lucero-palau@amd.com wrote:
> > > From: Alejandro Lucero <alucerop@amd.com>
> > > 
> > > Region creation involves finding available DPA (device-physical-address)
> > > capacity to map into HPA (host-physical-address) space. Define an API,
> > > cxl_request_dpa(), that tries to allocate the DPA memory the driver
> > > requires to operate. The memory requested should not be bigger than the
> > > max available HPA obtained previously with cxl_get_hpa_freespace.
> > > 
> > > Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
> > > 
> > > Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> > Hi Alejandro,
> 
> 
> Hi Simon,
> 
> 
> > 
> > As reported by the Kernel Test Robot, in some circumstances this
> > patch fails to build.
> > 
> > I did not see this with x86_64 or arm64 allmodconfig.
> > But I did see the problem on ARM and was able to reproduce it (quickly)
> > like this using the toolchain here [*].
> > 
> > $ PATH=.../gcc-12.3.0-nolibc/arm-linux-gnueabi/bin:$PATH
> > 
> > $ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make allmodconfig
> > $ echo CONFIG_GCC_PLUGINS=n >> .config
> > $ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make oldconfig
> > 
> > $ ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- make drivers/cxl/core/hdm.o
> > ...
> >    CC [M]  drivers/cxl/core/hdm.o
> > In file included from drivers/cxl/core/hdm.c:6:
> > ./include/cxl/cxl.h:150:22: error: field 'dpa_range' has incomplete type
> >    150 |         struct range dpa_range;
> >        |                      ^~~~~~~~~
> > ./include/cxl/cxl.h:221:30: error: field 'range' has incomplete type
> >    221 |                 struct range range;
> >        |
> > 
> > [*] https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/14.2.0/
> > 
> > ...
> 
> 
> Thanks for the references. I'll try it and figure out what is required.

Thanks,

I realised after posting that PATH above uses gcc-12.3 while the link
above is for gcc-14.2.0. Which is inconsistent. But in practice I tried both :)

...
diff mbox series

Patch

diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 70cae4ebf8a4..7b264e82440d 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -3,6 +3,7 @@ 
 #include <linux/seq_file.h>
 #include <linux/device.h>
 #include <linux/delay.h>
+#include <cxl/cxl.h>
 
 #include "cxlmem.h"
 #include "core.h"
@@ -572,6 +573,7 @@  int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
 	devm_cxl_dpa_release(cxled);
 	return 0;
 }
+EXPORT_SYMBOL_NS_GPL(cxl_dpa_free, "CXL");
 
 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
 		     enum cxl_partition_mode mode)
@@ -686,6 +688,87 @@  int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
 	return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
 }
 
+static int find_free_decoder(struct device *dev, const void *data)
+{
+	struct cxl_endpoint_decoder *cxled;
+	struct cxl_port *port;
+
+	if (!is_endpoint_decoder(dev))
+		return 0;
+
+	cxled = to_cxl_endpoint_decoder(dev);
+	port = cxled_to_port(cxled);
+
+	if (cxled->cxld.id != port->hdm_end + 1)
+		return 0;
+
+	return 1;
+}
+
+/**
+ * cxl_request_dpa - search and reserve DPA given input constraints
+ * @cxlmd: memdev with an endpoint port with available decoders
+ * @is_ram: DPA operation mode (ram vs pmem)
+ * @min: the minimum amount of capacity the call needs
+ *
+ * Given that a region needs to allocate from limited HPA capacity it
+ * may be the case that a device has more mappable DPA capacity than
+ * available HPA. So, the expectation is that @min is a driver known
+ * value for how much capacity is needed, and @max is the limit of
+ * how much HPA space is available for a new region.
+ *
+ * Returns a pinned cxl_decoder with at least @min bytes of capacity
+ * reserved, or an error pointer. The caller is also expected to own the
+ * lifetime of the memdev registration associated with the endpoint to
+ * pin the decoder registered as well.
+ */
+struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd,
+					     bool is_ram,
+					     resource_size_t alloc)
+{
+	struct cxl_port *endpoint = cxlmd->endpoint;
+	struct cxl_endpoint_decoder *cxled;
+	enum cxl_partition_mode mode;
+	struct device *cxled_dev;
+	int rc;
+
+	if (!IS_ALIGNED(alloc, SZ_256M))
+		return ERR_PTR(-EINVAL);
+
+	down_read(&cxl_dpa_rwsem);
+	cxled_dev = device_find_child(&endpoint->dev, NULL, find_free_decoder);
+	up_read(&cxl_dpa_rwsem);
+
+	if (!cxled_dev)
+		return ERR_PTR(-ENXIO);
+
+	cxled = to_cxl_endpoint_decoder(cxled_dev);
+
+	if (!cxled) {
+		rc = -ENODEV;
+		goto err;
+	}
+
+	if (is_ram)
+		mode = CXL_PARTMODE_RAM;
+	else
+		mode = CXL_PARTMODE_PMEM;
+
+	rc = cxl_dpa_set_part(cxled, mode);
+	if (rc)
+		goto err;
+
+	rc = cxl_dpa_alloc(cxled, alloc);
+	if (rc)
+		goto err;
+
+	return cxled;
+err:
+	put_device(cxled_dev);
+	return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, "CXL");
+
 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
 {
 	u16 eig;
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
index 6ca6230d1fe5..d6b2e803e20b 100644
--- a/include/cxl/cxl.h
+++ b/include/cxl/cxl.h
@@ -255,4 +255,8 @@  struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_memdev *cxlmd,
 					       unsigned long flags,
 					       resource_size_t *max);
 void cxl_put_root_decoder(struct cxl_root_decoder *cxlrd);
+struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_memdev *cxlmd,
+					     bool is_ram,
+					     resource_size_t alloc);
+int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
 #endif