[Qemu,RFC,0/7] Early enabling of DCD emulation in Qemu

Commit Message

Fan Ni May 11, 2023, 5:56 p.m. UTC

Since the early draft of DCD support in kernel is out
(https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/#t),
this patch series provide dcd emulation in qemu so people who are interested
can have an early try. It is noted that the patch series may need to be updated
accordingly if the kernel side implementation changes.

To support DCD emulation, the patch series add DCD related mailbox command
support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
with dynamic capacity extent and region representative.
To support read/write to the dynamic capacity of the device, a host backend
is provided and necessary check mechnism is added to ensure the dynamic
capacity accessed is backed with active dc extents.
Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
, but we add two qmp interfaces for adding/releasing dynamic capacity extents.
Also, the support for multiple hosts sharing the same DCD case is missing.

Things we can try with the patch series together with kernel dcd code:
1. Create DC regions to cover the address range of the dynamic capacity
regions.
2. Add/release dynamic capacity extents to the device and notify the
kernel.
3. Test kernel side code to accept added dc extents and create dax devices,
and release dc extents and notify the device
4. Online the memory range backed with dc extents and let application use
them.

The patch series is based on Jonathan's local qemu branch:
https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28

Simple tests peformed with the patch series:
1 Install cxl modules:

modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem

2 Create dc regions:

region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
echo 0x10000000 > /sys/bus/cxl/devices/$region/size
echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
echo 1 > /sys/bus/cxl/devices/$region/commit
echo $region > /sys/bus/cxl/drivers/cxl_region/bind

/home/fan/cxl/tools-and-scripts# cxl list
[
  {
    "memdevs":[
      {
        "memdev":"mem0",
        "pmem_size":536870912,
        "ram_size":0,
        "serial":0,
        "host":"0000:0d:00.0"
      }
    ]
  },
  {
    "regions":[
      {
        "region":"region0",
        "resource":45365592064,
        "size":268435456,
        "interleave_ways":1,
        "interleave_granularity":256,
        "decode_state":"commit"
      }
    ]
  }
]

3 Add two dc extents (128MB each) through qmp interface

{ "execute": "qmp_capabilities" }

{ "execute": "cxl-add-dynamic-capacity-event",
	"arguments": {
		 "path": "/machine/peripheral/cxl-pmem0",
		"region-id" : 0,
		 "num-extent": 2,
		"dpa":0,
		"extent-len": 128
	}
}

/home/fan/cxl/tools-and-scripts# lsmem
RANGE                                  SIZE   STATE REMOVABLE   BLOCK
0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339

Memory block size:       128M
Total online memory:       8G
Total offline memory:    256M


4.Online the momory with 'daxctl online-memory dax0.0' to online the memory

/home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
[  230.730553] Fallback order for Node 0: 0 1
[  230.730825] Fallback order for Node 1: 1 0
[  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
[  230.731110] Policy zone: Normal
onlined memory for 1 device

root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
RANGE                                  SIZE   STATE REMOVABLE BLOCK
0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
0x0000000a98000000-0x0000000a9fffffff  128M offline             339

Memory block size:       128M
Total online memory:     8.1G
Total offline memory:    128M

5 using dc extents as regular memory

/home/fan/cxl/ndctl# numactl --membind=1 ls
CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
Documentation	 build	    contrib	     daxctl	  meson.build		rhel
tags	topology.png LICENSES	 ccan	    cscope.files
git-version  meson_options.txt	rpmbuild.sh    test	util


QEMU command line cxl configuration:

RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
-object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
-object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
-device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
-device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
-device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
-M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"


Kernel DCD support used to test the changes

The code is tested with the posted kernel dcd support:
https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5/dcd-preview

commit: f425bc34c600e2a3721d6560202962ec41622815

To make the test work, we have made the following changes to the above kernel commit:




Fan Ni (7):
  hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
    payload of identify memory device command
  hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
    and mailbox command support
  hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
    device supports in qemu command line
  hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
  hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
    dynamic capacity response
  Add qmp interfaces to add/release dynamic capacity extents
  hw/mem/cxl_type3: add read/write support to dynamic capacity

 hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
 hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
 include/hw/cxl/cxl_device.h |  50 +++-
 include/hw/cxl/cxl_events.h |  16 ++
 qapi/cxl.json               |  44 ++++
 5 files changed, 924 insertions(+), 67 deletions(-)

Comments

Jonathan Cameron May 15, 2023, 1 p.m. UTC | #1

On Thu, 11 May 2023 17:56:40 +0000
Fan Ni <[e-mail]> wrote:

> Since the early draft of DCD support in kernel is out
> (https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/#t),
> this patch series provide dcd emulation in qemu so people who are interested
> can have an early try. It is noted that the patch series may need to be updated
> accordingly if the kernel side implementation changes.
> 
> To support DCD emulation, the patch series add DCD related mailbox command
> support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> with dynamic capacity extent and region representative.
> To support read/write to the dynamic capacity of the device, a host backend
> is provided and necessary check mechnism is added to ensure the dynamic
> capacity accessed is backed with active dc extents.
> Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> Also, the support for multiple hosts sharing the same DCD case is missing.
> 
> Things we can try with the patch series together with kernel dcd code:
> 1. Create DC regions to cover the address range of the dynamic capacity
> regions.
> 2. Add/release dynamic capacity extents to the device and notify the
> kernel.
> 3. Test kernel side code to accept added dc extents and create dax devices,
> and release dc extents and notify the device
> 4. Online the memory range backed with dc extents and let application use
> them.
> 
> The patch series is based on Jonathan's local qemu branch:
> https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28
> 
> Simple tests peformed with the patch series:
> 1 Install cxl modules:
> 
> modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> 
> 2 Create dc regions:
> 
> region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> echo 1 > /sys/bus/cxl/devices/$region/commit
> echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> 
> /home/fan/cxl/tools-and-scripts# cxl list
> [
>   {
>     "memdevs":[
>       {
>         "memdev":"mem0",
>         "pmem_size":536870912,
>         "ram_size":0,
>         "serial":0,
>         "host":"0000:0d:00.0"
>       }
>     ]
>   },
>   {
>     "regions":[
>       {
>         "region":"region0",
>         "resource":45365592064,
>         "size":268435456,
>         "interleave_ways":1,
>         "interleave_granularity":256,
>         "decode_state":"commit"
>       }
>     ]
>   }
> ]
> 
> 3 Add two dc extents (128MB each) through qmp interface
> 
> { "execute": "qmp_capabilities" }
> 
> { "execute": "cxl-add-dynamic-capacity-event",
> 	"arguments": {
> 		 "path": "/machine/peripheral/cxl-pmem0",
> 		"region-id" : 0,
> 		 "num-extent": 2,
> 		"dpa":0,
> 		"extent-len": 128
> 	}
> }
> 
> /home/fan/cxl/tools-and-scripts# lsmem
> RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> 
> Memory block size:       128M
> Total online memory:       8G
> Total offline memory:    256M
> 
> 
> 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> 
> /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> [  230.730553] Fallback order for Node 0: 0 1
> [  230.730825] Fallback order for Node 1: 1 0
> [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> [  230.731110] Policy zone: Normal
> onlined memory for 1 device
> 
> root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> RANGE                                  SIZE   STATE REMOVABLE BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> 
> Memory block size:       128M
> Total online memory:     8.1G
> Total offline memory:    128M
> 
> 5 using dc extents as regular memory
> 
> /home/fan/cxl/ndctl# numactl --membind=1 ls
> CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
> config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
> Documentation	 build	    contrib	     daxctl	  meson.build		rhel
> tags	topology.png LICENSES	 ccan	    cscope.files
> git-version  meson_options.txt	rpmbuild.sh    test	util
> 
> 
> QEMU command line cxl configuration:
> 
> RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> 
> 
> Kernel DCD support used to test the changes
> 
> The code is tested with the posted kernel dcd support:
> https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5/dcd-preview
> 

Very nice!  +CC Navneet who may want to comment on the below (and the
emulation as well)

I've not had a chance to look at the code on the kernel side yet.


> commit: f425bc34c600e2a3721d6560202962ec41622815
> 
> To make the test work, we have made the following changes to the above kernel commit:
> 
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> index 5f04bbc18af5..5f421d3c5cef 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
>  	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
>  	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
>  	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> +	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
>  };
>  
>  /*
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 291c716abd49..ae10e3cf43a1 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
>  		}
>  		cxlds->dc_list_gen_num = extent_gen_num;
>  		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> -		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> +		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/

Some race condition that means we need to enable the DCD event earlier?

>  	}
>  	return 0;
>  err:
> @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
>  				dev_dax->align, memremap_compat_align()))) {
>  		rc = alloc_dev_dax_range(dev_dax, hpa,
>  					resource_size(alloc_dpa_res));
> -		return rc;
> +		if (rc)
> +			return rc;

No idea on this one as it's in the code I haven't looked at yet!

>  	}
>  
>  	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 9e45b1056022..653bec203838 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
>  
>  	/* Driver enables DCD interrupt after creating the dc cxl_region */
>  	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> -					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> +					IRQF_SHARED | IRQF_ONESHOT);

This will be otherside of the removal of the enable above.

>  	if (rc) {
>  		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
>  		return rc;
> diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> index 6ca85861750c..910a48259239 100644
> --- a/include/uapi/linux/cxl_mem.h
> +++ b/include/uapi/linux/cxl_mem.h
> @@ -47,6 +47,7 @@
>  	___C(SCAN_MEDIA, "Scan Media"),                                   \
>  	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
>  	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> +	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
>  	___C(MAX, "invalid / last command")
>  
>  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> 
> 
> 
> Fan Ni (7):
>   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
>     payload of identify memory device command
>   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
>     and mailbox command support
>   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
>     device supports in qemu command line
>   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
>   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
>     dynamic capacity response
>   Add qmp interfaces to add/release dynamic capacity extents
>   hw/mem/cxl_type3: add read/write support to dynamic capacity
> 
>  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
>  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
>  include/hw/cxl/cxl_device.h |  50 +++-
>  include/hw/cxl/cxl_events.h |  16 ++
>  qapi/cxl.json               |  44 ++++
>  5 files changed, 924 insertions(+), 67 deletions(-)
>

Navneet Singh May 16, 2023, 2:39 p.m. UTC | #2

-----Original Message-----
From: Jonathan Cameron <[e-mail]> 
Sent: Monday, May 15, 2023 6:31 PM
To: Fan Ni <[e-mail]>
Cc: [e-mail]; [e-mail]; [e-mail]; [e-mail]; Browy, Christopher <[e-mail]>; Weiny, Ira <[e-mail]>; Williams, Dan J <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail]; [e-mail]; [e-mail]; Singh, Navneet <[e-mail]>
Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu

On Thu, 11 May 2023 17:56:40 +0000
Fan Ni <[e-mail]> wrote:

> Since the early draft of DCD support in kernel is out 
> (https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510
> -bm03/T/#t), this patch series provide dcd emulation in qemu so people 
> who are interested can have an early try. It is noted that the patch 
> series may need to be updated accordingly if the kernel side 
> implementation changes.
> 
> To support DCD emulation, the patch series add DCD related mailbox 
> command support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 
> memory device with dynamic capacity extent and region representative.
> To support read/write to the dynamic capacity of the device, a host 
> backend is provided and necessary check mechnism is added to ensure 
> the dynamic capacity accessed is backed with active dc extents.
> Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not 
> supported , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> Also, the support for multiple hosts sharing the same DCD case is missing.
> 
> Things we can try with the patch series together with kernel dcd code:
> 1. Create DC regions to cover the address range of the dynamic 
> capacity regions.
> 2. Add/release dynamic capacity extents to the device and notify the 
> kernel.
> 3. Test kernel side code to accept added dc extents and create dax 
> devices, and release dc extents and notify the device 4. Online the 
> memory range backed with dc extents and let application use them.
> 
> The patch series is based on Jonathan's local qemu branch:
> https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28
> 
> Simple tests peformed with the patch series:
> 1 Install cxl modules:
> 
> modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> 
> 2 Create dc regions:
> 
> region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> echo 0x10000000 > /sys/bus/cxl/devices/$region/size echo  "decoder2.0" 
> > /sys/bus/cxl/devices/$region/target0
> echo 1 > /sys/bus/cxl/devices/$region/commit
> echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> 
> /home/fan/cxl/tools-and-scripts# cxl list [
>   {
>     "memdevs":[
>       {
>         "memdev":"mem0",
>         "pmem_size":536870912,
>         "ram_size":0,
>         "serial":0,
>         "host":"0000:0d:00.0"
>       }
>     ]
>   },
>   {
>     "regions":[
>       {
>         "region":"region0",
>         "resource":45365592064,
>         "size":268435456,
>         "interleave_ways":1,
>         "interleave_granularity":256,
>         "decode_state":"commit"
>       }
>     ]
>   }
> ]
> 
> 3 Add two dc extents (128MB each) through qmp interface
> 
> { "execute": "qmp_capabilities" }
> 
> { "execute": "cxl-add-dynamic-capacity-event",
> 	"arguments": {
> 		 "path": "/machine/peripheral/cxl-pmem0",
> 		"region-id" : 0,
> 		 "num-extent": 2,
> 		"dpa":0,
> 		"extent-len": 128
> 	}
> }
> 
> /home/fan/cxl/tools-and-scripts# lsmem
> RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> 
> Memory block size:       128M
> Total online memory:       8G
> Total offline memory:    256M
> 
> 
> 4.Online the momory with 'daxctl online-memory dax0.0' to online the 
> memory
> 
> /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0 [  
> 230.730553] Fallback order for Node 0: 0 1 [  230.730825] Fallback 
> order for Node 1: 1 0 [  230.730953] Built 2 zonelists, mobility 
> grouping on.  Total pages: 2042541 [  230.731110] Policy zone: Normal 
> onlined memory for 1 device
> 
> root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> RANGE                                  SIZE   STATE REMOVABLE BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> 
> Memory block size:       128M
> Total online memory:     8.1G
> Total offline memory:    128M
> 
> 5 using dc extents as regular memory
> 
> /home/fan/cxl/ndctl# numactl --membind=1 ls
> CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
> config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
> Documentation	 build	    contrib	     daxctl	  meson.build		rhel
> tags	topology.png LICENSES	 ccan	    cscope.files
> git-version  meson_options.txt	rpmbuild.sh    test	util
> 
> 
> QEMU command line cxl configuration:
> 
> RP1="-object 
> memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,siz
> e=512M \ -object 
> memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,si
> ze=512M \ -object 
> memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=51
> 2M \ -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \ -device 
> cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \ -device 
> cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-m
> em2,id=cxl-pmem0,num-dc-regions=1\
> -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> 
> 
> Kernel DCD support used to test the changes
> 
> The code is tested with the posted kernel dcd support:
> https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for
> -6.5/dcd-preview
> 

Very nice!  +CC Navneet who may want to comment on the below (and the emulation as well)

I've not had a chance to look at the code on the kernel side yet.


> commit: f425bc34c600e2a3721d6560202962ec41622815
> 
> To make the test work, we have made the following changes to the above kernel commit:
> 
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 
> 5f04bbc18af5..5f421d3c5cef 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
>  	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
>  	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
>  	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> +	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
>  };
>  
>  /*
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c 
> index 291c716abd49..ae10e3cf43a1 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
>  		}
>  		cxlds->dc_list_gen_num = extent_gen_num;
>  		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> -		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> +		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/

Some race condition that means we need to enable the DCD event earlier?
Navneet - I have been working on the DCD feature last few weeks and this has been removed and will be handled like other Events.
>  	}
>  	return 0;
>  err:
> @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
>  				dev_dax->align, memremap_compat_align()))) {
>  		rc = alloc_dev_dax_range(dev_dax, hpa,
>  					resource_size(alloc_dpa_res));
> -		return rc;
> +		if (rc)
> +			return rc;

No idea on this one as it's in the code I haven't looked at yet!
Navneet - This is also fixed , in last moment changes this bug got introduced.

>  	}
>  
>  	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL); 
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 
> 9e45b1056022..653bec203838 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state 
> *cxlds)
>  
>  	/* Driver enables DCD interrupt after creating the dc cxl_region */
>  	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> -					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> +					IRQF_SHARED | IRQF_ONESHOT);

This will be otherside of the removal of the enable above.

>  	if (rc) {
>  		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
>  		return rc;
> diff --git a/include/uapi/linux/cxl_mem.h 
> b/include/uapi/linux/cxl_mem.h index 6ca85861750c..910a48259239 100644
> --- a/include/uapi/linux/cxl_mem.h
> +++ b/include/uapi/linux/cxl_mem.h
> @@ -47,6 +47,7 @@
>  	___C(SCAN_MEDIA, "Scan Media"),                                   \
>  	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
>  	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> +	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
>  	___C(MAX, "invalid / last command")
>  
>  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> 
> 
> 
> Fan Ni (7):
>   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
>     payload of identify memory device command
>   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
>     and mailbox command support
>   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
>     device supports in qemu command line
>   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
>   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
>     dynamic capacity response
>   Add qmp interfaces to add/release dynamic capacity extents
>   hw/mem/cxl_type3: add read/write support to dynamic capacity
> 
>  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
>  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
>  include/hw/cxl/cxl_device.h |  50 +++-  include/hw/cxl/cxl_events.h |  
> 16 ++
>  qapi/cxl.json               |  44 ++++
>  5 files changed, 924 insertions(+), 67 deletions(-)
>

Ira Weiny June 5, 2023, 5:35 p.m. UTC | #3

Fan Ni wrote:
> Since the early draft of DCD support in kernel is out
> (https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/#t),
> this patch series provide dcd emulation in qemu so people who are interested
> can have an early try. It is noted that the patch series may need to be updated
> accordingly if the kernel side implementation changes.

Fan,

Do you have a git tree we can pull this from which is updated to a more
recent CXL branch from Jonathan?

Thanks,
Ira

> 
> To support DCD emulation, the patch series add DCD related mailbox command
> support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> with dynamic capacity extent and region representative.
> To support read/write to the dynamic capacity of the device, a host backend
> is provided and necessary check mechnism is added to ensure the dynamic
> capacity accessed is backed with active dc extents.
> Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> Also, the support for multiple hosts sharing the same DCD case is missing.
> 
> Things we can try with the patch series together with kernel dcd code:
> 1. Create DC regions to cover the address range of the dynamic capacity
> regions.
> 2. Add/release dynamic capacity extents to the device and notify the
> kernel.
> 3. Test kernel side code to accept added dc extents and create dax devices,
> and release dc extents and notify the device
> 4. Online the memory range backed with dc extents and let application use
> them.
> 
> The patch series is based on Jonathan's local qemu branch:
> https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28
> 
> Simple tests peformed with the patch series:
> 1 Install cxl modules:
> 
> modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> 
> 2 Create dc regions:
> 
> region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> echo 1 > /sys/bus/cxl/devices/$region/commit
> echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> 
> /home/fan/cxl/tools-and-scripts# cxl list
> [
>   {
>     "memdevs":[
>       {
>         "memdev":"mem0",
>         "pmem_size":536870912,
>         "ram_size":0,
>         "serial":0,
>         "host":"0000:0d:00.0"
>       }
>     ]
>   },
>   {
>     "regions":[
>       {
>         "region":"region0",
>         "resource":45365592064,
>         "size":268435456,
>         "interleave_ways":1,
>         "interleave_granularity":256,
>         "decode_state":"commit"
>       }
>     ]
>   }
> ]
> 
> 3 Add two dc extents (128MB each) through qmp interface
> 
> { "execute": "qmp_capabilities" }
> 
> { "execute": "cxl-add-dynamic-capacity-event",
> 	"arguments": {
> 		 "path": "/machine/peripheral/cxl-pmem0",
> 		"region-id" : 0,
> 		 "num-extent": 2,
> 		"dpa":0,
> 		"extent-len": 128
> 	}
> }
> 
> /home/fan/cxl/tools-and-scripts# lsmem
> RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> 
> Memory block size:       128M
> Total online memory:       8G
> Total offline memory:    256M
> 
> 
> 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> 
> /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> [  230.730553] Fallback order for Node 0: 0 1
> [  230.730825] Fallback order for Node 1: 1 0
> [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> [  230.731110] Policy zone: Normal
> onlined memory for 1 device
> 
> root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> RANGE                                  SIZE   STATE REMOVABLE BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> 
> Memory block size:       128M
> Total online memory:     8.1G
> Total offline memory:    128M
> 
> 5 using dc extents as regular memory
> 
> /home/fan/cxl/ndctl# numactl --membind=1 ls
> CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
> config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
> Documentation	 build	    contrib	     daxctl	  meson.build		rhel
> tags	topology.png LICENSES	 ccan	    cscope.files
> git-version  meson_options.txt	rpmbuild.sh    test	util
> 
> 
> QEMU command line cxl configuration:
> 
> RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> 
> 
> Kernel DCD support used to test the changes
> 
> The code is tested with the posted kernel dcd support:
> https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5/dcd-preview
> 
> commit: f425bc34c600e2a3721d6560202962ec41622815
> 
> To make the test work, we have made the following changes to the above kernel commit:
> 
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> index 5f04bbc18af5..5f421d3c5cef 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
>  	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
>  	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
>  	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> +	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
>  };
>  
>  /*
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 291c716abd49..ae10e3cf43a1 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
>  		}
>  		cxlds->dc_list_gen_num = extent_gen_num;
>  		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> -		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> +		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
>  	}
>  	return 0;
>  err:
> @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
>  				dev_dax->align, memremap_compat_align()))) {
>  		rc = alloc_dev_dax_range(dev_dax, hpa,
>  					resource_size(alloc_dpa_res));
> -		return rc;
> +		if (rc)
> +			return rc;
>  	}
>  
>  	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 9e45b1056022..653bec203838 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
>  
>  	/* Driver enables DCD interrupt after creating the dc cxl_region */
>  	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> -					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> +					IRQF_SHARED | IRQF_ONESHOT);
>  	if (rc) {
>  		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
>  		return rc;
> diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> index 6ca85861750c..910a48259239 100644
> --- a/include/uapi/linux/cxl_mem.h
> +++ b/include/uapi/linux/cxl_mem.h
> @@ -47,6 +47,7 @@
>  	___C(SCAN_MEDIA, "Scan Media"),                                   \
>  	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
>  	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> +	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
>  	___C(MAX, "invalid / last command")
>  
>  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> 
> 
> 
> Fan Ni (7):
>   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
>     payload of identify memory device command
>   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
>     and mailbox command support
>   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
>     device supports in qemu command line
>   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
>   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
>     dynamic capacity response
>   Add qmp interfaces to add/release dynamic capacity extents
>   hw/mem/cxl_type3: add read/write support to dynamic capacity
> 
>  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
>  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
>  include/hw/cxl/cxl_device.h |  50 +++-
>  include/hw/cxl/cxl_events.h |  16 ++
>  qapi/cxl.json               |  44 ++++
>  5 files changed, 924 insertions(+), 67 deletions(-)
> 
> -- 
> 2.25.1

Fan Ni June 5, 2023, 5:51 p.m. UTC | #4

On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:
> Fan Ni wrote:
> > Since the early draft of DCD support in kernel is out
> > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > this patch series provide dcd emulation in qemu so people who are interested
> > can have an early try. It is noted that the patch series may need to be updated
> > accordingly if the kernel side implementation changes.
> 
> Fan,
> 
> Do you have a git tree we can pull this from which is updated to a more
> recent CXL branch from Jonathan?
> 
> Thanks,
> Ira

Hi Ira,

I have a git tree of the patch series based on Jonathan's branch
cxl-2023-02-28: https://github.com/moking/qemu-dev/tree/dcd-rfe.

That may be not new enough to include some of the recent patches, but I can
rebase it to a newer branch if you can tell me which branch you want to use.

Thanks,
Fan

> 
> > 
> > To support DCD emulation, the patch series add DCD related mailbox command
> > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > with dynamic capacity extent and region representative.
> > To support read/write to the dynamic capacity of the device, a host backend
> > is provided and necessary check mechnism is added to ensure the dynamic
> > capacity accessed is backed with active dc extents.
> > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > Also, the support for multiple hosts sharing the same DCD case is missing.
> > 
> > Things we can try with the patch series together with kernel dcd code:
> > 1. Create DC regions to cover the address range of the dynamic capacity
> > regions.
> > 2. Add/release dynamic capacity extents to the device and notify the
> > kernel.
> > 3. Test kernel side code to accept added dc extents and create dax devices,
> > and release dc extents and notify the device
> > 4. Online the memory range backed with dc extents and let application use
> > them.
> > 
> > The patch series is based on Jonathan's local qemu branch:
> > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > 
> > Simple tests peformed with the patch series:
> > 1 Install cxl modules:
> > 
> > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > 
> > 2 Create dc regions:
> > 
> > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > echo 1 > /sys/bus/cxl/devices/$region/commit
> > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > 
> > /home/fan/cxl/tools-and-scripts# cxl list
> > [
> >   {
> >     "memdevs":[
> >       {
> >         "memdev":"mem0",
> >         "pmem_size":536870912,
> >         "ram_size":0,
> >         "serial":0,
> >         "host":"0000:0d:00.0"
> >       }
> >     ]
> >   },
> >   {
> >     "regions":[
> >       {
> >         "region":"region0",
> >         "resource":45365592064,
> >         "size":268435456,
> >         "interleave_ways":1,
> >         "interleave_granularity":256,
> >         "decode_state":"commit"
> >       }
> >     ]
> >   }
> > ]
> > 
> > 3 Add two dc extents (128MB each) through qmp interface
> > 
> > { "execute": "qmp_capabilities" }
> > 
> > { "execute": "cxl-add-dynamic-capacity-event",
> > 	"arguments": {
> > 		 "path": "/machine/peripheral/cxl-pmem0",
> > 		"region-id" : 0,
> > 		 "num-extent": 2,
> > 		"dpa":0,
> > 		"extent-len": 128
> > 	}
> > }
> > 
> > /home/fan/cxl/tools-and-scripts# lsmem
> > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > 
> > Memory block size:       128M
> > Total online memory:       8G
> > Total offline memory:    256M
> > 
> > 
> > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > 
> > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > [  230.730553] Fallback order for Node 0: 0 1
> > [  230.730825] Fallback order for Node 1: 1 0
> > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > [  230.731110] Policy zone: Normal
> > onlined memory for 1 device
> > 
> > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > 
> > Memory block size:       128M
> > Total online memory:     8.1G
> > Total offline memory:    128M
> > 
> > 5 using dc extents as regular memory
> > 
> > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
> > config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
> > Documentation	 build	    contrib	     daxctl	  meson.build		rhel
> > tags	topology.png LICENSES	 ccan	    cscope.files
> > git-version  meson_options.txt	rpmbuild.sh    test	util
> > 
> > 
> > QEMU command line cxl configuration:
> > 
> > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > 
> > 
> > Kernel DCD support used to test the changes
> > 
> > The code is tested with the posted kernel dcd support:
> > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > 
> > commit: f425bc34c600e2a3721d6560202962ec41622815
> > 
> > To make the test work, we have made the following changes to the above kernel commit:
> > 
> > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > index 5f04bbc18af5..5f421d3c5cef 100644
> > --- a/drivers/cxl/core/mbox.c
> > +++ b/drivers/cxl/core/mbox.c
> > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> >  	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> >  	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> >  	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > +	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> >  };
> >  
> >  /*
> > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > index 291c716abd49..ae10e3cf43a1 100644
> > --- a/drivers/cxl/core/region.c
> > +++ b/drivers/cxl/core/region.c
> > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> >  		}
> >  		cxlds->dc_list_gen_num = extent_gen_num;
> >  		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > -		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > +		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> >  	}
> >  	return 0;
> >  err:
> > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> >  				dev_dax->align, memremap_compat_align()))) {
> >  		rc = alloc_dev_dax_range(dev_dax, hpa,
> >  					resource_size(alloc_dpa_res));
> > -		return rc;
> > +		if (rc)
> > +			return rc;
> >  	}
> >  
> >  	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > index 9e45b1056022..653bec203838 100644
> > --- a/drivers/cxl/pci.c
> > +++ b/drivers/cxl/pci.c
> > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> >  
> >  	/* Driver enables DCD interrupt after creating the dc cxl_region */
> >  	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > -					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > +					IRQF_SHARED | IRQF_ONESHOT);
> >  	if (rc) {
> >  		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> >  		return rc;
> > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > index 6ca85861750c..910a48259239 100644
> > --- a/include/uapi/linux/cxl_mem.h
> > +++ b/include/uapi/linux/cxl_mem.h
> > @@ -47,6 +47,7 @@
> >  	___C(SCAN_MEDIA, "Scan Media"),                                   \
> >  	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> >  	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > +	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> >  	___C(MAX, "invalid / last command")
> >  
> >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > 
> > 
> > 
> > Fan Ni (7):
> >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> >     payload of identify memory device command
> >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> >     and mailbox command support
> >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> >     device supports in qemu command line
> >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> >     dynamic capacity response
> >   Add qmp interfaces to add/release dynamic capacity extents
> >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > 
> >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> >  include/hw/cxl/cxl_device.h |  50 +++-
> >  include/hw/cxl/cxl_events.h |  16 ++
> >  qapi/cxl.json               |  44 ++++
> >  5 files changed, 924 insertions(+), 67 deletions(-)
> > 
> > -- 
> > 2.25.1
> 
>

Shesha Bhushan Sreenivasamurthy June 7, 2023, 6:13 p.m. UTC | #5

Hi Fan,
   I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?

https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25


From: Fan Ni <[e-mail]>
Sent: Monday, June 5, 2023 10:51 AM
To: Ira Weiny <[e-mail]>
Cc: [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
 
On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:
> Fan Ni wrote:
> > Since the early draft of DCD support in kernel is out
> > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > this patch series provide dcd emulation in qemu so people who are interested
> > can have an early try. It is noted that the patch series may need to be updated
> > accordingly if the kernel side implementation changes.
> 
> Fan,
> 
> Do you have a git tree we can pull this from which is updated to a more
> recent CXL branch from Jonathan?
> 
> Thanks,
> Ira

Hi Ira,

I have a git tree of the patch series based on Jonathan's branch
cxl-2023-02-28: https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddev_tree_dcd-2Drfe&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=w6dicn5kXEG4Imk6TpICIjdA6KJ-xt84dtHui-Y0fv5H13bijtzEvjxECKE5MHYf&s=3yeO9RN5FY3gPfO2y19X057YeqRTTQTQNfNA-Gfir_Q&e= .

That may be not new enough to include some of the recent patches, but I can
rebase it to a newer branch if you can tell me which branch you want to use.

Thanks,
Fan

> 
> > 
> > To support DCD emulation, the patch series add DCD related mailbox command
> > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > with dynamic capacity extent and region representative.
> > To support read/write to the dynamic capacity of the device, a host backend
> > is provided and necessary check mechnism is added to ensure the dynamic
> > capacity accessed is backed with active dc extents.
> > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > Also, the support for multiple hosts sharing the same DCD case is missing.
> > 
> > Things we can try with the patch series together with kernel dcd code:
> > 1. Create DC regions to cover the address range of the dynamic capacity
> > regions.
> > 2. Add/release dynamic capacity extents to the device and notify the
> > kernel.
> > 3. Test kernel side code to accept added dc extents and create dax devices,
> > and release dc extents and notify the device
> > 4. Online the memory range backed with dc extents and let application use
> > them.
> > 
> > The patch series is based on Jonathan's local qemu branch:
> > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > 
> > Simple tests peformed with the patch series:
> > 1 Install cxl modules:
> > 
> > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > 
> > 2 Create dc regions:
> > 
> > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > echo 1 > /sys/bus/cxl/devices/$region/commit
> > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > 
> > /home/fan/cxl/tools-and-scripts# cxl list
> > [
> >   {
> >     "memdevs":[
> >       {
> >         "memdev":"mem0",
> >         "pmem_size":536870912,
> >         "ram_size":0,
> >         "serial":0,
> >         "host":"0000:0d:00.0"
> >       }
> >     ]
> >   },
> >   {
> >     "regions":[
> >       {
> >         "region":"region0",
> >         "resource":45365592064,
> >         "size":268435456,
> >         "interleave_ways":1,
> >         "interleave_granularity":256,
> >         "decode_state":"commit"
> >       }
> >     ]
> >   }
> > ]
> > 
> > 3 Add two dc extents (128MB each) through qmp interface
> > 
> > { "execute": "qmp_capabilities" }
> > 
> > { "execute": "cxl-add-dynamic-capacity-event",
> >      "arguments": {
> >               "path": "/machine/peripheral/cxl-pmem0",
> >              "region-id" : 0,
> >               "num-extent": 2,
> >              "dpa":0,
> >              "extent-len": 128
> >      }
> > }
> > 
> > /home/fan/cxl/tools-and-scripts# lsmem
> > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > 
> > Memory block size:       128M
> > Total online memory:       8G
> > Total offline memory:    256M
> > 
> > 
> > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > 
> > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > [  230.730553] Fallback order for Node 0: 0 1
> > [  230.730825] Fallback order for Node 1: 1 0
> > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > [  230.731110] Policy zone: Normal
> > onlined memory for 1 device
> > 
> > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > 
> > Memory block size:       128M
> > Total online memory:     8.1G
> > Total offline memory:    128M
> > 
> > 5 using dc extents as regular memory
> > 
> > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > ndctl              scripts   test.h      version.h.in COPYING                 acpi.h
> > config.h.meson   cxl          make-git-snapshot.sh   ndctl.spec.in  sles     tools
> > Documentation        build       contrib           daxctl        meson.build            rhel
> > tags        topology.png LICENSES    ccan        cscope.files
> > git-version  meson_options.txt      rpmbuild.sh    test     util
> > 
> > 
> > QEMU command line cxl configuration:
> > 
> > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > 
> > 
> > Kernel DCD support used to test the changes
> > 
> > The code is tested with the posted kernel dcd support:
> > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > 
> > commit: f425bc34c600e2a3721d6560202962ec41622815
> > 
> > To make the test work, we have made the following changes to the above kernel commit:
> > 
> > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > index 5f04bbc18af5..5f421d3c5cef 100644
> > --- a/drivers/cxl/core/mbox.c
> > +++ b/drivers/cxl/core/mbox.c
> > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> >      CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> >      CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> >      CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > +   CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> >  };
> >  
> >  /*
> > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > index 291c716abd49..ae10e3cf43a1 100644
> > --- a/drivers/cxl/core/region.c
> > +++ b/drivers/cxl/core/region.c
> > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> >              }
> >              cxlds->dc_list_gen_num = extent_gen_num;
> >              dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > -           enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > +           /*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> >      }
> >      return 0;
> >  err:
> > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> >                              dev_dax->align, memremap_compat_align()))) {
> >              rc = alloc_dev_dax_range(dev_dax, hpa,
> >                                      resource_size(alloc_dpa_res));
> > -           return rc;
> > +           if (rc)
> > +                   return rc;
> >      }
> >  
> >      rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > index 9e45b1056022..653bec203838 100644
> > --- a/drivers/cxl/pci.c
> > +++ b/drivers/cxl/pci.c
> > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> >  
> >      /* Driver enables DCD interrupt after creating the dc cxl_region */
> >      rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > -                                   IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > +                                   IRQF_SHARED | IRQF_ONESHOT);
> >      if (rc) {
> >              dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> >              return rc;
> > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > index 6ca85861750c..910a48259239 100644
> > --- a/include/uapi/linux/cxl_mem.h
> > +++ b/include/uapi/linux/cxl_mem.h
> > @@ -47,6 +47,7 @@
> >      ___C(SCAN_MEDIA, "Scan Media"),                                   \
> >      ___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> >      ___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > +   ___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> >      ___C(MAX, "invalid / last command")
> >  
> >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > 
> > 
> > 
> > Fan Ni (7):
> >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> >     payload of identify memory device command
> >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> >     and mailbox command support
> >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> >     device supports in qemu command line
> >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> >     dynamic capacity response
> >   Add qmp interfaces to add/release dynamic capacity extents
> >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > 
> >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> >  include/hw/cxl/cxl_device.h |  50 +++-
> >  include/hw/cxl/cxl_events.h |  16 ++
> >  qapi/cxl.json               |  44 ++++
> >  5 files changed, 924 insertions(+), 67 deletions(-)
> > 
> > -- 
> > 2.25.1
> 
>

Fan Ni June 7, 2023, 6:31 p.m. UTC | #6

On Wed, Jun 07, 2023 at 06:13:01PM +0000, Shesha Bhushan Sreenivasamurthy wrote:
> Hi Fan,
>    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> 
> https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25__;!!EwVzqGoTKBqv-0DWAJBm!Vt5uIqwW-L4c4gh02ulI4M762JNQ3_aE9k9lb6QlwE2xm6T23ic7ig7Y77i1VN7l_RX_ySIQhre_z7Q0JA$ 

Can you push changes to the branch directly? I think it is Jonathan's private
branch. However, I can fork the branch and rebase my patch series atop and
share with you the new repo if that helps you move forward your
work.
Let me know your thought.

Fan

> 
> 
> From: Fan Ni <[e-mail]>
> Sent: Monday, June 5, 2023 10:51 AM
> To: Ira Weiny <[e-mail]>
> Cc: [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
> Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
>  
> On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:
> > Fan Ni wrote:
> > > Since the early draft of DCD support in kernel is out
> > > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > > this patch series provide dcd emulation in qemu so people who are interested
> > > can have an early try. It is noted that the patch series may need to be updated
> > > accordingly if the kernel side implementation changes.
> > 
> > Fan,
> > 
> > Do you have a git tree we can pull this from which is updated to a more
> > recent CXL branch from Jonathan?
> > 
> > Thanks,
> > Ira
> 
> Hi Ira,
> 
> I have a git tree of the patch series based on Jonathan's branch
> cxl-2023-02-28: https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddev_tree_dcd-2Drfe&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=w6dicn5kXEG4Imk6TpICIjdA6KJ-xt84dtHui-Y0fv5H13bijtzEvjxECKE5MHYf&s=3yeO9RN5FY3gPfO2y19X057YeqRTTQTQNfNA-Gfir_Q&e= .
> 
> That may be not new enough to include some of the recent patches, but I can
> rebase it to a newer branch if you can tell me which branch you want to use.
> 
> Thanks,
> Fan
> 
> > 
> > > 
> > > To support DCD emulation, the patch series add DCD related mailbox command
> > > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > > with dynamic capacity extent and region representative.
> > > To support read/write to the dynamic capacity of the device, a host backend
> > > is provided and necessary check mechnism is added to ensure the dynamic
> > > capacity accessed is backed with active dc extents.
> > > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > > Also, the support for multiple hosts sharing the same DCD case is missing.
> > > 
> > > Things we can try with the patch series together with kernel dcd code:
> > > 1. Create DC regions to cover the address range of the dynamic capacity
> > > regions.
> > > 2. Add/release dynamic capacity extents to the device and notify the
> > > kernel.
> > > 3. Test kernel side code to accept added dc extents and create dax devices,
> > > and release dc extents and notify the device
> > > 4. Online the memory range backed with dc extents and let application use
> > > them.
> > > 
> > > The patch series is based on Jonathan's local qemu branch:
> > > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > > 
> > > Simple tests peformed with the patch series:
> > > 1 Install cxl modules:
> > > 
> > > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > > 
> > > 2 Create dc regions:
> > > 
> > > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > > echo 1 > /sys/bus/cxl/devices/$region/commit
> > > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > > 
> > > /home/fan/cxl/tools-and-scripts# cxl list
> > > [
> > >   {
> > >     "memdevs":[
> > >       {
> > >         "memdev":"mem0",
> > >         "pmem_size":536870912,
> > >         "ram_size":0,
> > >         "serial":0,
> > >         "host":"0000:0d:00.0"
> > >       }
> > >     ]
> > >   },
> > >   {
> > >     "regions":[
> > >       {
> > >         "region":"region0",
> > >         "resource":45365592064,
> > >         "size":268435456,
> > >         "interleave_ways":1,
> > >         "interleave_granularity":256,
> > >         "decode_state":"commit"
> > >       }
> > >     ]
> > >   }
> > > ]
> > > 
> > > 3 Add two dc extents (128MB each) through qmp interface
> > > 
> > > { "execute": "qmp_capabilities" }
> > > 
> > > { "execute": "cxl-add-dynamic-capacity-event",
> > >      "arguments": {
> > >               "path": "/machine/peripheral/cxl-pmem0",
> > >              "region-id" : 0,
> > >               "num-extent": 2,
> > >              "dpa":0,
> > >              "extent-len": 128
> > >      }
> > > }
> > > 
> > > /home/fan/cxl/tools-and-scripts# lsmem
> > > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > > 
> > > Memory block size:       128M
> > > Total online memory:       8G
> > > Total offline memory:    256M
> > > 
> > > 
> > > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > > 
> > > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > > [  230.730553] Fallback order for Node 0: 0 1
> > > [  230.730825] Fallback order for Node 1: 1 0
> > > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > > [  230.731110] Policy zone: Normal
> > > onlined memory for 1 device
> > > 
> > > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > > 
> > > Memory block size:       128M
> > > Total online memory:     8.1G
> > > Total offline memory:    128M
> > > 
> > > 5 using dc extents as regular memory
> > > 
> > > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > > ndctl              scripts   test.h      version.h.in COPYING                 acpi.h
> > > config.h.meson   cxl          make-git-snapshot.sh   ndctl.spec.in  sles     tools
> > > Documentation        build       contrib           daxctl        meson.build            rhel
> > > tags        topology.png LICENSES    ccan        cscope.files
> > > git-version  meson_options.txt      rpmbuild.sh    test     util
> > > 
> > > 
> > > QEMU command line cxl configuration:
> > > 
> > > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > > 
> > > 
> > > Kernel DCD support used to test the changes
> > > 
> > > The code is tested with the posted kernel dcd support:
> > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > > 
> > > commit: f425bc34c600e2a3721d6560202962ec41622815
> > > 
> > > To make the test work, we have made the following changes to the above kernel commit:
> > > 
> > > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > > index 5f04bbc18af5..5f421d3c5cef 100644
> > > --- a/drivers/cxl/core/mbox.c
> > > +++ b/drivers/cxl/core/mbox.c
> > > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> > >      CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> > >      CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> > >      CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > > +   CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> > >  };
> > >  
> > >  /*
> > > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > > index 291c716abd49..ae10e3cf43a1 100644
> > > --- a/drivers/cxl/core/region.c
> > > +++ b/drivers/cxl/core/region.c
> > > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> > >              }
> > >              cxlds->dc_list_gen_num = extent_gen_num;
> > >              dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > > -           enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > > +           /*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> > >      }
> > >      return 0;
> > >  err:
> > > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> > >                              dev_dax->align, memremap_compat_align()))) {
> > >              rc = alloc_dev_dax_range(dev_dax, hpa,
> > >                                      resource_size(alloc_dpa_res));
> > > -           return rc;
> > > +           if (rc)
> > > +                   return rc;
> > >      }
> > >  
> > >      rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > index 9e45b1056022..653bec203838 100644
> > > --- a/drivers/cxl/pci.c
> > > +++ b/drivers/cxl/pci.c
> > > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> > >  
> > >      /* Driver enables DCD interrupt after creating the dc cxl_region */
> > >      rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > > -                                   IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > > +                                   IRQF_SHARED | IRQF_ONESHOT);
> > >      if (rc) {
> > >              dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> > >              return rc;
> > > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > > index 6ca85861750c..910a48259239 100644
> > > --- a/include/uapi/linux/cxl_mem.h
> > > +++ b/include/uapi/linux/cxl_mem.h
> > > @@ -47,6 +47,7 @@
> > >      ___C(SCAN_MEDIA, "Scan Media"),                                   \
> > >      ___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> > >      ___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > > +   ___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> > >      ___C(MAX, "invalid / last command")
> > >  
> > >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > > 
> > > 
> > > 
> > > Fan Ni (7):
> > >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> > >     payload of identify memory device command
> > >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> > >     and mailbox command support
> > >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> > >     device supports in qemu command line
> > >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> > >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> > >     dynamic capacity response
> > >   Add qmp interfaces to add/release dynamic capacity extents
> > >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > > 
> > >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> > >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> > >  include/hw/cxl/cxl_device.h |  50 +++-
> > >  include/hw/cxl/cxl_events.h |  16 ++
> > >  qapi/cxl.json               |  44 ++++
> > >  5 files changed, 924 insertions(+), 67 deletions(-)
> > > 
> > > -- 
> > > 2.25.1
> > 
> >

Shesha Bhushan Sreenivasamurthy June 7, 2023, 6:52 p.m. UTC | #7

From: Fan Ni <[e-mail]>
Sent: Wednesday, June 7, 2023 11:31 AM
To: Shesha Bhushan Sreenivasamurthy <[e-mail]>
Cc: Jonathan Cameron <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
 
External Email

----------------------------------------------------------------------
On Wed, Jun 07, 2023 at 06:13:01PM +0000, Shesha Bhushan Sreenivasamurthy wrote:
> Hi Fan,
>    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> 
> https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25__;!!EwVzqGoTKBqv-0DWAJBm!Vt5uIqwW-L4c4gh02ulI4M762JNQ3_aE9k9lb6QlwE2xm6T23ic7ig7Y77i1VN7l_RX_ySIQhre_z7Q0JA$ 

Can you push changes to the branch directly? I think it is Jonathan's private
branch. However, I can fork the branch and rebase my patch series atop and
share with you the new repo if that helps you move forward your
work.
Let me know your thought.

ss - I saw commits from others, so assumed you can. Since it is Jonathan's private repo, I will step back and let him answer.

Fan

> 
> 
> From: Fan Ni <[e-mail]>
> Sent: Monday, June 5, 2023 10:51 AM
> To: Ira Weiny <[e-mail]>
> Cc: [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
> Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
>  
> On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:
> > Fan Ni wrote:
> > > Since the early draft of DCD support in kernel is out
> > > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > > this patch series provide dcd emulation in qemu so people who are interested
> > > can have an early try. It is noted that the patch series may need to be updated
> > > accordingly if the kernel side implementation changes.
> > 
> > Fan,
> > 
> > Do you have a git tree we can pull this from which is updated to a more
> > recent CXL branch from Jonathan?
> > 
> > Thanks,
> > Ira
> 
> Hi Ira,
> 
> I have a git tree of the patch series based on Jonathan's branch
> cxl-2023-02-28: https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddev_tree_dcd-2Drfe&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=w6dicn5kXEG4Imk6TpICIjdA6KJ-xt84dtHui-Y0fv5H13bijtzEvjxECKE5MHYf&s=3yeO9RN5FY3gPfO2y19X057YeqRTTQTQNfNA-Gfir_Q&e= .
> 
> That may be not new enough to include some of the recent patches, but I can
> rebase it to a newer branch if you can tell me which branch you want to use.
> 
> Thanks,
> Fan
> 
> > 
> > > 
> > > To support DCD emulation, the patch series add DCD related mailbox command
> > > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > > with dynamic capacity extent and region representative.
> > > To support read/write to the dynamic capacity of the device, a host backend
> > > is provided and necessary check mechnism is added to ensure the dynamic
> > > capacity accessed is backed with active dc extents.
> > > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > > Also, the support for multiple hosts sharing the same DCD case is missing.
> > > 
> > > Things we can try with the patch series together with kernel dcd code:
> > > 1. Create DC regions to cover the address range of the dynamic capacity
> > > regions.
> > > 2. Add/release dynamic capacity extents to the device and notify the
> > > kernel.
> > > 3. Test kernel side code to accept added dc extents and create dax devices,
> > > and release dc extents and notify the device
> > > 4. Online the memory range backed with dc extents and let application use
> > > them.
> > > 
> > > The patch series is based on Jonathan's local qemu branch:
> > > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > > 
> > > Simple tests peformed with the patch series:
> > > 1 Install cxl modules:
> > > 
> > > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > > 
> > > 2 Create dc regions:
> > > 
> > > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > > echo 1 > /sys/bus/cxl/devices/$region/commit
> > > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > > 
> > > /home/fan/cxl/tools-and-scripts# cxl list
> > > [
> > >   {
> > >     "memdevs":[
> > >       {
> > >         "memdev":"mem0",
> > >         "pmem_size":536870912,
> > >         "ram_size":0,
> > >         "serial":0,
> > >         "host":"0000:0d:00.0"
> > >       }
> > >     ]
> > >   },
> > >   {
> > >     "regions":[
> > >       {
> > >         "region":"region0",
> > >         "resource":45365592064,
> > >         "size":268435456,
> > >         "interleave_ways":1,
> > >         "interleave_granularity":256,
> > >         "decode_state":"commit"
> > >       }
> > >     ]
> > >   }
> > > ]
> > > 
> > > 3 Add two dc extents (128MB each) through qmp interface
> > > 
> > > { "execute": "qmp_capabilities" }
> > > 
> > > { "execute": "cxl-add-dynamic-capacity-event",
> > >      "arguments": {
> > >               "path": "/machine/peripheral/cxl-pmem0",
> > >              "region-id" : 0,
> > >               "num-extent": 2,
> > >              "dpa":0,
> > >              "extent-len": 128
> > >      }
> > > }
> > > 
> > > /home/fan/cxl/tools-and-scripts# lsmem
> > > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > > 
> > > Memory block size:       128M
> > > Total online memory:       8G
> > > Total offline memory:    256M
> > > 
> > > 
> > > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > > 
> > > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > > [  230.730553] Fallback order for Node 0: 0 1
> > > [  230.730825] Fallback order for Node 1: 1 0
> > > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > > [  230.731110] Policy zone: Normal
> > > onlined memory for 1 device
> > > 
> > > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > > 
> > > Memory block size:       128M
> > > Total online memory:     8.1G
> > > Total offline memory:    128M
> > > 
> > > 5 using dc extents as regular memory
> > > 
> > > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > > ndctl              scripts   test.h      version.h.in COPYING                 acpi.h
> > > config.h.meson   cxl          make-git-snapshot.sh   ndctl.spec.in  sles     tools
> > > Documentation        build       contrib           daxctl        meson.build            rhel
> > > tags        topology.png LICENSES    ccan        cscope.files
> > > git-version  meson_options.txt      rpmbuild.sh    test     util
> > > 
> > > 
> > > QEMU command line cxl configuration:
> > > 
> > > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > > 
> > > 
> > > Kernel DCD support used to test the changes
> > > 
> > > The code is tested with the posted kernel dcd support:
> > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > > 
> > > commit: f425bc34c600e2a3721d6560202962ec41622815
> > > 
> > > To make the test work, we have made the following changes to the above kernel commit:
> > > 
> > > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > > index 5f04bbc18af5..5f421d3c5cef 100644
> > > --- a/drivers/cxl/core/mbox.c
> > > +++ b/drivers/cxl/core/mbox.c
> > > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> > >      CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> > >      CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> > >      CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > > +   CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> > >  };
> > >  
> > >  /*
> > > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > > index 291c716abd49..ae10e3cf43a1 100644
> > > --- a/drivers/cxl/core/region.c
> > > +++ b/drivers/cxl/core/region.c
> > > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> > >              }
> > >              cxlds->dc_list_gen_num = extent_gen_num;
> > >              dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > > -           enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > > +           /*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> > >      }
> > >      return 0;
> > >  err:
> > > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> > >                              dev_dax->align, memremap_compat_align()))) {
> > >              rc = alloc_dev_dax_range(dev_dax, hpa,
> > >                                      resource_size(alloc_dpa_res));
> > > -           return rc;
> > > +           if (rc)
> > > +                   return rc;
> > >      }
> > >  
> > >      rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > index 9e45b1056022..653bec203838 100644
> > > --- a/drivers/cxl/pci.c
> > > +++ b/drivers/cxl/pci.c
> > > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> > >  
> > >      /* Driver enables DCD interrupt after creating the dc cxl_region */
> > >      rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > > -                                   IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > > +                                   IRQF_SHARED | IRQF_ONESHOT);
> > >      if (rc) {
> > >              dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> > >              return rc;
> > > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > > index 6ca85861750c..910a48259239 100644
> > > --- a/include/uapi/linux/cxl_mem.h
> > > +++ b/include/uapi/linux/cxl_mem.h
> > > @@ -47,6 +47,7 @@
> > >      ___C(SCAN_MEDIA, "Scan Media"),                                   \
> > >      ___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> > >      ___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > > +   ___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> > >      ___C(MAX, "invalid / last command")
> > >  
> > >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > > 
> > > 
> > > 
> > > Fan Ni (7):
> > >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> > >     payload of identify memory device command
> > >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> > >     and mailbox command support
> > >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> > >     device supports in qemu command line
> > >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> > >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> > >     dynamic capacity response
> > >   Add qmp interfaces to add/release dynamic capacity extents
> > >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > > 
> > >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> > >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> > >  include/hw/cxl/cxl_device.h |  50 +++-
> > >  include/hw/cxl/cxl_events.h |  16 ++
> > >  qapi/cxl.json               |  44 ++++
> > >  5 files changed, 924 insertions(+), 67 deletions(-)
> > > 
> > > -- 
> > > 2.25.1
> > 
> >

Jonathan Cameron June 8, 2023, 9:43 a.m. UTC | #8

On Wed, 7 Jun 2023 18:52:14 +0000
Shesha Bhushan Sreenivasamurthy <[e-mail]> wrote:

> From: Fan Ni <[e-mail]>
> Sent: Wednesday, June 7, 2023 11:31 AM
> To: Shesha Bhushan Sreenivasamurthy <[e-mail]>
> Cc: Jonathan Cameron <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
> Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
>  
> External Email
> 
> ----------------------------------------------------------------------
> On Wed, Jun 07, 2023 at 06:13:01PM +0000, Shesha Bhushan Sreenivasamurthy wrote:
> > Hi Fan,
> >    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> > 
> > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25__;!!EwVzqGoTKBqv-0DWAJBm!Vt5uIqwW-L4c4gh02ulI4M762JNQ3_aE9k9lb6QlwE2xm6T23ic7ig7Y77i1VN7l_RX_ySIQhre_z7Q0JA$   
> 
> Can you push changes to the branch directly? I think it is Jonathan's private
> branch. However, I can fork the branch and rebase my patch series atop and
> share with you the new repo if that helps you move forward your
> work.
> Let me know your thought.
> 
> ss - I saw commits from others, so assumed you can. Since it is Jonathan's private repo, I will step back and let him answer.

I tend to apply stuff in manually rather than given more people commit access
to that particular gitlab tree. 

Easiest option is to fork on gitlab and share the path of your own fork.

I normally queue reasonably mature stuff up on my tree, but that's about managing
the series sent with intent of being applied upstream + providing a fairly stable test
branch.  It's not intended as a general place for stuff in development (though I might
sneak out an extra branch myself from time to time if I want to talk about it :)

Jonathan

> 
> Fan
> 
> > 
> > 
> > From: Fan Ni <[e-mail]>
> > Sent: Monday, June 5, 2023 10:51 AM
> > To: Ira Weiny <[e-mail]>
> > Cc: [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
> > Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
> >  
> > On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:  
> > > Fan Ni wrote:  
> > > > Since the early draft of DCD support in kernel is out
> > > > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > > > this patch series provide dcd emulation in qemu so people who are interested
> > > > can have an early try. It is noted that the patch series may need to be updated
> > > > accordingly if the kernel side implementation changes.  
> > > 
> > > Fan,
> > > 
> > > Do you have a git tree we can pull this from which is updated to a more
> > > recent CXL branch from Jonathan?
> > > 
> > > Thanks,
> > > Ira  
> > 
> > Hi Ira,
> > 
> > I have a git tree of the patch series based on Jonathan's branch
> > cxl-2023-02-28: https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddev_tree_dcd-2Drfe&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=w6dicn5kXEG4Imk6TpICIjdA6KJ-xt84dtHui-Y0fv5H13bijtzEvjxECKE5MHYf&s=3yeO9RN5FY3gPfO2y19X057YeqRTTQTQNfNA-Gfir_Q&e= .
> > 
> > That may be not new enough to include some of the recent patches, but I can
> > rebase it to a newer branch if you can tell me which branch you want to use.
> > 
> > Thanks,
> > Fan
> >   
> > >   
> > > > 
> > > > To support DCD emulation, the patch series add DCD related mailbox command
> > > > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > > > with dynamic capacity extent and region representative.
> > > > To support read/write to the dynamic capacity of the device, a host backend
> > > > is provided and necessary check mechnism is added to ensure the dynamic
> > > > capacity accessed is backed with active dc extents.
> > > > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > > > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > > > Also, the support for multiple hosts sharing the same DCD case is missing.
> > > > 
> > > > Things we can try with the patch series together with kernel dcd code:
> > > > 1. Create DC regions to cover the address range of the dynamic capacity
> > > > regions.
> > > > 2. Add/release dynamic capacity extents to the device and notify the
> > > > kernel.
> > > > 3. Test kernel side code to accept added dc extents and create dax devices,
> > > > and release dc extents and notify the device
> > > > 4. Online the memory range backed with dc extents and let application use
> > > > them.
> > > > 
> > > > The patch series is based on Jonathan's local qemu branch:
> > > > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > > > 
> > > > Simple tests peformed with the patch series:
> > > > 1 Install cxl modules:
> > > > 
> > > > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > > > 
> > > > 2 Create dc regions:
> > > > 
> > > > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > > > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > > > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > > > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > > > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > > > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > > > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > > > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > > > echo 1 > /sys/bus/cxl/devices/$region/commit
> > > > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > > > 
> > > > /home/fan/cxl/tools-and-scripts# cxl list
> > > > [
> > > >   {
> > > >     "memdevs":[
> > > >       {
> > > >         "memdev":"mem0",
> > > >         "pmem_size":536870912,
> > > >         "ram_size":0,
> > > >         "serial":0,
> > > >         "host":"0000:0d:00.0"
> > > >       }
> > > >     ]
> > > >   },
> > > >   {
> > > >     "regions":[
> > > >       {
> > > >         "region":"region0",
> > > >         "resource":45365592064,
> > > >         "size":268435456,
> > > >         "interleave_ways":1,
> > > >         "interleave_granularity":256,
> > > >         "decode_state":"commit"
> > > >       }
> > > >     ]
> > > >   }
> > > > ]
> > > > 
> > > > 3 Add two dc extents (128MB each) through qmp interface
> > > > 
> > > > { "execute": "qmp_capabilities" }
> > > > 
> > > > { "execute": "cxl-add-dynamic-capacity-event",
> > > >      "arguments": {
> > > >               "path": "/machine/peripheral/cxl-pmem0",
> > > >              "region-id" : 0,
> > > >               "num-extent": 2,
> > > >              "dpa":0,
> > > >              "extent-len": 128
> > > >      }
> > > > }
> > > > 
> > > > /home/fan/cxl/tools-and-scripts# lsmem
> > > > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > > > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > > > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > > > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > > > 
> > > > Memory block size:       128M
> > > > Total online memory:       8G
> > > > Total offline memory:    256M
> > > > 
> > > > 
> > > > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > > > 
> > > > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > > > [  230.730553] Fallback order for Node 0: 0 1
> > > > [  230.730825] Fallback order for Node 1: 1 0
> > > > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > > > [  230.731110] Policy zone: Normal
> > > > onlined memory for 1 device
> > > > 
> > > > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > > > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > > > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > > > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > > > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > > > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > > > 
> > > > Memory block size:       128M
> > > > Total online memory:     8.1G
> > > > Total offline memory:    128M
> > > > 
> > > > 5 using dc extents as regular memory
> > > > 
> > > > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > > > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > > > ndctl              scripts   test.h      version.h.in COPYING                 acpi.h
> > > > config.h.meson   cxl          make-git-snapshot.sh   ndctl.spec.in  sles     tools
> > > > Documentation        build       contrib           daxctl        meson.build            rhel
> > > > tags        topology.png LICENSES    ccan        cscope.files
> > > > git-version  meson_options.txt      rpmbuild.sh    test     util
> > > > 
> > > > 
> > > > QEMU command line cxl configuration:
> > > > 
> > > > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > > > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > > > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > > > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > > > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > > > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > > > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > > > 
> > > > 
> > > > Kernel DCD support used to test the changes
> > > > 
> > > > The code is tested with the posted kernel dcd support:
> > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > > > 
> > > > commit: f425bc34c600e2a3721d6560202962ec41622815
> > > > 
> > > > To make the test work, we have made the following changes to the above kernel commit:
> > > > 
> > > > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > > > index 5f04bbc18af5..5f421d3c5cef 100644
> > > > --- a/drivers/cxl/core/mbox.c
> > > > +++ b/drivers/cxl/core/mbox.c
> > > > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> > > >      CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> > > >      CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> > > >      CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > > > +   CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> > > >  };
> > > >  
> > > >  /*
> > > > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > > > index 291c716abd49..ae10e3cf43a1 100644
> > > > --- a/drivers/cxl/core/region.c
> > > > +++ b/drivers/cxl/core/region.c
> > > > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> > > >              }
> > > >              cxlds->dc_list_gen_num = extent_gen_num;
> > > >              dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > > > -           enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > > > +           /*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> > > >      }
> > > >      return 0;
> > > >  err:
> > > > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> > > >                              dev_dax->align, memremap_compat_align()))) {
> > > >              rc = alloc_dev_dax_range(dev_dax, hpa,
> > > >                                      resource_size(alloc_dpa_res));
> > > > -           return rc;
> > > > +           if (rc)
> > > > +                   return rc;
> > > >      }
> > > >  
> > > >      rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > > index 9e45b1056022..653bec203838 100644
> > > > --- a/drivers/cxl/pci.c
> > > > +++ b/drivers/cxl/pci.c
> > > > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> > > >  
> > > >      /* Driver enables DCD interrupt after creating the dc cxl_region */
> > > >      rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > > > -                                   IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > > > +                                   IRQF_SHARED | IRQF_ONESHOT);
> > > >      if (rc) {
> > > >              dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> > > >              return rc;
> > > > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > > > index 6ca85861750c..910a48259239 100644
> > > > --- a/include/uapi/linux/cxl_mem.h
> > > > +++ b/include/uapi/linux/cxl_mem.h
> > > > @@ -47,6 +47,7 @@
> > > >      ___C(SCAN_MEDIA, "Scan Media"),                                   \
> > > >      ___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> > > >      ___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > > > +   ___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> > > >      ___C(MAX, "invalid / last command")
> > > >  
> > > >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > > > 
> > > > 
> > > > 
> > > > Fan Ni (7):
> > > >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> > > >     payload of identify memory device command
> > > >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> > > >     and mailbox command support
> > > >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> > > >     device supports in qemu command line
> > > >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> > > >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> > > >     dynamic capacity response
> > > >   Add qmp interfaces to add/release dynamic capacity extents
> > > >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > > > 
> > > >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> > > >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> > > >  include/hw/cxl/cxl_device.h |  50 +++-
> > > >  include/hw/cxl/cxl_events.h |  16 ++
> > > >  qapi/cxl.json               |  44 ++++
> > > >  5 files changed, 924 insertions(+), 67 deletions(-)
> > > > 
> > > > -- 
> > > > 2.25.1  
> > > 
> > >

Shesha Bhushan Sreenivasamurthy June 8, 2023, 3:20 p.m. UTC | #9

From: Jonathan Cameron <[e-mail]>
Sent: Thursday, June 8, 2023 2:43 AM
To: Shesha Bhushan Sreenivasamurthy <[e-mail]>
Cc: Fan Ni <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
 
External Email

----------------------------------------------------------------------
On Wed, 7 Jun 2023 18:52:14 +0000
Shesha Bhushan Sreenivasamurthy <[e-mail]> wrote:

> From: Fan Ni <[e-mail]>
> Sent: Wednesday, June 7, 2023 11:31 AM
> To: Shesha Bhushan Sreenivasamurthy <[e-mail]>
> Cc: Jonathan Cameron <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
> Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
>  
> External Email
> 
> ----------------------------------------------------------------------
> On Wed, Jun 07, 2023 at 06:13:01PM +0000, Shesha Bhushan Sreenivasamurthy wrote:
> > Hi Fan,
> >    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> > 
> > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25__;!!EwVzqGoTKBqv-0DWAJBm!Vt5uIqwW-L4c4gh02ulI4M762JNQ3_aE9k9lb6QlwE2xm6T23ic7ig7Y77i1VN7l_RX_ySIQhre_z7Q0JA$   
> 
> Can you push changes to the branch directly? I think it is Jonathan's private
> branch. However, I can fork the branch and rebase my patch series atop and
> share with you the new repo if that helps you move forward your
> work.
> Let me know your thought.
> 
> ss - I saw commits from others, so assumed you can. Since it is Jonathan's private repo, I will step back and let him answer.

I tend to apply stuff in manually rather than given more people commit access
to that particular gitlab tree. 

Easiest option is to fork on gitlab and share the path of your own fork.

I normally queue reasonably mature stuff up on my tree, but that's about managing
the series sent with intent of being applied upstream + providing a fairly stable test
branch.  It's not intended as a general place for stuff in development (though I might
sneak out an extra branch myself from time to time if I want to talk about it :)

Jonathan

ss - Thanks ! I have forked Jonathan's repo on Gitlab and will work of cxl-2023-05-25 branch. I will apply Fan's patch manually and continue ..

> 
> Fan
> 
> > 
> > 
> > From: Fan Ni <[e-mail]>
> > Sent: Monday, June 5, 2023 10:51 AM
> > To: Ira Weiny <[e-mail]>
> > Cc: [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
> > Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
> >  
> > On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:  
> > > Fan Ni wrote:  
> > > > Since the early draft of DCD support in kernel is out
> > > > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > > > this patch series provide dcd emulation in qemu so people who are interested
> > > > can have an early try. It is noted that the patch series may need to be updated
> > > > accordingly if the kernel side implementation changes.  
> > > 
> > > Fan,
> > > 
> > > Do you have a git tree we can pull this from which is updated to a more
> > > recent CXL branch from Jonathan?
> > > 
> > > Thanks,
> > > Ira  
> > 
> > Hi Ira,
> > 
> > I have a git tree of the patch series based on Jonathan's branch
> > cxl-2023-02-28: https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddev_tree_dcd-2Drfe&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=w6dicn5kXEG4Imk6TpICIjdA6KJ-xt84dtHui-Y0fv5H13bijtzEvjxECKE5MHYf&s=3yeO9RN5FY3gPfO2y19X057YeqRTTQTQNfNA-Gfir_Q&e= .
> > 
> > That may be not new enough to include some of the recent patches, but I can
> > rebase it to a newer branch if you can tell me which branch you want to use.
> > 
> > Thanks,
> > Fan
> >   
> > >   
> > > > 
> > > > To support DCD emulation, the patch series add DCD related mailbox command
> > > > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > > > with dynamic capacity extent and region representative.
> > > > To support read/write to the dynamic capacity of the device, a host backend
> > > > is provided and necessary check mechnism is added to ensure the dynamic
> > > > capacity accessed is backed with active dc extents.
> > > > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > > > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > > > Also, the support for multiple hosts sharing the same DCD case is missing.
> > > > 
> > > > Things we can try with the patch series together with kernel dcd code:
> > > > 1. Create DC regions to cover the address range of the dynamic capacity
> > > > regions.
> > > > 2. Add/release dynamic capacity extents to the device and notify the
> > > > kernel.
> > > > 3. Test kernel side code to accept added dc extents and create dax devices,
> > > > and release dc extents and notify the device
> > > > 4. Online the memory range backed with dc extents and let application use
> > > > them.
> > > > 
> > > > The patch series is based on Jonathan's local qemu branch:
> > > > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > > > 
> > > > Simple tests peformed with the patch series:
> > > > 1 Install cxl modules:
> > > > 
> > > > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > > > 
> > > > 2 Create dc regions:
> > > > 
> > > > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > > > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > > > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > > > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > > > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > > > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > > > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > > > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > > > echo 1 > /sys/bus/cxl/devices/$region/commit
> > > > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > > > 
> > > > /home/fan/cxl/tools-and-scripts# cxl list
> > > > [
> > > >   {
> > > >     "memdevs":[
> > > >       {
> > > >         "memdev":"mem0",
> > > >         "pmem_size":536870912,
> > > >         "ram_size":0,
> > > >         "serial":0,
> > > >         "host":"0000:0d:00.0"
> > > >       }
> > > >     ]
> > > >   },
> > > >   {
> > > >     "regions":[
> > > >       {
> > > >         "region":"region0",
> > > >         "resource":45365592064,
> > > >         "size":268435456,
> > > >         "interleave_ways":1,
> > > >         "interleave_granularity":256,
> > > >         "decode_state":"commit"
> > > >       }
> > > >     ]
> > > >   }
> > > > ]
> > > > 
> > > > 3 Add two dc extents (128MB each) through qmp interface
> > > > 
> > > > { "execute": "qmp_capabilities" }
> > > > 
> > > > { "execute": "cxl-add-dynamic-capacity-event",
> > > >      "arguments": {
> > > >               "path": "/machine/peripheral/cxl-pmem0",
> > > >              "region-id" : 0,
> > > >               "num-extent": 2,
> > > >              "dpa":0,
> > > >              "extent-len": 128
> > > >      }
> > > > }
> > > > 
> > > > /home/fan/cxl/tools-and-scripts# lsmem
> > > > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > > > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > > > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > > > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > > > 
> > > > Memory block size:       128M
> > > > Total online memory:       8G
> > > > Total offline memory:    256M
> > > > 
> > > > 
> > > > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > > > 
> > > > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > > > [  230.730553] Fallback order for Node 0: 0 1
> > > > [  230.730825] Fallback order for Node 1: 1 0
> > > > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > > > [  230.731110] Policy zone: Normal
> > > > onlined memory for 1 device
> > > > 
> > > > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > > > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > > > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > > > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > > > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > > > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > > > 
> > > > Memory block size:       128M
> > > > Total online memory:     8.1G
> > > > Total offline memory:    128M
> > > > 
> > > > 5 using dc extents as regular memory
> > > > 
> > > > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > > > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > > > ndctl              scripts   test.h      version.h.in COPYING                 acpi.h
> > > > config.h.meson   cxl          make-git-snapshot.sh   ndctl.spec.in  sles     tools
> > > > Documentation        build       contrib           daxctl        meson.build            rhel
> > > > tags        topology.png LICENSES    ccan        cscope.files
> > > > git-version  meson_options.txt      rpmbuild.sh    test     util
> > > > 
> > > > 
> > > > QEMU command line cxl configuration:
> > > > 
> > > > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > > > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > > > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > > > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > > > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > > > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > > > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > > > 
> > > > 
> > > > Kernel DCD support used to test the changes
> > > > 
> > > > The code is tested with the posted kernel dcd support:
> > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > > > 
> > > > commit: f425bc34c600e2a3721d6560202962ec41622815
> > > > 
> > > > To make the test work, we have made the following changes to the above kernel commit:
> > > > 
> > > > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > > > index 5f04bbc18af5..5f421d3c5cef 100644
> > > > --- a/drivers/cxl/core/mbox.c
> > > > +++ b/drivers/cxl/core/mbox.c
> > > > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> > > >      CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> > > >      CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> > > >      CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > > > +   CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> > > >  };
> > > >  
> > > >  /*
> > > > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > > > index 291c716abd49..ae10e3cf43a1 100644
> > > > --- a/drivers/cxl/core/region.c
> > > > +++ b/drivers/cxl/core/region.c
> > > > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> > > >              }
> > > >              cxlds->dc_list_gen_num = extent_gen_num;
> > > >              dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > > > -           enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > > > +           /*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> > > >      }
> > > >      return 0;
> > > >  err:
> > > > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> > > >                              dev_dax->align, memremap_compat_align()))) {
> > > >              rc = alloc_dev_dax_range(dev_dax, hpa,
> > > >                                      resource_size(alloc_dpa_res));
> > > > -           return rc;
> > > > +           if (rc)
> > > > +                   return rc;
> > > >      }
> > > >  
> > > >      rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > > index 9e45b1056022..653bec203838 100644
> > > > --- a/drivers/cxl/pci.c
> > > > +++ b/drivers/cxl/pci.c
> > > > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> > > >  
> > > >      /* Driver enables DCD interrupt after creating the dc cxl_region */
> > > >      rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > > > -                                   IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > > > +                                   IRQF_SHARED | IRQF_ONESHOT);
> > > >      if (rc) {
> > > >              dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> > > >              return rc;
> > > > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > > > index 6ca85861750c..910a48259239 100644
> > > > --- a/include/uapi/linux/cxl_mem.h
> > > > +++ b/include/uapi/linux/cxl_mem.h
> > > > @@ -47,6 +47,7 @@
> > > >      ___C(SCAN_MEDIA, "Scan Media"),                                   \
> > > >      ___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> > > >      ___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > > > +   ___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> > > >      ___C(MAX, "invalid / last command")
> > > >  
> > > >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > > > 
> > > > 
> > > > 
> > > > Fan Ni (7):
> > > >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> > > >     payload of identify memory device command
> > > >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> > > >     and mailbox command support
> > > >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> > > >     device supports in qemu command line
> > > >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> > > >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> > > >     dynamic capacity response
> > > >   Add qmp interfaces to add/release dynamic capacity extents
> > > >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > > > 
> > > >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> > > >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> > > >  include/hw/cxl/cxl_device.h |  50 +++-
> > > >  include/hw/cxl/cxl_events.h |  16 ++
> > > >  qapi/cxl.json               |  44 ++++
> > > >  5 files changed, 924 insertions(+), 67 deletions(-)
> > > > 
> > > > -- 
> > > > 2.25.1  
> > > 
> > >

Ira Weiny June 8, 2023, 3:43 p.m. UTC | #10

Shesha Bhushan Sreenivasamurthy wrote:
> Hi Fan,
>    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> 
> https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25

This is the branch I'm trying to use as well.

Thanks,
Ira

Shesha Bhushan Sreenivasamurthy June 8, 2023, 4:55 p.m. UTC | #11

Here is the repo where I am committing my changes

https://gitlab.com/sheshas/qemu-fmapi/-/commits/cxl-2023-05-25


From: Shesha Bhushan Sreenivasamurthy <[e-mail]>
Sent: Thursday, June 8, 2023 8:20 AM
To: Jonathan Cameron <[e-mail]>
Cc: Fan Ni <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
Subject: Re: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
 



From: Jonathan Cameron <[e-mail]>
Sent: Thursday, June 8, 2023 2:43 AM
To: Shesha Bhushan Sreenivasamurthy <[e-mail]>
Cc: Fan Ni <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
 
External Email

----------------------------------------------------------------------
On Wed, 7 Jun 2023 18:52:14 +0000
Shesha Bhushan Sreenivasamurthy <[e-mail]> wrote:

> From: Fan Ni <[e-mail]>
> Sent: Wednesday, June 7, 2023 11:31 AM
> To: Shesha Bhushan Sreenivasamurthy <[e-mail]>
> Cc: Jonathan Cameron <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>
> Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
>  
> External Email
> 
> ----------------------------------------------------------------------
> On Wed, Jun 07, 2023 at 06:13:01PM +0000, Shesha Bhushan Sreenivasamurthy wrote:
> > Hi Fan,
> >    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> > 
> > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25__;!!EwVzqGoTKBqv-0DWAJBm!Vt5uIqwW-L4c4gh02ulI4M762JNQ3_aE9k9lb6QlwE2xm6T23ic7ig7Y77i1VN7l_RX_ySIQhre_z7Q0JA$   
> 
> Can you push changes to the branch directly? I think it is Jonathan's private
> branch. However, I can fork the branch and rebase my patch series atop and
> share with you the new repo if that helps you move forward your
> work.
> Let me know your thought.
> 
> ss - I saw commits from others, so assumed you can. Since it is Jonathan's private repo, I will step back and let him answer.

I tend to apply stuff in manually rather than given more people commit access
to that particular gitlab tree. 

Easiest option is to fork on gitlab and share the path of your own fork.

I normally queue reasonably mature stuff up on my tree, but that's about managing
the series sent with intent of being applied upstream + providing a fairly stable test
branch.  It's not intended as a general place for stuff in development (though I might
sneak out an extra branch myself from time to time if I want to talk about it :)

Jonathan

ss - Thanks ! I have forked Jonathan's repo on Gitlab and will work of cxl-2023-05-25 branch. I will apply Fan's patch manually and continue ..

> 
> Fan
> 
> > 
> > 
> > From: Fan Ni <[e-mail]>
> > Sent: Monday, June 5, 2023 10:51 AM
> > To: Ira Weiny <[e-mail]>
> > Cc: [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
> > Subject: Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
> >  
> > On Mon, Jun 05, 2023 at 10:35:48AM -0700, Ira Weiny wrote:  
> > > Fan Ni wrote:  
> > > > Since the early draft of DCD support in kernel is out
> > > > (https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21LzAGNOk$ ),
> > > > this patch series provide dcd emulation in qemu so people who are interested
> > > > can have an early try. It is noted that the patch series may need to be updated
> > > > accordingly if the kernel side implementation changes.  
> > > 
> > > Fan,
> > > 
> > > Do you have a git tree we can pull this from which is updated to a more
> > > recent CXL branch from Jonathan?
> > > 
> > > Thanks,
> > > Ira  
> > 
> > Hi Ira,
> > 
> > I have a git tree of the patch series based on Jonathan's branch
> > cxl-2023-02-28: https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddev_tree_dcd-2Drfe&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=w6dicn5kXEG4Imk6TpICIjdA6KJ-xt84dtHui-Y0fv5H13bijtzEvjxECKE5MHYf&s=3yeO9RN5FY3gPfO2y19X057YeqRTTQTQNfNA-Gfir_Q&e= .
> > 
> > That may be not new enough to include some of the recent patches, but I can
> > rebase it to a newer branch if you can tell me which branch you want to use.
> > 
> > Thanks,
> > Fan
> >   
> > >   
> > > > 
> > > > To support DCD emulation, the patch series add DCD related mailbox command
> > > > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > > > with dynamic capacity extent and region representative.
> > > > To support read/write to the dynamic capacity of the device, a host backend
> > > > is provided and necessary check mechnism is added to ensure the dynamic
> > > > capacity accessed is backed with active dc extents.
> > > > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > > > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > > > Also, the support for multiple hosts sharing the same DCD case is missing.
> > > > 
> > > > Things we can try with the patch series together with kernel dcd code:
> > > > 1. Create DC regions to cover the address range of the dynamic capacity
> > > > regions.
> > > > 2. Add/release dynamic capacity extents to the device and notify the
> > > > kernel.
> > > > 3. Test kernel side code to accept added dc extents and create dax devices,
> > > > and release dc extents and notify the device
> > > > 4. Online the memory range backed with dc extents and let application use
> > > > them.
> > > > 
> > > > The patch series is based on Jonathan's local qemu branch:
> > > > https://urldefense.com/v3/__https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28__;!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21OO3UHEM$ 
> > > > 
> > > > Simple tests peformed with the patch series:
> > > > 1 Install cxl modules:
> > > > 
> > > > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > > > 
> > > > 2 Create dc regions:
> > > > 
> > > > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > > > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > > > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > > > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > > > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > > > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > > > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > > > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > > > echo 1 > /sys/bus/cxl/devices/$region/commit
> > > > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > > > 
> > > > /home/fan/cxl/tools-and-scripts# cxl list
> > > > [
> > > >   {
> > > >     "memdevs":[
> > > >       {
> > > >         "memdev":"mem0",
> > > >         "pmem_size":536870912,
> > > >         "ram_size":0,
> > > >         "serial":0,
> > > >         "host":"0000:0d:00.0"
> > > >       }
> > > >     ]
> > > >   },
> > > >   {
> > > >     "regions":[
> > > >       {
> > > >         "region":"region0",
> > > >         "resource":45365592064,
> > > >         "size":268435456,
> > > >         "interleave_ways":1,
> > > >         "interleave_granularity":256,
> > > >         "decode_state":"commit"
> > > >       }
> > > >     ]
> > > >   }
> > > > ]
> > > > 
> > > > 3 Add two dc extents (128MB each) through qmp interface
> > > > 
> > > > { "execute": "qmp_capabilities" }
> > > > 
> > > > { "execute": "cxl-add-dynamic-capacity-event",
> > > >      "arguments": {
> > > >               "path": "/machine/peripheral/cxl-pmem0",
> > > >              "region-id" : 0,
> > > >               "num-extent": 2,
> > > >              "dpa":0,
> > > >              "extent-len": 128
> > > >      }
> > > > }
> > > > 
> > > > /home/fan/cxl/tools-and-scripts# lsmem
> > > > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > > > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > > > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > > > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > > > 
> > > > Memory block size:       128M
> > > > Total online memory:       8G
> > > > Total offline memory:    256M
> > > > 
> > > > 
> > > > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > > > 
> > > > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > > > [  230.730553] Fallback order for Node 0: 0 1
> > > > [  230.730825] Fallback order for Node 1: 1 0
> > > > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > > > [  230.731110] Policy zone: Normal
> > > > onlined memory for 1 device
> > > > 
> > > > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > > > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > > > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > > > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > > > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > > > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > > > 
> > > > Memory block size:       128M
> > > > Total online memory:     8.1G
> > > > Total offline memory:    128M
> > > > 
> > > > 5 using dc extents as regular memory
> > > > 
> > > > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > > > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > > > ndctl              scripts   test.h      version.h.in COPYING                 acpi.h
> > > > config.h.meson   cxl          make-git-snapshot.sh   ndctl.spec.in  sles     tools
> > > > Documentation        build       contrib           daxctl        meson.build            rhel
> > > > tags        topology.png LICENSES    ccan        cscope.files
> > > > git-version  meson_options.txt      rpmbuild.sh    test     util
> > > > 
> > > > 
> > > > QEMU command line cxl configuration:
> > > > 
> > > > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > > > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > > > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > > > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > > > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > > > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > > > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > > > 
> > > > 
> > > > Kernel DCD support used to test the changes
> > > > 
> > > > The code is tested with the posted kernel dcd support:
> > > > https://urldefense.com/v3/__https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5*dcd-preview__;Lw!!EwVzqGoTKBqv-0DWAJBm!RHzXPIcSiGsqUciUIH6HnlG_W--4L5CHfvcOIeUFdwKFhAujXuFDxjymmpCdOu7SLr61rww7lr21q5Iza3M$ 
> > > > 
> > > > commit: f425bc34c600e2a3721d6560202962ec41622815
> > > > 
> > > > To make the test work, we have made the following changes to the above kernel commit:
> > > > 
> > > > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > > > index 5f04bbc18af5..5f421d3c5cef 100644
> > > > --- a/drivers/cxl/core/mbox.c
> > > > +++ b/drivers/cxl/core/mbox.c
> > > > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> > > >      CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> > > >      CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> > > >      CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > > > +   CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> > > >  };
> > > >  
> > > >  /*
> > > > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > > > index 291c716abd49..ae10e3cf43a1 100644
> > > > --- a/drivers/cxl/core/region.c
> > > > +++ b/drivers/cxl/core/region.c
> > > > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> > > >              }
> > > >              cxlds->dc_list_gen_num = extent_gen_num;
> > > >              dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > > > -           enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > > > +           /*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> > > >      }
> > > >      return 0;
> > > >  err:
> > > > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> > > >                              dev_dax->align, memremap_compat_align()))) {
> > > >              rc = alloc_dev_dax_range(dev_dax, hpa,
> > > >                                      resource_size(alloc_dpa_res));
> > > > -           return rc;
> > > > +           if (rc)
> > > > +                   return rc;
> > > >      }
> > > >  
> > > >      rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > > index 9e45b1056022..653bec203838 100644
> > > > --- a/drivers/cxl/pci.c
> > > > +++ b/drivers/cxl/pci.c
> > > > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> > > >  
> > > >      /* Driver enables DCD interrupt after creating the dc cxl_region */
> > > >      rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > > > -                                   IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > > > +                                   IRQF_SHARED | IRQF_ONESHOT);
> > > >      if (rc) {
> > > >              dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> > > >              return rc;
> > > > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > > > index 6ca85861750c..910a48259239 100644
> > > > --- a/include/uapi/linux/cxl_mem.h
> > > > +++ b/include/uapi/linux/cxl_mem.h
> > > > @@ -47,6 +47,7 @@
> > > >      ___C(SCAN_MEDIA, "Scan Media"),                                   \
> > > >      ___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> > > >      ___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > > > +   ___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> > > >      ___C(MAX, "invalid / last command")
> > > >  
> > > >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > > > 
> > > > 
> > > > 
> > > > Fan Ni (7):
> > > >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> > > >     payload of identify memory device command
> > > >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> > > >     and mailbox command support
> > > >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> > > >     device supports in qemu command line
> > > >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> > > >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> > > >     dynamic capacity response
> > > >   Add qmp interfaces to add/release dynamic capacity extents
> > > >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > > > 
> > > >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> > > >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> > > >  include/hw/cxl/cxl_device.h |  50 +++-
> > > >  include/hw/cxl/cxl_events.h |  16 ++
> > > >  qapi/cxl.json               |  44 ++++
> > > >  5 files changed, 924 insertions(+), 67 deletions(-)
> > > > 
> > > > -- 
> > > > 2.25.1  
> > > 
> > >

[e-mail] June 8, 2023, 6:10 p.m. UTC | #12

The 06/08/2023 08:43, Ira Weiny wrote:
> Shesha Bhushan Sreenivasamurthy wrote:
> > Hi Fan,
> >    I am implementing DCD FMAPI commands and planning to start pushing changes to the below branch. That requires the contributions you have made. Can your changes be pushed to the below branch ?
> > 
> > https://gitlab.com/jic23/qemu/-/tree/cxl-2023-05-25
> 
> This is the branch I'm trying to use as well.
> 
> Thanks,
> Ira

Hi Ira & Shesha,
FYI. I reabased my patch series on top of the above branch and created a new
branch here:

https://github.com/moking/qemu-dcd-preview-latest/tree/dcd-preview

It passes the same tests as shown here:
https://lore.kernel.org/linux-cxl/[e-mail]/T/#m76f6e85ce3d7292b1982960eb22086ee03922166

Ira Weiny June 9, 2023, 9:06 p.m. UTC | #13

[e-mail] wrote:
> The 06/08/2023 08:43, Ira Weiny wrote:
> > Shesha Bhushan Sreenivasamurthy wrote:

[snip]

> 
> Hi Ira & Shesha,
> FYI. I reabased my patch series on top of the above branch and created a new
> branch here:
> 
> https://github.com/moking/qemu-dcd-preview-latest/tree/dcd-preview

Thanks!

> 
> It passes the same tests as shown here:
> https://lore.kernel.org/linux-cxl/[e-mail]/T/#m76f6e85ce3d7292b1982960eb22086ee03922166

I've not gotten very far with this testing.  But I did find that regular
type 3 devices don't work with this change.  I used the patch below to get
this working.  Was there something I was missing to configure a non-DCD
device?

I don't particularly like adding another bool to this call stack.  Seems
like this calls for a flags field but I want to move on to DCD work so I
hacked this in.

Ira

commit ed27935044dcbd2c6ba71f8411b218621f3f4167
Author: Ira Weiny <[e-mail]>
Date:   Fri Jun 9 13:56:33 2023 -0700

    hw/mem/cxl_type3: Exclude DCD from CEL when type3 is not DCD
    
    Per CXL 3.0 9.13.3 Dynamic Capacity Device (DCD) when the type 3 memory
    device does not have DCD support the CEL should not include DCD
    configuration commands.
    
    If the number of DC regions supported is 0 skip the DCD commands in the
    CEL.
    
    Applies on top of Fan Ni's work here:
    https://github.com/moking/qemu-dcd-preview-latest/tree/dcd-preview
    
    Not-yet-Signed-off-by: Ira Weiny <[e-mail]>

diff --git a/hw/cxl/cxl-device-utils.c b/hw/cxl/cxl-device-utils.c
index a4a2c6a80004..262e35935563 100644
--- a/hw/cxl/cxl-device-utils.c
+++ b/hw/cxl/cxl-device-utils.c
@@ -288,7 +288,7 @@ static void mailbox_reg_init_common(CXLDeviceState *cxl_dstate)
 
 static void memdev_reg_init_common(CXLDeviceState *cxl_dstate) { }
 
-void cxl_device_register_init_common(CXLDeviceState *cxl_dstate)
+void cxl_device_register_init_common(CXLDeviceState *cxl_dstate, bool is_dcd)
 {
     uint64_t *cap_hdrs = cxl_dstate->caps_reg_state64;
     const int cap_count = 3;
@@ -307,7 +307,7 @@ void cxl_device_register_init_common(CXLDeviceState *cxl_dstate)
     cxl_device_cap_init(cxl_dstate, MEMORY_DEVICE, 0x4000, 1);
     memdev_reg_init_common(cxl_dstate);
 
-    cxl_initialize_mailbox(cxl_dstate, false);
+    cxl_initialize_mailbox(cxl_dstate, false, is_dcd);
 }
 
 void cxl_device_register_init_swcci(CXLDeviceState *cxl_dstate)
@@ -329,7 +329,7 @@ void cxl_device_register_init_swcci(CXLDeviceState *cxl_dstate)
     cxl_device_cap_init(cxl_dstate, MEMORY_DEVICE, 0x4000, 1);
     memdev_reg_init_common(cxl_dstate);
 
-    cxl_initialize_mailbox(cxl_dstate, true);
+    cxl_initialize_mailbox(cxl_dstate, true, false);
 }
 
 uint64_t cxl_device_get_timestamp(CXLDeviceState *cxl_dstate)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index 93b26e717c94..80e9cb9a8f04 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -1526,7 +1526,8 @@ static void bg_timercb(void *opaque)
     }
 }
 
-void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci)
+void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci,
+                            bool is_dcd)
 {
     if (!switch_cci) {
         cxl_dstate->cxl_cmd_set = cxl_cmd_set;
@@ -1534,6 +1535,9 @@ void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci)
         cxl_dstate->cxl_cmd_set = cxl_cmd_set_sw;
     }
     for (int set = 0; set < 256; set++) {
+        if (!is_dcd && set == DCD_CONFIG) {
+            continue;
+        }
         for (int cmd = 0; cmd < 256; cmd++) {
             if (cxl_dstate->cxl_cmd_set[set][cmd].handler) {
                 struct cxl_cmd *c = &cxl_dstate->cxl_cmd_set[set][cmd];
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 329e8b5915b3..e6e6e125990c 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -1276,9 +1276,11 @@ static void ct3d_reset(DeviceState *dev)
     CXLType3Dev *ct3d = CXL_TYPE3(dev);
     uint32_t *reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
     uint32_t *write_msk = ct3d->cxl_cstate.crb.cache_mem_regs_write_mask;
+    bool is_dcd;
 
     cxl_component_register_init_common(reg_state, write_msk, CXL2_TYPE3_DEVICE);
-    cxl_device_register_init_common(&ct3d->cxl_dstate);
+    is_dcd = (ct3d->dc.num_regions != 0);
+    cxl_device_register_init_common(&ct3d->cxl_dstate, is_dcd);
 }
 
 static Property ct3_props[] = {
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index 1ccddcca7d0d..4621bba4f533 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -233,7 +233,7 @@ typedef struct cxl_device_state {
 void cxl_device_register_block_init(Object *obj, CXLDeviceState *dev);
 
 /* Set up default values for the register block */
-void cxl_device_register_init_common(CXLDeviceState *dev);
+void cxl_device_register_init_common(CXLDeviceState *dev, bool is_dcd);
 void cxl_device_register_init_swcci(CXLDeviceState *dev);
 
 /*
@@ -280,7 +280,7 @@ CXL_DEVICE_CAPABILITY_HEADER_REGISTER(MEMORY_DEVICE,
                                       CXL_DEVICE_CAP_HDR1_OFFSET +
                                           CXL_DEVICE_CAP_REG_SIZE * 2)
 
-void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci);
+void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci, bool is_dcd);
 void cxl_process_mailbox(CXLDeviceState *cxl_dstate);
 
 #define cxl_device_cap_init(dstate, reg, cap_id, ver)                      \

Shesha Bhushan Sreenivasamurthy June 10, 2023, 12:28 a.m. UTC | #14

From: Ira Weiny <[e-mail]>
Sent: Friday, June 9, 2023 2:06 PM
To: [e-mail] <[e-mail]>; Ira Weiny <[e-mail]>; Shesha Bhushan Sreenivasamurthy <[e-mail]>
Cc: Shesha Bhushan Sreenivasamurthy <[e-mail]>; Fan Ni <[e-mail]>; Jonathan Cameron <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>; Adam Manzanares <[e-mail]>; [e-mail] <[e-mail]>; [e-mail] <[e-mail]>
Subject: [EXT] Re: [Qemu RFC 0/7] Early enabling of DCD emulation in Qemu 
 
External Email

----------------------------------------------------------------------
[e-mail] wrote:
> The 06/08/2023 08:43, Ira Weiny wrote:
> > Shesha Bhushan Sreenivasamurthy wrote:

[snip]

> 
> Hi Ira & Shesha,
> FYI. I reabased my patch series on top of the above branch and created a new
> branch here:
> 
> https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddcd-2Dpreview-2Dlatest_tree_dcd-2Dpreview&d=DwIBAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=SvyB_49EIFUT8-ZEVgIEYYjU6-zGTX4wb30kuNLUhkTSHYZK5-C0Gxr7uvefhtj4&s=MFD7qlSaTuy-w6aDmavIMbSP_aeaqZmSML7IVOX5jLs&e= 

Thanks!

> 
> It passes the same tests as shown here:
> https://urldefense.proofpoint.com/v2/url?u=https-3A__lore.kernel.org_linux-2Dcxl_6481f70fca5c2-5Fc82be29440-40iweiny-2Dmobl.notmuch_T_-23m76f6e85ce3d7292b1982960eb22086ee03922166&d=DwIBAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=SvyB_49EIFUT8-ZEVgIEYYjU6-zGTX4wb30kuNLUhkTSHYZK5-C0Gxr7uvefhtj4&s=e-fQOi0RzSZXxfSz37Bpz1sKtp7Yy0MWqonZnswK0RU&e= 

I've not gotten very far with this testing.  But I did find that regular
type 3 devices don't work with this change.  I used the patch below to get
this working.  Was there something I was missing to configure a non-DCD
device?

I don't particularly like adding another bool to this call stack.  Seems
like this calls for a flags field but I want to move on to DCD work so I
hacked this in.

I am working on the DCD FM-API commands here -
https://gitlab.com/sheshas/qemu-fmapi/-/tree/cxl-2023-05-25
-Shesha

Ira

commit ed27935044dcbd2c6ba71f8411b218621f3f4167
Author: Ira Weiny <[e-mail]>
Date:   Fri Jun 9 13:56:33 2023 -0700

    hw/mem/cxl_type3: Exclude DCD from CEL when type3 is not DCD
    
    Per CXL 3.0 9.13.3 Dynamic Capacity Device (DCD) when the type 3 memory
    device does not have DCD support the CEL should not include DCD
    configuration commands.
    
    If the number of DC regions supported is 0 skip the DCD commands in the
    CEL.
    
    Applies on top of Fan Ni's work here:
    https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_moking_qemu-2Ddcd-2Dpreview-2Dlatest_tree_dcd-2Dpreview&d=DwIBAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=Zta64bwn4nurTRpD4LY2OGr8KklkMRPn7Z_Qy0o4unU&m=SvyB_49EIFUT8-ZEVgIEYYjU6-zGTX4wb30kuNLUhkTSHYZK5-C0Gxr7uvefhtj4&s=MFD7qlSaTuy-w6aDmavIMbSP_aeaqZmSML7IVOX5jLs&e= 
    
    Not-yet-Signed-off-by: Ira Weiny <[e-mail]>

diff --git a/hw/cxl/cxl-device-utils.c b/hw/cxl/cxl-device-utils.c
index a4a2c6a80004..262e35935563 100644
--- a/hw/cxl/cxl-device-utils.c
+++ b/hw/cxl/cxl-device-utils.c
@@ -288,7 +288,7 @@ static void mailbox_reg_init_common(CXLDeviceState *cxl_dstate)
 
 static void memdev_reg_init_common(CXLDeviceState *cxl_dstate) { }
 
-void cxl_device_register_init_common(CXLDeviceState *cxl_dstate)
+void cxl_device_register_init_common(CXLDeviceState *cxl_dstate, bool is_dcd)
 {
     uint64_t *cap_hdrs = cxl_dstate->caps_reg_state64;
     const int cap_count = 3;
@@ -307,7 +307,7 @@ void cxl_device_register_init_common(CXLDeviceState *cxl_dstate)
     cxl_device_cap_init(cxl_dstate, MEMORY_DEVICE, 0x4000, 1);
     memdev_reg_init_common(cxl_dstate);
 
-    cxl_initialize_mailbox(cxl_dstate, false);
+    cxl_initialize_mailbox(cxl_dstate, false, is_dcd);
 }
 
 void cxl_device_register_init_swcci(CXLDeviceState *cxl_dstate)
@@ -329,7 +329,7 @@ void cxl_device_register_init_swcci(CXLDeviceState *cxl_dstate)
     cxl_device_cap_init(cxl_dstate, MEMORY_DEVICE, 0x4000, 1);
     memdev_reg_init_common(cxl_dstate);
 
-    cxl_initialize_mailbox(cxl_dstate, true);
+    cxl_initialize_mailbox(cxl_dstate, true, false);
 }
 
 uint64_t cxl_device_get_timestamp(CXLDeviceState *cxl_dstate)
diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c
index 93b26e717c94..80e9cb9a8f04 100644
--- a/hw/cxl/cxl-mailbox-utils.c
+++ b/hw/cxl/cxl-mailbox-utils.c
@@ -1526,7 +1526,8 @@ static void bg_timercb(void *opaque)
     }
 }
 
-void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci)
+void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci,
+                            bool is_dcd)
 {
     if (!switch_cci) {
         cxl_dstate->cxl_cmd_set = cxl_cmd_set;
@@ -1534,6 +1535,9 @@ void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci)
         cxl_dstate->cxl_cmd_set = cxl_cmd_set_sw;
     }
     for (int set = 0; set < 256; set++) {
+        if (!is_dcd && set == DCD_CONFIG) {
+            continue;
+        }
         for (int cmd = 0; cmd < 256; cmd++) {
             if (cxl_dstate->cxl_cmd_set[set][cmd].handler) {
                 struct cxl_cmd *c = &cxl_dstate->cxl_cmd_set[set][cmd];
diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index 329e8b5915b3..e6e6e125990c 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -1276,9 +1276,11 @@ static void ct3d_reset(DeviceState *dev)
     CXLType3Dev *ct3d = CXL_TYPE3(dev);
     uint32_t *reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
     uint32_t *write_msk = ct3d->cxl_cstate.crb.cache_mem_regs_write_mask;
+    bool is_dcd;
 
     cxl_component_register_init_common(reg_state, write_msk, CXL2_TYPE3_DEVICE);
-    cxl_device_register_init_common(&ct3d->cxl_dstate);
+    is_dcd = (ct3d->dc.num_regions != 0);
+    cxl_device_register_init_common(&ct3d->cxl_dstate, is_dcd);
 }
 
 static Property ct3_props[] = {
diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h
index 1ccddcca7d0d..4621bba4f533 100644
--- a/include/hw/cxl/cxl_device.h
+++ b/include/hw/cxl/cxl_device.h
@@ -233,7 +233,7 @@ typedef struct cxl_device_state {
 void cxl_device_register_block_init(Object *obj, CXLDeviceState *dev);
 
 /* Set up default values for the register block */
-void cxl_device_register_init_common(CXLDeviceState *dev);
+void cxl_device_register_init_common(CXLDeviceState *dev, bool is_dcd);
 void cxl_device_register_init_swcci(CXLDeviceState *dev);
 
 /*
@@ -280,7 +280,7 @@ CXL_DEVICE_CAPABILITY_HEADER_REGISTER(MEMORY_DEVICE,
                                       CXL_DEVICE_CAP_HDR1_OFFSET +
                                           CXL_DEVICE_CAP_REG_SIZE * 2)
 
-void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci);
+void cxl_initialize_mailbox(CXLDeviceState *cxl_dstate, bool switch_cci, bool is_dcd);
 void cxl_process_mailbox(CXLDeviceState *cxl_dstate);
 
 #define cxl_device_cap_init(dstate, reg, cap_id, ver)                      \

[e-mail] June 27, 2023, 8:52 p.m. UTC | #15

The 05/15/2023 14:00, Jonathan Cameron wrote:
> On Thu, 11 May 2023 17:56:40 +0000
> Fan Ni <[e-mail]> wrote:
> 
> > Since the early draft of DCD support in kernel is out
> > (https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/#t),
> > this patch series provide dcd emulation in qemu so people who are interested
> > can have an early try. It is noted that the patch series may need to be updated
> > accordingly if the kernel side implementation changes.
> > 
> > To support DCD emulation, the patch series add DCD related mailbox command
> > support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> > with dynamic capacity extent and region representative.
> > To support read/write to the dynamic capacity of the device, a host backend
> > is provided and necessary check mechnism is added to ensure the dynamic
> > capacity accessed is backed with active dc extents.
> > Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> > , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> > Also, the support for multiple hosts sharing the same DCD case is missing.
> > 
> > Things we can try with the patch series together with kernel dcd code:
> > 1. Create DC regions to cover the address range of the dynamic capacity
> > regions.
> > 2. Add/release dynamic capacity extents to the device and notify the
> > kernel.
> > 3. Test kernel side code to accept added dc extents and create dax devices,
> > and release dc extents and notify the device
> > 4. Online the memory range backed with dc extents and let application use
> > them.
> > 
> > The patch series is based on Jonathan's local qemu branch:
> > https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28
> > 
> > Simple tests peformed with the patch series:
> > 1 Install cxl modules:
> > 
> > modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> > 
> > 2 Create dc regions:
> > 
> > region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> > echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> > echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> > echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> > echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> > echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> > echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> > echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> > echo 1 > /sys/bus/cxl/devices/$region/commit
> > echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> > 
> > /home/fan/cxl/tools-and-scripts# cxl list
> > [
> >   {
> >     "memdevs":[
> >       {
> >         "memdev":"mem0",
> >         "pmem_size":536870912,
> >         "ram_size":0,
> >         "serial":0,
> >         "host":"0000:0d:00.0"
> >       }
> >     ]
> >   },
> >   {
> >     "regions":[
> >       {
> >         "region":"region0",
> >         "resource":45365592064,
> >         "size":268435456,
> >         "interleave_ways":1,
> >         "interleave_granularity":256,
> >         "decode_state":"commit"
> >       }
> >     ]
> >   }
> > ]
> > 
> > 3 Add two dc extents (128MB each) through qmp interface
> > 
> > { "execute": "qmp_capabilities" }
> > 
> > { "execute": "cxl-add-dynamic-capacity-event",
> > 	"arguments": {
> > 		 "path": "/machine/peripheral/cxl-pmem0",
> > 		"region-id" : 0,
> > 		 "num-extent": 2,
> > 		"dpa":0,
> > 		"extent-len": 128
> > 	}
> > }
> > 
> > /home/fan/cxl/tools-and-scripts# lsmem
> > RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> > 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> > 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> > 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> > 
> > Memory block size:       128M
> > Total online memory:       8G
> > Total offline memory:    256M
> > 
> > 
> > 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> > 
> > /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> > [  230.730553] Fallback order for Node 0: 0 1
> > [  230.730825] Fallback order for Node 1: 1 0
> > [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> > [  230.731110] Policy zone: Normal
> > onlined memory for 1 device
> > 
> > root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> > RANGE                                  SIZE   STATE REMOVABLE BLOCK
> > 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> > 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> > 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> > 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> > 
> > Memory block size:       128M
> > Total online memory:     8.1G
> > Total offline memory:    128M
> > 
> > 5 using dc extents as regular memory
> > 
> > /home/fan/cxl/ndctl# numactl --membind=1 ls
> > CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> > ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
> > config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
> > Documentation	 build	    contrib	     daxctl	  meson.build		rhel
> > tags	topology.png LICENSES	 ccan	    cscope.files
> > git-version  meson_options.txt	rpmbuild.sh    test	util
> > 
> > 
> > QEMU command line cxl configuration:
> > 
> > RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> > -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> > -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> > -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> > -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> > -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> > -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> > 
> > 
> > Kernel DCD support used to test the changes
> > 
> > The code is tested with the posted kernel dcd support:
> > https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5/dcd-preview
> > 
> 
> Very nice!  +CC Navneet who may want to comment on the below (and the
> emulation as well)
> 
> I've not had a chance to look at the code on the kernel side yet.

Thanks Jonathan for all the comments for the series, will reflect them
in the next version.

Fan

> 
> 
> > commit: f425bc34c600e2a3721d6560202962ec41622815
> > 
> > To make the test work, we have made the following changes to the above kernel commit:
> > 
> > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> > index 5f04bbc18af5..5f421d3c5cef 100644
> > --- a/drivers/cxl/core/mbox.c
> > +++ b/drivers/cxl/core/mbox.c
> > @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
> >  	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
> >  	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
> >  	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> > +	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
> >  };
> >  
> >  /*
> > diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> > index 291c716abd49..ae10e3cf43a1 100644
> > --- a/drivers/cxl/core/region.c
> > +++ b/drivers/cxl/core/region.c
> > @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
> >  		}
> >  		cxlds->dc_list_gen_num = extent_gen_num;
> >  		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> > -		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> > +		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
> 
> Some race condition that means we need to enable the DCD event earlier?
> 
> >  	}
> >  	return 0;
> >  err:
> > @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
> >  				dev_dax->align, memremap_compat_align()))) {
> >  		rc = alloc_dev_dax_range(dev_dax, hpa,
> >  					resource_size(alloc_dpa_res));
> > -		return rc;
> > +		if (rc)
> > +			return rc;
> 
> No idea on this one as it's in the code I haven't looked at yet!
> 
> >  	}
> >  
> >  	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > index 9e45b1056022..653bec203838 100644
> > --- a/drivers/cxl/pci.c
> > +++ b/drivers/cxl/pci.c
> > @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
> >  
> >  	/* Driver enables DCD interrupt after creating the dc cxl_region */
> >  	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> > -					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> > +					IRQF_SHARED | IRQF_ONESHOT);
> 
> This will be otherside of the removal of the enable above.
> 
> >  	if (rc) {
> >  		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
> >  		return rc;
> > diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> > index 6ca85861750c..910a48259239 100644
> > --- a/include/uapi/linux/cxl_mem.h
> > +++ b/include/uapi/linux/cxl_mem.h
> > @@ -47,6 +47,7 @@
> >  	___C(SCAN_MEDIA, "Scan Media"),                                   \
> >  	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
> >  	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> > +	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
> >  	___C(MAX, "invalid / last command")
> >  
> >  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> > 
> > 
> > 
> > Fan Ni (7):
> >   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
> >     payload of identify memory device command
> >   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
> >     and mailbox command support
> >   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
> >     device supports in qemu command line
> >   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
> >   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
> >     dynamic capacity response
> >   Add qmp interfaces to add/release dynamic capacity extents
> >   hw/mem/cxl_type3: add read/write support to dynamic capacity
> > 
> >  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
> >  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
> >  include/hw/cxl/cxl_device.h |  50 +++-
> >  include/hw/cxl/cxl_events.h |  16 ++
> >  qapi/cxl.json               |  44 ++++
> >  5 files changed, 924 insertions(+), 67 deletions(-)
> > 
>

Fan Ni July 24, 2023, 5:19 p.m. UTC | #16

On Thu, May 11, 2023 at 05:56:40PM +0000, Fan Ni wrote:

FYI.

I have updated the patch series and sent out again.

I suggested anyone who are interested in DCD and using this patch series to
use the new series. Quite a few things has been fixed.

https://lore.kernel.org/linux-cxl/[e-mail]/T/#t

Also, if you want to use the code repo directly, you can try

https://github.com/moking/qemu-dcd-preview-latest/tree/dcd-dev

Fan


> Since the early draft of DCD support in kernel is out
> (https://lore.kernel.org/linux-cxl/20230417164126.GA1904906@bgt-140510-bm03/T/#t),
> this patch series provide dcd emulation in qemu so people who are interested
> can have an early try. It is noted that the patch series may need to be updated
> accordingly if the kernel side implementation changes.
> 
> To support DCD emulation, the patch series add DCD related mailbox command
> support (CXL Spec 3.0: 8.2.9.8.9), and extend the cxl type3 memory device
> with dynamic capacity extent and region representative.
> To support read/write to the dynamic capacity of the device, a host backend
> is provided and necessary check mechnism is added to ensure the dynamic
> capacity accessed is backed with active dc extents.
> Currently FM related mailbox commands (cxl spec 3.0: 7.6.7.6) is not supported
> , but we add two qmp interfaces for adding/releasing dynamic capacity extents.
> Also, the support for multiple hosts sharing the same DCD case is missing.
> 
> Things we can try with the patch series together with kernel dcd code:
> 1. Create DC regions to cover the address range of the dynamic capacity
> regions.
> 2. Add/release dynamic capacity extents to the device and notify the
> kernel.
> 3. Test kernel side code to accept added dc extents and create dax devices,
> and release dc extents and notify the device
> 4. Online the memory range backed with dc extents and let application use
> them.
> 
> The patch series is based on Jonathan's local qemu branch:
> https://gitlab.com/jic23/qemu/-/tree/cxl-2023-02-28
> 
> Simple tests peformed with the patch series:
> 1 Install cxl modules:
> 
> modprobe -a cxl_acpi cxl_core cxl_pci cxl_port cxl_mem
> 
> 2 Create dc regions:
> 
> region=$(cat /sys/bus/cxl/devices/decoder0.0/create_dc_region)
> echo $region> /sys/bus/cxl/devices/decoder0.0/create_dc_region
> echo 256 > /sys/bus/cxl/devices/$region/interleave_granularity
> echo 1 > /sys/bus/cxl/devices/$region/interleave_ways
> echo "dc" >/sys/bus/cxl/devices/decoder2.0/mode
> echo 0x10000000 >/sys/bus/cxl/devices/decoder2.0/dpa_size
> echo 0x10000000 > /sys/bus/cxl/devices/$region/size
> echo  "decoder2.0" > /sys/bus/cxl/devices/$region/target0
> echo 1 > /sys/bus/cxl/devices/$region/commit
> echo $region > /sys/bus/cxl/drivers/cxl_region/bind
> 
> /home/fan/cxl/tools-and-scripts# cxl list
> [
>   {
>     "memdevs":[
>       {
>         "memdev":"mem0",
>         "pmem_size":536870912,
>         "ram_size":0,
>         "serial":0,
>         "host":"0000:0d:00.0"
>       }
>     ]
>   },
>   {
>     "regions":[
>       {
>         "region":"region0",
>         "resource":45365592064,
>         "size":268435456,
>         "interleave_ways":1,
>         "interleave_granularity":256,
>         "decode_state":"commit"
>       }
>     ]
>   }
> ]
> 
> 3 Add two dc extents (128MB each) through qmp interface
> 
> { "execute": "qmp_capabilities" }
> 
> { "execute": "cxl-add-dynamic-capacity-event",
> 	"arguments": {
> 		 "path": "/machine/peripheral/cxl-pmem0",
> 		"region-id" : 0,
> 		 "num-extent": 2,
> 		"dpa":0,
> 		"extent-len": 128
> 	}
> }
> 
> /home/fan/cxl/tools-and-scripts# lsmem
> RANGE                                  SIZE   STATE REMOVABLE   BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes    0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes   32-79
> 0x0000000a90000000-0x0000000a9fffffff  256M offline           338-339
> 
> Memory block size:       128M
> Total online memory:       8G
> Total offline memory:    256M
> 
> 
> 4.Online the momory with 'daxctl online-memory dax0.0' to online the memory
> 
> /home/fan/cxl/ndctl# ./build/daxctl/daxctl online-memory dax0.0
> [  230.730553] Fallback order for Node 0: 0 1
> [  230.730825] Fallback order for Node 1: 1 0
> [  230.730953] Built 2 zonelists, mobility grouping on.  Total pages: 2042541
> [  230.731110] Policy zone: Normal
> onlined memory for 1 device
> 
> root@bgt-140510-bm03:/home/fan/cxl/ndctl# lsmem
> RANGE                                  SIZE   STATE REMOVABLE BLOCK
> 0x0000000000000000-0x000000007fffffff    2G  online       yes  0-15
> 0x0000000100000000-0x000000027fffffff    6G  online       yes 32-79
> 0x0000000a90000000-0x0000000a97ffffff  128M  online       yes   338
> 0x0000000a98000000-0x0000000a9fffffff  128M offline             339
> 
> Memory block size:       128M
> Total online memory:     8.1G
> Total offline memory:    128M
> 
> 5 using dc extents as regular memory
> 
> /home/fan/cxl/ndctl# numactl --membind=1 ls
> CONTRIBUTING.md  README.md  clean_config.sh  cscope.out   git-version-gen
> ndctl	       scripts	test.h      version.h.in COPYING		 acpi.h
> config.h.meson   cxl	  make-git-snapshot.sh	ndctl.spec.in  sles	tools
> Documentation	 build	    contrib	     daxctl	  meson.build		rhel
> tags	topology.png LICENSES	 ccan	    cscope.files
> git-version  meson_options.txt	rpmbuild.sh    test	util
> 
> 
> QEMU command line cxl configuration:
> 
> RP1="-object memory-backend-file,id=cxl-mem1,share=on,mem-path=/tmp/cxltest.raw,size=512M \
> -object memory-backend-file,id=cxl-mem2,share=on,mem-path=/tmp/cxltest2.raw,size=512M \
> -object memory-backend-file,id=cxl-lsa1,share=on,mem-path=/tmp/lsa.raw,size=512M \
> -device pxb-cxl,bus_nr=12,bus=pcie.0,id=cxl.1 \
> -device cxl-rp,port=0,bus=cxl.1,id=root_port13,chassis=0,slot=2 \
> -device cxl-type3,bus=root_port13,memdev=cxl-mem1,lsa=cxl-lsa1,dc-memdev=cxl-mem2,id=cxl-pmem0,num-dc-regions=1\
> -M cxl-fmw.0.targets.0=cxl.1,cxl-fmw.0.size=4G,cxl-fmw.0.interleave-granularity=8k"
> 
> 
> Kernel DCD support used to test the changes
> 
> The code is tested with the posted kernel dcd support:
> https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git/log/?h=for-6.5/dcd-preview
> 
> commit: f425bc34c600e2a3721d6560202962ec41622815
> 
> To make the test work, we have made the following changes to the above kernel commit:
> 
> diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
> index 5f04bbc18af5..5f421d3c5cef 100644
> --- a/drivers/cxl/core/mbox.c
> +++ b/drivers/cxl/core/mbox.c
> @@ -68,6 +68,7 @@ static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
>  	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
>  	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
>  	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
> +	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
>  };
>  
>  /*
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 291c716abd49..ae10e3cf43a1 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -194,7 +194,7 @@ static int cxl_region_manage_dc(struct cxl_region *cxlr)
>  		}
>  		cxlds->dc_list_gen_num = extent_gen_num;
>  		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
> -		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
> +		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
>  	}
>  	return 0;
>  err:
> @@ -2810,7 +2810,8 @@ int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
>  				dev_dax->align, memremap_compat_align()))) {
>  		rc = alloc_dev_dax_range(dev_dax, hpa,
>  					resource_size(alloc_dpa_res));
> -		return rc;
> +		if (rc)
> +			return rc;
>  	}
>  
>  	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 9e45b1056022..653bec203838 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -659,7 +659,7 @@ static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
>  
>  	/* Driver enables DCD interrupt after creating the dc cxl_region */
>  	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
> -					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
> +					IRQF_SHARED | IRQF_ONESHOT);
>  	if (rc) {
>  		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
>  		return rc;
> diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
> index 6ca85861750c..910a48259239 100644
> --- a/include/uapi/linux/cxl_mem.h
> +++ b/include/uapi/linux/cxl_mem.h
> @@ -47,6 +47,7 @@
>  	___C(SCAN_MEDIA, "Scan Media"),                                   \
>  	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
>  	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
> +	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
>  	___C(MAX, "invalid / last command")
>  
>  #define ___C(a, b) CXL_MEM_COMMAND_ID_##a
> 
> 
> 
> Fan Ni (7):
>   hw/cxl/cxl-mailbox-utils: Add dc_event_log_size field to output
>     payload of identify memory device command
>   hw/cxl/cxl-mailbox-utils: Add dynamic capacity region representative
>     and mailbox command support
>   hw/mem/cxl_type3: Add a parameter to pass number of DC regions the
>     device supports in qemu command line
>   hw/mem/cxl_type3: Add DC extent representative to cxl type3 device
>   hw/cxl/cxl-mailbox-utils: Add mailbox commands to support add/release
>     dynamic capacity response
>   Add qmp interfaces to add/release dynamic capacity extents
>   hw/mem/cxl_type3: add read/write support to dynamic capacity
> 
>  hw/cxl/cxl-mailbox-utils.c  | 389 +++++++++++++++++++++++++++-
>  hw/mem/cxl_type3.c          | 492 +++++++++++++++++++++++++++++++-----
>  include/hw/cxl/cxl_device.h |  50 +++-
>  include/hw/cxl/cxl_events.h |  16 ++
>  qapi/cxl.json               |  44 ++++
>  5 files changed, 924 insertions(+), 67 deletions(-)
> 
> -- 
> 2.25.1
>

Ira Weiny July 25, 2023, 3:18 p.m. UTC | #17

Fan Ni wrote:
> On Thu, May 11, 2023 at 05:56:40PM +0000, Fan Ni wrote:
> 
> FYI.
> 
> I have updated the patch series and sent out again.
> 
> I suggested anyone who are interested in DCD and using this patch series to
> use the new series. Quite a few things has been fixed.
> 
> https://lore.kernel.org/linux-cxl/[e-mail]/T/#t
> 
> Also, if you want to use the code repo directly, you can try
> 
> https://github.com/moking/qemu-dcd-preview-latest/tree/dcd-dev

Thanks for the branch!

I took a quick look and I don't see a resolution to the problem I
mentioned with non DCD devices being supported.[1]

[1] https://lore.kernel.org/all/[e-mail]/

Did you fix this in a different way?  If I don't add DC to my mem devices they
don't get probed properly.  I'm still looking into this with your new branch,
but I don't think DC commands should be in the CEL if the device does not
support it.

Also I get a build warning on this branch I had to fix[3] as my build is
treating warnings as errors.[2]

I don't think this fix is technically necessary as 'list' should never be NULL
that I can see.  But might be nice to check or just use my fix.

I'll try and get to a review once I get the DCD stuff out on the list again.

Ira


[2]
../hw/mem/cxl_type3.c: In function ‘qmp_cxl_process_dynamic_capacity_event.constprop’:
../hw/mem/cxl_type3.c:2063:28: error: ‘rid’ may be used uninitialized [-Werror=maybe-uninitialized]
 2063 |     dCap.updated_region_id = rid;
      |     ~~~~~~~~~~~~~~~~~~~~~~~^~~~~
../hw/mem/cxl_type3.c:1987:13: note: ‘rid’ was declared here
 1987 |     uint8_t rid;
      |             ^~~
cc1: all warnings being treated as errors

[3]

diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
index e67328780407..d25e6064f6c9 100644
--- a/hw/mem/cxl_type3.c
+++ b/hw/mem/cxl_type3.c
@@ -1984,7 +1984,7 @@ static void qmp_cxl_process_dynamic_capacity_event(const char *path,
     CXLDCExtentRecordList *list = records;
     CXLDCExtent_raw *extents;
     uint64_t dpa, len;
-    uint8_t rid;
+    uint8_t rid = 0;
     int i;
 
     if (!obj) {

Fan Ni July 25, 2023, 4:46 p.m. UTC | #18

On Tue, Jul 25, 2023 at 08:18:08AM -0700, Ira Weiny wrote:

> Fan Ni wrote:
> > On Thu, May 11, 2023 at 05:56:40PM +0000, Fan Ni wrote:
> > 
> > FYI.
> > 
> > I have updated the patch series and sent out again.
> > 
> > I suggested anyone who are interested in DCD and using this patch series to
> > use the new series. Quite a few things has been fixed.
> > 
> > https://urldefense.com/v3/__https://lore.kernel.org/linux-cxl/[e-mail]/T/*t__;Iw!!EwVzqGoTKBqv-0DWAJBm!V8kDTpT5yLUAyWm3sFm7XIgN0QdNUyQYZd9vYLHjUVkMkhDT14F8avgNBh23KPAtsS_dGm2LZuHJ102mgIg$ 
> > 
> > Also, if you want to use the code repo directly, you can try
> > 
> > https://urldefense.com/v3/__https://protect2.fireeye.com/v1/url?k=0600fd6b-678be820-06017624-74fe485fb305-f529279062b02b73&q=1&e=5df65010-e62f-40a1-9a21-609eb1400921&u=https*3A*2F*2Fgithub.com*2Fmoking*2Fqemu-dcd-preview-latest*2Ftree*2Fdcd-dev__;JSUlJSUlJQ!!EwVzqGoTKBqv-0DWAJBm!V8kDTpT5yLUAyWm3sFm7XIgN0QdNUyQYZd9vYLHjUVkMkhDT14F8avgNBh23KPAtsS_dGm2LZuHJpB0SmNs$ 
> 
> Thanks for the branch!
> 
> I took a quick look and I don't see a resolution to the problem I
> mentioned with non DCD devices being supported.[1]
> 
> [1] https://urldefense.com/v3/__https://lore.kernel.org/all/[e-mail]/__;!!EwVzqGoTKBqv-0DWAJBm!V8kDTpT5yLUAyWm3sFm7XIgN0QdNUyQYZd9vYLHjUVkMkhDT14F8avgNBh23KPAtsS_dGm2LZuHJ4geSEAg$ 
> 
> Did you fix this in a different way?  If I don't add DC to my mem devices they
> don't get probed properly.  I'm still looking into this with your new branch,
> but I don't think DC commands should be in the CEL if the device does not
> support it.
> 
> Also I get a build warning on this branch I had to fix[3] as my build is
> treating warnings as errors.[2]
> 
> I don't think this fix is technically necessary as 'list' should never be NULL
> that I can see.  But might be nice to check or just use my fix.
> 
> I'll try and get to a review once I get the DCD stuff out on the list again.
> 
> Ira
> 

Oh, I missed your previous comments, let me look into it and fix
accordingly and send out a new version.

Btw, when I did the DCD test with the last DCD kernel code, I found
some issue there.

When I add a DCD extent for the first time, it will be recognized as
system RAM automatically and show up with command lsmem.

However, when I release it and try to re-add the same extent again.
The adding seems normal and the device will show up under /dev/ as
dax0.X. But it will not show up with lsmem command and I have to use
daxctl reconfigure command to turn it to system ram and then it can
show up with lsmem command. I would expect the behavior for the
first add and second add be the same.

Fan.


> 
> [2]
> ../hw/mem/cxl_type3.c: In function ‘qmp_cxl_process_dynamic_capacity_event.constprop’:
> ../hw/mem/cxl_type3.c:2063:28: error: ‘rid’ may be used uninitialized [-Werror=maybe-uninitialized]
>  2063 |     dCap.updated_region_id = rid;
>       |     ~~~~~~~~~~~~~~~~~~~~~~~^~~~~
> ../hw/mem/cxl_type3.c:1987:13: note: ‘rid’ was declared here
>  1987 |     uint8_t rid;
>       |             ^~~
> cc1: all warnings being treated as errors
> 
> [3]
> 
> diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c
> index e67328780407..d25e6064f6c9 100644
> --- a/hw/mem/cxl_type3.c
> +++ b/hw/mem/cxl_type3.c
> @@ -1984,7 +1984,7 @@ static void qmp_cxl_process_dynamic_capacity_event(const char *path,
>      CXLDCExtentRecordList *list = records;
>      CXLDCExtent_raw *extents;
>      uint64_t dpa, len;
> -    uint8_t rid;
> +    uint8_t rid = 0;
>      int i;
>  
>      if (!obj) {

Patch

diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 5f04bbc18af5..5f421d3c5cef 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -68,6 +68,7 @@  static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
 	CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
 	CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
 	CXL_CMD(GET_DC_EXTENT_LIST, 0x8, CXL_VARIABLE_PAYLOAD, 0),
+	CXL_CMD(GET_DC_CONFIG, 0x2, CXL_VARIABLE_PAYLOAD, 0),
 };
 
 /*
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 291c716abd49..ae10e3cf43a1 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -194,7 +194,7 @@  static int cxl_region_manage_dc(struct cxl_region *cxlr)
 		}
 		cxlds->dc_list_gen_num = extent_gen_num;
 		dev_dbg(cxlds->dev, "No of preallocated extents :%d\n", rc);
-		enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);
+		/*enable_irq(cxlds->cxl_irq[CXL_EVENT_TYPE_DCD]);*/
 	}
 	return 0;
 err:
@@ -2810,7 +2810,8 @@  int cxl_add_dc_extent(struct cxl_dev_state *cxlds, struct resource *alloc_dpa_re
 				dev_dax->align, memremap_compat_align()))) {
 		rc = alloc_dev_dax_range(dev_dax, hpa,
 					resource_size(alloc_dpa_res));
-		return rc;
+		if (rc)
+			return rc;
 	}
 
 	rc = xa_insert(&cxlr_dc->dax_dev_list, hpa, dev_dax, GFP_KERNEL);
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 9e45b1056022..653bec203838 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -659,7 +659,7 @@  static int cxl_event_irqsetup(struct cxl_dev_state *cxlds)
 
 	/* Driver enables DCD interrupt after creating the dc cxl_region */
 	rc = cxl_event_req_irq(cxlds, policy.dyncap_settings, CXL_EVENT_TYPE_DCD,
-					IRQF_SHARED | IRQF_ONESHOT | IRQF_NO_AUTOEN);
+					IRQF_SHARED | IRQF_ONESHOT);
 	if (rc) {
 		dev_err(cxlds->dev, "Failed to get interrupt for event dc log\n");
 		return rc;
diff --git a/include/uapi/linux/cxl_mem.h b/include/uapi/linux/cxl_mem.h
index 6ca85861750c..910a48259239 100644
--- a/include/uapi/linux/cxl_mem.h
+++ b/include/uapi/linux/cxl_mem.h
@@ -47,6 +47,7 @@ 
 	___C(SCAN_MEDIA, "Scan Media"),                                   \
 	___C(GET_SCAN_MEDIA, "Get Scan Media Results"),                   \
 	___C(GET_DC_EXTENT_LIST, "Get dynamic capacity extents"),         \
+	___C(GET_DC_CONFIG, "Get dynamic capacity configuration"),         \
 	___C(MAX, "invalid / last command")
 
 #define ___C(a, b) CXL_MEM_COMMAND_ID_##a