diff mbox series

[1/3] cxl: Wait Memory_Info_Valid before access memory related info

Message ID 168428295907.2205351.12793990978386772732.stgit@djiang5-mobl3
State Superseded
Headers show
Series cxl: Move operations after memory is ready | expand

Commit Message

Dave Jiang May 17, 2023, 12:22 a.m. UTC 
CXL rev3.0 8.1.3.8.2 Memory_Info_valid field

The Memory_Info_Valid bit indicates that the CXL Range Size High and Size
Low registers are valid. The bit must be set within 1 second of reset
deassertion to the device. Check valid bit before we check the
Memory_Active bit when waiting for cxl_await_media_ready() to ensure that
the memory info is valid for consumption.

Fixes: 2e4ba0ec9783 ("cxl/pci: Move cxl_await_media_ready() to the core")
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>

---
v2:
- Check both ranges. (Jonathan)
---
 drivers/cxl/core/pci.c |   83 +++++++++++++++++++++++++++++++++++++++++++-----
 drivers/cxl/cxlpci.h   |    2 +
 2 files changed, 77 insertions(+), 8 deletions(-)

Comments

Ira Weiny May 18, 2023, 3:39 a.m. UTC | #1 
Dave Jiang wrote:
> CXL rev3.0 8.1.3.8.2 Memory_Info_valid field
> 
> The Memory_Info_Valid bit indicates that the CXL Range Size High and Size
> Low registers are valid. The bit must be set within 1 second of reset
> deassertion to the device. Check valid bit before we check the
> Memory_Active bit when waiting for cxl_await_media_ready() to ensure that
> the memory info is valid for consumption.

FWIW this also ensures both DVSEC ranges 1 and 2 are ready if the DVSEC
Capability indicates they are both supported.

> 
> Fixes: 2e4ba0ec9783 ("cxl/pci: Move cxl_await_media_ready() to the core")
> Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> 
> ---
> v2:
> - Check both ranges. (Jonathan)
> ---
>  drivers/cxl/core/pci.c |   83 +++++++++++++++++++++++++++++++++++++++++++-----
>  drivers/cxl/cxlpci.h   |    2 +
>  2 files changed, 77 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 63f2f0b86fbc..484a8b0b96b9 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -101,21 +101,55 @@ int devm_cxl_port_enumerate_dports(struct cxl_port *port)
>  }
>  EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);
>  
> -/*
> - * Wait up to @media_ready_timeout for the device to report memory
> - * active.
> - */
> -int cxl_await_media_ready(struct cxl_dev_state *cxlds)
> +static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id)
> +{
> +	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
> +	int d = cxlds->cxl_dvsec;
> +	bool valid = false;
> +	int rc, i;
> +	u32 temp;
> +
> +	if (id > CXL_DVSEC_RANGE_MAX)
> +		return -EINVAL;
> +
> +	/* Check MEM INFO VALID bit first, give up after 1s */
> +	i = 1;
> +	do {
> +		rc = pci_read_config_dword(pdev,
> +					   d + CXL_DVSEC_RANGE_SIZE_LOW(id),
> +					   &temp);
> +		if (rc)
> +			return rc;
> +
> +		valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp);
> +		if (valid)
> +			break;
> +		msleep(1000);
> +	} while (i--);
> +
> +	if (!valid) {
> +		dev_err(&pdev->dev,
> +			"Timeout awaiting memory range %d valid after 1s.\n",
> +			id);
> +		return -ETIMEDOUT;
> +	}
> +
> +	return 0;
> +}
> +
> +static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
>  {
>  	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
>  	int d = cxlds->cxl_dvsec;
>  	bool active = false;
> -	u64 md_status;
>  	int rc, i;
> +	u32 temp;
>  
> -	for (i = media_ready_timeout; i; i--) {
> -		u32 temp;
> +	if (id > CXL_DVSEC_RANGE_MAX)
> +		return -EINVAL;
>  
> +	/* Check MEM ACTIVE bit, up to 60s timeout by default */
> +	for (i = media_ready_timeout; i; i--) {
>  		rc = pci_read_config_dword(
>  			pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
				  ^^^^^^^^^^^^^^^^^^^^^^^^^^^

CXL_DVSEC_RANGE_SIZE_LOW(id)?

Ira

>  		if (rc)
> @@ -134,6 +168,39 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds)
>  		return -ETIMEDOUT;
>  	}
>  
> +	return 0;
> +}
> +
> +/*
> + * Wait up to @media_ready_timeout for the device to report memory
> + * active.
> + */
> +int cxl_await_media_ready(struct cxl_dev_state *cxlds)
> +{
> +	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
> +	int d = cxlds->cxl_dvsec;
> +	int rc, i, hdm_count;
> +	u64 md_status;
> +	u16 cap;
> +
> +	rc = pci_read_config_word(pdev,
> +				  d + CXL_DVSEC_CAP_OFFSET, &cap);
> +	if (rc)
> +		return rc;
> +
> +	hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
> +	for (i = 0; i < hdm_count; i++) {
> +		rc = cxl_dvsec_mem_range_valid(cxlds, i);
> +		if (rc)
> +			return rc;
> +	}
> +
> +	for (i = 0; i < hdm_count; i++) {
> +		rc = cxl_dvsec_mem_range_active(cxlds, i);
> +		if (rc)
> +			return rc;
> +	}
> +
>  	md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
>  	if (!CXLMDEV_READY(md_status))
>  		return -EIO;
> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> index 0465ef963cd6..7c02e55b8042 100644
> --- a/drivers/cxl/cxlpci.h
> +++ b/drivers/cxl/cxlpci.h
> @@ -31,6 +31,8 @@
>  #define   CXL_DVSEC_RANGE_BASE_LOW(i)	(0x24 + (i * 0x10))
>  #define     CXL_DVSEC_MEM_BASE_LOW_MASK	GENMASK(31, 28)
>  
> +#define CXL_DVSEC_RANGE_MAX		2
> +
>  /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
>  #define CXL_DVSEC_FUNCTION_MAP					2
>  
> 
>
Dan Williams May 18, 2023, 3:40 a.m. UTC | #2 
Dave Jiang wrote:
> CXL rev3.0 8.1.3.8.2 Memory_Info_valid field
> 
> The Memory_Info_Valid bit indicates that the CXL Range Size High and Size
> Low registers are valid. The bit must be set within 1 second of reset
> deassertion to the device. Check valid bit before we check the
> Memory_Active bit when waiting for cxl_await_media_ready() to ensure that
> the memory info is valid for consumption.
> 
> Fixes: 2e4ba0ec9783 ("cxl/pci: Move cxl_await_media_ready() to the core")

Please be careful about picking Fixes tags. This one is just code
movement. The lack of valid checking for both ranges was from:

523e594d9cc0 ("cxl/pci: Implement wait for media active")

I can fix this up on applying.

I assume no need for flagging stable@ at this point since this has
always been this way with no end user reports?
Dave Jiang May 18, 2023, 3:21 p.m. UTC | #3 
On 5/17/23 8:39 PM, Ira Weiny wrote:
> Dave Jiang wrote:
>> CXL rev3.0 8.1.3.8.2 Memory_Info_valid field
>>
>> The Memory_Info_Valid bit indicates that the CXL Range Size High and Size
>> Low registers are valid. The bit must be set within 1 second of reset
>> deassertion to the device. Check valid bit before we check the
>> Memory_Active bit when waiting for cxl_await_media_ready() to ensure that
>> the memory info is valid for consumption.
> 
> FWIW this also ensures both DVSEC ranges 1 and 2 are ready if the DVSEC
> Capability indicates they are both supported.
> 
>>
>> Fixes: 2e4ba0ec9783 ("cxl/pci: Move cxl_await_media_ready() to the core")
>> Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>>
>> ---
>> v2:
>> - Check both ranges. (Jonathan)
>> ---
>>   drivers/cxl/core/pci.c |   83 +++++++++++++++++++++++++++++++++++++++++++-----
>>   drivers/cxl/cxlpci.h   |    2 +
>>   2 files changed, 77 insertions(+), 8 deletions(-)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index 63f2f0b86fbc..484a8b0b96b9 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -101,21 +101,55 @@ int devm_cxl_port_enumerate_dports(struct cxl_port *port)
>>   }
>>   EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);
>>   
>> -/*
>> - * Wait up to @media_ready_timeout for the device to report memory
>> - * active.
>> - */
>> -int cxl_await_media_ready(struct cxl_dev_state *cxlds)
>> +static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id)
>> +{
>> +	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
>> +	int d = cxlds->cxl_dvsec;
>> +	bool valid = false;
>> +	int rc, i;
>> +	u32 temp;
>> +
>> +	if (id > CXL_DVSEC_RANGE_MAX)
>> +		return -EINVAL;
>> +
>> +	/* Check MEM INFO VALID bit first, give up after 1s */
>> +	i = 1;
>> +	do {
>> +		rc = pci_read_config_dword(pdev,
>> +					   d + CXL_DVSEC_RANGE_SIZE_LOW(id),
>> +					   &temp);
>> +		if (rc)
>> +			return rc;
>> +
>> +		valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp);
>> +		if (valid)
>> +			break;
>> +		msleep(1000);
>> +	} while (i--);
>> +
>> +	if (!valid) {
>> +		dev_err(&pdev->dev,
>> +			"Timeout awaiting memory range %d valid after 1s.\n",
>> +			id);
>> +		return -ETIMEDOUT;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
>>   {
>>   	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
>>   	int d = cxlds->cxl_dvsec;
>>   	bool active = false;
>> -	u64 md_status;
>>   	int rc, i;
>> +	u32 temp;
>>   
>> -	for (i = media_ready_timeout; i; i--) {
>> -		u32 temp;
>> +	if (id > CXL_DVSEC_RANGE_MAX)
>> +		return -EINVAL;
>>   
>> +	/* Check MEM ACTIVE bit, up to 60s timeout by default */
>> +	for (i = media_ready_timeout; i; i--) {
>>   		rc = pci_read_config_dword(
>>   			pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
> 				  ^^^^^^^^^^^^^^^^^^^^^^^^^^^
> 
> CXL_DVSEC_RANGE_SIZE_LOW(id)?

Yup. I missed that. Thanks.

> 
> Ira
> 
>>   		if (rc)
>> @@ -134,6 +168,39 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds)
>>   		return -ETIMEDOUT;
>>   	}
>>   
>> +	return 0;
>> +}
>> +
>> +/*
>> + * Wait up to @media_ready_timeout for the device to report memory
>> + * active.
>> + */
>> +int cxl_await_media_ready(struct cxl_dev_state *cxlds)
>> +{
>> +	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
>> +	int d = cxlds->cxl_dvsec;
>> +	int rc, i, hdm_count;
>> +	u64 md_status;
>> +	u16 cap;
>> +
>> +	rc = pci_read_config_word(pdev,
>> +				  d + CXL_DVSEC_CAP_OFFSET, &cap);
>> +	if (rc)
>> +		return rc;
>> +
>> +	hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
>> +	for (i = 0; i < hdm_count; i++) {
>> +		rc = cxl_dvsec_mem_range_valid(cxlds, i);
>> +		if (rc)
>> +			return rc;
>> +	}
>> +
>> +	for (i = 0; i < hdm_count; i++) {
>> +		rc = cxl_dvsec_mem_range_active(cxlds, i);
>> +		if (rc)
>> +			return rc;
>> +	}
>> +
>>   	md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
>>   	if (!CXLMDEV_READY(md_status))
>>   		return -EIO;
>> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
>> index 0465ef963cd6..7c02e55b8042 100644
>> --- a/drivers/cxl/cxlpci.h
>> +++ b/drivers/cxl/cxlpci.h
>> @@ -31,6 +31,8 @@
>>   #define   CXL_DVSEC_RANGE_BASE_LOW(i)	(0x24 + (i * 0x10))
>>   #define     CXL_DVSEC_MEM_BASE_LOW_MASK	GENMASK(31, 28)
>>   
>> +#define CXL_DVSEC_RANGE_MAX		2
>> +
>>   /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
>>   #define CXL_DVSEC_FUNCTION_MAP					2
>>   
>>
>>
> 
>
diff mbox series

Patch

diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 63f2f0b86fbc..484a8b0b96b9 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -101,21 +101,55 @@  int devm_cxl_port_enumerate_dports(struct cxl_port *port)
 }
 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);
 
-/*
- * Wait up to @media_ready_timeout for the device to report memory
- * active.
- */
-int cxl_await_media_ready(struct cxl_dev_state *cxlds)
+static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id)
+{
+	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
+	int d = cxlds->cxl_dvsec;
+	bool valid = false;
+	int rc, i;
+	u32 temp;
+
+	if (id > CXL_DVSEC_RANGE_MAX)
+		return -EINVAL;
+
+	/* Check MEM INFO VALID bit first, give up after 1s */
+	i = 1;
+	do {
+		rc = pci_read_config_dword(pdev,
+					   d + CXL_DVSEC_RANGE_SIZE_LOW(id),
+					   &temp);
+		if (rc)
+			return rc;
+
+		valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp);
+		if (valid)
+			break;
+		msleep(1000);
+	} while (i--);
+
+	if (!valid) {
+		dev_err(&pdev->dev,
+			"Timeout awaiting memory range %d valid after 1s.\n",
+			id);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
 {
 	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
 	int d = cxlds->cxl_dvsec;
 	bool active = false;
-	u64 md_status;
 	int rc, i;
+	u32 temp;
 
-	for (i = media_ready_timeout; i; i--) {
-		u32 temp;
+	if (id > CXL_DVSEC_RANGE_MAX)
+		return -EINVAL;
 
+	/* Check MEM ACTIVE bit, up to 60s timeout by default */
+	for (i = media_ready_timeout; i; i--) {
 		rc = pci_read_config_dword(
 			pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
 		if (rc)
@@ -134,6 +168,39 @@  int cxl_await_media_ready(struct cxl_dev_state *cxlds)
 		return -ETIMEDOUT;
 	}
 
+	return 0;
+}
+
+/*
+ * Wait up to @media_ready_timeout for the device to report memory
+ * active.
+ */
+int cxl_await_media_ready(struct cxl_dev_state *cxlds)
+{
+	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
+	int d = cxlds->cxl_dvsec;
+	int rc, i, hdm_count;
+	u64 md_status;
+	u16 cap;
+
+	rc = pci_read_config_word(pdev,
+				  d + CXL_DVSEC_CAP_OFFSET, &cap);
+	if (rc)
+		return rc;
+
+	hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
+	for (i = 0; i < hdm_count; i++) {
+		rc = cxl_dvsec_mem_range_valid(cxlds, i);
+		if (rc)
+			return rc;
+	}
+
+	for (i = 0; i < hdm_count; i++) {
+		rc = cxl_dvsec_mem_range_active(cxlds, i);
+		if (rc)
+			return rc;
+	}
+
 	md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
 	if (!CXLMDEV_READY(md_status))
 		return -EIO;
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 0465ef963cd6..7c02e55b8042 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -31,6 +31,8 @@ 
 #define   CXL_DVSEC_RANGE_BASE_LOW(i)	(0x24 + (i * 0x10))
 #define     CXL_DVSEC_MEM_BASE_LOW_MASK	GENMASK(31, 28)
 
+#define CXL_DVSEC_RANGE_MAX		2
+
 /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
 #define CXL_DVSEC_FUNCTION_MAP					2