| Message ID | 20230607221651.2454764-2-terry.bowman@amd.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | cxl/pci: Add support for RCH RAS error handling | expand |
Terry Bowman wrote: > From: Robert Richter <rrichter@amd.com> > > The RCRB is extracted already during ACPI CEDT table parsing while the > data of this is needed not earlier than dport creation. This > implementation comes with drawbacks: During ACPI table scan there is > already MMIO access including mapping and unmapping, but only ACPI > data should be collected here. The collected data must be transferred > through a couple of interfaces until it is finally consumed when > creating the dport. This causes complex data structures and function > interfaces. Additionally, RCRB parsing will be extended to also > extract AER data, it would be much easier do this at a later point > during port and dport creation when the data structures are available > to hold that data. > > To simplify all that, probe the RCRB at a later point during RCH > downstream port creation. Change ACPI table parser to only extract the > base address of either the component registers or the RCRB. Parse and > extract the RCRB in devm_cxl_add_rch_dport(). > > This is in preparation to centralize all RCRB scanning. I really like the approach of this patch, the cleanups just make sense, the changelog is great... ...just the small matter of massive cxl_test breakages. Given that QEMU only supports CXL VH topologies I would like to keep cxl_test emulation of RCH topologies to regression test ongoing core reworks against the RCH case. The problem with cxl_test though is that due to how it uses linker tricks (--wrap=) to inject mock topology data, it cannot support a symbol that is exported by the core *and* consumed by the core. This patch moved cxl_rcrb_to_component() from being called by cxl_acpi and cxl_mem to being called internally by cxl_core and cxl_mem. The fix path I chose is to add a new cxl_rcd_component_reg_phys() helper for retrieving the upstream-port register block out of the RCRB, and add a mock devm_cxl_add_rch_dport() that fakes the discovery of an RCH. Below is the result for patch1. My plan is to go through the rest of the series and ensure cxl_test keeps working. Perhaps you can fold in the cxl_test fixups as I send them to be incorporated in your v6? To be clear, I feel this is my maintenance burden to bear, I don't fault you for not using cxl_test, but I am invested in keeping it operational. -- >8 -- diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 39227070da9b..1f4ae1b24cc1 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -372,7 +372,9 @@ static int add_host_bridge_uport(struct device *match, void *arg) return 0; } + /* Note, @dev is used by mock_acpi_table_parse_cedt() */ struct cxl_chbs_context { + struct device *dev; unsigned long long uid; resource_size_t base; u32 cxl_version; @@ -431,6 +433,7 @@ static int add_host_bridge_dport(struct device *match, void *arg) dev_dbg(match, "UID found: %lld\n", uid); ctx = (struct cxl_chbs_context) { + .dev = match, .uid = uid, }; acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbs, &ctx); diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 27f0968449de..bd0a5788c696 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -63,6 +63,14 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size); int cxl_dpa_free(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled); + +enum cxl_rcrb { + CXL_RCRB_DOWNSTREAM, + CXL_RCRB_UPSTREAM, +}; +resource_size_t __rcrb_to_component(struct device *dev, resource_size_t rcrb, + enum cxl_rcrb which); + extern struct rw_semaphore cxl_dpa_rwsem; int cxl_memdev_init(void); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 1a3f8729a616..45f5299af7a6 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -939,8 +939,8 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, return ERR_PTR(-ENOMEM); if (rcrb != CXL_RESOURCE_NONE) { - component_reg_phys = cxl_rcrb_to_component(dport_dev, - rcrb, CXL_RCRB_DOWNSTREAM); + component_reg_phys = __rcrb_to_component(dport_dev, rcrb, + CXL_RCRB_DOWNSTREAM); if (component_reg_phys == CXL_RESOURCE_NONE) { dev_warn(dport_dev, "Invalid Component Registers in RCRB"); return ERR_PTR(-ENXIO); diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 1476a0299c9b..564dd430258a 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -332,9 +332,8 @@ int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, } EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL); -resource_size_t cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) +resource_size_t __rcrb_to_component(struct device *dev, resource_size_t rcrb, + enum cxl_rcrb which) { resource_size_t component_reg_phys; void __iomem *addr; @@ -395,4 +394,12 @@ resource_size_t cxl_rcrb_to_component(struct device *dev, return component_reg_phys; } -EXPORT_SYMBOL_NS_GPL(cxl_rcrb_to_component, CXL); + +resource_size_t cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport) +{ + if (!dport->rch) + return CXL_RESOURCE_NONE; + return __rcrb_to_component(dev, dport->rcrb, CXL_RCRB_UPSTREAM); +} +EXPORT_SYMBOL_NS_GPL(cxl_rcd_component_reg_phys, CXL); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index a5cd661face2..28888bb0c088 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -262,14 +262,9 @@ int cxl_map_device_regs(struct device *dev, struct cxl_device_regs *regs, enum cxl_regloc_type; int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, struct cxl_register_map *map); - -enum cxl_rcrb { - CXL_RCRB_DOWNSTREAM, - CXL_RCRB_UPSTREAM, -}; -resource_size_t cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which); +struct cxl_dport; +resource_size_t cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport); #define CXL_RESOURCE_NONE ((resource_size_t) -1) #define CXL_TARGET_STRLEN 20 diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 519edd0eb196..45d4c32d78b0 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -72,8 +72,8 @@ static int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd, * typical register locator mechanism. */ if (parent_dport->rch && cxlds->component_reg_phys == CXL_RESOURCE_NONE) - component_reg_phys = cxl_rcrb_to_component( - &cxlmd->dev, parent_dport->rcrb, CXL_RCRB_UPSTREAM); + component_reg_phys = + cxl_rcd_component_reg_phys(&cxlmd->dev, parent_dport); else component_reg_phys = cxlds->component_reg_phys; endpoint = devm_cxl_add_port(host, &cxlmd->dev, component_reg_phys, diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 6f9347ade82c..8a87d7d5f7f8 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -12,7 +12,8 @@ ldflags-y += --wrap=devm_cxl_enumerate_decoders ldflags-y += --wrap=cxl_await_media_ready ldflags-y += --wrap=cxl_hdm_decode_init ldflags-y += --wrap=cxl_dvsec_rr_decode -ldflags-y += --wrap=cxl_rcrb_to_component +ldflags-y += --wrap=devm_cxl_add_rch_dport +ldflags-y += --wrap=cxl_rcd_component_reg_phys DRIVERS := ../../../drivers CXL_SRC := $(DRIVERS)/cxl diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index bf00dc52fe96..f5c04787bcc8 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -971,15 +971,6 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port) return 0; } -resource_size_t mock_cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) -{ - dev_dbg(dev, "rcrb: %pa which: %d\n", &rcrb, which); - - return (resource_size_t) which + 1; -} - static struct cxl_mock_ops cxl_mock_ops = { .is_mock_adev = is_mock_adev, .is_mock_bridge = is_mock_bridge, @@ -988,7 +979,6 @@ static struct cxl_mock_ops cxl_mock_ops = { .is_mock_dev = is_mock_dev, .acpi_table_parse_cedt = mock_acpi_table_parse_cedt, .acpi_evaluate_integer = mock_acpi_evaluate_integer, - .cxl_rcrb_to_component = mock_cxl_rcrb_to_component, .acpi_pci_find_root = mock_acpi_pci_find_root, .devm_cxl_port_enumerate_dports = mock_cxl_port_enumerate_dports, .devm_cxl_setup_hdm = mock_cxl_setup_hdm, diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index 284416527644..30119a16ae85 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -259,24 +259,44 @@ int __wrap_cxl_dvsec_rr_decode(struct device *dev, int dvsec, } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_dvsec_rr_decode, CXL); -resource_size_t __wrap_cxl_rcrb_to_component(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which) +struct cxl_dport *__wrap_devm_cxl_add_rch_dport(struct cxl_port *port, + struct device *dport_dev, + int port_id, + resource_size_t rcrb) +{ + int index; + struct cxl_dport *dport; + struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); + + if (ops && ops->is_mock_port(dport_dev)) { + dport = devm_cxl_add_dport(port, dport_dev, port_id, + CXL_RESOURCE_NONE); + if (!IS_ERR(dport)) + dport->rch = true; + } else + dport = devm_cxl_add_rch_dport(port, dport_dev, port_id, rcrb); + put_cxl_mock_ops(index); + + return dport; +} +EXPORT_SYMBOL_NS_GPL(__wrap_devm_cxl_add_rch_dport, CXL); + +resource_size_t __wrap_cxl_rcd_component_reg_phys(struct device *dev, + struct cxl_dport *dport) { int index; resource_size_t component_reg_phys; struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); if (ops && ops->is_mock_port(dev)) - component_reg_phys = - ops->cxl_rcrb_to_component(dev, rcrb, which); + component_reg_phys = CXL_RESOURCE_NONE; else - component_reg_phys = cxl_rcrb_to_component(dev, rcrb, which); + component_reg_phys = cxl_rcd_component_reg_phys(dev, dport); put_cxl_mock_ops(index); return component_reg_phys; } -EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcrb_to_component, CXL); +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_rcd_component_reg_phys, CXL); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(ACPI); diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h index bef8817b01f2..a94223750346 100644 --- a/tools/testing/cxl/test/mock.h +++ b/tools/testing/cxl/test/mock.h @@ -15,9 +15,6 @@ struct cxl_mock_ops { acpi_string pathname, struct acpi_object_list *arguments, unsigned long long *data); - resource_size_t (*cxl_rcrb_to_component)(struct device *dev, - resource_size_t rcrb, - enum cxl_rcrb which); struct acpi_pci_root *(*acpi_pci_find_root)(acpi_handle handle); bool (*is_mock_bus)(struct pci_bus *bus); bool (*is_mock_port)(struct device *dev);
Dan Williams wrote: > Terry Bowman wrote: > > From: Robert Richter <rrichter@amd.com> > > > > The RCRB is extracted already during ACPI CEDT table parsing while the > > data of this is needed not earlier than dport creation. This > > implementation comes with drawbacks: During ACPI table scan there is > > already MMIO access including mapping and unmapping, but only ACPI > > data should be collected here. The collected data must be transferred > > through a couple of interfaces until it is finally consumed when > > creating the dport. This causes complex data structures and function > > interfaces. Additionally, RCRB parsing will be extended to also > > extract AER data, it would be much easier do this at a later point > > during port and dport creation when the data structures are available > > to hold that data. > > > > To simplify all that, probe the RCRB at a later point during RCH > > downstream port creation. Change ACPI table parser to only extract the > > base address of either the component registers or the RCRB. Parse and > > extract the RCRB in devm_cxl_add_rch_dport(). > > > > This is in preparation to centralize all RCRB scanning. > > I really like the approach of this patch, the cleanups just make sense, > the changelog is great... [..] > To be clear, I feel this is my maintenance burden to bear, I don't fault > you for not using cxl_test, but I am invested in keeping it operational. On to patch2... $ stg push Pushing patch "cxl-rch-prepare-for-caching" ... done (conflict) Error: 8 merge conflict(s) CONFLICT (content): Merge conflict in tools/testing/cxl/test/mock.h CONFLICT (content): Merge conflict in tools/testing/cxl/test/mock.c CONFLICT (content): Merge conflict in tools/testing/cxl/test/cxl.c CONFLICT (content): Merge conflict in tools/testing/cxl/Kbuild CONFLICT (content): Merge conflict in drivers/cxl/mem.c CONFLICT (content): Merge conflict in drivers/cxl/cxl.h CONFLICT (content): Merge conflict in drivers/cxl/core/regs.c CONFLICT (content): Merge conflict in drivers/cxl/core/port.c Now at patch "cxl-rch-prepare-for-caching" ...oh, looks like you are trying to keep cxl_test going. I had stopped at patch1 because that one already raised a circular dependency build error: depmod: ERROR: Cycle detected: cxl_mock -> cxl_core -> cxl_mock Let me see what's going on here...
diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 7e1765b09e04..39227070da9b 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -373,20 +373,18 @@ static int add_host_bridge_uport(struct device *match, void *arg) } struct cxl_chbs_context { - struct device *dev; unsigned long long uid; - resource_size_t rcrb; - resource_size_t chbcr; + resource_size_t base; u32 cxl_version; }; -static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg, +static int cxl_get_chbs(union acpi_subtable_headers *header, void *arg, const unsigned long end) { struct cxl_chbs_context *ctx = arg; struct acpi_cedt_chbs *chbs; - if (ctx->chbcr) + if (ctx->base) return 0; chbs = (struct acpi_cedt_chbs *) header; @@ -395,23 +393,16 @@ static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg, return 0; ctx->cxl_version = chbs->cxl_version; - ctx->rcrb = CXL_RESOURCE_NONE; - ctx->chbcr = CXL_RESOURCE_NONE; + ctx->base = CXL_RESOURCE_NONE; if (!chbs->base) return 0; - if (chbs->cxl_version != ACPI_CEDT_CHBS_VERSION_CXL11) { - ctx->chbcr = chbs->base; + if (chbs->cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11 && + chbs->length != CXL_RCRB_SIZE) return 0; - } - if (chbs->length != CXL_RCRB_SIZE) - return 0; - - ctx->rcrb = chbs->base; - ctx->chbcr = cxl_rcrb_to_component(ctx->dev, chbs->base, - CXL_RCRB_DOWNSTREAM); + ctx->base = chbs->base; return 0; } @@ -440,36 +431,33 @@ static int add_host_bridge_dport(struct device *match, void *arg) dev_dbg(match, "UID found: %lld\n", uid); ctx = (struct cxl_chbs_context) { - .dev = match, .uid = uid, }; - acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbcr, &ctx); + acpi_table_parse_cedt(ACPI_CEDT_TYPE_CHBS, cxl_get_chbs, &ctx); - if (!ctx.chbcr) { + if (!ctx.base) { dev_warn(match, "No CHBS found for Host Bridge (UID %lld)\n", uid); return 0; } - if (ctx.rcrb != CXL_RESOURCE_NONE) - dev_dbg(match, "RCRB found for UID %lld: %pa\n", uid, &ctx.rcrb); - - if (ctx.chbcr == CXL_RESOURCE_NONE) { - dev_warn(match, "CHBCR invalid for Host Bridge (UID %lld)\n", + if (ctx.base == CXL_RESOURCE_NONE) { + dev_warn(match, "CHBS invalid for Host Bridge (UID %lld)\n", uid); return 0; } - dev_dbg(match, "CHBCR found: %pa\n", &ctx.chbcr); - pci_root = acpi_pci_find_root(hb->handle); bridge = pci_root->bus->bridge; - if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11) - dport = devm_cxl_add_rch_dport(root_port, bridge, uid, - ctx.chbcr, ctx.rcrb); - else - dport = devm_cxl_add_dport(root_port, bridge, uid, - ctx.chbcr); + + if (ctx.cxl_version == ACPI_CEDT_CHBS_VERSION_CXL11) { + dev_dbg(match, "RCRB found for UID %lld: %pa\n", uid, &ctx.base); + dport = devm_cxl_add_rch_dport(root_port, bridge, uid, ctx.base); + } else { + dev_dbg(match, "CHBCR found for UID %lld: %pa\n", uid, &ctx.base); + dport = devm_cxl_add_dport(root_port, bridge, uid, ctx.base); + } + if (IS_ERR(dport)) return PTR_ERR(dport); diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index e7c284c890bc..1a3f8729a616 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -938,12 +938,25 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev, if (!dport) return ERR_PTR(-ENOMEM); + if (rcrb != CXL_RESOURCE_NONE) { + component_reg_phys = cxl_rcrb_to_component(dport_dev, + rcrb, CXL_RCRB_DOWNSTREAM); + if (component_reg_phys == CXL_RESOURCE_NONE) { + dev_warn(dport_dev, "Invalid Component Registers in RCRB"); + return ERR_PTR(-ENXIO); + } + + dport->rch = true; + } + + if (component_reg_phys != CXL_RESOURCE_NONE) + dev_dbg(dport_dev, "Component Registers found for dport: %pa\n", + &component_reg_phys); + dport->dport = dport_dev; dport->port_id = port_id; dport->component_reg_phys = component_reg_phys; dport->port = port; - if (rcrb != CXL_RESOURCE_NONE) - dport->rch = true; dport->rcrb = rcrb; cond_cxl_root_lock(port); @@ -1004,14 +1017,12 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL); * @port: the cxl_port that references this dport * @dport_dev: firmware or PCI device representing the dport * @port_id: identifier for this dport in a decoder's target list - * @component_reg_phys: optional location of CXL component registers * @rcrb: mandatory location of a Root Complex Register Block * * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH */ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, - resource_size_t component_reg_phys, resource_size_t rcrb) { struct cxl_dport *dport; @@ -1022,7 +1033,7 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, } dport = __devm_cxl_add_dport(port, dport_dev, port_id, - component_reg_phys, rcrb); + CXL_RESOURCE_NONE, rcrb); if (IS_ERR(dport)) { dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n", dev_name(&port->dev), PTR_ERR(dport)); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index f93a28538962..a5cd661face2 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -671,7 +671,6 @@ struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, resource_size_t component_reg_phys); struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, - resource_size_t component_reg_phys, resource_size_t rcrb); struct cxl_decoder *to_cxl_decoder(struct device *dev);