diff mbox series

[RFC,05/29] nvkm/vgpu: populate GSP_VF_INFO when NVIDIA vGPU is enabled

Message ID 20240922124951.1946072-6-zhiw@nvidia.com (mailing list archive)
State New, archived
Headers show
Series Introduce NVIDIA GPU Virtualization (vGPU) Support | expand

Commit Message

Zhi Wang Sept. 22, 2024, 12:49 p.m. UTC 
GSP firmware needs to know the VF BAR offsets to correctly calculate the
VF events.

The VF BAR information is stored in GSP_VF_INFO, which needs to be
initialized and uploaded together with the GSP_SYSTEM_INFO.

Populate GSP_VF_INFO when nvkm uploads the GSP_SYSTEM_INFO if NVIDIA
vGPU is enabled.

Cc: Surath Mitra <smitra@nvidia.com>
Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 .../nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h  |  2 +
 .../gpu/drm/nouveau/nvkm/subdev/gsp/r535.c    |  3 ++
 .../gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c  | 50 +++++++++++++++++++
 3 files changed, 55 insertions(+)

Comments

Jason Gunthorpe Sept. 26, 2024, 10:52 p.m. UTC | #1 
On Sun, Sep 22, 2024 at 05:49:27AM -0700, Zhi Wang wrote:
> +void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device,
> +					void *info)
> +{
> +	struct pci_dev *pdev = nvkm_to_pdev(device);
> +	GspSystemInfo *gsp_info = info;
> +	GSP_VF_INFO *vf_info = &gsp_info->gspVFInfo;
> +	u32 lo, hi;
> +	u16 v;
> +	int pos;
> +
> +	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
> +
> +	pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF, &v);
> +	vf_info->totalVFs = v;
> +
> +	pci_read_config_word(pdev, pos + PCI_SRIOV_VF_OFFSET, &v);
> +	vf_info->firstVFOffset = v;
> +
> +	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR, &lo);
> +	vf_info->FirstVFBar0Address = lo & 0xFFFFFFF0;
> +
> +	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 4, &lo);
> +	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 8, &hi);
> +
> +	vf_info->FirstVFBar1Address = (((u64)hi) << 32) + (lo & 0xFFFFFFF0);
> +
> +	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 12, &lo);
> +	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 16, &hi);
> +
> +	vf_info->FirstVFBar2Address = (((u64)hi) << 32) + (lo & 0xFFFFFFF0);
> +
> +#define IS_BAR_64(i) (((i) & 0x00000006) == 0x00000004)

This should come from the PCI core not be re-read with pci_read_config
and hand rolled macros.

From a mlx perspective this is really weird, I'd expect the FW to be
able to read its own config space.

Jason
diff mbox series

Patch

diff --git a/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h b/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
index 9e10e18306b0..6bc10fa40cde 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/vgpu_mgr/vgpu_mgr.h
@@ -21,5 +21,7 @@  bool nvkm_vgpu_mgr_is_supported(struct nvkm_device *device);
 bool nvkm_vgpu_mgr_is_enabled(struct nvkm_device *device);
 int nvkm_vgpu_mgr_init(struct nvkm_device *device);
 void nvkm_vgpu_mgr_fini(struct nvkm_device *device);
+void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device,
+					void *info);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
index 14fc152d6859..49552d7df88f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c
@@ -1717,6 +1717,9 @@  r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)
 	info->pciConfigMirrorSize = 0x001000;
 	r535_gsp_acpi_info(gsp, &info->acpiMethodData);
 
+	if (nvkm_vgpu_mgr_is_supported(device))
+		nvkm_vgpu_mgr_populate_gsp_vf_info(device, info);
+
 	return nvkm_gsp_rpc_wr(gsp, info, false);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
index 0639596f8a96..d6ddb1f02275 100644
--- a/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/vgpu_mgr/vgpu_mgr.c
@@ -3,6 +3,10 @@ 
 #include <core/driver.h>
 #include <nvif/driverif.h>
 #include <core/pci.h>
+
+#include <nvrm/nvtypes.h>
+#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>
+
 #include <vgpu_mgr/vgpu_mgr.h>
 
 static bool support_vgpu_mgr = false;
@@ -120,3 +124,49 @@  void nvkm_vgpu_mgr_fini(struct nvkm_device *device)
 	detach_nvkm(vgpu_mgr);
 	vgpu_mgr->enabled = false;
 }
+
+/**
+ * nvkm_vgpu_mgr_populate_vf_info - populate GSP_VF_INFO when vGPU
+ * is enabled
+ * @device: the nvkm_device pointer
+ * @info: GSP_VF_INFO data structure
+ */
+void nvkm_vgpu_mgr_populate_gsp_vf_info(struct nvkm_device *device,
+					void *info)
+{
+	struct pci_dev *pdev = nvkm_to_pdev(device);
+	GspSystemInfo *gsp_info = info;
+	GSP_VF_INFO *vf_info = &gsp_info->gspVFInfo;
+	u32 lo, hi;
+	u16 v;
+	int pos;
+
+	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
+
+	pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF, &v);
+	vf_info->totalVFs = v;
+
+	pci_read_config_word(pdev, pos + PCI_SRIOV_VF_OFFSET, &v);
+	vf_info->firstVFOffset = v;
+
+	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR, &lo);
+	vf_info->FirstVFBar0Address = lo & 0xFFFFFFF0;
+
+	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 4, &lo);
+	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 8, &hi);
+
+	vf_info->FirstVFBar1Address = (((u64)hi) << 32) + (lo & 0xFFFFFFF0);
+
+	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 12, &lo);
+	pci_read_config_dword(pdev, pos + PCI_SRIOV_BAR + 16, &hi);
+
+	vf_info->FirstVFBar2Address = (((u64)hi) << 32) + (lo & 0xFFFFFFF0);
+
+#define IS_BAR_64(i) (((i) & 0x00000006) == 0x00000004)
+
+	v = nvkm_rd32(device, 0x88000 + 0xbf4);
+	vf_info->b64bitBar1 = IS_BAR_64(v);
+
+	v = nvkm_rd32(device, 0x88000 + 0xbfc);
+	vf_info->b64bitBar2 = IS_BAR_64(v);
+}