| Message ID | 1666288635-72591-12-git-send-email-mikelley@microsoft.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Add PCI pass-thru support to Hyper-V Confidential VMs | expand |
On Thu, Oct 20, 2022 at 10:57:14AM -0700, Michael Kelley wrote: > To support PCI pass-thru devices in Confidential VMs, Hyper-V > has added hypercalls to read and write MMIO space. Add the > appropriate definitions to hyperv-tlfs.h and implement > functions to make the hypercalls. These functions are used > in a subsequent patch. > > Co-developed-by: Dexuan Cui <decui@microsoft.com> > Signed-off-by: Dexuan Cui <decui@microsoft.com> > Signed-off-by: Michael Kelley <mikelley@microsoft.com> > --- > arch/x86/include/asm/hyperv-tlfs.h | 3 ++ > drivers/pci/controller/pci-hyperv.c | 62 +++++++++++++++++++++++++++++++++++++ > include/asm-generic/hyperv-tlfs.h | 22 +++++++++++++ > 3 files changed, 87 insertions(+) > > diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h > index 3089ec3..f769b9d 100644 > --- a/arch/x86/include/asm/hyperv-tlfs.h > +++ b/arch/x86/include/asm/hyperv-tlfs.h > @@ -117,6 +117,9 @@ > /* Recommend using enlightened VMCS */ > #define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED BIT(14) > > +/* Use hypercalls for MMIO config space access */ > +#define HV_X64_USE_MMIO_HYPERCALLS BIT(21) > + > /* > * CPU management features identification. > * These are HYPERV_CPUID_CPU_MANAGEMENT_FEATURES.EAX bits. > diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c > index e7c6f66..02ebf3e 100644 > --- a/drivers/pci/controller/pci-hyperv.c > +++ b/drivers/pci/controller/pci-hyperv.c > @@ -1054,6 +1054,68 @@ static int wslot_to_devfn(u32 wslot) > return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func); > } > > +static void hv_pci_read_mmio(phys_addr_t gpa, int size, u32 *val) > +{ > + struct hv_mmio_read_input *in; > + struct hv_mmio_read_output *out; > + u64 ret; > + > + /* > + * Must be called with interrupts disabled so it is safe > + * to use the per-cpu input argument page. Use it for > + * both input and output. > + */ > + in = *this_cpu_ptr(hyperv_pcpu_input_arg); > + out = *this_cpu_ptr(hyperv_pcpu_input_arg) + sizeof(*in); > + in->gpa = gpa; > + in->size = size; > + > + ret = hv_do_hypercall(HVCALL_MMIO_READ, in, out); > + if (hv_result_success(ret)) { > + switch (size) { > + case 1: > + *val = *(u8 *)(out->data); > + break; > + case 2: > + *val = *(u16 *)(out->data); > + break; > + default: > + *val = *(u32 *)(out->data); > + break; > + } > + } else > + pr_err("MMIO read hypercall failed with status %llx\n", ret); Too bad there's not more information to give the user/administrator here. Seeing "MMIO read hypercall failed with status -5" in the log doesn't give many clues about where to look or who to notify. I don't know what's even feasible, but driver name, device, address (gpa), size would all be possibilities. Bjorn
From: Bjorn Helgaas <helgaas@kernel.org> Sent: Thursday, October 20, 2022 12:04 PM > > On Thu, Oct 20, 2022 at 10:57:14AM -0700, Michael Kelley wrote: > > To support PCI pass-thru devices in Confidential VMs, Hyper-V > > has added hypercalls to read and write MMIO space. Add the > > appropriate definitions to hyperv-tlfs.h and implement > > functions to make the hypercalls. These functions are used > > in a subsequent patch. > > > > Co-developed-by: Dexuan Cui <decui@microsoft.com> > > Signed-off-by: Dexuan Cui <decui@microsoft.com> > > Signed-off-by: Michael Kelley <mikelley@microsoft.com> > > --- > > arch/x86/include/asm/hyperv-tlfs.h | 3 ++ > > drivers/pci/controller/pci-hyperv.c | 62 > +++++++++++++++++++++++++++++++++++++ > > include/asm-generic/hyperv-tlfs.h | 22 +++++++++++++ > > 3 files changed, 87 insertions(+) > > > > diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv- > tlfs.h > > index 3089ec3..f769b9d 100644 > > --- a/arch/x86/include/asm/hyperv-tlfs.h > > +++ b/arch/x86/include/asm/hyperv-tlfs.h > > @@ -117,6 +117,9 @@ > > /* Recommend using enlightened VMCS */ > > #define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED BIT(14) > > > > +/* Use hypercalls for MMIO config space access */ > > +#define HV_X64_USE_MMIO_HYPERCALLS BIT(21) > > + > > /* > > * CPU management features identification. > > * These are HYPERV_CPUID_CPU_MANAGEMENT_FEATURES.EAX bits. > > diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c > > index e7c6f66..02ebf3e 100644 > > --- a/drivers/pci/controller/pci-hyperv.c > > +++ b/drivers/pci/controller/pci-hyperv.c > > @@ -1054,6 +1054,68 @@ static int wslot_to_devfn(u32 wslot) > > return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func); > > } > > > > +static void hv_pci_read_mmio(phys_addr_t gpa, int size, u32 *val) > > +{ > > + struct hv_mmio_read_input *in; > > + struct hv_mmio_read_output *out; > > + u64 ret; > > + > > + /* > > + * Must be called with interrupts disabled so it is safe > > + * to use the per-cpu input argument page. Use it for > > + * both input and output. > > + */ > > + in = *this_cpu_ptr(hyperv_pcpu_input_arg); > > + out = *this_cpu_ptr(hyperv_pcpu_input_arg) + sizeof(*in); > > + in->gpa = gpa; > > + in->size = size; > > + > > + ret = hv_do_hypercall(HVCALL_MMIO_READ, in, out); > > + if (hv_result_success(ret)) { > > + switch (size) { > > + case 1: > > + *val = *(u8 *)(out->data); > > + break; > > + case 2: > > + *val = *(u16 *)(out->data); > > + break; > > + default: > > + *val = *(u32 *)(out->data); > > + break; > > + } > > + } else > > + pr_err("MMIO read hypercall failed with status %llx\n", ret); > > Too bad there's not more information to give the user/administrator > here. Seeing "MMIO read hypercall failed with status -5" in the log > doesn't give many clues about where to look or who to notify. I don't > know what's even feasible, but driver name, device, address (gpa), > size would all be possibilities. > Good point. We can pass the device as an input argument to hv_pci_read_mmio() and hv_pci_write_mmio(), use dev_err() instead of pr_err(), and provide the gpa and size in the message. This is one of those errors that should never happen, but when it does, having the additional info will be helpful for debugging. I'll do this in v2. Michael
diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 3089ec3..f769b9d 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -117,6 +117,9 @@ /* Recommend using enlightened VMCS */ #define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED BIT(14) +/* Use hypercalls for MMIO config space access */ +#define HV_X64_USE_MMIO_HYPERCALLS BIT(21) + /* * CPU management features identification. * These are HYPERV_CPUID_CPU_MANAGEMENT_FEATURES.EAX bits. diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index e7c6f66..02ebf3e 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -1054,6 +1054,68 @@ static int wslot_to_devfn(u32 wslot) return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func); } +static void hv_pci_read_mmio(phys_addr_t gpa, int size, u32 *val) +{ + struct hv_mmio_read_input *in; + struct hv_mmio_read_output *out; + u64 ret; + + /* + * Must be called with interrupts disabled so it is safe + * to use the per-cpu input argument page. Use it for + * both input and output. + */ + in = *this_cpu_ptr(hyperv_pcpu_input_arg); + out = *this_cpu_ptr(hyperv_pcpu_input_arg) + sizeof(*in); + in->gpa = gpa; + in->size = size; + + ret = hv_do_hypercall(HVCALL_MMIO_READ, in, out); + if (hv_result_success(ret)) { + switch (size) { + case 1: + *val = *(u8 *)(out->data); + break; + case 2: + *val = *(u16 *)(out->data); + break; + default: + *val = *(u32 *)(out->data); + break; + } + } else + pr_err("MMIO read hypercall failed with status %llx\n", ret); +} + +static void hv_pci_write_mmio(phys_addr_t gpa, int size, u32 val) +{ + struct hv_mmio_write_input *in; + u64 ret; + + /* + * Must be called with interrupts disabled so it is safe + * to use the per-cpu input argument memory. + */ + in = *this_cpu_ptr(hyperv_pcpu_input_arg); + in->gpa = gpa; + in->size = size; + switch (size) { + case 1: + *(u8 *)(in->data) = val; + break; + case 2: + *(u16 *)(in->data) = val; + break; + default: + *(u32 *)(in->data) = val; + break; + } + + ret = hv_do_hypercall(HVCALL_MMIO_WRITE, in, NULL); + if (!hv_result_success(ret)) + pr_err("MMIO write hypercall failed with status %llx\n", ret); +} + /* * PCI Configuration Space for these root PCI buses is implemented as a pair * of pages in memory-mapped I/O space. Writing to the first page chooses diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h index fdce7a4..383b620 100644 --- a/include/asm-generic/hyperv-tlfs.h +++ b/include/asm-generic/hyperv-tlfs.h @@ -159,6 +159,8 @@ struct ms_hyperv_tsc_page { #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 #define HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY 0x00db +#define HVCALL_MMIO_READ 0x0106 +#define HVCALL_MMIO_WRITE 0x0107 /* Extended hypercalls */ #define HV_EXT_CALL_QUERY_CAPABILITIES 0x8001 @@ -781,4 +783,24 @@ struct hv_memory_hint { union hv_gpa_page_range ranges[]; } __packed; +/* Data structures for HVCALL_MMIO_READ and HVCALL_MMIO_WRITE */ +#define HV_HYPERCALL_MMIO_MAX_DATA_LENGTH 64 + +struct hv_mmio_read_input { + u64 gpa; + u32 size; + u32 reserved; +} __packed; + +struct hv_mmio_read_output { + u8 data[HV_HYPERCALL_MMIO_MAX_DATA_LENGTH]; +} __packed; + +struct hv_mmio_write_input { + u64 gpa; + u32 size; + u32 reserved; + u8 data[HV_HYPERCALL_MMIO_MAX_DATA_LENGTH]; +} __packed; + #endif