@@ -115,6 +115,13 @@ static inline uint64_t ntohll(uint64_t x
#define wmb() mb()
#define wc_wmb() wmb()
+#elif defined(__s390x__)
+
+#define mb() { asm volatile("" : : : "memory"); } /* for s390x */
+#define rmb() mb() /* for s390x */
+#define wmb() mb() /* for s390x */
+#define wc_wmb() wmb() /* for s390x */
+
#else
#warning No architecture specific defines found. Using generic implementation.
@@ -91,7 +91,8 @@ enum {
IB_USER_VERBS_CMD_OPEN_XRCD,
IB_USER_VERBS_CMD_CLOSE_XRCD,
IB_USER_VERBS_CMD_CREATE_XSRQ,
- IB_USER_VERBS_CMD_OPEN_QP
+ IB_USER_VERBS_CMD_OPEN_QP,
+ IB_USER_VERBS_CMD_KWRITE_MMIO
};
#define IB_USER_VERBS_CMD_COMMAND_MASK 0xff
@@ -904,7 +905,8 @@ enum {
IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1,
- IB_USER_VERBS_CMD_OPEN_QP_V2 = -1
+ IB_USER_VERBS_CMD_OPEN_QP_V2 = -1,
+ IB_USER_VERBS_CMD_KWRITE_MMIO_V2 = -1
};
struct ibv_modify_srq_v3 {
The current implementation of some Infiniband verbs uses mapped memory to directly access the PCI I/O memory from userspace programs to perform I/O operations. On the s390x platform the PCI I/O memory can be accessed only using special privileged CPU instructions and since those instructions are privileged, they cannot be used from userspace programs. This makes impossible to use mapped memory areas to access the PCI I/O memory from userspace on s390x platform and aplications cannot use the Infiniband verbs on that platform without modification. There are two approaches could be implemented to solve this problem: * using a page fault handler to intercept mapped memory area access errors, and handle them in the handler by issuing the appropriate privileged CPU instructions; * modification of the existing verbs to avoid the mapped memory areas usage on the s390x platform. The page fault handler solution is the most complex one because it requires not only modifcation of the virtual memory handling in Linux kernel but also makes the developer to provide code for all the CPU instrutions which work with memory program interpretation. This approcach requires lots of lines of code and noticable overhead during the program execution. The modification of the existing verbs solution is much simpler and more realible. It requires modification of the libraries provided in the DAPL support packages to replace the usage of mapped memory areas used to access the device UAR and Blueflame page with the device driver write primitive calls supplying a special verb command to kernelspace. The modification of the existing verbs solution has been choosen for implementation. Userspace library libibverbs contains platform independent part of the Infiniband verb implementation. To allow the privileged CPU instruction execution in kernel context on request from the userspace programs, this patch adds a new user verb command IB_USER_VERBS_CMD_KWRITE_MMIO and its parameter list structure, which is used to identify the PCI I/O operation request in the kernelspace driver. Also, it adds s390x platform specific definitions for the macro mb, rmb, wmb, and wc_wmb. Signed-off-by: Alexey Ishchuk <alexey_ishchuk@ru.ibm.com> --- include/infiniband/arch.h | 7 +++++++ include/infiniband/kern-abi.h | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html