@@ -398,6 +398,7 @@ configure_file("${BUILDLIB}/config.h.in" "${BUILD_INCLUDE}/config.h" ESCAPE_QUOT
add_subdirectory(ccan)
add_subdirectory(util)
add_subdirectory(Documentation)
+add_subdirectory(kernel-boot)
# Libraries
add_subdirectory(libibumad)
add_subdirectory(libibumad/man)
@@ -1,5 +1,9 @@
+etc/rdma/rdma.conf
lib/systemd/system/rdma-ndd.service
+lib/systemd/system/rdma.service
+lib/udev/rules.d/98-rdma.rules
lib/udev/rules.d/rdma-ndd.rules
+usr/lib/rdma-init-kernel
usr/sbin/rdma-ndd
usr/share/doc/rdma-core/MAINTAINERS
usr/share/doc/rdma-core/README.md
@@ -58,7 +58,6 @@ rdma_man_pages(
)
# FIXME: update the .init.in
-set(rdmascript "openibd")
set(prefix "${CMAKE_INSTALL_PREFIX}")
rdma_subst_install(FILES "ibacm.init.in"
DESTINATION "${CMAKE_INSTALL_INITDDIR}"
@@ -10,8 +10,8 @@
# Provides: ibacm
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
-# Required-Start: @rdmascript@ $network $remote_fs
-# Required-Stop: @rdmascript@ $network $remote_fs
+# Required-Start: rdma $network $remote_fs
+# Required-Stop: rdma $network $remote_fs
# Should-Start:
# Should-Stop:
# Short-Description: Starts and stops the InfiniBand ACM service
new file mode 100644
@@ -0,0 +1,22 @@
+# Copyright (c) 2017 Benjamin Drung <benjamin.drung@profitbricks.com>
+# Licensed under BSD (MIT variant) or GPLv2. See COPYING.
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+
+install(FILES rdma.conf DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/rdma/"
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ)
+
+rdma_subst_install(FILES "rdma-init-kernel.in"
+ DESTINATION "${CMAKE_INSTALL_FULL_LIBEXECDIR}"
+ RENAME "rdma-init-kernel"
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ OWNER_EXECUTE GROUP_EXECUTE WORLD_EXECUTE)
+
+install(FILES "rdma.udev-rules"
+ DESTINATION "${CMAKE_INSTALL_UDEV_RULESDIR}"
+ RENAME "98-rdma.rules"
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ)
+
+rdma_subst_install(FILES "rdma.service.in"
+ DESTINATION "${CMAKE_INSTALL_SYSTEMD_SERVICEDIR}"
+ RENAME "rdma.service"
+ PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ)
new file mode 100644
@@ -0,0 +1,143 @@
+#!/bin/bash
+#
+# Bring up the kernel RDMA stack
+#
+# This is usually run automatically by systemd after a hardware activation
+# event in udev has triggered a start of the rdma.service unit
+#
+
+set -eu
+shopt -s nullglob
+
+CONFIG=@CMAKE_INSTALL_SYSCONFDIR@/rdma/rdma.conf
+
+LOAD_ULP_MODULES=""
+LOAD_CORE_USER_MODULES="ib_umad ib_uverbs ib_ucm rdma_ucm"
+LOAD_CORE_CM_MODULES="iw_cm ib_cm rdma_cm"
+LOAD_CORE_MODULES="ib_core"
+
+if [ -f $CONFIG ]; then
+ . $CONFIG
+
+ if [ "${RDS_LOAD-}" = "yes" ]; then
+ IPOIB_LOAD=yes
+ fi
+
+ if [ "${IPOIB_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_ipoib"
+ fi
+
+ if [ "${RDS_LOAD-}" = "yes" ] && [ -f "/lib/modules/$(uname -r)/kernel/net/rds/rds.ko" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES rds"
+ if [ -f "/lib/modules/$(uname -r)/kernel/net/rds/rds_tcp.ko" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES rds_tcp"
+ fi
+ if [ -f "/lib/modules/$(uname -r)/kernel/net/rds/rds_rdma.ko" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES rds_rdma"
+ fi
+ fi
+
+ if [ "${SRP_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_srp"
+ fi
+
+ if [ "${SRPT_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_srpt"
+ fi
+
+ if [ "${ISER_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_iser"
+ fi
+
+ if [ "${ISERT_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_isert"
+ fi
+
+ if [ "${XPRTRDMA_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES xprtrdma"
+ fi
+
+ if [ "${SVCRDMA_LOAD-}" = "yes" ]; then
+ LOAD_ULP_MODULES="$LOAD_ULP_MODULES svcrdma"
+ fi
+else
+ LOAD_ULP_MODULES="ib_ipoib"
+fi
+
+# If module $1 is loaded return - 0 else - 1
+is_loaded()
+{
+ /sbin/lsmod | grep -w "$1" > /dev/null 2>&1
+}
+
+load_modules()
+{
+ local RC=0
+
+ for module in "$@"; do
+ if ! /sbin/modinfo "$module" > /dev/null 2>&1; then
+ # do not attempt to load modules which do not exist
+ continue
+ fi
+ if ! is_loaded "$module"; then
+ if ! /sbin/modprobe "$module"; then
+ RC=$(( RC + 1 ))
+ echo
+ echo "Failed to load module $module"
+ fi
+ fi
+ done
+ return $RC
+}
+
+load_hardware_modules()
+{
+ local -i RC=0
+
+ # We match both class NETWORK and class INFINIBAND devices since our
+ # iWARP hardware is listed under class NETWORK. The side effect of
+ # this is that we might cause a non-iWARP network driver to be loaded.
+ udevadm trigger --subsystem-match=pci --attr-nomatch=driver --attr-match=class=0x020000 --attr-match=class=0x0c0600
+ udevadm settle
+ if [ -r /proc/device-tree ]; then
+ if ls /proc/device-tree/*lhca* >/dev/null 2>&1; then
+ if ! is_loaded ib_ehca; then
+ load_modules ib_ehca || RC=$(( RC + $? ))
+ fi
+ fi
+ fi
+ if is_loaded mlx4_core -a ! is_loaded mlx4_ib; then
+ load_modules mlx4_ib || RC=$(( RC + $? ))
+ fi
+ if is_loaded mlx4_core -a ! is_loaded mlx4_en; then
+ load_modules mlx4_en || RC=$(( RC + $? ))
+ fi
+ if is_loaded mlx5_core -a ! is_loaded mlx5_ib; then
+ load_modules mlx5_ib || RC=$(( RC + $? ))
+ fi
+ if is_loaded cxgb3 -a ! is_loaded iw_cxgb3; then
+ load_modules iw_cxgb3 || RC=$(( RC + $? ))
+ fi
+ if is_loaded cxgb4 -a ! is_loaded iw_cxgb4; then
+ load_modules iw_cxgb4 || RC=$(( RC + $? ))
+ fi
+ if is_loaded be2net -a ! is_loaded ocrdma; then
+ load_modules ocrdma || RC=$(( RC + $? ))
+ fi
+ if is_loaded enic -a ! is_loaded usnic_verbs; then
+ load_modules usnic_verbs || RC=$(( RC + $? ))
+ fi
+ if is_loaded i40e -a ! is_loaded i40iw; then
+ load_modules i40iw || RC=$(( RC + $? ))
+ fi
+ return $RC
+}
+
+RC=0
+load_hardware_modules || RC=$(( RC + $? ))
+load_modules $LOAD_CORE_MODULES || RC=$(( RC + $? ))
+load_modules $LOAD_CORE_CM_MODULES || RC=$(( RC + $? ))
+load_modules $LOAD_CORE_USER_MODULES || RC=$(( RC + $? ))
+load_modules $LOAD_ULP_MODULES || RC=$(( RC + $? ))
+
+exit $RC
new file mode 100644
@@ -0,0 +1,23 @@
+# Load IP over InfiniBand (IPoIB) network driver module
+IPOIB_LOAD=yes
+
+# Load SCSI Remote Protocol (SRP) initiator support module
+SRP_LOAD=yes
+
+# Load SCSI Remote Protocol target (SRPT) support module
+SRPT_LOAD=yes
+
+# Load iSCSI Extensions for RDMA (iSER) initiator support module
+ISER_LOAD=yes
+
+# Load iSCSI Extensions for RDMA target (iSERT) support module
+ISERT_LOAD=yes
+
+# Load Reliable Datagram Service (RDS) network protocol
+RDS_LOAD=no
+
+# Load NFS over RDMA (NFSoRDMA) client transport module
+XPRTRDMA_LOAD=yes
+
+# Load NFS over RDMA (NFSoRDMA) server transport module
+SVCRDMA_LOAD=no
similarity index 59%
rename from redhat/rdma.service
rename to kernel-boot/rdma.service.in
@@ -1,15 +1,15 @@
[Unit]
Description=Initialize the iWARP/InfiniBand/RDMA stack in the kernel
-Documentation=file:/etc/rdma/rdma.conf
+Documentation=file:@CMAKE_INSTALL_SYSCONFDIR@/rdma/rdma.conf
RefuseManualStop=true
DefaultDependencies=false
Conflicts=emergency.target emergency.service
-Before=network.target remote-fs-pre.target
+Before=network-pre.target remote-fs-pre.target
[Service]
Type=oneshot
RemainAfterExit=yes
-ExecStart=/usr/libexec/rdma-init-kernel
+ExecStart=@CMAKE_INSTALL_FULL_LIBEXECDIR@/rdma-init-kernel
[Install]
WantedBy=sysinit.target
similarity index 100%
rename from redhat/rdma.udev-rules
rename to kernel-boot/rdma.udev-rules
@@ -4,9 +4,9 @@
# enable the IB stack, so do so unilaterally) and on load of any of that
# hardware, we trigger the rdma.service load in systemd
+SUBSYSTEM=="module", KERNEL=="be2net", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
SUBSYSTEM=="module", KERNEL=="cxgb*", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
+SUBSYSTEM=="module", KERNEL=="enic", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
SUBSYSTEM=="module", KERNEL=="ib_*", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
-SUBSYSTEM=="module", KERNEL=="mlx*", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
SUBSYSTEM=="module", KERNEL=="iw_*", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
-SUBSYSTEM=="module", KERNEL=="be2net", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
-SUBSYSTEM=="module", KERNEL=="enic", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
+SUBSYSTEM=="module", KERNEL=="mlx*", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}="rdma.service"
@@ -250,7 +250,6 @@ mkdir -p %{buildroot}%{_libexecdir}
mkdir -p %{buildroot}%{_udevrulesdir}
mkdir -p %{buildroot}%{dracutlibdir}/modules.d/05rdma
mkdir -p %{buildroot}%{sysmodprobedir}
-install -D -m0644 redhat/rdma.conf %{buildroot}/%{_sysconfdir}/rdma/rdma.conf
install -D -m0644 redhat/rdma.sriov-vfs %{buildroot}/%{_sysconfdir}/rdma/sriov-vfs
install -D -m0644 redhat/rdma.mlx4.conf %{buildroot}/%{_sysconfdir}/rdma/mlx4.conf
install -D -m0755 redhat/rdma.ifup-ib %{buildroot}/%{_sysconfdir}/sysconfig/network-scripts/ifup-ib
@@ -259,7 +258,6 @@ install -D -m0644 redhat/rdma.service %{buildroot}%{_unitdir}/rdma.service
install -D -m0644 redhat/rdma.udev-ipoib-naming.rules %{buildroot}%{_sysconfdir}/udev/rules.d/70-persistent-ipoib.rules
install -D -m0644 redhat/rdma.mlx4.user.modprobe %{buildroot}%{_sysconfdir}/modprobe.d/mlx4.conf
install -D -m0755 redhat/rdma.modules-setup.sh %{buildroot}%{dracutlibdir}/modules.d/05rdma/module-setup.sh
-install -D -m0644 redhat/rdma.udev-rules %{buildroot}%{_udevrulesdir}/98-rdma.rules
install -D -m0644 redhat/rdma.mlx4.sys.modprobe %{buildroot}%{sysmodprobedir}/libmlx4.conf
install -D -m0644 redhat/rdma.cxgb3.sys.modprobe %{buildroot}%{sysmodprobedir}/cxgb3.conf
install -D -m0644 redhat/rdma.cxgb4.sys.modprobe %{buildroot}%{sysmodprobedir}/cxgb4.conf
deleted file mode 100644
@@ -1,25 +0,0 @@
-# Load IPoIB
-IPOIB_LOAD=yes
-# Load SRP (SCSI Remote Protocol initiator support) module
-SRP_LOAD=yes
-# Load SRPT (SCSI Remote Protocol target support) module
-SRPT_LOAD=yes
-# Load iSER (iSCSI over RDMA initiator support) module
-ISER_LOAD=yes
-# Load iSERT (iSCSI over RDMA target support) module
-ISERT_LOAD=yes
-# Load RDS (Reliable Datagram Service) network protocol
-RDS_LOAD=no
-# Load NFSoRDMA client transport module
-XPRTRDMA_LOAD=yes
-# Load NFSoRDMA server transport module
-SVCRDMA_LOAD=no
-# Load Tech Preview device driver modules
-TECH_PREVIEW_LOAD=no
-# Should we modify the system mtrr registers? We may need to do this if you
-# get messages from the ib_ipath driver saying that it couldn't enable
-# write combining for the PIO buffs on the card.
-#
-# Note: recent kernels should do this for us, but in case they don't, we'll
-# leave this option
-FIXUP_MTRR_REGS=no
@@ -45,8 +45,6 @@ install(FILES srp_daemon.conf DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}")
install(FILES srp_daemon.rules DESTINATION "${CMAKE_INSTALL_UDEV_RULESDIR}")
-# FIXME: The ib init.d file should really be included in rdma-core as well.
-set(RDMA_SERVICE "openibd" CACHE STRING "init.d file service name to order srpd after")
# NOTE: These defaults are for CentOS, packagers should override.
set(SRP_DEFAULT_START "2 3 4 5" CACHE STRING "Default-Start service data for srpd")
set(SRP_DEFAULT_STOP "0 1 6" CACHE STRING "Default-Stop service data for srpd")
@@ -9,8 +9,8 @@
#
### BEGIN INIT INFO
# Provides: srpd
-# Required-Start: $syslog @RDMA_SERVICE@
-# Required-Stop: $syslog @RDMA_SERVICE@
+# Required-Start: $syslog rdma
+# Required-Stop: $syslog rdma
# Default-Start: @SRP_DEFAULT_START@
# Default-Stop: @SRP_DEFAULT_STOP@
# Should-Start:
Currently upstream does not provide a rdma or openibd service. Only the RedHat package ships a rdma service and rdma modules configuration files. The ibacm and srp_daemon services depend on the openibd service. Make RedHat's rdma service available to all distros by cherry-picking the basic files for the rdma service to run and trim them down to the minimum. Do not pick workarounds or other quirk that might not needed any more. Then replace the openibd service dependency by the rdma service. Signed-off-by: Benjamin Drung <benjamin.drung@profitbricks.com> --- CMakeLists.txt | 1 + debian/rdma-core.install | 4 + ibacm/CMakeLists.txt | 1 - ibacm/ibacm.init.in | 4 +- kernel-boot/CMakeLists.txt | 22 ++++ kernel-boot/rdma-init-kernel.in | 143 +++++++++++++++++++++ kernel-boot/rdma.conf | 23 ++++ redhat/rdma.service => kernel-boot/rdma.service.in | 6 +- {redhat => kernel-boot}/rdma.udev-rules | 6 +- redhat/rdma-core.spec | 2 - redhat/rdma.conf | 25 ---- srp_daemon/CMakeLists.txt | 2 - srp_daemon/srpd.in | 4 +- 13 files changed, 203 insertions(+), 40 deletions(-) create mode 100644 kernel-boot/CMakeLists.txt create mode 100644 kernel-boot/rdma-init-kernel.in create mode 100644 kernel-boot/rdma.conf rename redhat/rdma.service => kernel-boot/rdma.service.in (59%) rename {redhat => kernel-boot}/rdma.udev-rules (100%) delete mode 100644 redhat/rdma.conf