From patchwork Thu Aug 20 16:46:29 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Achiad Shochat X-Patchwork-Id: 7046211 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork2.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork2.web.kernel.org (Postfix) with ESMTP id 27E86C05AC for ; Thu, 20 Aug 2015 16:50:26 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id 383E120552 for ; Thu, 20 Aug 2015 16:50:25 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 4D7902041E for ; Thu, 20 Aug 2015 16:50:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751878AbbHTQuX (ORCPT ); Thu, 20 Aug 2015 12:50:23 -0400 Received: from [193.47.165.129] ([193.47.165.129]:46824 "EHLO mellanox.co.il" rhost-flags-FAIL-FAIL-OK-FAIL) by vger.kernel.org with ESMTP id S1751818AbbHTQuX (ORCPT ); Thu, 20 Aug 2015 12:50:23 -0400 Received: from Internal Mail-Server by MTLPINE1 (envelope-from achiad@mellanox.com) with ESMTPS (AES256-SHA encrypted); 20 Aug 2015 19:49:15 +0300 Received: from rsws33.mtr.labs.mlnx (dev-r-vrt-064.mtr.labs.mlnx [10.212.64.1]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id t7KGnBJ4020546; Thu, 20 Aug 2015 19:49:12 +0300 From: Achiad Shochat To: Doug Ledford Cc: linux-rdma@vger.kernel.org, Matan Barak , Jason Gunthorpe , Or Gerlitz , Haggai Eran , Somnath Kotur , Achiad Shochat Subject: [PATCH for-next 10/10] IB/mlx5: Support RoCE Date: Thu, 20 Aug 2015 19:46:29 +0300 Message-Id: <1440089189-3361-11-git-send-email-achiad@mellanox.com> X-Mailer: git-send-email 2.1.0 In-Reply-To: <1440089189-3361-1-git-send-email-achiad@mellanox.com> References: <1440089189-3361-1-git-send-email-achiad@mellanox.com> Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-7.5 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=ham version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Advertise RoCE support for IB/core layer and set the hardware to work in RoCE mode. Signed-off-by: Achiad Shochat --- drivers/infiniband/hw/mlx5/main.c | 48 +++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 27dab5d..3e2e24e8 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1586,6 +1586,32 @@ static void destroy_dev_resources(struct mlx5_ib_resources *devr) mlx5_ib_dealloc_pd(devr->p0); } +static u32 get_core_cap_flags(struct ib_device *ibdev) +{ + struct mlx5_ib_dev *dev = to_mdev(ibdev); + enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1); + u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type); + u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version); + u32 ret = 0; + + if (ll == IB_LINK_LAYER_INFINIBAND) + return RDMA_CORE_PORT_IBA_IB; + + if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP)) + return 0; + + if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP)) + return 0; + + if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP) + ret |= RDMA_CORE_PORT_IBA_ROCE; + + if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP) + ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + + return ret; +} + static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { @@ -1598,7 +1624,7 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, immutable->pkey_tbl_len = attr.pkey_tbl_len; immutable->gid_tbl_len = attr.gid_tbl_len; - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; + immutable->core_cap_flags = get_core_cap_flags(ibdev); immutable->max_mad_size = IB_MGMT_MAD_SIZE; return 0; @@ -1606,13 +1632,28 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num, static int mlx5_enable_roce(struct mlx5_ib_dev *dev) { + int err; + rwlock_init(&dev->roce.netdev_lock); dev->roce.nb.notifier_call = mlx5_netdev_event; - return register_netdevice_notifier(&dev->roce.nb); + err = register_netdevice_notifier(&dev->roce.nb); + if (err) + return err; + + err = mlx5_nic_vport_enable_roce(dev->mdev); + if (err) + goto err_unregister_netdevice_notifier; + + return 0; + +err_unregister_netdevice_notifier: + unregister_netdevice_notifier(&dev->roce.nb); + return err; } static void mlx5_disable_roce(struct mlx5_ib_dev *dev) { + mlx5_nic_vport_disable_roce(dev->mdev); unregister_netdevice_notifier(&dev->roce.nb); } @@ -1627,8 +1668,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) port_type_cap = MLX5_CAP_GEN(mdev, port_type); ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap); - /* don't create IB instance over Eth ports, no RoCE yet! */ - if (ll == IB_LINK_LAYER_ETHERNET) + if ((ll == IB_LINK_LAYER_ETHERNET) && !MLX5_CAP_GEN(mdev, roce)) return NULL; printk_once(KERN_INFO "%s", mlx5_version);