[openib-general] [PATCH 05/17] ehca: InfiniBand query and multicast functionality
Heiko J Schick
schihei at de.ibm.com
Thu Mar 2 00:37:35 PST 2006
Adds InfiniBand query and multicast functionality.
Signed-off-by: Heiko J Schick <schickhj at de.ibm.com>
ehca_hca.c | 317 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
ehca_mcast.c | 196 ++++++++++++++++++++++++++++++++++++
2 files changed, 513 insertions(+)
--- linux-2.6.16-rc4-orig/drivers/infiniband/hw/ehca/ehca_hca.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.16-rc4/drivers/infiniband/hw/ehca/ehca_hca.c 2006-02-28 08:41:04.000000000 +0100
@@ -0,0 +1,317 @@
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HCA query functions
+ *
+ * Authors: Heiko J Schick <schickhj at de.ibm.com>
+ * Christoph Raisch <raisch at de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $Id: ehca_hca.c,v 1.4 2006/02/28 07:41:04 schickhj Exp $
+ */
+
+#undef DEB_PREFIX
+#define DEB_PREFIX "shca"
+
+#include "ehca_kernel.h"
+#include "ehca_tools.h"
+
+#include "hcp_if.h" /* TODO: later via hipz_* header file */
+
+#define TO_MAX_INT(dest, src) \
+ if (src >= INT_MAX) \
+ dest = INT_MAX; \
+ else \
+ dest = src
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
+{
+ int ret = 0;
+ struct ehca_shca *shca;
+ struct hipz_query_hca *rblock;
+
+ EDEB_EN(7, "");
+
+ memset(props, 0, sizeof(struct ib_device_attr));
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+
+ rblock = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (rblock == NULL) {
+ EDEB_ERR(4, "Can't allocate rblock memory.");
+ ret = -ENOMEM;
+ goto query_device0;
+ }
+
+ memset(rblock, 0, PAGE_SIZE);
+
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_Success) {
+ EDEB_ERR(4, "Can't query device properties");
+ ret = -EINVAL;
+ goto query_device1;
+ }
+ props->fw_ver = rblock->hw_ver;
+ /* TODO: memcpy(&props->sys_image_guid, ...); */
+ props->max_mr_size = rblock->max_mr_size;
+ /* TODO: props->page_size_cap */
+ props->vendor_id = rblock->vendor_id >> 8;
+ props->vendor_part_id = rblock->vendor_part_id >> 16;
+ props->hw_ver = rblock->hw_ver;
+ TO_MAX_INT(props->max_qp, (rblock->max_qp - rblock->cur_qp));
+ /* TODO: props->max_qp_wr = */
+ /* TODO: props->device_cap_flags */
+ props->max_sge = rblock->max_sge;
+ props->max_sge_rd = rblock->max_sge_rd;
+ TO_MAX_INT(props->max_qp, (rblock->max_cq - rblock->cur_cq));
+ props->max_cqe = rblock->max_cqe;
+ TO_MAX_INT(props->max_mr, (rblock->max_cq - rblock->cur_mr));
+ TO_MAX_INT(props->max_pd, rblock->max_pd);
+ /* TODO: props->max_qp_rd_atom */
+ /* TODO: props->max_qp_init_rd_atom */
+ /* TODO: props->atomic_cap */
+ /* TODO: props->max_ee */
+ /* TODO: props->max_rdd */
+ props->max_mw = rblock->max_mw;
+ TO_MAX_INT(props->max_mr, (rblock->max_mw - rblock->cur_mw));
+ props->max_raw_ipv6_qp = rblock->max_raw_ipv6_qp;
+ props->max_raw_ethy_qp = rblock->max_raw_ethy_qp;
+ props->max_mcast_grp = rblock->max_mcast_grp;
+ props->max_mcast_qp_attach = rblock->max_qps_attached_mcast_grp;
+ props->max_total_mcast_qp_attach = rblock->max_qps_attached_all_mcast_grp;
+
+ TO_MAX_INT(props->max_ah, rblock->max_ah);
+
+ props->max_fmr = rblock->max_mr;
+ /* TODO: props->max_map_per_fmr */
+
+ /* TODO: props->max_srq */
+ /* TODO: props->max_srq_wr */
+ /* TODO: props->max_srq_sge */
+ props->max_srq = 0;
+ props->max_srq_wr = 0;
+ props->max_srq_sge = 0;
+
+ /* TODO: props->max_pkeys */
+ props->max_pkeys = 16;
+
+ props->local_ca_ack_delay = rblock->local_ca_ack_delay;
+
+ query_device1:
+ kfree(rblock);
+
+ query_device0:
+ EDEB_EX(7, "ret=%x", ret);
+
+ return ret;
+}
+
+int ehca_query_port(struct ib_device *ibdev,
+ u8 port, struct ib_port_attr *props)
+{
+ int ret = 0;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+
+ EDEB_EN(7, "port=%x", port);
+
+ memset(props, 0, sizeof(struct ib_port_attr));
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+
+ rblock = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (rblock == NULL) {
+ EDEB_ERR(4, "Can't allocate rblock memory.");
+ ret = -ENOMEM;
+ goto query_port0;
+ }
+
+ memset(rblock, 0, PAGE_SIZE);
+
+ if (hipz_h_query_port(shca->ipz_hca_handle, port, rblock) != H_Success) {
+ EDEB_ERR(4, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_port1;
+ }
+
+ props->state = rblock->state;
+
+ switch (rblock->max_mtu) {
+ case 0x1:
+ props->active_mtu = props->max_mtu = IB_MTU_256;
+ break;
+ case 0x2:
+ props->active_mtu = props->max_mtu = IB_MTU_512;
+ break;
+ case 0x3:
+ props->active_mtu = props->max_mtu = IB_MTU_1024;
+ break;
+ case 0x4:
+ props->active_mtu = props->max_mtu = IB_MTU_2048;
+ break;
+ case 0x5:
+ props->active_mtu = props->max_mtu = IB_MTU_4096;
+ break;
+ default:
+ EDEB_ERR(4, "Unknown MTU size: %x.", rblock->max_mtu);
+ }
+
+ props->gid_tbl_len = rblock->gid_tbl_len;
+ /* TODO: props->port_cap_flags */
+ props->max_msg_sz = rblock->max_msg_sz;
+ props->bad_pkey_cntr = rblock->bad_pkey_cntr;
+ props->qkey_viol_cntr = rblock->qkey_viol_cntr;
+ props->pkey_tbl_len = rblock->pkey_tbl_len;
+ props->lid = rblock->lid;
+ props->sm_lid = rblock->sm_lid;
+ props->lmc = rblock->lmc;
+ /* TODO: max_vl_num */
+ props->sm_sl = rblock->sm_sl;
+ props->subnet_timeout = rblock->subnet_timeout;
+ props->init_type_reply = rblock->init_type_reply;
+
+ /* TODO: props->active_width */
+ props->active_width = IB_WIDTH_12X;
+ /* TODO: props->active_speed */
+
+ /* TODO: props->phys_state */
+
+ query_port1:
+ kfree(rblock);
+
+ query_port0:
+ EDEB_EX(7, "ret=%x", ret);
+
+ return ret;
+}
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
+{
+ int ret = 0;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+
+ EDEB_EN(7, "port=%x index=%x", port, index);
+
+ if (index > 16) {
+ EDEB_ERR(4, "Invalid index: %x.", index);
+ ret = -EINVAL;
+ goto query_pkey0;
+ }
+
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+
+ rblock = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (rblock == NULL) {
+ EDEB_ERR(4, "Can't allocate rblock memory.");
+ ret = -ENOMEM;
+ goto query_pkey0;
+ }
+
+ memset(rblock, 0, PAGE_SIZE);
+
+ if (hipz_h_query_port(shca->ipz_hca_handle, port, rblock) != H_Success) {
+ EDEB_ERR(4, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_pkey1;
+ }
+
+ memcpy(pkey, &rblock->pkey_entries + index, sizeof(u16));
+
+ query_pkey1:
+ kfree(rblock);
+
+ query_pkey0:
+ EDEB_EX(7, "ret=%x", ret);
+
+ return ret;
+}
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port,
+ int index, union ib_gid *gid)
+{
+ int ret = 0;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+
+ EDEB_EN(7, "port=%x index=%x", port, index);
+
+ if (index > 255) {
+ EDEB_ERR(4, "Invalid index: %x.", index);
+ ret = -EINVAL;
+ goto query_gid0;
+ }
+
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+
+ rblock = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (rblock == NULL) {
+ EDEB_ERR(4, "Can't allocate rblock memory.");
+ ret = -ENOMEM;
+ goto query_gid0;
+ }
+
+ memset(rblock, 0, PAGE_SIZE);
+
+ if (hipz_h_query_port(shca->ipz_hca_handle, port, rblock) != H_Success) {
+ EDEB_ERR(4, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_gid1;
+ }
+
+ memcpy(&gid->raw[0], &rblock->gid_prefix, sizeof(u64));
+ memcpy(&gid->raw[8], &rblock->guid_entries[index], sizeof(u64));
+
+ query_gid1:
+ kfree(rblock);
+
+ query_gid0:
+ EDEB_EX(7, "ret=%x GID=%lx%lx", ret,
+ *(u64 *) & gid->raw[0],
+ *(u64 *) & gid->raw[8]);
+
+ return ret;
+}
+
+int ehca_modify_port(struct ib_device *ibdev,
+ u8 port, int port_modify_mask,
+ struct ib_port_modify *props)
+{
+ int ret = 0;
+
+ EDEB_EN(7, "port=%x", port);
+
+ /* Not implemented yet. */
+
+ EDEB_EX(7, "ret=%x", ret);
+
+ return ret;
+}
--- linux-2.6.16-rc4-orig/drivers/infiniband/hw/ehca/ehca_mcast.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.16-rc4/drivers/infiniband/hw/ehca/ehca_mcast.c 2006-02-28 12:41:24.000000000 +0100
@@ -0,0 +1,196 @@
+
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * mcast functions
+ *
+ * Authors: Khadija Souissi <souissik at de.ibm.com>
+ * Waleri Fomin <fomin at de.ibm.com>
+ * Reinhard Ernst <rernst at de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen at de.ibm.com>
+ * Heiko J Schick <schickhj at de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $Id: ehca_mcast.c,v 1.5 2006/02/28 11:41:24 souissi Exp $
+ */
+
+#define DEB_PREFIX "mcas"
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include "ehca_kernel.h"
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+
+#include "hcp_if.h"
+
+#define MAX_MC_LID 0xFFFE
+#define MIN_MC_LID 0xC000 /* Multicast limits */
+#define EHCA_VALID_MULTICAST_GID(gid) ((gid)[0] == 0xFF)
+#define EHCA_VALID_MULTICAST_LID(lid) (((lid) >= MIN_MC_LID) && ((lid) <= MIN_MC_LID))
+
+int ehca_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+ struct ehca_qp *my_qp = NULL;
+ struct ehca_shca *shca = NULL;
+ union ib_gid my_gid;
+ u64 hipz_rc = H_Success;
+ int retcode = 0;
+
+ EHCA_CHECK_ADR(ibqp);
+ EHCA_CHECK_ADR(gid);
+
+ my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+
+ EHCA_CHECK_QP(my_qp);
+ if (ibqp->qp_type != IB_QPT_UD) {
+ EDEB_ERR(4, "invalid qp_type %x gid, retcode=%x",
+ ibqp->qp_type, EINVAL);
+ return (-EINVAL);
+ }
+
+ shca = container_of(ibqp->pd->device, struct ehca_shca, ib_device);
+ EHCA_CHECK_ADR(shca);
+
+ if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+ EDEB_ERR(4, "gid is not valid mulitcast gid retcode=%x",
+ EINVAL);
+ return (-EINVAL);
+ } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+ EDEB_ERR(4, "lid=%x is not valid mulitcast lid retcode=%x",
+ lid, EINVAL);
+ return (-EINVAL);
+ }
+
+ memcpy(&my_gid.raw, gid->raw, sizeof(union ib_gid));
+
+ hipz_rc = hipz_h_attach_mcqp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ my_qp->ehca_qp_core.galpas.kernel,
+ lid, my_gid);
+ if (H_Success != hipz_rc) {
+ EDEB_ERR(4,
+ "ehca_qp=%p qp_num=%x hipz_h_attach_mcqp() failed "
+ "hipz_rc=%lx", my_qp, ibqp->qp_num, hipz_rc);
+ }
+ retcode = ehca2ib_return_code(hipz_rc);
+
+ EDEB_EX(7, "mcast attach retcode=%x\n"
+ "ehca_qp=%p qp_num=%x lid=%x\n"
+ "my_gid= %x %x %x %x\n"
+ " %x %x %x %x\n"
+ " %x %x %x %x\n"
+ " %x %x %x %x\n",
+ retcode, my_qp, ibqp->qp_num, lid,
+ my_gid.raw[0], my_gid.raw[1],
+ my_gid.raw[2], my_gid.raw[3],
+ my_gid.raw[4], my_gid.raw[5],
+ my_gid.raw[6], my_gid.raw[7],
+ my_gid.raw[8], my_gid.raw[9],
+ my_gid.raw[10], my_gid.raw[11],
+ my_gid.raw[12], my_gid.raw[13],
+ my_gid.raw[14], my_gid.raw[15]);
+
+ return retcode;
+}
+
+int ehca_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+ struct ehca_qp *my_qp = NULL;
+ struct ehca_shca *shca = NULL;
+ union ib_gid my_gid;
+ u64 hipz_rc = H_Success;
+ int retcode = 0;
+
+ EHCA_CHECK_ADR(ibqp);
+ EHCA_CHECK_ADR(gid);
+
+ my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+
+ EHCA_CHECK_QP(my_qp);
+ if (ibqp->qp_type != IB_QPT_UD) {
+ EDEB_ERR(4, "invalid qp_type %x gid, retcode=%x",
+ ibqp->qp_type, EINVAL);
+ return (-EINVAL);
+ }
+
+ shca = container_of(ibqp->pd->device, struct ehca_shca, ib_device);
+ EHCA_CHECK_ADR(shca);
+
+ if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+ EDEB_ERR(4, "gid is not valid mulitcast gid retcode=%x",
+ EINVAL);
+ return (-EINVAL);
+ } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+ EDEB_ERR(4, "lid=%x is not valid mulitcast lid retcode=%x",
+ lid, EINVAL);
+ return (-EINVAL);
+ }
+
+ EDEB_EN(7, "dgid=%p qp_numl=%x lid=%x",
+ gid, ibqp->qp_num, lid);
+
+ memcpy(&my_gid.raw, gid->raw, sizeof(union ib_gid));
+
+ hipz_rc = hipz_h_detach_mcqp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ my_qp->ehca_qp_core.galpas.kernel,
+ lid, my_gid);
+ if (H_Success != hipz_rc) {
+ EDEB_ERR(4,
+ "ehca_qp=%p qp_num=%x hipz_h_detach_mcqp() failed "
+ "hipz_rc=%lx", my_qp, ibqp->qp_num, hipz_rc);
+ }
+ retcode = ehca2ib_return_code(hipz_rc);
+
+ EDEB_EX(7, "mcast detach retcode=%x\n"
+ "ehca_qp=%p qp_num=%x lid=%x\n"
+ "my_gid= %x %x %x %x\n"
+ " %x %x %x %x\n"
+ " %x %x %x %x\n"
+ " %x %x %x %x\n",
+ retcode, my_qp, ibqp->qp_num, lid,
+ my_gid.raw[0], my_gid.raw[1],
+ my_gid.raw[2], my_gid.raw[3],
+ my_gid.raw[4], my_gid.raw[5],
+ my_gid.raw[6], my_gid.raw[7],
+ my_gid.raw[8], my_gid.raw[9],
+ my_gid.raw[10], my_gid.raw[11],
+ my_gid.raw[12], my_gid.raw[13],
+ my_gid.raw[14], my_gid.raw[15]);
+
+ return retcode;
+}
More information about the general
mailing list