[openfabrics-ewg] [Patch ofed1.2 1/3]libehca: cleanup and adjust mmap
Stefan Roscher
ossrosch at linux.vnet.ibm.com
Fri Jan 26 08:47:07 PST 2007
Signed-off-by: Stefan Roscher <stefan.roscher at de.ibm.com>
---
diff -Nurp libehca_old/Makefile.am libehca_new/Makefile.am
--- libehca_old/Makefile.am 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/Makefile.am 2007-01-26 14:27:43.000000000 +0100
@@ -48,6 +48,9 @@ if HAVE_IBV_DEVICE_LIBRARY_EXTENSION
src_libehca_la_LDFLAGS = -avoid-version -release @IBV_DEVICE_LIBRARY_EXTENSION@ \
-Wl,--version-script=$(srcdir)/src/libehca.map \
-lpthread -libverbs -nostdlib
+ ehcaconfdir = $(sysconfdir)/libibverbs.d
+ ehcaconf_DATA = ehca.driver
+
else
ehcalibdir = $(libdir)/infiniband
ehcalib_LTLIBRARIES = src/libehca.la
diff -Nurp libehca_old/src/ehca_asm.h libehca_new/src/ehca_asm.h
--- libehca_old/src/ehca_asm.h 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_asm.h 2007-01-26 14:27:43.000000000 +0100
@@ -37,8 +37,6 @@
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
- *
- * $Id: ehca_asm.h,v 1.1 2006/02/22 12:26:55 nguyen Exp $
*/
@@ -48,9 +46,14 @@
#if defined(CONFIG_PPC_PSERIES) || defined (__PPC64__) || defined (__PPC__)
#define clear_cacheline(adr) __asm__ __volatile("dcbz 0,%0"::"r"(adr))
+/* rmb() in 32-bit mode does a full sync, while we need a lwsync */
+#define lwsync() __asm__ __volatile__ ("lwsync" : : : "memory")
+
+#define mftb() ({ unsigned long rval; \
+ asm volatile("mftb %0" : "=r" (rval)); rval; })
+
+#define asm_sync_mem() __asm__ __volatile__ ("sync" : : : "memory")
-#elif defined(CONFIG_ARCH_S390)
-#error "unsupported yet"
#else
#error "invalid platform"
#endif
diff -Nurp libehca_old/src/ehca_everbs.h libehca_new/src/ehca_everbs.h
--- libehca_old/src/ehca_everbs.h 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_everbs.h 2007-01-26 14:27:43.000000000 +0100
@@ -44,44 +44,47 @@
#include <linux/types.h>
-/** direct access qp and send/recv completion flags (can be OR-ed)
+/*
+ * direct access qp and send/recv completion flags (can be OR-ed)
* set this option in ibv_qp_init_attr.qp_type when creating a daqp
*/
enum ehcau_completion_flag {
- DA_SEND_COMPLETION=0x20,
- DA_RECV_COMPLETION=0x40,
- DA_QP_ENABLE=0x80
+ DA_SEND_COMPLETION = 0x20,
+ DA_RECV_COMPLETION = 0x40,
+ DA_QP_ENABLE = 0x80
};
-/** da send/recv message size
+/*
+ * da send/recv message size
* set send/recv message size in ibv_ap_init_attr.cap.max_send/recv_sge
* respectively when creating a daqp
*/
enum ehcau_msg_size {
- DA_MSG_SIZE_128=0,
- DA_MSG_SIZE_256=1,
- DA_MSG_SIZE_512=2,
- DA_MSG_SIZE_1024=3,
- DA_MSG_SIZE_2048=4,
- DA_MSG_SIZE_4096=5
+ DA_MSG_SIZE_128 = 0,
+ DA_MSG_SIZE_256 = 1,
+ DA_MSG_SIZE_512 = 2,
+ DA_MSG_SIZE_1024 = 3,
+ DA_MSG_SIZE_2048 = 4,
+ DA_MSG_SIZE_4096 = 5
};
-/**
+/*
+ *
* ehcau_qp_attr_da - enhanced qp attr containing send/recv queue data
*/
struct ehcau_qp_attr_da {
- /* wqe size in bytes for send */
- __u16 send_wqe_size;
- /* total length of send queue */
- __u64 send_queue_length;
- /* send queue pointer, i.e. first wqe address */
- void *send_queue_ptr;
- /* wqe size in bytes for recv */
- __u16 recv_wqe_size;
- /* total length of recv queue */
- __u64 recv_queue_length;
- /* recv queue pointer, i.e. first wqe address */
- void *recv_queue_ptr;
+ /* wqe size in bytes for send */
+ __u16 send_wqe_size;
+ /* total length of send queue */
+ __u64 send_queue_length;
+ /* send queue pointer, i.e. first wqe address */
+ void *send_queue_ptr;
+ /* wqe size in bytes for recv */
+ __u16 recv_wqe_size;
+ /* total length of recv queue */
+ __u64 recv_queue_length;
+ /* recv queue pointer, i.e. first wqe address */
+ void *recv_queue_ptr;
};
/*
@@ -117,14 +120,18 @@ int ehcau_write_rwqe(void *wqe, struct i
* returns send queue's next entry ptr
*/
#define GET_SQUEUE_NEXT_ENTRY_PTR(current_ptr, qp_attr) \
- (((void*)current_ptr+(qp_attr).send_wqe_size>=(qp_attr).send_queue_ptr+(qp_attr).send_queue_length) ? \
- (qp_attr).send_queue_ptr : (void*)current_ptr+(qp_attr).send_wqe_size)
+ (((void *)current_ptr + (qp_attr).send_wqe_size >= \
+ (qp_attr).send_queue_ptr + (qp_attr).send_queue_length) ? \
+ (qp_attr).send_queue_ptr : (void *)current_ptr + \
+ (qp_attr).send_wqe_size)
/*
* returns recv queue's next entry ptr
*/
#define GET_RQUEUE_NEXT_ENTRY_PTR(current_ptr, qp_attr) \
- (((void*)current_ptr+(qp_attr).recv_wqe_size>=(void*)(qp_attr).recv_queue_ptr+(qp_attr).recv_queue_length) ? \
- (qp_attr).recv_queue_ptr : (void*)current_ptr+(qp_attr).recv_wqe_size)
+ (((void *)current_ptr + (qp_attr).recv_wqe_size >= \
+ (void *)(qp_attr).recv_queue_ptr + (qp_attr).recv_queue_length) ? \
+ (qp_attr).recv_queue_ptr : (void *)current_ptr + \
+ (qp_attr).recv_wqe_size)
#endif /* __EHCA_EVERBS_H__ */
diff -Nurp libehca_old/src/ehca_galpa.h libehca_new/src/ehca_galpa.h
--- libehca_old/src/ehca_galpa.h 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_galpa.h 2007-01-26 14:27:43.000000000 +0100
@@ -37,38 +37,36 @@
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
- *
- * $Id: ehca_galpa.h,v 1.1 2006/02/22 12:26:55 nguyen Exp $
*/
#ifndef __EHCA_GALPA_H__
#define __EHCA_GALPA_H__
-/* eHCA page (mapped into p-memory)
- resource to access eHCA register pages in CPU address space
-*/
+/*
+ * eHCA page (mapped into p-memory)
+ * resource to access eHCA register pages in CPU address space
+ */
struct h_galpa {
u64 fw_handle;
/* for pSeries this is a 64bit memory address where
- I/O memory is mapped into CPU address space (kv) */
+ * I/O memory is mapped into CPU address space (kv)
+ */
};
-/**
- resource to access eHCA address space registers, all types
-*/
+/* resource to access eHCA address space registers, all types */
struct h_galpas {
u32 pid; /*PID of userspace galpa checking */
struct h_galpa user; /* user space accessible resource,
- set to 0 if unused */
+ * set to 0 if unused
+ */
struct h_galpa kernel; /* kernel space accessible resource,
- set to 0 if unused */
+ * set to 0 if unused
+ */
};
-/** @brief store value at offset into galpa, will be inline function
- */
+/* store value at offset into galpa, will be inline function */
void hipz_galpa_store(struct h_galpa galpa, u32 offset, u64 value);
-/** @brief return value from offset in galpa, will be inline function
- */
+/* return value from offset in galpa, will be inline function */
u64 hipz_galpa_load(struct h_galpa galpa, u32 offset);
-#endif /* __EHCA_GALPA_H__ */
+#endif /* __EHCA_GALPA_H__ */
diff -Nurp libehca_old/src/ehca_qes.h libehca_new/src/ehca_qes.h
--- libehca_old/src/ehca_qes.h 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_qes.h 2007-01-26 14:27:43.000000000 +0100
@@ -38,36 +38,33 @@
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
- *
- * $Id: ehca_qes.h,v 1.1 2006/02/22 12:26:55 nguyen Exp $
*/
#ifndef _EHCA_QES_H_
#define _EHCA_QES_H_
-/** DON'T include any kernel related files here!!!
+/*
+ * DON'T include any kernel related files here!!!
* This file is used commonly in user and kernel space!!!
*/
-/**
- * virtual scatter gather entry to specify remote adresses with length
- */
+/* virtual scatter gather entry to specify remote adresses with length */
struct ehca_vsgentry {
u64 vaddr;
u32 lkey;
u32 length;
};
-#define GRH_FLAG_MASK EHCA_BMASK_IBM(7,7)
-#define GRH_IPVERSION_MASK EHCA_BMASK_IBM(0,3)
-#define GRH_TCLASS_MASK EHCA_BMASK_IBM(4,12)
-#define GRH_FLOWLABEL_MASK EHCA_BMASK_IBM(13,31)
-#define GRH_PAYLEN_MASK EHCA_BMASK_IBM(32,47)
-#define GRH_NEXTHEADER_MASK EHCA_BMASK_IBM(48,55)
-#define GRH_HOPLIMIT_MASK EHCA_BMASK_IBM(56,63)
+#define GRH_FLAG_MASK EHCA_BMASK_IBM(7, 7)
+#define GRH_IPVERSION_MASK EHCA_BMASK_IBM(0, 3)
+#define GRH_TCLASS_MASK EHCA_BMASK_IBM(4, 12)
+#define GRH_FLOWLABEL_MASK EHCA_BMASK_IBM(13, 31)
+#define GRH_PAYLEN_MASK EHCA_BMASK_IBM(32, 47)
+#define GRH_NEXTHEADER_MASK EHCA_BMASK_IBM(48, 55)
+#define GRH_HOPLIMIT_MASK EHCA_BMASK_IBM(56, 63)
-/**
+/*
* Unreliable Datagram Address Vector Format
* see IBTA Vol1 chapter 8.3 Global Routing Header
*/
@@ -183,8 +180,7 @@ struct ehca_wqe {
} ud_avp;
struct {
struct ehca_ud_av ud_av;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES -
- 2];
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES - 2];
} ud_av;
struct {
u64 reserved0;
@@ -224,10 +220,10 @@ struct ehca_wqe {
};
-#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0,0)
-#define WC_IMM_DATA EHCA_BMASK_IBM(1,1)
-#define WC_GRH_PRESENT EHCA_BMASK_IBM(2,2)
-#define WC_SE_BIT EHCA_BMASK_IBM(3,3)
+#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0, 0)
+#define WC_IMM_DATA EHCA_BMASK_IBM(1, 1)
+#define WC_GRH_PRESENT EHCA_BMASK_IBM(2, 2)
+#define WC_SE_BIT EHCA_BMASK_IBM(3, 3)
struct ehca_cqe {
u64 work_request_id;
@@ -262,7 +258,7 @@ struct ehca_eqe {
struct ehca_mrte {
u64 starting_va;
- u64 length; /* length of memory region in bytes*/
+ u64 length; /* length of memory region in bytes */
u32 pd;
u8 key_instance;
u8 pagesize;
diff -Nurp libehca_old/src/ehca_u_mrmw.c libehca_new/src/ehca_u_mrmw.c
--- libehca_old/src/ehca_u_mrmw.c 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_u_mrmw.c 2007-01-26 14:27:43.000000000 +0100
@@ -38,30 +38,26 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-
-#define DEB_PREFIX "umrw"
-
#include <stdlib.h>
#include <infiniband/driver.h>
#include "ehca_utools.h"
-/*----------------------------------------------------------------------*/
struct ibv_mr *ehcau_reg_mr(struct ibv_pd *pd,
void *addr,
size_t length,
enum ibv_access_flags access)
{
- int ret = 0;
- struct ibv_mr *mr = NULL;
+ int ret;
+ struct ibv_mr *mr;
struct ibv_reg_mr cmd;
- EDEB_EN(7, "pd=%p addr=%p length=%lx access=%x",
- pd, addr, (unsigned long)length, access);
+ ehca_dbg(pd->context->device, "pd=%p addr=%p length=%lx access=%x",
+ pd, addr, (unsigned long)length, access);
mr = malloc(sizeof(struct ibv_mr));
if (!mr) {
- EDEB_ERR(4, "no memory");
+ ehca_err(pd->context->device, "no memory");
return NULL;
}
@@ -78,30 +74,30 @@ struct ibv_mr *ehcau_reg_mr(struct ibv_p
access, mr, &cmd, sizeof(struct ibv_reg_mr));
#endif
if (ret) {
- EDEB_ERR(4, "ibv_cmd_reg_mr ret=%x", ret);
+ ehca_err(pd->context->device, "ibv_cmd_reg_mr ret=%x", ret);
free(mr);
return NULL;
}
- EDEB_EX(7, "mr=%p", mr);
+ ehca_dbg(pd->context->device, "mr=%p", mr);
return mr;
}
-/*----------------------------------------------------------------------*/
int ehcau_dereg_mr(struct ibv_mr *mr)
{
- int ret = 0;
+ int ret;
- EDEB_EN(7, "mr=%p", mr);
+ ehca_dbg(mr->context->device, "mr=%p", mr);
ret = ibv_cmd_dereg_mr(mr);
if (ret) {
- EDEB_ERR(4, "ibv_cmd_dereg_mr failed, ret=%x", ret);
+ ehca_err(mr->context->device, "ibv_cmd_dereg_mr failed, ret=%x",
+ ret);
return ret;
}
free(mr);
- EDEB_EX(7, "");
+ ehca_gen_dbg("mr=%p", mr);
return 0;
}
diff -Nurp libehca_old/src/ehca_uclasses.h libehca_new/src/ehca_uclasses.h
--- libehca_old/src/ehca_uclasses.h 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_uclasses.h 2007-01-26 14:27:43.000000000 +0100
@@ -118,7 +118,8 @@ struct ibv_mr *ehcau_reg_mr(struct ibv_p
int ehcau_dereg_mr(struct ibv_mr *mr);
struct ibv_cq *ehcau_create_cq(struct ibv_context *context, int cqe,
- struct ibv_comp_channel *channel, int comp_vector);
+ struct ibv_comp_channel *channel,
+ int comp_vector);
int ehcau_destroy_cq(struct ibv_cq *cq);
@@ -149,13 +150,13 @@ int ehcau_attach_mcast(struct ibv_qp *qp
int ehcau_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
int ehcau_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
- enum ibv_qp_attr_mask attr_mask, struct ibv_qp_init_attr* init_attr);
+ enum ibv_qp_attr_mask attr_mask,
+ struct ibv_qp_init_attr* init_attr);
-/**
+/*
* resp structs from kernel space
*/
struct ipzu_queue_resp {
- u64 queue; /* points to first queue entry */
u32 qe_size; /* queue entry size */
u32 act_nr_of_sg;
u32 queue_length; /* queue length allocated in bytes */
@@ -169,7 +170,6 @@ struct ehcau_create_cq_resp {
u32 cq_number;
u32 token;
struct ipzu_queue_resp ipz_queue;
- struct h_galpas galpas;
};
struct ehcau_create_qp_resp {
@@ -183,7 +183,6 @@ struct ehcau_create_qp_resp {
u32 dummy; /* padding for 8 byte alignment */
struct ipzu_queue_resp ipz_squeue;
struct ipzu_queue_resp ipz_rqueue;
- struct h_galpas galpas;
};
struct ehcau_qp* ehca_cq_get_qp(struct ehcau_cq *cq, int qp_num);
@@ -198,8 +197,8 @@ inline static void ehcau_lock(unsigned i
" stwcx. %1,0,%2\n"
" bne- 1b\n"
" lwsync\n"
- : "=&r" (temp1), "+&r" (setval) : "r" (lock_var) : "cr0", "memo\
-ry");
+ : "=&r" (temp1), "+&r" (setval) : "r" (lock_var) : "cr0",
+ "memory");
}
inline static void ehcau_unlock(unsigned int * lock_var) {
diff -Nurp libehca_old/src/ehca_uinit.c libehca_new/src/ehca_uinit.c
--- libehca_old/src/ehca_uinit.c 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_uinit.c 2007-01-26 14:27:43.000000000 +0100
@@ -40,20 +40,24 @@
*/
#if HAVE_CONFIG_H
-# include <config.h>
+#include <config.h>
#endif /* HAVE_CONFIG_H */
-#include <infiniband/driver.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <pthread.h>
+#ifndef HAVE_IBV_READ_SYSFS_FILE
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
+#endif
+
+#ifdef HAVE_SYSFS_LIBSYSFS_H
#include <sysfs/libsysfs.h>
+#endif
#include "ehca_uclasses.h"
@@ -81,19 +85,18 @@ static struct ibv_context_ops ehcau_ctx_
.detach_mcast = ehcau_detach_mcast
};
-static struct ibv_context *ehcau_alloc_context(struct ibv_device *ibdev, int cmd_fd)
+static struct ibv_context *ehcau_alloc_context(struct ibv_device *ibdev,
+ int cmd_fd)
{
- struct ehcau_context *my_context = NULL;
+ struct ehcau_context *my_context;
struct ibv_get_context cmd;
struct ibv_get_context_resp resp;
- EDEB_EN(7, "device=%p cmd_fd=%x",
- ibdev, cmd_fd);
+ ehca_dbg(ibdev, "device=%p cmd_fd=%x", ibdev, cmd_fd);
my_context = (struct ehcau_context *)malloc(sizeof(*my_context));
- if (my_context == NULL) {
+ if (!my_context)
return NULL;
- }
memset(my_context, 0, sizeof(*my_context));
my_context->ibv_ctx.cmd_fd = cmd_fd;
@@ -102,7 +105,8 @@ static struct ibv_context *ehcau_alloc_c
memset(&resp, 0, sizeof(resp));
if (ibv_cmd_get_context(&my_context->ibv_ctx, &cmd,
sizeof(cmd), &resp, sizeof(resp))) {
- EDEB_ERR(4, "ibv_cmd_get_context() failed device=%p", ibdev);
+ ehca_err(ibdev, "ibv_cmd_get_context() failed device=%p",
+ ibdev);
goto alloc_context_exit0;
}
@@ -110,42 +114,42 @@ static struct ibv_context *ehcau_alloc_c
my_context->ibv_ctx.device = ibdev;
my_context->ibv_ctx.ops = ehcau_ctx_ops;
- EDEB_EX(7, "retcode=context=%p device=%p", &my_context->ibv_ctx, ibdev);
+ ehca_dbg(ibdev, "retcode=context=%p device=%p",
+ &my_context->ibv_ctx, ibdev);
return &my_context->ibv_ctx;
alloc_context_exit0:
free(my_context);
- EDEB_EX(7, "retcode=context=NULL device=%p", ibdev);
+ ehca_dbg(ibdev, "retcode=context=NULL device=%p", ibdev);
return NULL;
}
static void ehcau_free_context(struct ibv_context *ibctx)
{
- struct ehcau_context *my_context = NULL;
- EDEB_EN(7, "context=%p", ibctx);
- if (ibctx == NULL) {
- EDEB_ERR(4, "context pointer is NULL");
- } else {
+ struct ehcau_context *my_context;
+ ehca_dbg(ibctx->device, "context=%p", ibctx);
+ if (!ibctx)
+ ehca_err(ibctx->device, "context pointer is NULL");
+ else {
my_context = container_of(ibctx, struct ehcau_context, ibv_ctx);
free(my_context);
}
- EDEB_EX(7, "context=%p", ibctx);
+ ehca_dbg(ibctx->device, "context=%p", ibctx);
}
static char *str_strip(char *str)
{
- char *last=NULL;
+ char *last;
/* strip left white spaces */
- while (*str==' ' || *str=='\t' || *str=='\n' || *str=='\r') {
+ while (*str == ' ' || *str == '\t' || *str == '\n' || *str == '\r')
str++;
- }
/* strip right white spaces */
- last=str+(strlen(str)-1);
+ last = str + (strlen(str) - 1);
while (last>=str &&
- (*last==' ' || *last=='\t' || *last=='\n' || *last=='\r')) {
+ (*last == ' ' || *last == '\t' ||
+ *last == '\n' || *last == '\r'))
last--;
- }
- *(last+1)=0;
+ *(last + 1) = 0;
return str;
}
@@ -185,19 +189,19 @@ static int ibv_read_sysfs_file(const cha
#endif /* HAVE_IBV_READ_SYSFS_FILE */
static struct ibv_device *ehca_driver_init(const char *uverbs_sys_path,
- int abi_version)
+ int abi_version)
{
struct ehcau_device *my_dev = NULL;
char value[64];
int num_ports = 0;
- EDEB_EN(7, "");
+ ehca_gen_dbg("uverbs_sys_path=%s", uverbs_sys_path);
if (ibv_read_sysfs_file(uverbs_sys_path, "device/name",
value, sizeof value) < 0)
return NULL;
- if (strcmp("lhca", str_strip(value)) != 0)
+ if (strcmp("lhca", str_strip(value)))
return NULL;
if (ibv_read_sysfs_file(uverbs_sys_path, "device/num_ports",
@@ -206,9 +210,8 @@ static struct ibv_device *ehca_driver_in
sscanf(value, "%i", &num_ports);
- if (num_ports < 1) {
+ if (num_ports < 1)
return NULL;
- }
/* here we've got our dev */
my_dev = malloc(sizeof *my_dev);
@@ -220,11 +223,19 @@ static struct ibv_device *ehca_driver_in
my_dev->ibv_dev.ops = ehcau_dev_ops;
- EDEB_EX(7, "ehcau_device=%p", my_dev);
+ ehca_gen_dbg("ehcau_device=%p", my_dev);
return &my_dev->ibv_dev;
}
+/*
+ * Export the old libsysfs sysfs_class_device-based driver entry point
+ * if libsysfs headers are installed. It doesn't hurt to export it,
+ * even if libibverbs is new enough not to use it; but if libsysfs
+ * headers are not installed, we can assume that the version of
+ * libibverbs we are building against is new enough not to use
+ * openib_driver_init().
+ */
#ifndef HAVE_IBV_REGISTER_DRIVER
struct ibv_device *openib_driver_init(struct sysfs_class_device *sysdev)
{
@@ -237,98 +248,17 @@ struct ibv_device *openib_driver_init(st
return ehca_driver_init(sysdev->path, abi_ver);
}
-#endif /* HAVE_IBV_REGISTER_DRIVER */
+#endif
-/** @brief module initialization
- */
-int libehca_trlevel = 5;
-FILE *libehca_fh = NULL;
-#define LIBEHCA_DEFAULT_CFGFILE "/usr/local/libehca/etc/libehca.conf"
-#define CFG_TOKEN_TRLEVEL "log.trlevel"
-#define CFG_TOKEN_FILENAME "log.filename"
+int libehca_trace_on = 0; /* default is no debug */
+#define CFG_VAR_TRACE_ON "LIBEHCA_TRACE_ON"
void __attribute__ ((constructor)) ehcau_init(void)
{
- char *cfgfilename = LIBEHCA_DEFAULT_CFGFILE;
- FILE *cfg_fh = NULL;
- char linebuf[1024];
- char *cur_line = NULL;
- char *logfilename = NULL;
-
- cfg_fh = fopen(cfgfilename, "r");
- if (cfg_fh == NULL) {
- libehca_trlevel = -1;
- fprintf(stderr, "Could not read config file \"%s\"\n",
- cfgfilename);
- return;
- }
- /* parse for log cfg entries per line:
- "log.trlevel <digit>"
- "log.filename <log file path>"
- lines starting with # will be ignored
- */
- while ((cur_line=fgets(linebuf, sizeof(linebuf)-1, cfg_fh))!=NULL) {
- /* strip white spaces, LF. CR */
- cur_line = str_strip(cur_line);
- if (*cur_line==0 || *cur_line=='#') {
- continue;
- }
- /* parse for token and value */
- if (strstr(cur_line, CFG_TOKEN_TRLEVEL)==cur_line) {
- char *trlevel=cur_line+strlen(CFG_TOKEN_TRLEVEL);
- if (*trlevel!=' ' && *trlevel!='\t') {
- fprintf(stderr, "Invalid token \"%s\"\n",
- cur_line);
- continue;
- }
- trlevel=str_strip(trlevel);
- if (*trlevel!=0 && *trlevel>='0' && *trlevel<='9') {
- libehca_trlevel=(*trlevel)-'0';
- } else {
- fprintf(stderr, "Invalid trace level \"%s\"\n",
- cur_line);
- }
- } else if (strstr(cur_line, CFG_TOKEN_FILENAME)==cur_line) {
- char *filename=cur_line+strlen(CFG_TOKEN_FILENAME);
- if (*filename!=' ' && *filename!='\t') {
- fprintf(stderr, "Invalid token \"%s\"\n",
- cur_line);
- continue;
- }
- filename=str_strip(filename);
- if (*filename==0) {
- continue;
- }
- logfilename=malloc(strlen(filename)+1);
- if (logfilename==NULL) {
- fprintf(stderr, "Out of memory error");
- exit(-ENOMEM);
- }
- strcpy(logfilename, filename);
- } else { /* invalid token */
- fprintf(stderr, "Invalid token \"%s\"\n", cur_line);
- }
- } /* eof while current_line */
- fclose(cfg_fh);
- /* open logfile if given and found */
- if (logfilename!=NULL) {
- if (strcmp(logfilename, "STDOUT")==0) {
- libehca_fh = stdout;
- } else if (strcmp(logfilename, "STDERR")==0) {
- libehca_fh = stderr;
- } else {
- libehca_fh = fopen(logfilename, "a");
- if (libehca_fh == NULL) {
- fprintf(stderr, "Could not open "
- "logfile \"%s\"\n", logfilename);
- }
- }
- free(logfilename);
- }
- if (libehca_fh==NULL) {
- libehca_trlevel = -1;
- } else {
- fprintf(libehca_fh, "tracelevel is:%i\n", libehca_trlevel);
+ char *value = getenv(CFG_VAR_TRACE_ON);
+ if (value) {
+ value = str_strip(value);
+ libehca_trace_on = (*value) - '0';
}
#ifdef HAVE_IBV_REGISTER_DRIVER
diff -Nurp libehca_old/src/ehca_umain.c libehca_new/src/ehca_umain.c
--- libehca_old/src/ehca_umain.c 2007-01-26 14:27:03.000000000 +0100
+++ libehca_new/src/ehca_umain.c 2007-01-26 14:27:43.000000000 +0100
@@ -40,7 +40,6 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
- * $Id: ehca_umain.c,v 1.14 2006/06/07 10:38:00 nguyen Exp $
*/
#include <infiniband/driver.h>
@@ -48,7 +47,7 @@
#include "ehca_utools.h"
#include "ehca_everbs.h"
#include "ipzu_pt_fn.h"
-
+#include "hipz_hw.h"
#include <stdio.h>
#include <stdlib.h>
@@ -57,49 +56,50 @@
#include <sys/mman.h>
#include <netinet/in.h>
+#define EHCA_PAGESIZE 4096
+
static int cq_assign_qp(struct ehcau_cq *cq, struct ehcau_qp *qp)
{
unsigned int qp_num = qp->real_qp_num;
- unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
+ unsigned int key = qp_num & (QP_HASHTAB_LEN - 1);
ehcau_lock(&cq->lockvar);
- LIST_INSERT_HEAD(&cq->qp_hashtab[key],
- qp, list_entries);
+ LIST_INSERT_HEAD(&cq->qp_hashtab[key], qp, list_entries);
ehcau_unlock(&cq->lockvar);
- EDEB(7, "cq_num=%x real_qp_num=%x", cq->cq_number, qp_num);
+ ehca_gen_dbg("cq_num=%x real_qp_num=%x", cq->cq_number, qp_num);
return 0;
}
static int cq_unassign_qp(struct ehcau_cq *cq, unsigned int real_qp_num)
{
int ret = -EINVAL;
- unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
- struct ehcau_qp *qp = NULL;
+ unsigned int key = real_qp_num & (QP_HASHTAB_LEN - 1);
+ struct ehcau_qp *qp;
ehcau_lock(&cq->lockvar);
- for (qp = cq->qp_hashtab[key].lh_first;
- qp!=NULL; qp = qp->list_entries.le_next) {
+ for (qp = cq->qp_hashtab[key].lh_first; qp;
+ qp = qp->list_entries.le_next) {
if (qp->real_qp_num == real_qp_num) {
LIST_REMOVE(qp, list_entries);
- EDEB(7, "removed qp from cq .cq_num=%x real_qp_num=%x",
- cq->cq_number, real_qp_num);
+ ehca_gen_dbg("removed qp from cq .cq_num=%x "
+ "real_qp_num=%x",
+ cq->cq_number, real_qp_num);
ret = 0;
break;
}
}
ehcau_unlock(&cq->lockvar);
- if (ret!=0) {
- EDEB_ERR(4, "qp not found cq_num=%x real_qp_num=%x",
- cq->cq_number, real_qp_num);
- }
+ if (ret)
+ ehca_gen_err("qp not found cq_num=%x real_qp_num=%x",
+ cq->cq_number, real_qp_num);
return ret;
}
struct ehcau_qp* ehca_cq_get_qp(struct ehcau_cq *cq, int real_qp_num)
{
struct ehcau_qp *ret = NULL;
- unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
- struct ehcau_qp *qp = NULL;
- for (qp = cq->qp_hashtab[key].lh_first;
- qp!=NULL; qp = qp->list_entries.le_next) {
+ unsigned int key = real_qp_num & (QP_HASHTAB_LEN - 1);
+ struct ehcau_qp *qp;
+ for (qp = cq->qp_hashtab[key].lh_first; qp;
+ qp = qp->list_entries.le_next) {
if (qp->real_qp_num == real_qp_num) {
ret = qp;
break;
@@ -111,59 +111,45 @@ struct ehcau_qp* ehca_cq_get_qp(struct e
int ehcau_query_device(struct ibv_context *context,
struct ibv_device_attr *device_attr)
{
- int ret = 0;
+ int ret;
struct ibv_query_device cmd;
uint64_t raw_fw_ver;
- EDEB_EN(7, "context=%p", context);
-
memset(&cmd, 0, sizeof(cmd));
ret = ibv_cmd_query_device(context, device_attr, &raw_fw_ver, &cmd,
sizeof(cmd));
+ if (ret)
+ ehca_gen_err("ibv_cmd_query_device() failed, ret=%x", ret);
- if (ret) {
- EDEB_ERR(4, "ibv_cmd_query_device() failed, ret=%x", ret);
- }
-
- EDEB_EX(7, "context=%p", context);
return ret;
}
int ehcau_query_port(struct ibv_context *context, uint8_t port,
struct ibv_port_attr *attr)
{
- int ret = 0;
+ int ret;
struct ibv_query_port cmd;
- EDEB_EN(7, "context=%p port=%x", context, port);
-
memset(&cmd, 0, sizeof(cmd));
ret = ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
+ if (ret)
+ ehca_gen_err("ibv_cmd_query_port failed ret=%x context=%p "
+ "port=%x", ret, context, port);
- if (ret) {
- EDEB_ERR(4,
- "ibv_cmd_query_port failed ret=%x context=%p port=%x",
- ret, context, port);
- }
-
- EDEB_EX(7, "ret=%x context=%p port=%x", ret, context, port);
return ret;
-
}
struct ibv_pd *ehcau_alloc_pd(struct ibv_context *context)
{
- struct ibv_pd *pd = NULL;
+ struct ibv_pd *pd;
struct ibv_alloc_pd cmd;
struct ibv_alloc_pd_resp resp;
- int ret = 0;
-
- EDEB_EN(7, "context=%p", context);
+ int ret;
pd = malloc(sizeof(*pd));
- if (pd == NULL) {
- EDEB_ERR(4, "Out of memory to alloc ehcau_pd "
- "context=%p", context);
+ if (!pd) {
+ ehca_gen_err("Out of memory to alloc ehcau_pd "
+ "context=%p", context);
return NULL;
}
@@ -173,31 +159,29 @@ struct ibv_pd *ehcau_alloc_pd(struct ibv
ret = ibv_cmd_alloc_pd(context, pd,
&cmd, sizeof(cmd), &resp, sizeof(resp));
- if (ret != 0) {
- EDEB_ERR(4, "ibv_cmd_alloc_pd() failed ret=%x context=%p",
- ret, context);
+ if (ret) {
+ ehca_err(context->device, "ibv_cmd_alloc_pd() failed ret=%x "
+ "context=%p", ret, context);
free(pd);
return NULL;
}
- EDEB_EX(7, "context=%p ret=ehcau_pd=%p", context, pd);
- return (pd);
+ return pd;
}
int ehcau_dealloc_pd(struct ibv_pd *pd)
{
- int ret = 0;
+ int ret;
- EHCA_CHECK_ADR(pd);
- EDEB_EN(7, "pd=%p", pd);
ret = ibv_cmd_dealloc_pd(pd);
if (ret) {
- EDEB_ERR(4, "ibv_cmd_dealloc_pd failed ret=%x pd=%p", ret, pd);
+ ehca_err(pd->context->device,
+ "ibv_cmd_dealloc_pd failed ret=%x pd=%p", ret, pd);
return ret;
}
free(pd);
- EDEB_EX(7, "pd=%p", pd);
- return (0);
+
+ return 0;
}
struct ibv_cq *ehcau_create_cq(struct ibv_context *context, int cqe,
@@ -205,14 +189,14 @@ struct ibv_cq *ehcau_create_cq(struct ib
{
struct ibv_create_cq cmd;
struct ehcau_create_cq_resp resp;
- struct ehcau_cq *my_cq = NULL;
- int ret = 0;
+ struct ehcau_cq *my_cq;
+ int ret;
int i;
- EDEB_EN(7, "context=%p cqe=%x", context, cqe);
my_cq = malloc(sizeof(*my_cq));
if (!my_cq) {
- EDEB_ERR(4, "Out of memory context=%p cqe=%x", context, cqe);
+ ehca_err(context->device, "Out of memory context=%p cqe=%x",
+ context, cqe);
return NULL;
}
@@ -223,83 +207,119 @@ struct ibv_cq *ehcau_create_cq(struct ib
&cmd, sizeof(cmd),
&resp.ibv_resp, sizeof(resp));
if (ret) {
- EDEB_ERR(4, "ibv_cmd_create_cq() failed "
+ ehca_err(context->device, "ibv_cmd_create_cq() failed "
"ret=%x context=%p cqe=%x", ret, context, cqe);
goto create_cq_exit0;
}
- for (i=0; i<QP_HASHTAB_LEN; i++) {
+ for (i = 0; i < QP_HASHTAB_LEN; i++)
LIST_INIT(&my_cq->qp_hashtab[i]);
- }
/* copy data returned from kernel */
my_cq->cq_number = resp.cq_number;
my_cq->token = resp.token;
- /* right most cast is required to avoid gcc warning in 32 bit mode */
- my_cq->ipz_queue.queue = (u8*)(long)resp.ipz_queue.queue;
- my_cq->ipz_queue.current_q_addr = (u8*)(long)resp.ipz_queue.queue;
+ my_cq->ipz_queue.queue = mmap64(NULL, resp.ipz_queue.queue_length,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ context->cmd_fd,
+ ((u64)my_cq->token << 32) | 0x12000000);
+ if (!my_cq->ipz_queue.queue) {
+ ehca_err(context->device, "mmap64() failed cq_num=%x",
+ my_cq->cq_number);
+ goto create_cq_exit1;
+ }
+ my_cq->ipz_queue.current_q_addr = my_cq->ipz_queue.queue;
my_cq->ipz_queue.qe_size = resp.ipz_queue.qe_size;
my_cq->ipz_queue.act_nr_of_sg = resp.ipz_queue.act_nr_of_sg;
my_cq->ipz_queue.queue_length = resp.ipz_queue.queue_length;
my_cq->ipz_queue.pagesize = resp.ipz_queue.pagesize;
my_cq->ipz_queue.toggle_state = resp.ipz_queue.toggle_state;
- my_cq->galpas = resp.galpas;
+ /* right most cast is required to avoid gcc warning in 32 bit mode */
+ my_cq->galpas.kernel.fw_handle = (u64)(unsigned long)
+ mmap64(NULL, EHCA_PAGESIZE,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ context->cmd_fd,
+ ((u64)my_cq->token << 32) | 0x11000000);
+ if (!my_cq->galpas.kernel.fw_handle) {
+ ehca_err(context->device, "mmap64() failed cq_num=%x",
+ my_cq->cq_number);
+ goto create_cq_exit2;
+ }
/* access queue mem to fill page cache */
- memset(my_cq->ipz_queue.queue, 0,
- my_cq->ipz_queue.queue_length);
+ memset(my_cq->ipz_queue.queue, 0, my_cq->ipz_queue.queue_length);
- EDEB_EX(7, "ehcau_cq=%p cqn=%x token=%x "
- "ipz_queue.galpa=%p ipz_queue.adr=%p", my_cq,
- my_cq->cq_number, my_cq->token,
- (u64 *) (unsigned long)my_cq->galpas.kernel.fw_handle,
- (u64 *) my_cq->ipz_queue.queue);
+ ehca_dbg(context->device, "ehcau_cq=%p cqn=%x token=%x "
+ "ipz_queue.galpa=%p ipz_queue.adr=%p", my_cq,
+ my_cq->cq_number, my_cq->token,
+ (u64*)(unsigned long)my_cq->galpas.kernel.fw_handle,
+ (u64*)my_cq->ipz_queue.queue);
return &my_cq->ib_cq;
- create_cq_exit0:
- EDEB_EX(4, "An error has occured context=%p cqe=%x",
- context, cqe);
+create_cq_exit2:
+ ret = munmap(my_cq->ipz_queue.queue, my_cq->ipz_queue.queue_length);
+ if (ret)
+ ehca_err(context->device, "munmap() failed rc=%x cq_num=%x queue=%p",
+ ret, my_cq->cq_number, my_cq->ipz_queue.queue);
+
+create_cq_exit1:
+ ret = ibv_cmd_destroy_cq(&my_cq->ib_cq);
+ if (ret)
+ ehca_err(context->device, "ibv_cmd_destroy_cq() failed "
+ "ret=%x ehcau_cq=%p cq_num=%x",
+ ret, my_cq, my_cq->cq_number);
+
+create_cq_exit0:
+ ehca_err(context->device, "An error has occured context=%p cqe=%x",
+ context, cqe);
free(my_cq);
return NULL;
}
int ehcau_destroy_cq(struct ibv_cq *cq)
{
- struct ehcau_cq *my_cq = NULL;
- int cq_num = 0;
- int ret = 0;
- EHCA_CHECK_ADR(cq);
+ struct ehcau_cq *my_cq;
+ int cq_num;
+ int ret;
+
my_cq = container_of(cq, struct ehcau_cq, ib_cq);
cq_num = my_cq->cq_number;
- EDEB_EN(7, "ehcau_cq=%p cq_num=%x", my_cq, cq_num);
+ ret = munmap(my_cq->ipz_queue.queue, my_cq->ipz_queue.queue_length);
+ if (ret) {
+ ehca_err(cq->context->device, "munmap() failed rc=%x cq_num=%x "
+ "queue=%p", ret, cq_num, my_cq->ipz_queue.queue);
+ return ret;
+ }
+ ret = munmap((void*)(unsigned long)my_cq->galpas.kernel.fw_handle, EHCA_PAGESIZE);
+ if (ret) {
+ ehca_err(cq->context->device, "munmap() failed rc=%x cq_num=%x "
+ "fw_handle=%p", ret, cq_num,
+ (void*)(unsigned long)my_cq->galpas.kernel.fw_handle);
+ return ret;
+ }
ret = ibv_cmd_destroy_cq(cq);
if (ret) {
- EDEB_ERR(4, "ibv_cmd_destroy_cq() failed ret=%x "
- "ehcau_cq=%p cq_num=%x", ret, my_cq, cq_num);
+ ehca_err(cq->context->device, "ibv_cmd_destroy_cq() failed "
+ "ret=%x ehcau_cq=%p cq_num=%x", ret, my_cq, cq_num);
return ret;
}
free(my_cq);
- EDEB_EX(7, "ehcau_cq=%p cq_num=%x", my_cq, cq_num);
+ ehca_gen_dbg("ehcau_cq=%p cq_num=%x", my_cq, cq_num);
return 0;
}
struct ibv_qp *ehcau_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
{
- int ret = 0;
- struct ehcau_qp *my_qp = NULL;
+ int ret;
+ struct ehcau_qp *my_qp;
struct ibv_create_qp cmd;
struct ehcau_create_qp_resp resp;
- struct ibv_context *context = NULL;
- int ret2 = 0;
+ struct ibv_context *context;
- EHCA_CHECK_ADR_P(pd);
- EHCA_CHECK_ADR_P(pd->context);
context = pd->context;
- EDEB_EN(7, "pd=%p attr=%p", pd, attr);
-
my_qp = malloc(sizeof(*my_qp));
if (!my_qp) {
- EDEB_ERR(4, "Out of memory to alloc qp pd=%p", pd);
+ ehca_err(pd->context->device, "Out of memory to alloc qp pd=%p",
+ pd);
return NULL;
}
@@ -311,9 +331,9 @@ struct ibv_qp *ehcau_create_qp(struct ib
&cmd, sizeof(cmd),
&resp.ibv_resp, sizeof resp);
- if (ret != 0) {
- EDEB_ERR(4, "ibv_cmd_create_qp() failed ret=%x pd=%p",
- ret, pd);
+ if (ret) {
+ ehca_err(pd->context->device, "ibv_cmd_create_qp() failed "
+ "ret=%x pd=%p", ret, pd);
goto create_qp_exit0;
}
/* copy data returned from kernel */
@@ -323,22 +343,48 @@ struct ibv_qp *ehcau_create_qp(struct ib
my_qp->qkey = resp.qkey;
my_qp->real_qp_num = resp.real_qp_num;
/* rqueue properties */
- my_qp->ipz_rqueue.queue = (u8*)(long)resp.ipz_rqueue.queue;
- my_qp->ipz_rqueue.current_q_addr = (u8*)(long)resp.ipz_rqueue.queue;
+ my_qp->ipz_rqueue.queue = mmap64(NULL, resp.ipz_rqueue.queue_length,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ context->cmd_fd,
+ ((u64)my_qp->token << 32) | 0x22000000);
+ if (!my_qp->ipz_rqueue.queue) {
+ ehca_err(pd->context->device, "mmap64() failed qp_num=%x",
+ my_qp->qp_num);
+ goto create_qp_exit1;
+ }
+ my_qp->ipz_rqueue.current_q_addr = my_qp->ipz_rqueue.queue;
my_qp->ipz_rqueue.qe_size = resp.ipz_rqueue.qe_size;
my_qp->ipz_rqueue.act_nr_of_sg = resp.ipz_rqueue.act_nr_of_sg;
my_qp->ipz_rqueue.queue_length = resp.ipz_rqueue.queue_length;
my_qp->ipz_rqueue.pagesize = resp.ipz_rqueue.pagesize;
my_qp->ipz_rqueue.toggle_state = resp.ipz_rqueue.toggle_state;
/* squeue properties */
- my_qp->ipz_squeue.queue = (u8*)(long)resp.ipz_squeue.queue;
- my_qp->ipz_squeue.current_q_addr = (u8*)(long)resp.ipz_squeue.queue;
+ my_qp->ipz_squeue.queue = mmap64(NULL, resp.ipz_squeue.queue_length,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ context->cmd_fd,
+ ((u64)my_qp->token << 32) | 0x23000000);
+ if (!my_qp->ipz_squeue.queue) {
+ ehca_err(pd->context->device, "mmap64() failed qp_num=%x",
+ my_qp->qp_num);
+ goto create_qp_exit2;
+ }
+ my_qp->ipz_squeue.current_q_addr = my_qp->ipz_squeue.queue;
my_qp->ipz_squeue.qe_size = resp.ipz_squeue.qe_size;
my_qp->ipz_squeue.act_nr_of_sg = resp.ipz_squeue.act_nr_of_sg;
my_qp->ipz_squeue.queue_length = resp.ipz_squeue.queue_length;
my_qp->ipz_squeue.pagesize = resp.ipz_squeue.pagesize;
my_qp->ipz_squeue.toggle_state = resp.ipz_squeue.toggle_state;
- my_qp->galpas = resp.galpas;
+ /* right most cast is required to avoid gcc warning in 32 bit mode */
+ my_qp->galpas.kernel.fw_handle = (u64)(unsigned long)
+ mmap64(NULL, EHCA_PAGESIZE,
+ PROT_READ | PROT_WRITE, MAP_SHARED,
+ context->cmd_fd,
+ ((u64)my_qp->token << 32) | 0x21000000);
+ if (!my_qp->galpas.kernel.fw_handle) {
+ ehca_err(pd->context->device, "mmap64() failed qp_num=%x",
+ my_qp->qp_num);
+ goto create_qp_exit3;
+ }
/* access queue mem to fill page cache */
memset(my_qp->ipz_squeue.queue, 0,
@@ -346,33 +392,53 @@ struct ibv_qp *ehcau_create_qp(struct ib
memset(my_qp->ipz_rqueue.queue, 0,
my_qp->ipz_rqueue.queue_length);
- if (attr->send_cq!=NULL) {
- struct ehcau_cq *cq = container_of(attr->send_cq, struct ehcau_cq, ib_cq);
+ if (attr->send_cq) {
+ struct ehcau_cq *cq = container_of(attr->send_cq,
+ struct ehcau_cq, ib_cq);
ret = cq_assign_qp(cq, my_qp);
- if (ret !=0) {
- EDEB_ERR(4, "Couldn't assign qp to send_cq ret=%x", ret);
- goto create_qp_exit1;
+ if (ret) {
+ ehca_err(pd->context->device,
+ "Couldn't assign qp to send_cq ret=%x", ret);
+ goto create_qp_exit4;
}
my_qp->send_cq = cq;
}
- EDEB_EX(7, "ehcau_qp=%p "
- "ipz_queue.galpa=%p ipz_rqueue.adr=%p ipz_squeue.adr=%p",
- my_qp,
- (u64 *) (unsigned long)my_qp->galpas.kernel.fw_handle,
- (u64 *) my_qp->ipz_rqueue.queue,
- (u64 *) my_qp->ipz_squeue.queue);
+ ehca_dbg(pd->context->device, "ehcau_qp=%p "
+ "ipz_queue.galpa=%p ipz_rqueue.adr=%p ipz_squeue.adr=%p",
+ my_qp,
+ (u64*)(unsigned long)my_qp->galpas.kernel.fw_handle,
+ (u64*)my_qp->ipz_rqueue.queue,
+ (u64*)my_qp->ipz_squeue.queue);
return &my_qp->ib_qp;
- create_qp_exit1:
- ret2 = ibv_cmd_destroy_qp(&my_qp->ib_qp);
- if (ret2) {
- EDEB_ERR(4, "ibv_cmd_destroy_qp() failed ret=%x "
- "qp=%p qp_num=%x", ret2, my_qp, my_qp->qp_num);
- }
- create_qp_exit0:
- EDEB_EX(4, "An error has occured pd=%p", pd);
+create_qp_exit4:
+ ret = munmap((void*)(unsigned long)my_qp->galpas.kernel.fw_handle, EHCA_PAGESIZE);
+ if (ret)
+ ehca_err(pd->context->device, "munmap() failed rc=%x qp_num=%x "
+ "fw_handle=%p", ret, my_qp->qp_num,
+ (void*)(unsigned long)my_qp->galpas.kernel.fw_handle);
+
+create_qp_exit3:
+ ret = munmap(my_qp->ipz_squeue.queue, my_qp->ipz_squeue.queue_length);
+ if (ret)
+ ehca_err(pd->context->device, "munmap() failed rc=%x qp_num=%x "
+ "squeue=%p", ret, my_qp->qp_num, my_qp->ipz_squeue.queue);
+
+create_qp_exit2:
+ ret = munmap(my_qp->ipz_rqueue.queue, my_qp->ipz_rqueue.queue_length);
+ if (ret)
+ ehca_err(pd->context->device, "munmap() failed rc=%x qp_num=%x "
+ "rqueue=%p", ret, my_qp->qp_num, my_qp->ipz_rqueue.queue);
+
+create_qp_exit1:
+ ret = ibv_cmd_destroy_qp(&my_qp->ib_qp);
+ if (ret)
+ ehca_err(pd->context->device, "ibv_cmd_destroy_qp() failed "
+ "ret=%x qp=%p qp_num=%x", ret, my_qp, my_qp->qp_num);
+
+create_qp_exit0:
free(my_qp);
return NULL;
}
@@ -380,23 +446,17 @@ struct ibv_qp *ehcau_create_qp(struct ib
int ehcau_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
enum ibv_qp_attr_mask attr_mask)
{
- int ret = 0;
+ int ret;
struct ibv_modify_qp cmd;
- struct ehcau_qp *my_qp = NULL;
+ struct ehcau_qp *my_qp;
struct ehca_wqe *wqe = NULL;
int sq_locked = 0;
- EHCA_CHECK_ADR(qp);
- EHCA_CHECK_ADR(attr);
- if (attr_mask == 0) { /* nothing to modify */
- return ret;
- }
+ if (!attr_mask) /* nothing to modify */
+ return 0;
my_qp = container_of(qp, struct ehcau_qp, ib_qp);
- EDEB_EN(7, "qp=%p qp_num=%x attr=%p attr_mask=%x",
- qp, my_qp->qp_num, attr, attr_mask);
-
- if ((attr_mask & IBV_QP_STATE)!=0 && attr->qp_state==IBV_QPS_RTS) {
+ if ((attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RTS) {
unsigned int qp_type = -1;
qp_type = my_qp->qp_type;
if (IBV_QPT_UD == qp_type) {
@@ -407,16 +467,17 @@ int ehcau_modify_qp(struct ibv_qp *qp, s
wqe=(struct ehca_wqe*)
my_qp->ipz_squeue.current_q_addr;
wqe->optype = wqe->wqef = 0xff;
- EDEB(7, "qp_num=%x next_free_wqe=%p",
- my_qp->qp_num, wqe);
+ ehca_dbg(qp->context->device,
+ "qp_num=%x next_free_wqe=%p",
+ my_qp->qp_num, wqe);
}
}
ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd));
- if (ret) {
- EDEB_ERR(4, "ibv_cmd_modify_qp() failed ret=%x "
- "qp=%p qp_num=%x", ret, qp, my_qp->qp_num);
- } else if (attr_mask & IBV_QP_STATE) {
+ if (ret)
+ ehca_err(qp->context->device, "ibv_cmd_modify_qp() failed "
+ "ret=%x qp=%p qp_num=%x", ret, qp, my_qp->qp_num);
+ else if (attr_mask & IBV_QP_STATE) {
switch (attr->qp_state) {
case IBV_QPS_RESET:
/* reset s/r queue pointers */
@@ -424,77 +485,85 @@ int ehcau_modify_qp(struct ibv_qp *qp, s
ipzu_qeit_reset(&my_qp->ipz_squeue);
break;
case IBV_QPS_RTS:
- if (sq_locked && wqe!=NULL) {
+ if (sq_locked && wqe)
my_qp->sqerr_purgeflag = ~wqe->wqef;
- }
break;
default: /* nothing to do */
break;
}
}
- if (sq_locked) { /* unlock send queue */
+ if (sq_locked) /* unlock send queue */
ehcau_unlock(&my_qp->lockvar_s);
- }
- if (attr_mask & IBV_QP_QKEY) {
+ if (attr_mask & IBV_QP_QKEY)
my_qp->qkey = attr->qkey;
- }
- EDEB_EX(7, "ret=%x qp=%p qp_num=%x", ret, qp, my_qp->qp_num);
return ret;
}
int ehcau_destroy_qp(struct ibv_qp *qp)
{
- int ret = 0;
- struct ehcau_qp *my_qp = NULL;
- u32 qp_num = 0;
+ int ret;
+ struct ehcau_qp *my_qp;
+ u32 qp_num;
- EHCA_CHECK_ADR(qp);
my_qp = container_of(qp, struct ehcau_qp, ib_qp);
qp_num = my_qp->qp_num;
- EDEB_EN(7, "qp=%p qp_num=%x", qp, qp_num);
-
- if (my_qp->send_cq!=NULL) {
+ if (my_qp->send_cq) {
ret = cq_unassign_qp(my_qp->send_cq,
my_qp->real_qp_num);
- if (ret !=0) {
- EDEB_ERR(4, "Couldn't unassign qp from send_cq "
+ if (ret) {
+ ehca_err(qp->context->device,
+ "Couldn't unassign qp from send_cq "
"ret=%x real_qp_num=%x cq_num=%x",
ret, my_qp->real_qp_num,
my_qp->send_cq->cq_number);
- goto destroy_qp_exit0;
+ return ret;
}
}
- ret = ibv_cmd_destroy_qp(qp);
+ ret = munmap(my_qp->ipz_rqueue.queue, my_qp->ipz_rqueue.queue_length);
if (ret) {
- EDEB_ERR(4, "ibv_cmd_destroy_qp() failed ret=%x "
+ ehca_err(qp->context->device, "munmap() failed rc=%x qp_num=%x "
+ "rqueue=%p", ret, qp_num, my_qp->ipz_rqueue.queue);
+ return ret;
+ }
+ ret = munmap(my_qp->ipz_squeue.queue, my_qp->ipz_squeue.queue_length);
+ if (ret) {
+ ehca_err(qp->context->device, "munmap() failed rc=%x qp_num=%x "
+ "squeue=%p", ret, qp_num, my_qp->ipz_squeue.queue);
+ return ret;
+ }
+ ret = munmap((void*)(unsigned long)my_qp->galpas.kernel.fw_handle, EHCA_PAGESIZE);
+ if (ret) {
+ ehca_err(qp->context->device, "munmap() failed rc=%x qp_num=%x "
+ "fw_handle=%p", ret, qp_num,
+ (void*)(unsigned long)my_qp->galpas.kernel.fw_handle);
+ return ret;
+ }
+ ret = ibv_cmd_destroy_qp(qp);
+ if (ret)
+ ehca_err(qp->context->device,
+ "ibv_cmd_destroy_qp() failed ret=%x "
"qp=%p qp_num=%x", ret, qp, qp_num);
- } else {
+ else {
+ ehca_gen_dbg("ret=%x qp=%p qp_num=%x", ret, qp, qp_num);
free(my_qp);
}
- destroy_qp_exit0:
- EDEB_EX(7, "ret=%x qp=%p qp_num=%x", ret, qp, qp_num);
return ret;
}
struct ibv_ah *ehcau_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
{
- struct ehcau_av *my_av = NULL;
-
- EHCA_CHECK_ADR_P(pd);
- EHCA_CHECK_ADR_P(attr);
-
- EDEB_EN(7, "pd=%p attr=%p", pd, attr);
+ struct ehcau_av *my_av;
my_av = malloc(sizeof *my_av);
if (!my_av) {
- EDEB_ERR(4, "no address handle");
- return (NULL);
+ ehca_err(pd->context->device, "no address handle");
+ return NULL;
}
memset(my_av, 0, sizeof(*my_av));
@@ -518,15 +587,15 @@ struct ibv_ah *ehcau_create_ah(struct ib
my_av->av.grh.word_0 = be64_to_cpu(my_av->av.grh.word_0);
/* set sgid in grh.word_1 */
- if (attr->is_global != 0) {
+ if (attr->is_global) {
struct ibv_context *context = pd->context;
struct ibv_port_attr port_attr;
union ibv_gid gid;
- int rc = 0;
+ int rc;
memset(&port_attr, 0, sizeof(port_attr));
rc = ibv_query_port(context, attr->port_num, &port_attr);
- if (rc != 0) { /* port number invalid */
- EDEB_ERR(4, "ibv_query_port() failed "
+ if (rc) { /* port number invalid */
+ ehca_err(pd->context->device, "ibv_query_port() failed "
"rc=%x context=%p port_num=%x",
rc, context, attr->port_num);
free(my_av);
@@ -536,8 +605,8 @@ struct ibv_ah *ehcau_create_ah(struct ib
rc = ibv_query_gid(context,
attr->port_num,
attr->grh.sgid_index, &gid);
- if (rc != 0) {
- EDEB_ERR(4, "ibv_query_gid() failed "
+ if (rc) {
+ ehca_err(pd->context->device, "ibv_query_gid() failed "
"rc=%x context=%p port_num=%x "
"sgid_index=%x",
rc, context, attr->port_num,
@@ -548,129 +617,87 @@ struct ibv_ah *ehcau_create_ah(struct ib
memcpy(&my_av->av.grh.word_1, &gid, sizeof(gid));
}
- /* see also ehca_av.c
- * For the time beeing we use a hard coded PMTU of 2048 Bytes.
- */
- my_av->av.pmtu = 4; /* TODO */
+ /* see also ehca_av.c */
+ my_av->av.pmtu = EHCA_MAX_MTU;
/* dgid comes in grh.word_3 */
memcpy(&my_av->av.grh.word_3, &attr->grh.dgid, sizeof(attr->grh.dgid));
- EDEB_EX(7, "pd=%p attr=%p my_av=%p", pd, attr, my_av);
-
- return (&my_av->ib_ah);
+ return &my_av->ib_ah;
}
int ehcau_destroy_ah(struct ibv_ah *ah)
{
- EHCA_CHECK_ADR(ah);
-
- EDEB_EN(7, "ah=%p", ah);
+ ehca_dbg(ah->context->device, "ah=%p", ah);
free(ah);
- EDEB_EX(7, "ah=%p", ah);
return 0;
}
int ehcau_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
{
- int ret = 0;
- struct ehcau_qp *my_qp = NULL;
- u32 qp_num = 0;
+ int ret;
+ struct ehcau_qp *my_qp;
+ u32 qp_num;
- EHCA_CHECK_ADR(qp);
- EHCA_CHECK_ADR(gid);
my_qp = container_of(qp, struct ehcau_qp, ib_qp);
-
- EHCA_CHECK_QP(my_qp);
-
qp_num = my_qp->qp_num;
- EDEB_EN(7, "qp=%p qp_num=%x", qp, qp_num);
-
ret = ibv_cmd_attach_mcast(qp, gid, lid);
-
- if (ret) {
-
- EDEB_ERR(4, "ehcau_qp=%p qp_num=%x "
+ if (ret)
+ ehca_err(qp->context->device, "ehcau_qp=%p qp_num=%x "
"ibv_cmd_attach_mcast() failed "
"ret=%x", my_qp, my_qp->qp_num, ret);
- }
-
- EDEB_EX(7, "qp=%p qp_num=%x ret=%x", qp, qp_num, ret);
return ret;
}
int ehcau_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
{
- int ret = 0;
- struct ehcau_qp *my_qp = NULL;
- u32 qp_num = 0;
+ int ret;
+ struct ehcau_qp *my_qp;
+ u32 qp_num;
- EHCA_CHECK_ADR(qp);
- EHCA_CHECK_ADR(gid);
my_qp = container_of(qp, struct ehcau_qp, ib_qp);
-
- EHCA_CHECK_QP(my_qp);
-
qp_num = my_qp->qp_num;
- EDEB_EN(7, "qp=%p qp_num=%x", qp, qp_num);
-
ret = ibv_cmd_detach_mcast(qp, gid, lid);
-
- if (ret) {
-
- EDEB_ERR(4, "ehcau_qp=%p qp_num=%x "
+ if (ret)
+ ehca_err(qp->context->device, "ehcau_qp=%p qp_num=%x "
"ibv_cmd_detach_mcast() failed "
"ret=%x", my_qp, my_qp->qp_num, ret);
- }
-
- EDEB_EX(7, "qp=%p qp_num=%x ret=%x", qp, qp_num, ret);
return ret;
}
int ehcau_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
- enum ibv_qp_attr_mask attr_mask, struct ibv_qp_init_attr* init_attr)
+ enum ibv_qp_attr_mask attr_mask,
+ struct ibv_qp_init_attr* init_attr)
{
- int ret = 0;
- struct ehcau_qp *my_qp = NULL;
+ int ret;
+ struct ehcau_qp *my_qp;
struct ibv_query_qp cmd;
- EHCA_CHECK_ADR(qp);
- EHCA_CHECK_ADR(qp_attr);
-
my_qp = container_of(qp, struct ehcau_qp, ib_qp);
- EDEB_EN(7, "qp=%p qp_num=%x qp_attr=%p", qp, my_qp->qp_num, qp_attr);
-
ret = ibv_cmd_query_qp(qp, qp_attr, attr_mask, init_attr,
&cmd, sizeof(cmd));
- if (ret) {
- EDEB_ERR(4, "ehcau_qp=%p qp_num=%x "
+ if (ret)
+ ehca_err(qp->context->device, "ehcau_qp=%p qp_num=%x "
"ibv_cmd_query_() failed "
"ret=%x", my_qp, my_qp->qp_num, ret);
- }
- EDEB_EX(7, "ret=%x qp=%p qp_num=%x", ret, qp, my_qp->qp_num);
return ret;
}
int ehcau_query_qp_da(struct ibv_qp *qp, struct ehcau_qp_attr_da *qp_attr)
{
- int ret = 0;
- struct ehcau_qp *my_qp = NULL;
+ struct ehcau_qp *my_qp;
- EHCA_CHECK_ADR(qp);
- EHCA_CHECK_ADR(qp_attr);
my_qp = container_of(qp, struct ehcau_qp, ib_qp);
- EDEB_EN(7, "qp=%p qp_num=%x qp_attr=%p", qp, my_qp->qp_num, qp_attr);
-
qp_attr->send_wqe_size = my_qp->ipz_squeue.qe_size;
qp_attr->send_queue_length = my_qp->ipz_squeue.queue_length;
qp_attr->send_queue_ptr = my_qp->ipz_squeue.queue;
@@ -678,17 +705,17 @@ int ehcau_query_qp_da(struct ibv_qp *qp,
qp_attr->recv_queue_length = my_qp->ipz_rqueue.queue_length;
qp_attr->recv_queue_ptr = my_qp->ipz_rqueue.queue;
- EDEB_EX(7, "ret=%x qp=%p qp_num=%x "
- "send_wqe_size=%x send_queue_size=%lx send_queue_ptr=%p "
- "recv_wqe_size=%x recv_queue_size=%lx recv_queue_ptr=%p",
- ret, qp, my_qp->qp_num,
- qp_attr->send_wqe_size,
- (unsigned long)qp_attr->send_queue_length,
- qp_attr->send_queue_ptr,
- qp_attr->recv_wqe_size,
- (unsigned long)qp_attr->recv_queue_length,
- qp_attr->recv_queue_ptr);
- return ret;
+ ehca_dbg(qp->context->device, "qp=%p qp_num=%x "
+ "send_wqe_size=%x send_queue_size=%lx send_queue_ptr=%p "
+ "recv_wqe_size=%x recv_queue_size=%lx recv_queue_ptr=%p",
+ qp, my_qp->qp_num,
+ qp_attr->send_wqe_size,
+ (unsigned long)qp_attr->send_queue_length,
+ qp_attr->send_queue_ptr,
+ qp_attr->recv_wqe_size,
+ (unsigned long)qp_attr->recv_queue_length,
+ qp_attr->recv_queue_ptr);
+ return 0;
}
/* eof ehca_umain.c */
More information about the ewg
mailing list