[ofw] [PATCH] libibverbs: fix event reporting and minor fixes

Sean Hefty sean.hefty at intel.com
Fri Apr 17 11:32:23 PDT 2009


Use the latest comp_channel changes to fix event reporting and avoid
hangs when destroying resources.  We need to track when closing
devices to make sure that events are canceled, and avoid issuing
new wait calls.

Rename windows specific calls to include 'w' after the ibv prefix to
avoid any potential future conflicts and clearly indicate to a caller
that they're using a windows only call.

Use the common ntohll definition.

Device names are changed from ibv_device_<guid> to ibv_device_X, where
X is an index (0, 1, 2, etc.).  This gives devices across the cluster
the same name, which is closer to IBAL and OFED device naming.

Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/examples/devinfo/devinfo.c
branches\winverbs\ulp\libibverbs/examples/devinfo/devinfo.c
--- trunk\ulp\libibverbs/examples/devinfo/devinfo.c	2008-06-02 16:03:24.087187200 -0700
+++ branches\winverbs\ulp\libibverbs/examples/devinfo/devinfo.c	2009-03-23 13:12:47.832624500 -0700
@@ -35,8 +35,8 @@
 
 #include "..\..\..\..\etc\user\getopt.c"
 #include <infiniband/verbs.h>
+#include <netinet/in.h>
 
-#define ntohll _byteswap_uint64
 static int verbose = 0;
 
 static int null_gid(union ibv_gid *gid)
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/examples/devinfo/SOURCES
branches\winverbs\ulp\libibverbs/examples/devinfo/SOURCES
--- trunk\ulp\libibverbs/examples/devinfo/SOURCES	2008-12-17 11:06:18.617375000 -0800
+++ branches\winverbs\ulp\libibverbs/examples/devinfo/SOURCES	2009-03-23 13:24:56.797354500 -0700
@@ -14,7 +14,8 @@ SOURCES =			\
 	devinfo.rc		\
 	devinfo.c
 	
-INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;
+INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;\
+		   ..\..\..\..\inc\user\linux;
 
 TARGETLIBS =						\
 	$(SDK_LIB_PATH)\kernel32.lib	\
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/examples/srq_pingpong/SOURCES
branches\winverbs\ulp\libibverbs/examples/srq_pingpong/SOURCES
--- trunk\ulp\libibverbs/examples/srq_pingpong/SOURCES	2008-12-17 11:06:18.336125000 -0800
+++ branches\winverbs\ulp\libibverbs/examples/srq_pingpong/SOURCES	2008-06-16 15:09:51.062500000 -0700
@@ -11,7 +11,6 @@ USE_NATIVE_EH = 1
 USE_IOSTREAM = 1
 
 SOURCES =			\
-	srq_pingpong.rc	\
 	srq_pingpong.c
 	
 INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/include/infiniband/verbs.h
branches\winverbs\ulp\libibverbs/include/infiniband/verbs.h
--- trunk\ulp\libibverbs/include/infiniband/verbs.h	2009-03-10 15:52:23.913500000 -0700
+++ branches\winverbs\ulp\libibverbs/include/infiniband/verbs.h	2009-04-08 11:51:39.087642900 -0700
@@ -1117,19 +1117,19 @@ const char *ibv_event_type_str(enum ibv_
 /*
  * Windows specific structures and interfaces
  */
-struct ibv_windata
+struct ibvw_windata
 {
 	IWVProvider		*prov;
 	COMP_MANAGER	*comp_mgr;
 };
 
-#define IBV_WINDATA_VERSION 1
+#define IBVW_WINDATA_VERSION 1
 
 __declspec(dllexport)
-int ibv_get_windata(struct ibv_windata *windata, int version);
+int ibvw_get_windata(struct ibvw_windata *windata, int version);
 
 __declspec(dllexport)
-void ibv_release_windata(struct ibv_windata *windata, int version);
+void ibvw_release_windata(struct ibvw_windata *windata, int version);
 
 #ifdef __cplusplus
 }
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/device.cpp
branches\winverbs\ulp\libibverbs/src/device.cpp
--- trunk\ulp\libibverbs/src/device.cpp	2009-03-10 15:52:33.397875000 -0700
+++ branches\winverbs\ulp\libibverbs/src/device.cpp	2009-04-15 15:29:20.089441100 -0700
@@ -32,9 +32,12 @@
 #include <infiniband/verbs.h>
 #include <rdma/winverbs.h>
 #include "..\..\..\etc\user\comp_channel.cpp"
+#include "..\..\..\etc\user\dlist.c"
 
+CRITICAL_SECTION lock;
 IWVProvider *prov;
 COMP_MANAGER comp_mgr;
+static DWORD ref;
 
 struct verbs_device
 {
@@ -54,31 +57,59 @@ struct verbs_context
 {
 	struct ibv_context	context;
 	struct verbs_device	device;
+	uint8_t				closing;
 	struct verbs_port	*port;
 	verbs_port			*event_port;
 };
 
-static int ibv_init(void)
+static int ibv_acquire(void)
 {
 	HRESULT hr;
 
-	if (prov == NULL) {
+	EnterCriticalSection(&lock);
+	if (ref++ == 0) {
 		hr = WvGetObject(IID_IWVProvider, (LPVOID*) &prov);
 		if (FAILED(hr)) {
-			return -1;
+			goto err1;
+		}
+		hr = CompManagerOpen(&comp_mgr);
+		if (FAILED(hr)) {
+			goto err2;
+		}
+		hr = CompManagerMonitor(&comp_mgr, prov->GetFileHandle(), 0);
+		if (FAILED(hr)) {
+			goto err3;
 		}
-		CompManagerOpen(&comp_mgr);
-		CompManagerMonitor(&comp_mgr, prov->GetFileHandle(), 0);
 	}
+	LeaveCriticalSection(&lock);
 	return 0;
+
+err3:
+	CompManagerClose(&comp_mgr);
+err2:
+	prov->Release();
+err1:
+	ref--;
+	LeaveCriticalSection(&lock);
+	return hr;
+}
+
+static void ibv_release(void)
+{
+	EnterCriticalSection(&lock);
+	if (--ref == 0) {
+		CompManagerClose(&comp_mgr);
+		prov->Release();
+	}
+	LeaveCriticalSection(&lock);
 }
 
 __declspec(dllexport)
-int ibv_get_windata(struct ibv_windata *windata, int version)
+int ibvw_get_windata(struct ibvw_windata *windata, int version)
 {
 	int ret;
 
-	if (version != IBV_WINDATA_VERSION || ibv_init()) {
+	if (version != IBVW_WINDATA_VERSION || ibv_acquire()) {
 		return -1;
 	}
 
@@ -89,9 +120,9 @@ int ibv_get_windata(struct ibv_windata *
 }
 
 __declspec(dllexport)
-void ibv_release_windata(struct ibv_windata *windata, int version)
+void ibvw_release_windata(struct ibvw_windata *windata, int version)
 {
-	windata->prov->Release();
+	ibv_release();
 }
 
 __declspec(dllexport)
@@ -104,7 +135,7 @@ struct ibv_device **ibv_get_device_list(
 	SIZE_T size, cnt;
 	HRESULT hr;
 
-	if (ibv_init()) {
+	if (ibv_acquire()) {
 		goto err1;
 	}
 
@@ -142,7 +173,7 @@ struct ibv_device **ibv_get_device_list(
 			goto err3;
 		}
 
-		sprintf(dev_array[cnt].device.name, "ibv_device_0x%I64x", guid[cnt]);
+		sprintf(dev_array[cnt].device.name, "ibv_device%d", cnt);
 		dev_array[cnt].device.node_type = IBV_NODE_UNKNOWN;
 		dev_array[cnt].device.transport_type = (ibv_transport_type) attr.DeviceType;
 		dev_array[cnt].guid = guid[cnt];
@@ -166,6 +197,7 @@ err1:
 __declspec(dllexport)
 void ibv_free_device_list(struct ibv_device **list)
 {
+	ibv_release();
 	delete CONTAINING_RECORD(list[0], struct verbs_device, device);
 	delete list;
 }
@@ -195,8 +227,12 @@ struct ibv_context *ibv_open_device(stru
 	if (vcontext == NULL) {
 		return NULL;
 	}
+
+	ibv_acquire();
 	memcpy(&vcontext->device, vdev, sizeof(struct verbs_device));
+	vcontext->context.device = &vcontext->device.device;
 	vcontext->event_port = NULL;
+	vcontext->closing = 0;
 	CompChannelInit(&comp_mgr, &vcontext->context.channel, INFINITE);
 
 	vcontext->port = new struct verbs_port[vdev->phys_port_cnt];
@@ -213,6 +249,7 @@ struct ibv_context *ibv_open_device(stru
 		vcontext->port[i].port_num = (uint8_t) i + 1;
 		vcontext->port[i].event_flag = 0;
 		CompEntryInit(&vcontext->context.channel, &vcontext->port[i].comp_entry);
+		vcontext->port[i].comp_entry.Busy = 1;
 		vcontext->context.cmd_if->Notify(vcontext->port[i].port_num,
 										 &vcontext->port[i].comp_entry.Overlap,
 										 &vcontext->port[i].event_flag);
@@ -224,6 +261,7 @@ err2:
 	delete vcontext->port;
 err1:
 	delete vcontext;
+	ibv_release();
 	return NULL;
 }
 
@@ -234,13 +272,15 @@ int ibv_close_device(struct ibv_context 
 	int i;
 
 	vcontext = CONTAINING_RECORD(context, struct verbs_context, context);
+	vcontext->closing = 1;
 	context->cmd_if->CancelOverlappedRequests();
 
 	for (i = 0; i < vcontext->device.phys_port_cnt; i++) {
-		CompChannelRemoveEntry(&context->channel, &vcontext->port[i].comp_entry);
+		CompEntryCancel(&vcontext->port[i].comp_entry);
 	}
 
 	context->cmd_if->Release();
+	ibv_release();
 	delete vcontext->port;
 	delete vcontext;
 	return 0;
@@ -289,7 +329,8 @@ static int ibv_report_port_event(struct 
 		ret = -1;
 	}
 	
-	if (port->event_flag == 0) {
+	if (port->event_flag == 0 && !vcontext->closing) {
+		port->comp_entry.Busy = 1;
 		vcontext->context.cmd_if->Notify(vcontext->event_port->port_num,
 										 &port->comp_entry.Overlap,
 										 &port->event_flag);
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/ibv_exports.src
branches\winverbs\ulp\libibverbs/src/ibv_exports.src
--- trunk\ulp\libibverbs/src/ibv_exports.src	2009-03-10 15:49:59.647875000 -0700
+++ branches\winverbs\ulp\libibverbs/src/ibv_exports.src	2009-03-26 11:13:34.375120500 -0700
@@ -51,6 +51,6 @@ ibv_detach_mcast
 ibv_node_type_str
 ibv_port_state_str
 ibv_event_type_str
-ibv_get_windata
-ibv_release_windata
+ibvw_get_windata
+ibvw_release_windata
 #endif
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/ibv_main.cpp
branches\winverbs\ulp\libibverbs/src/ibv_main.cpp
--- trunk\ulp\libibverbs/src/ibv_main.cpp	2008-05-22 23:59:26.698750000 -0700
+++ branches\winverbs\ulp\libibverbs/src/ibv_main.cpp	2009-04-10 10:08:18.203720100 -0700
@@ -29,11 +29,14 @@
 
 #include <windows.h>
 
+extern CRITICAL_SECTION lock;
+
 BOOLEAN WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID lpReserved)
 {
 	UNREFERENCED_PARAMETER(hInstance);
 	UNREFERENCED_PARAMETER(dwReason);
 	UNREFERENCED_PARAMETER(lpReserved);
 
+	InitializeCriticalSection(&lock);
 	return TRUE;
 }
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/Sources
branches\winverbs\ulp\libibverbs/src/Sources
--- trunk\ulp\libibverbs/src/Sources	2009-03-10 02:31:17.625000000 -0700
+++ branches\winverbs\ulp\libibverbs/src/Sources	2009-03-23 12:50:51.655306300 -0700
@@ -23,7 +23,7 @@ SOURCES = \
 	device.cpp		\
 	enum_strs.cpp
 
-INCLUDES = ..\include;..\..\..\inc;..\..\..\inc\user;
+INCLUDES = ..\include;..\..\..\inc;..\..\..\inc\user;..\..\..\inc\user\linux;
 
 USER_C_FLAGS = $(USER_C_FLAGS) -DEXPORT_IBV_SYMBOLS
 
diff -up -r -X \mshefty\scm\winof\trunk\docs\dontdiff.txt -I '\$Id:' trunk\ulp\libibverbs/src/verbs.cpp
branches\winverbs\ulp\libibverbs/src/verbs.cpp
--- trunk\ulp\libibverbs/src/verbs.cpp	2009-03-10 00:51:37.552875000 -0700
+++ branches\winverbs\ulp\libibverbs/src/verbs.cpp	2009-04-08 13:33:19.307495000 -0700
@@ -32,11 +32,13 @@
 #include <windows.h>
 #include <winsock2.h>
 #include <stdio.h>
+#include <netinet/in.h>
 
 #include <infiniband/verbs.h>
 #include <comp_channel.h>
 #include "ibverbs.h"
 
+
 __declspec(dllexport)
 int ibv_rate_to_mult(enum ibv_rate rate)
 {
@@ -296,6 +298,7 @@ struct ibv_mr *ibv_reg_mr(struct ibv_pd 
 		delete mr;
 		return NULL;
 	}
+	mr->rkey = ntohl(mr->rkey);
 	return mr;
 }
 
@@ -383,13 +386,16 @@ int ibv_req_notify_cq(struct ibv_cq *cq,
 {
 	HRESULT hr;
 
-	hr = cq->handle->Notify(solicited_only ? WvCqSolicited : WvCqNextCompletion,
-							&cq->comp_entry.Overlap);
-	if (SUCCEEDED(hr) || hr == WV_IO_PENDING) {
-		return 0;
+	if (InterlockedCompareExchange(&cq->comp_entry.Busy, 1, 0) == 0) {
+		hr = cq->handle->Notify(solicited_only ? WvCqSolicited : WvCqNextCompletion,
+								&cq->comp_entry.Overlap);
+		if (SUCCEEDED(hr) || hr == WV_IO_PENDING) {
+			hr = 0;
+		}
 	} else {
-		return hr;
+		hr = 0;
 	}
+	return hr;
 }
 
 __declspec(dllexport)
@@ -410,11 +416,9 @@ __declspec(dllexport)
 int ibv_destroy_cq(struct ibv_cq *cq)
 {
 	cq->handle->CancelOverlappedRequests();
-
 	if (cq->channel != NULL) {
-		CompChannelRemoveEntry(&cq->channel->comp_channel, &cq->comp_entry);
+		CompEntryCancel(&cq->comp_entry);
 	}
-
 	cq->handle->Release();
 	delete cq;
 	return 0;
@@ -729,21 +733,29 @@ int ibv_post_send(struct ibv_qp *qp, str
 			cur_wr->opcode = (ibv_wr_opcode) (cur_wr->opcode & ~0x80000000);
 			cur_wr->send_flags = (ibv_send_flags) (cur_wr->send_flags | WV_SEND_IMMEDIATE);
 		}
+
+		if (cur_wr->opcode != 0) {
+			cur_wr->wr.rdma.rkey = htonl(cur_wr->wr.rdma.rkey);
+		}
 	}
 
 	hr = qp->handle->PostSend((WV_SEND_REQUEST *) wr, (WV_SEND_REQUEST **) bad_wr);
 
 	for (cur_wr = wr; cur_wr != NULL; cur_wr = cur_wr->next) {
-		if (qp->qp_type == IBV_QPT_UD) {
-			cur_wr->wr.ud.ah = ah;
-			cur_wr->wr.ud.remote_qkey = ntohl(cur_wr->wr.ud.remote_qkey);
-			cur_wr->wr.ud.remote_qpn = ntohl(cur_wr->wr.ud.remote_qpn);
+		if (cur_wr->opcode != 0) {
+			cur_wr->wr.rdma.rkey = ntohl(cur_wr->wr.rdma.rkey);
 		}
 
 		if ((cur_wr->send_flags & WV_SEND_IMMEDIATE) != 0) {
 			cur_wr->send_flags = (ibv_send_flags) (cur_wr->send_flags & ~WV_SEND_IMMEDIATE);
 			cur_wr->opcode = (ibv_wr_opcode) (cur_wr->opcode | 0x80000000);
 		}
+
+		if (qp->qp_type == IBV_QPT_UD) {
+			cur_wr->wr.ud.ah = ah;
+			cur_wr->wr.ud.remote_qkey = ntohl(cur_wr->wr.ud.remote_qkey);
+			cur_wr->wr.ud.remote_qpn = ntohl(cur_wr->wr.ud.remote_qpn);
+		}
 	}
 
 	return hr;




More information about the ofw mailing list