[openib-general] [PATCH v3] ib_sa: require SA registration

Sean Hefty sean.hefty at intel.com
Mon Aug 21 16:40:12 PDT 2006


Require registration with SA module, to prevent module text
from going away while sa query callback is still running,
and update all users.

Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Changes from the previous post include:

* Move struct ib_sa_client definition external.

Index: include/rdma/ib_sa.h
===================================================================
--- include/rdma/ib_sa.h	(revision 8928)
+++ include/rdma/ib_sa.h	(working copy)
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
+ * Copyright (c) 2006 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -36,6 +37,9 @@
 #ifndef IB_SA_H
 #define IB_SA_H
 
+#include <asm/atomic.h>
+
+#include <linux/completion.h>
 #include <linux/compiler.h>
 
 #include <rdma/ib_verbs.h>
@@ -250,11 +254,28 @@ struct ib_sa_service_rec {
 	u64		data64[2];
 };
 
+struct ib_sa_client {
+	atomic_t users;
+	struct completion comp;
+};
+
+/**
+ * ib_sa_register_client - Register an SA client.
+ */
+void ib_sa_register_client(struct ib_sa_client *client);
+
+/**
+ * ib_sa_unregister_client - Deregister an SA client.
+ * @client: Client object to deregister.
+ */
+void ib_sa_unregister_client(struct ib_sa_client *client);
+
 struct ib_sa_query;
 
 void ib_sa_cancel_query(int id, struct ib_sa_query *query);
 
-int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
+int ib_sa_path_rec_get(struct ib_sa_client *client,
+		       struct ib_device *device, u8 port_num,
 		       struct ib_sa_path_rec *rec,
 		       ib_sa_comp_mask comp_mask,
 		       int timeout_ms, int retries, gfp_t gfp_mask,
@@ -264,7 +285,8 @@ int ib_sa_path_rec_get(struct ib_device 
 		       void *context,
 		       struct ib_sa_query **query);
 
-int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
+			     struct ib_device *device, u8 port_num,
 			     u8 method,
 			     struct ib_sa_mcmember_rec *rec,
 			     ib_sa_comp_mask comp_mask,
@@ -275,7 +297,8 @@ int ib_sa_mcmember_rec_query(struct ib_d
 			     void *context,
 			     struct ib_sa_query **query);
 
-int ib_sa_service_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_service_rec_query(struct ib_sa_client *client,
+			 struct ib_device *device, u8 port_num,
 			 u8 method,
 			 struct ib_sa_service_rec *rec,
 			 ib_sa_comp_mask comp_mask,
@@ -288,6 +311,7 @@ int ib_sa_service_rec_query(struct ib_de
 
 /**
  * ib_sa_mcmember_rec_set - Start an MCMember set query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:MCMember Record to send in query
@@ -312,7 +336,8 @@ int ib_sa_service_rec_query(struct ib_de
  * cancel the query.
  */
 static inline int
-ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
+ib_sa_mcmember_rec_set(struct ib_sa_client *client,
+		       struct ib_device *device, u8 port_num,
 		       struct ib_sa_mcmember_rec *rec,
 		       ib_sa_comp_mask comp_mask,
 		       int timeout_ms, int retries, gfp_t gfp_mask,
@@ -322,7 +347,7 @@ ib_sa_mcmember_rec_set(struct ib_device 
 		       void *context,
 		       struct ib_sa_query **query)
 {
-	return ib_sa_mcmember_rec_query(device, port_num,
+	return ib_sa_mcmember_rec_query(client, device, port_num,
 					IB_MGMT_METHOD_SET,
 					rec, comp_mask,
 					timeout_ms, retries, gfp_mask, callback,
@@ -331,6 +356,7 @@ ib_sa_mcmember_rec_set(struct ib_device 
 
 /**
  * ib_sa_mcmember_rec_delete - Start an MCMember delete query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:MCMember Record to send in query
@@ -355,7 +381,8 @@ ib_sa_mcmember_rec_set(struct ib_device 
  * cancel the query.
  */
 static inline int
-ib_sa_mcmember_rec_delete(struct ib_device *device, u8 port_num,
+ib_sa_mcmember_rec_delete(struct ib_sa_client *client,
+			  struct ib_device *device, u8 port_num,
 			  struct ib_sa_mcmember_rec *rec,
 			  ib_sa_comp_mask comp_mask,
 			  int timeout_ms, int retries, gfp_t gfp_mask,
@@ -365,7 +392,7 @@ ib_sa_mcmember_rec_delete(struct ib_devi
 			  void *context,
 			  struct ib_sa_query **query)
 {
-	return ib_sa_mcmember_rec_query(device, port_num,
+	return ib_sa_mcmember_rec_query(client, device, port_num,
 					IB_SA_METHOD_DELETE,
 					rec, comp_mask,
 					timeout_ms, retries, gfp_mask, callback,
Index: core/sa_query.c
===================================================================
--- core/sa_query.c	(revision 8928)
+++ core/sa_query.c	(working copy)
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
+ * Copyright (c) 2006 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -74,6 +75,7 @@ struct ib_sa_device {
 struct ib_sa_query {
 	void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
 	void (*release)(struct ib_sa_query *);
+	struct ib_sa_client    *client;
 	struct ib_sa_port      *port;
 	struct ib_mad_send_buf *mad_buf;
 	struct ib_sa_sm_ah     *sm_ah;
@@ -413,6 +415,38 @@ static void ib_sa_event(struct ib_event_
 	}
 }
 
+void ib_sa_register_client(struct ib_sa_client *client)
+{
+	atomic_set(&client->users, 1);
+	init_completion(&client->comp);
+}
+EXPORT_SYMBOL(ib_sa_register_client);
+
+static inline void ib_sa_client_get(struct ib_sa_query *query,
+				    struct ib_sa_client *client)
+{
+	atomic_inc(&client->users);
+	query->client = client;
+}
+
+static inline void deref_client(struct ib_sa_client *client)
+{
+	if (atomic_dec_and_test(&client->users))
+		complete(&client->comp);
+}
+
+static inline void ib_sa_client_put(struct ib_sa_query *query)
+{
+	deref_client(query->client);
+}
+
+void ib_sa_unregister_client(struct ib_sa_client *client)
+{
+	deref_client(client);
+	wait_for_completion(&client->comp);
+}
+EXPORT_SYMBOL(ib_sa_unregister_client);
+
 /**
  * ib_sa_cancel_query - try to cancel an SA query
  * @id:ID of query to cancel
@@ -613,6 +647,7 @@ static void ib_sa_path_rec_release(struc
 
 /**
  * ib_sa_path_rec_get - Start a Path get query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:Path Record to send in query
@@ -636,7 +671,8 @@ static void ib_sa_path_rec_release(struc
  * error code.  Otherwise it is a query ID that can be used to cancel
  * the query.
  */
-int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
+int ib_sa_path_rec_get(struct ib_sa_client *client,
+		       struct ib_device *device, u8 port_num,
 		       struct ib_sa_path_rec *rec,
 		       ib_sa_comp_mask comp_mask,
 		       int timeout_ms, int retries, gfp_t gfp_mask,
@@ -671,6 +707,7 @@ int ib_sa_path_rec_get(struct ib_device 
 		goto err1;
 	}
 
+	ib_sa_client_get(&query->sa_query, client);
 	query->callback = callback;
 	query->context  = context;
 
@@ -696,6 +733,7 @@ int ib_sa_path_rec_get(struct ib_device 
 
 err2:
 	*sa_query = NULL;
+	ib_sa_client_put(&query->sa_query);
 	ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -728,6 +766,7 @@ static void ib_sa_service_rec_release(st
 
 /**
  * ib_sa_service_rec_query - Start Service Record operation
+ * @client:SA client
  * @device:device to send request on
  * @port_num: port number to send request on
  * @method:SA method - should be get, set, or delete
@@ -753,7 +792,8 @@ static void ib_sa_service_rec_release(st
  * error code.  Otherwise it is a request ID that can be used to cancel
  * the query.
  */
-int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
+int ib_sa_service_rec_query(struct ib_sa_client *client,
+			    struct ib_device *device, u8 port_num, u8 method,
 			    struct ib_sa_service_rec *rec,
 			    ib_sa_comp_mask comp_mask,
 			    int timeout_ms, int retries, gfp_t gfp_mask,
@@ -793,6 +833,7 @@ int ib_sa_service_rec_query(struct ib_de
 		goto err1;
 	}
 
+	ib_sa_client_get(&query->sa_query, client);
 	query->callback = callback;
 	query->context  = context;
 
@@ -819,6 +860,7 @@ int ib_sa_service_rec_query(struct ib_de
 
 err2:
 	*sa_query = NULL;
+	ib_sa_client_put(&query->sa_query);
 	ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -849,7 +891,8 @@ static void ib_sa_mcmember_rec_release(s
 	kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query));
 }
 
-int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
+			     struct ib_device *device, u8 port_num,
 			     u8 method,
 			     struct ib_sa_mcmember_rec *rec,
 			     ib_sa_comp_mask comp_mask,
@@ -885,6 +928,7 @@ int ib_sa_mcmember_rec_query(struct ib_d
 		goto err1;
 	}
 
+	ib_sa_client_get(&query->sa_query, client);
 	query->callback = callback;
 	query->context  = context;
 
@@ -911,6 +955,7 @@ int ib_sa_mcmember_rec_query(struct ib_d
 
 err2:
 	*sa_query = NULL;
+	ib_sa_client_put(&query->sa_query);
 	ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -947,6 +992,7 @@ static void send_handler(struct ib_mad_a
 
         ib_free_send_mad(mad_send_wc->send_buf);
 	kref_put(&query->sm_ah->ref, free_sm_ah);
+	ib_sa_client_put(query);
 	query->release(query);
 }
 
Index: core/multicast.c
===================================================================
--- core/multicast.c	(revision 8928)
+++ core/multicast.c	(working copy)
@@ -63,6 +63,7 @@ static struct ib_client mcast_client = {
 	.remove = mcast_remove_one
 };
 
+static struct ib_sa_client	sa_client;
 static struct ib_event_handler	event_handler;
 static struct workqueue_struct	*mcast_wq;
 static union ib_gid mgid0;
@@ -305,8 +306,8 @@ static int send_join(struct mcast_group 
 	int ret;
 
 	group->last_join = member;
-	ret = ib_sa_mcmember_rec_set(port->dev->device, port->port_num,
-				     &member->multicast.rec,
+	ret = ib_sa_mcmember_rec_set(&sa_client, port->dev->device,
+				     port->port_num, &member->multicast.rec,
 				     member->multicast.comp_mask,
 				     retry_timer, retries, GFP_KERNEL,
 				     join_handler, group, &group->query);
@@ -326,7 +327,8 @@ static int send_leave(struct mcast_group
 	rec = group->rec;
 	rec.join_state = leave_state;
 
-	ret = ib_sa_mcmember_rec_delete(port->dev->device, port->port_num, &rec,
+	ret = ib_sa_mcmember_rec_delete(&sa_client, port->dev->device,
+					port->port_num, &rec,
 					IB_SA_MCMEMBER_REC_MGID     |
 					IB_SA_MCMEMBER_REC_PORT_GID |
 					IB_SA_MCMEMBER_REC_JOIN_STATE,
@@ -770,12 +772,15 @@ static int __init mcast_init(void)
 	if (!mcast_wq)
 		return -ENOMEM;
 
+	ib_sa_register_client(&sa_client);
+
 	ret = ib_register_client(&mcast_client);
 	if (ret)
 		goto err;
 	return 0;
 
 err:
+	ib_sa_unregister_client(&sa_client);
 	destroy_workqueue(mcast_wq);
 	return ret;
 }
@@ -783,6 +788,7 @@ err:
 static void __exit mcast_cleanup(void)
 {
 	ib_unregister_client(&mcast_client);
+	ib_sa_unregister_client(&sa_client);
 	destroy_workqueue(mcast_wq);
 }
 
Index: core/cma.c
===================================================================
--- core/cma.c	(revision 8928)
+++ core/cma.c	(working copy)
@@ -61,6 +61,7 @@ static struct ib_client cma_client = {
 	.remove = cma_remove_one
 };
 
+struct ib_sa_client sa_client;
 static LIST_HEAD(dev_list);
 static LIST_HEAD(listen_any_list);
 static DEFINE_MUTEX(lock);
@@ -1272,7 +1273,7 @@ static int cma_query_ib_route(struct rdm
 	path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr));
 	path_rec.numb_path = 1;
 
-	id_priv->query_id = ib_sa_path_rec_get(id_priv->id.device,
+	id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
 				id_priv->id.port_num, &path_rec,
 				IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
 				IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH,
@@ -2367,12 +2368,15 @@ static int cma_init(void)
 	if (!cma_wq)
 		return -ENOMEM;
 
+	ib_sa_register_client(&sa_client);
+
 	ret = ib_register_client(&cma_client);
 	if (ret)
 		goto err;
 	return 0;
 
 err:
+	ib_sa_unregister_client(&sa_client);
 	destroy_workqueue(cma_wq);
 	return ret;
 }
@@ -2380,6 +2384,7 @@ err:
 static void cma_cleanup(void)
 {
 	ib_unregister_client(&cma_client);
+	ib_sa_unregister_client(&sa_client);
 	destroy_workqueue(cma_wq);
 	idr_destroy(&sdp_ps);
 	idr_destroy(&tcp_ps);
Index: ulp/ipoib/ipoib_main.c
===================================================================
--- ulp/ipoib/ipoib_main.c	(revision 8928)
+++ ulp/ipoib/ipoib_main.c	(working copy)
@@ -91,6 +91,8 @@ static struct ib_client ipoib_client = {
 	.remove = ipoib_remove_one
 };
 
+static struct ib_sa_client ipoib_sa_client;
+
 int ipoib_open(struct net_device *dev)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -462,7 +464,7 @@ static int path_rec_start(struct net_dev
 	init_completion(&path->done);
 
 	path->query_id =
-		ib_sa_path_rec_get(priv->ca, priv->port,
+		ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
 				   &path->pathrec,
 				   IB_SA_PATH_REC_DGID		|
 				   IB_SA_PATH_REC_SGID		|
@@ -1187,13 +1189,16 @@ static int __init ipoib_init_module(void
 		goto err_fs;
 	}
 
+	ib_sa_register_client(&ipoib_sa_client);
+
 	ret = ib_register_client(&ipoib_client);
 	if (ret)
-		goto err_wq;
+		goto err_sa;
 
 	return 0;
 
-err_wq:
+err_sa:
+	ib_sa_unregister_client(&ipoib_sa_client);
 	destroy_workqueue(ipoib_workqueue);
 
 err_fs:
@@ -1205,6 +1210,7 @@ err_fs:
 static void __exit ipoib_cleanup_module(void)
 {
 	ib_unregister_client(&ipoib_client);
+	ib_sa_unregister_client(&ipoib_sa_client);
 	ipoib_unregister_debugfs();
 	destroy_workqueue(ipoib_workqueue);
 }
Index: ulp/srp/ib_srp.c
===================================================================
--- ulp/srp/ib_srp.c	(revision 8928)
+++ ulp/srp/ib_srp.c	(working copy)
@@ -103,6 +103,8 @@ static struct ib_client srp_client = {
 	.remove = srp_remove_one
 };
 
+static struct ib_sa_client srp_sa_client;
+
 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 {
 	return (struct srp_target_port *) host->hostdata;
@@ -274,7 +276,8 @@ static int srp_lookup_path(struct srp_ta
 
 	init_completion(&target->done);
 
-	target->path_query_id = ib_sa_path_rec_get(target->srp_host->dev->dev,
+	target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
+						   target->srp_host->dev->dev,
 						   target->srp_host->port,
 						   &target->path,
 						   IB_SA_PATH_REC_DGID		|
@@ -2002,9 +2005,12 @@ static int __init srp_init_module(void)
 		return ret;
 	}
 
+	ib_sa_register_client(&srp_sa_client);
+
 	ret = ib_register_client(&srp_client);
 	if (ret) {
 		printk(KERN_ERR PFX "couldn't register IB client\n");
+		ib_sa_unregister_client(&srp_sa_client);
 		class_unregister(&srp_class);
 		return ret;
 	}
@@ -2015,6 +2021,7 @@ static int __init srp_init_module(void)
 static void __exit srp_cleanup_module(void)
 {
 	ib_unregister_client(&srp_client);
+	ib_sa_unregister_client(&srp_sa_client);
 	class_unregister(&srp_class);
 }
 





More information about the general mailing list