[Fwd: [openib-general] [PATCH] mad/agent: Modify receive buffer allocation strategy]

Hal Rosenstock halr at voltaire.com
Tue Nov 9 09:30:50 PST 2004


One more thing on this I forgot to post:

As I am not yet set up with Kegel cross tools (and don't have a machine
where the pci_ macros are non trivial), I would appreciate it if someone
could verify these changes (or latest code) on some architecture where
the pci_ macros are non trivial.

Thanks.

-- Hal

-----Forwarded Message-----

From: Hal Rosenstock <halr at voltaire.com>
To: openib-general at openib.org
Subject: [openib-general] [PATCH] mad/agent: Modify receive buffer       allocation strategy
Date: 09 Nov 2004 11:49:21 -0500

mad/agent: Modify receive buffer allocation strategy
(Inefficiency pointed out by Sean; algorithm described by Roland)

Problem: Currently, if the underlying driver provides a process_mad
routine, a response MAD is allocated every time a MAD is received on
QP 0 or 1.

Solution: The MAD layer can allocate a response MAD when a MAD is
received, and if the process_mad call doesn't actually generate a
response the MAD layer just stashes the response MAD away to use for
the next receive. This should keep the number of allocations within 1
of the number of responses actually generated, but save us from
tracking allocations between two layers.

Index: agent.h
===================================================================
--- agent.h	(revision 1180)
+++ agent.h	(working copy)
@@ -31,7 +31,7 @@
 
 extern int ib_agent_port_close(struct ib_device *device, int port_num);
 
-extern int agent_send(struct ib_mad *mad,
+extern int agent_send(struct ib_mad_private *mad,
 		      struct ib_grh *grh,
 		      struct ib_wc *wc,
 		      struct ib_device *device,
Index: agent_priv.h
===================================================================
--- agent_priv.h	(revision 1180)
+++ agent_priv.h	(working copy)
@@ -33,7 +33,7 @@
 struct ib_agent_send_wr {
 	struct list_head send_list;
 	struct ib_ah *ah;
-	struct ib_mad *mad;
+	struct ib_mad_private *mad;
 	DECLARE_PCI_UNMAP_ADDR(mapping)
 };
 
Index: agent.c
===================================================================
--- agent.c	(revision 1182)
+++ agent.c	(working copy)
@@ -33,7 +33,9 @@
 static spinlock_t ib_agent_port_list_lock = SPIN_LOCK_UNLOCKED;
 static LIST_HEAD(ib_agent_port_list);
 
+extern kmem_cache_t *ib_mad_cache;
 
+
 static inline struct ib_agent_port_private *
 __ib_get_agent_port(struct ib_device *device, int port_num,
 		    struct ib_mad_agent *mad_agent)
@@ -95,7 +97,7 @@
 
 static int agent_mad_send(struct ib_mad_agent *mad_agent,
 			  struct ib_agent_port_private *port_priv,
-			  struct ib_mad *mad,
+			  struct ib_mad_private *mad,
 			  struct ib_grh *grh,
 			  struct ib_wc *wc)
 {
@@ -114,10 +116,10 @@
 
 	/* PCI mapping */
 	gather_list.addr = pci_map_single(mad_agent->device->dma_device,
-					  mad,
-					  sizeof(struct ib_mad),
+					  &mad->grh,
+					  sizeof *mad - sizeof mad->header,
 					  PCI_DMA_TODEVICE);
-	gather_list.length = sizeof(struct ib_mad);
+	gather_list.length = sizeof *mad - sizeof mad->header;
 	gather_list.lkey = (*port_priv->mr).lkey;
 
 	send_wr.next = NULL;
@@ -133,7 +135,7 @@
 	ah_attr.src_path_bits = wc->dlid_path_bits;
 	ah_attr.sl = wc->sl;
 	ah_attr.static_rate = 0;
-	if (mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT) {
+	if (mad->mad.mad.mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT) {
 		if (wc->wc_flags & IB_WC_GRH) {
 			ah_attr.ah_flags = IB_AH_GRH;
 			/* Should sgid be looked up ? */
@@ -162,14 +164,14 @@
 	}
 
 	send_wr.wr.ud.ah = agent_send_wr->ah;
-	if (mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT) {
+	if (mad->mad.mad.mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT) {
 		send_wr.wr.ud.pkey_index = wc->pkey_index;
 		send_wr.wr.ud.remote_qkey = IB_QP1_QKEY;
 	} else {
 		send_wr.wr.ud.pkey_index = 0; /* Should only matter for GMPs */
 		send_wr.wr.ud.remote_qkey = 0; /* for SMPs */
 	}
-	send_wr.wr.ud.mad_hdr = (struct ib_mad_hdr *)mad;
+	send_wr.wr.ud.mad_hdr = &mad->mad.mad.mad_hdr;
 	send_wr.wr_id = ++port_priv->wr_id;
 
 	pci_unmap_addr_set(agent_send_wr, mapping, gather_list.addr);
@@ -180,7 +182,8 @@
 		spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
 		pci_unmap_single(mad_agent->device->dma_device,
 				 pci_unmap_addr(agent_send_wr, mapping),
-				 sizeof(struct ib_mad),
+				 sizeof(struct ib_mad_private) -
+				 sizeof(struct ib_mad_private_header),
 				 PCI_DMA_TODEVICE);
 		ib_destroy_ah(agent_send_wr->ah);
 		kfree(agent_send_wr);
@@ -195,7 +198,7 @@
 	return ret;
 }
 
-int agent_send(struct ib_mad *mad,
+int agent_send(struct ib_mad_private *mad,
 	       struct ib_grh *grh,
 	       struct ib_wc *wc,
 	       struct ib_device *device,
@@ -212,7 +215,7 @@
 	}
 
 	/* Get mad agent based on mgmt_class in MAD */
-	switch (mad->mad_hdr.mgmt_class) {
+	switch (mad->mad.mad.mad_hdr.mgmt_class) {
 		case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
 			mad_agent = port_priv->dr_smp_agent;
 			break;
@@ -269,13 +272,14 @@
 	/* Unmap PCI */
 	pci_unmap_single(mad_agent->device->dma_device,
 			 pci_unmap_addr(agent_send_wr, mapping),
-			 sizeof(struct ib_mad),
+			 sizeof(struct ib_mad_private) -
+			 sizeof(struct ib_mad_private_header),
 			 PCI_DMA_TODEVICE);
 
 	ib_destroy_ah(agent_send_wr->ah);
 
 	/* Release allocated memory */
-	kfree(agent_send_wr->mad);
+	kmem_cache_free(ib_mad_cache, agent_send_wr->mad);
 	kfree(agent_send_wr);
 }
 
Index: mad.c
===================================================================
--- mad.c	(revision 1181)
+++ mad.c	(working copy)
@@ -69,7 +69,7 @@
 MODULE_AUTHOR("Sean Hefty");
 

-static kmem_cache_t *ib_mad_cache;
+kmem_cache_t *ib_mad_cache;
 static struct list_head ib_mad_port_list;
 static u32 ib_mad_client_id = 0;
 
@@ -83,7 +83,8 @@
 static int add_mad_reg_req(struct ib_mad_reg_req *mad_reg_req,
 			   struct ib_mad_agent_private *priv);
 static void remove_mad_reg_req(struct ib_mad_agent_private *priv); 
-static int ib_mad_post_receive_mad(struct ib_mad_qp_info *qp_info);
+static int ib_mad_post_receive_mad(struct ib_mad_qp_info *qp_info,
+				   struct ib_mad_private *mad);
 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info);
 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
 static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
@@ -1067,12 +1068,17 @@
 {
 	struct ib_mad_qp_info *qp_info;
 	struct ib_mad_private_header *mad_priv_hdr;
-	struct ib_mad_private *recv;
+	struct ib_mad_private *recv, *response;
 	struct ib_mad_list_head *mad_list;
 	struct ib_mad_agent_private *mad_agent;
 	struct ib_smp *smp;
 	int solicited;
 
+	response = kmem_cache_alloc(ib_mad_cache, GFP_KERNEL);
+	if (!response)
+		printk(KERN_ERR PFX "ib_mad_recv_done_handler no memory "
+		       "for response buffer\n");
+
 	mad_list = (struct ib_mad_list_head *)(unsigned long)wc->wr_id;
 	qp_info = mad_list->mad_queue->qp_info;
 	dequeue_mad(mad_list);
@@ -1119,11 +1125,9 @@
 
 	/* Give driver "right of first refusal" on incoming MAD */
 	if (port_priv->device->process_mad) {
-		struct ib_mad *response;
 		struct ib_grh *grh;
 		int ret;
 
-		response = kmalloc(sizeof(struct ib_mad), GFP_KERNEL);
 		if (!response) {
 			printk(KERN_ERR PFX "No memory for response MAD\n");
 			/*
@@ -1137,32 +1141,29 @@
 						     port_priv->port_num,
 						     wc->slid,
 						     recv->header.recv_buf.mad,
-						     response);
+						     &response->mad.mad);
 		if (ret & IB_MAD_RESULT_SUCCESS) {
 			if (ret & IB_MAD_RESULT_REPLY) {
-				if (response->mad_hdr.mgmt_class ==
+				if (response->mad.mad.mad_hdr.mgmt_class ==
 				    IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
 					if (!smi_handle_dr_smp_recv(
-					    (struct ib_smp *)response,
+					    (struct ib_smp *)&response->mad.mad,
 					    port_priv->device->node_type,
 					    port_priv->port_num,
 					    port_priv->device->phys_port_cnt)) {
-						kfree(response);
 						goto out;
 					}
 				}
 				/* Send response */
 				grh = (void *)recv->header.recv_buf.mad -
 				      sizeof(struct ib_grh);
-				if (agent_send(response, grh, wc,
-					       port_priv->device,
-					       port_priv->port_num)) {
-					kfree(response);
-				}
+				if (!agent_send(response, grh, wc,
+						port_priv->device,
+						port_priv->port_num))
+					response = NULL;
 				goto out;
 			}
-		} else
-			kfree(response);
+		} 
 	}
 
 	/* Determine corresponding MAD agent for incoming receive MAD */
@@ -1183,7 +1184,7 @@
 		kmem_cache_free(ib_mad_cache, recv);
 
 	/* Post another receive request for this QP */
-	ib_mad_post_receive_mad(qp_info);
+	ib_mad_post_receive_mad(qp_info, response);
 }
 
 static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
@@ -1491,7 +1492,8 @@
 	queue_work(port_priv->wq, &port_priv->work);
 }
 
-static int ib_mad_post_receive_mad(struct ib_mad_qp_info *qp_info)
+static int ib_mad_post_receive_mad(struct ib_mad_qp_info *qp_info,
+				   struct ib_mad_private *mad)
 {
 	struct ib_mad_private *mad_priv;
 	struct ib_sge sg_list;
@@ -1499,19 +1501,23 @@
 	struct ib_recv_wr *bad_recv_wr;
 	int ret;
 
-	/* 
-	 * Allocate memory for receive buffer.
-	 * This is for both MAD and private header
-	 * which contains the receive tracking structure.
-	 * By prepending this header, there is one rather 
-	 * than two memory allocations.
-	 */
-	mad_priv = kmem_cache_alloc(ib_mad_cache,
-				    (in_atomic() || irqs_disabled()) ?
-				    GFP_ATOMIC : GFP_KERNEL);
-	if (!mad_priv) {
-		printk(KERN_ERR PFX "No memory for receive buffer\n");
-		return -ENOMEM;
+	if (mad)
+		mad_priv = mad;
+	else {
+		/* 
+		 * Allocate memory for receive buffer.
+		 * This is for both MAD and private header
+		 * which contains the receive tracking structure.
+		 * By prepending this header, there is one rather 
+		 * than two memory allocations.
+		 */
+		mad_priv = kmem_cache_alloc(ib_mad_cache,
+					    (in_atomic() || irqs_disabled()) ?
+					    GFP_ATOMIC : GFP_KERNEL);
+		if (!mad_priv) {
+			printk(KERN_ERR PFX "No memory for receive buffer\n");
+			return -ENOMEM;
+		}
 	}
 
 	/* Setup scatter list */
@@ -1559,7 +1565,7 @@
 	int i, ret;
 
 	for (i = 0; i < IB_MAD_QP_RECV_SIZE; i++) {
-		ret = ib_mad_post_receive_mad(qp_info);
+		ret = ib_mad_post_receive_mad(qp_info, NULL);
 		if (ret) {
 			printk(KERN_ERR PFX "receive post %d failed "
 				"on %s port %d\n", i + 1,



_______________________________________________
openib-general mailing list
openib-general at openib.org
http://openib.org/mailman/listinfo/openib-general

To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general




More information about the general mailing list