[ofa-general] Re: [PATCH 2/3] remove ib pkey gid and lmc cache

[Roland Dreier]

 > @@ -279,7 +278,7 @@ int mthca_read_ah(struct mthca_dev *dev,
 >  			(be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 20) & 0xff;
 >  		header->grh.flow_label    =
 >  			ah->av->sl_tclass_flowlabel & cpu_to_be32(0xfffff);
 > -		ib_get_cached_gid(&dev->ib_dev,
 > +		ib_query_gid(&dev->ib_dev,
 >  				  be32_to_cpu(ah->av->port_pd) >> 24,
 >  				  ah->av->gid_index % dev->limits.gid_table_len,
 >  				  &header->grh.source_gid);
 > Index: b/drivers/infiniband/hw/mthca/mthca_qp.c
 > ===================================================================
 > --- a/drivers/infiniband/hw/mthca/mthca_qp.c	2007-05-02 17:47:53.153873064 +0300
 > +++ b/drivers/infiniband/hw/mthca/mthca_qp.c	2007-05-02 18:04:14.123981858 +0300
 > @@ -40,9 +40,8 @@
 >  
 >  #include <asm/io.h>
 >  
 > -#include <rdma/ib_verbs.h>
 > -#include <rdma/ib_cache.h>
 >  #include <rdma/ib_pack.h>
 > +#include <rdma/ib_verbs.h>
 >  
 >  #include "mthca_dev.h"
 >  #include "mthca_cmd.h"
 > @@ -1485,11 +1484,10 @@ static int build_mlx_header(struct mthca
 >  		sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
 >  	sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
 >  	if (!sqp->qp.ibqp.qp_num)
 > -		ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port,
 > -				   sqp->pkey_index, &pkey);
 > +		ib_query_pkey(&dev->ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
 >  	else
 > -		ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port,
 > -				   wr->wr.ud.pkey_index, &pkey);
 > +		ib_query_pkey(&dev->ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey);
 > +
 >  	sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
 >  	sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
 >  	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));

These mthca changes can't be correct -- you are adding calls to
sleeping functions from build_mlx_header(), which is always called
with a spinlock held.  You'll have to update mthca to keep track of
the GID and P_Key tables internally to fix this.

Please test your code with CONFIG_DEBUG_SPINLOCK_SLEEP=y at least to
see if there are any other places that are using the cache because
they're not allowed to sleep.

 - R.