<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
<HTML>
<HEAD>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=us-ascii">
<META NAME="Generator" CONTENT="MS Exchange Server version 6.0.6618.4">
<TITLE>IBV_WC_LOC_PROT_ERR on RDMA read into huge paged buffers acquiered with shmget.... </TITLE>
</HEAD>
<BODY>
<!-- Converted from text/rtf format -->
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">Our application uses huge paged</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">(shmget</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">…</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">..)</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">buffers in order to get highest transfer speeds.</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">The buffers (></FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">=</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> 16 MByte) span over multiple of the</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">2 MByte</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">pages</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">.</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial"></FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> </SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">In cases where my read buffer is smaller then a certain size let</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">’</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">s say 128 MByte I get </FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">IBV_WC_LOC_PROT_ERR</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> into the completion queue.</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">This happens for any RDMA transmission size</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">.</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">T</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">he</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> only</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">visible</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">difference</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">in user space</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">between the buffers that fail and the ones that work seem</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">s</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> to be</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">their</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> address returned from the shmat command</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">The ones that work are out of the range 0x75E00000</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"></FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">(>= 128MByte)</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">the ones that fail are in the range 0xFC8</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">00000.</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> (< 128 MByte)</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">I</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">’</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">m using OFED 1.1</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> here.</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">Any help</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">/</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">idea welcome.</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">Thomas</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">P.S.:</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> </SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">On Mellanox gen1 I already discovered a bug where on</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">ly</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> the first page of the buffer was filled</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">correctly</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">.</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial"> This was fixed</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">in a patch for IBGD-1.8.2</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">P.P.S:</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Arial">To register my buffers I</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">’</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">m using</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Arial">:</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Courier New">accessFlags = (ibv_access_flags)(IBV_ACCESS_LOCAL_WRITE |</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Courier New"> </FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Courier New"> IBV_ACCESS_REMOTE_WRITE |</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Courier New"> </FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Courier New"></FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Courier New"> IBV_ACCESS_REMOTE_READ |</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Courier New"> IBV_ACCESS_REMOTE_ATOMIC|</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"> <FONT SIZE=2 FACE="Courier New"> IBV_ACCESS_MW_BIND);</FONT></SPAN></P>
<P DIR=LTR><SPAN LANG="en-gb"><FONT SIZE=2 FACE="Courier New">mr = ibv_reg_mr(_pd, (void*)(uintptr_t)bufferPtr, size, accessFlags);</FONT></SPAN><SPAN LANG="de"></SPAN><SPAN LANG="en-gb"></SPAN></P>
</BODY>
</HTML>