<div dir="ltr"><div dir="ltr"><div dir="ltr"><div dir="ltr"><div dir="ltr"><div class="gmail_quote">Hi Libfabric community,</div><div class="gmail_quote"><br></div><div class="gmail_quote">I have been "playing" with libfabric for a while wishing to write a provider agnostic communication interface for our application - to the extend possible. In doing tests for how to best do this, I have stumble on an issue that is either due to a misunderstanding on my side or a possible bug when using udp;ofi_rxd. I have been able to reproduce the issue with the code below. Basically with the tcp and the udp provider, I am able to fi_av_insert an ip address and fi_av_lookup it up afterwards. However if I select the upd;ofi_rxd provider, I can fi_av_insert the address with success, but fi_av_lookup fails to find the address afterward, error -61 -> No data available. Please see code and output below.</div><div class="gmail_quote"><br></div><div class="gmail_quote">Any feedback is most welcomed, even if it is that I have simply misunderstood something. </div><div class="gmail_quote"><br></div><div class="gmail_quote">Thanks & Best Regards, Anders<br><br><div dir="ltr"><div style="background-color:rgb(30,31,34);color:rgb(188,190,196)"><pre style="font-family:"JetBrains Mono",monospace"><span style="color:rgb(179,174,96)">#include </span><span style="color:rgb(106,171,115)"><iostream><br></span><span style="color:rgb(106,171,115)"><br></span><span style="color:rgb(179,174,96)">#include </span><span style="color:rgb(106,171,115)"><rdma/fabric.h><br></span><span style="color:rgb(179,174,96)">#include </span><span style="color:rgb(106,171,115)"><rdma/fi_domain.h><br></span><span style="color:rgb(179,174,96)">#include </span><span style="color:rgb(106,171,115)"><cstring><br></span><span style="color:rgb(106,171,115)"><br></span><span style="color:rgb(106,171,115)"><br></span><span style="color:rgb(207,142,109)">void </span><span style="color:rgb(86,168,245)">result</span>( <span style="color:rgb(207,142,109)">int </span>err, <span style="color:rgb(207,142,109)">const char </span>*name )<br>{<br> printf( <span style="color:rgb(106,171,115)">"%s: %i -> %s</span><span style="color:rgb(207,142,109)">\n</span><span style="color:rgb(106,171,115)">"</span>, name, err, fi_strerror( err ) );<br>}<br><br><span style="color:rgb(207,142,109)">void </span><span style="color:rgb(86,168,245)">av_test</span>( <span style="color:rgb(181,182,227)">fi_info </span>*hints )<br>{<br> <span style="color:rgb(181,182,227)">fi_info </span>*fi;<br> <span style="color:rgb(181,182,227)">fid_fabric </span>*fabric;<br> <span style="color:rgb(181,182,227)">fid_domain </span>*domain;<br> <span style="color:rgb(181,182,227)">fid_av </span>*av;<br> <span style="color:rgb(185,188,209)">fi_addr_t </span>peer_addr {};<br><br><br> <span style="color:rgb(207,142,109)">int </span>err = fi_getinfo( <span style="color:rgb(144,139,37)">FI_VERSION</span>(<span style="color:rgb(42,172,184)">1</span>, <span style="color:rgb(42,172,184)">21</span>), <span style="color:rgb(207,142,109)">nullptr</span>, <span style="color:rgb(106,171,115)">"5000"</span>, <span style="color:rgb(144,139,37)">FI_SOURCE</span>, hints, &fi);<br><br> result( err, <span style="color:rgb(106,171,115)">"fi_getinfo" </span>);<br><br> printf(<span style="color:rgb(106,171,115)">"Provider: %s</span><span style="color:rgb(207,142,109)">\n</span><span style="color:rgb(106,171,115)">"</span>, fi-><span style="color:rgb(147,115,165)">fabric_attr</span>-><span style="color:rgb(147,115,165)">prov_name</span>);<br><br> err = fi_fabric(fi-><span style="color:rgb(147,115,165)">fabric_attr</span>, &fabric, <span style="color:rgb(207,142,109)">nullptr</span>);<br> result( err, <span style="color:rgb(106,171,115)">"fi_fabric"</span>);<br><br> err = fi_domain(fabric, fi, &domain, <span style="color:rgb(207,142,109)">nullptr </span>);<br> result( err, <span style="color:rgb(106,171,115)">"fi_domain" </span>);<br><br> <span style="color:rgb(181,182,227)">fi_av_attr </span>av_attr {};<br><br> av_attr.<span style="color:rgb(147,115,165)">type </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_AV_UNSPEC</span>;<br><br> err = fi_av_open(domain, &av_attr, &av, <span style="color:rgb(207,142,109)">nullptr </span>);<br> result( err, <span style="color:rgb(106,171,115)">"fi_av_open" </span>);<br><br> <span style="color:rgb(207,142,109)">char </span>dst_addr[] = {<span style="color:rgb(42,172,184)">2</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">19</span>, -<span style="color:rgb(42,172,184)">120</span>, <span style="color:rgb(42,172,184)">10</span>, <span style="color:rgb(42,172,184)">0</span>, -<span style="color:rgb(42,172,184)">1</span>, <span style="color:rgb(42,172,184)">2</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(42,172,184)">0 </span>};<br><br> err = fi_av_insert(av, dst_addr, <span style="color:rgb(42,172,184)">1</span>, &peer_addr, <span style="color:rgb(42,172,184)">0</span>, <span style="color:rgb(144,139,37)">NULL</span>);<br> <span style="color:rgb(207,142,109)">if</span>( err == <span style="color:rgb(42,172,184)">1 </span>) err = <span style="color:rgb(42,172,184)">0</span>; <span style="color:rgb(122,126,133)">// fi_av_insert return number of successful insert upon success, i.e. 1<br></span><span style="color:rgb(122,126,133)"> </span>result( err, <span style="color:rgb(106,171,115)">"fi_av_insert" </span>);<br><br> <span style="color:rgb(207,142,109)">char </span>lookupAddr[ <span style="color:rgb(42,172,184)">64 </span>]; <span style="color:rgb(185,188,209)">size_t </span>len = <span style="color:rgb(42,172,184)">64</span>;<br> err = fi_av_lookup(av, peer_addr, lookupAddr, &len);<br> result( err, <span style="color:rgb(106,171,115)">"fi_av_lookup" </span>);<br><br> printf( <span style="color:rgb(106,171,115)">"</span><span style="color:rgb(207,142,109)">\n</span><span style="color:rgb(106,171,115)">" </span>);<br> <br> fi_close((<span style="color:rgb(185,188,209)">fid_t</span>)av);<br> fi_close((<span style="color:rgb(185,188,209)">fid_t</span>)domain);<br> fi_close((<span style="color:rgb(185,188,209)">fid_t</span>)fabric);<br>}<br><br><span style="color:rgb(207,142,109)">void </span><span style="color:rgb(86,168,245)">test_tcp</span>()<br>{<br> <span style="color:rgb(181,182,227)">fi_info </span>*hints;<br> hints = fi_allocinfo();<br><br> hints-><span style="color:rgb(147,115,165)">ep_attr</span>-><span style="color:rgb(147,115,165)">type </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_EP_RDM</span>;<br> hints-><span style="color:rgb(147,115,165)">caps </span>= <span style="color:rgb(144,139,37)">FI_MSG</span>;<br> hints-><span style="color:rgb(147,115,165)">addr_format </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_SOCKADDR</span>;<br> hints-><span style="color:rgb(147,115,165)">fabric_attr</span>-><span style="color:rgb(147,115,165)">prov_name </span>= strdup(<span style="color:rgb(106,171,115)">"tcp"</span>);<br><br> av_test( hints );<br><br> fi_freeinfo( hints );<br>}<br><br><span style="color:rgb(207,142,109)">void </span><span style="color:rgb(86,168,245)">test_dgram_udp</span>()<br>{<br> <span style="color:rgb(181,182,227)">fi_info </span>*hints;<br> hints = fi_allocinfo();<br><br> hints-><span style="color:rgb(147,115,165)">ep_attr</span>-><span style="color:rgb(147,115,165)">type </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_EP_DGRAM</span>;<br> hints-><span style="color:rgb(147,115,165)">caps </span>= <span style="color:rgb(144,139,37)">FI_MSG</span>;<br> hints-><span style="color:rgb(147,115,165)">addr_format </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_SOCKADDR</span>;<br> hints-><span style="color:rgb(147,115,165)">fabric_attr</span>-><span style="color:rgb(147,115,165)">prov_name </span>= strdup(<span style="color:rgb(106,171,115)">"udp"</span>);<br><br> av_test( hints );<br><br> fi_freeinfo( hints );<br>}<br><br><span style="color:rgb(207,142,109)">void </span><span style="color:rgb(86,168,245)">test_rdm_udp</span>()<br>{<br> <span style="color:rgb(181,182,227)">fi_info </span>*hints;<br> hints = fi_allocinfo();<br><br> hints-><span style="color:rgb(147,115,165)">ep_attr</span>-><span style="color:rgb(147,115,165)">type </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_EP_RDM</span>;<br> hints-><span style="color:rgb(147,115,165)">caps </span>= <span style="color:rgb(144,139,37)">FI_MSG</span>;<br> hints-><span style="color:rgb(147,115,165)">addr_format </span>= <span style="color:rgb(199,125,187);font-style:italic">FI_SOCKADDR</span>;<br> hints-><span style="color:rgb(147,115,165)">fabric_attr</span>-><span style="color:rgb(147,115,165)">prov_name </span>= strdup(<span style="color:rgb(106,171,115)">"udp"</span>);<br><br> av_test( hints );<br><br> fi_freeinfo( hints );<br>}<br><br><br><span style="color:rgb(207,142,109)">int </span><span style="color:rgb(86,168,245)">main</span>()<br>{<br> test_tcp();<br> test_dgram_udp();<br> test_rdm_udp();<br>}</pre><pre style="font-family:"JetBrains Mono",monospace"><br></pre><pre style="font-family:"JetBrains Mono",monospace">Output:</pre><pre style="font-family:"JetBrains Mono",monospace">fi_getinfo: 0 -> Success<br>Provider: tcp<br>fi_fabric: 0 -> Success<br>fi_domain: 0 -> Success<br>fi_av_open: 0 -> Success<br>fi_av_insert: 0 -> Success<br>fi_av_lookup: 0 -> Success<br><br>fi_getinfo: 0 -> Success<br>Provider: udp<br>fi_fabric: 0 -> Success<br>fi_domain: 0 -> Success<br>fi_av_open: 0 -> Success<br>fi_av_insert: 0 -> Success<br>fi_av_lookup: 0 -> Success<br><br>fi_getinfo: 0 -> Success<br>Provider: udp;ofi_rxd<br>fi_fabric: 0 -> Success<br>fi_domain: 0 -> Success<br>fi_av_open: 0 -> Success<br>fi_av_insert: 0 -> Success<br>fi_av_lookup: -61 -> No data available<br><br><br>Process finished with exit code 0<br></pre></div></div>
</div></div></div></div></div></div>