<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Aptos;
panose-1:2 11 0 4 2 2 2 2 2 4;}
@font-face
{font-family:Menlo;
panose-1:2 11 6 9 3 8 4 2 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
font-size:12.0pt;
font-family:"Aptos",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
p.emailquote, li.emailquote, div.emailquote
{mso-style-name:emailquote;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:1.0pt;
border:none;
padding:0in;
font-size:12.0pt;
font-family:"Aptos",sans-serif;}
span.EmailStyle19
{mso-style-type:personal-reply;
font-family:"Aptos",sans-serif;
color:windowtext;}
p.p1, li.p1, div.p1
{mso-style-name:p1;
margin:0in;
font-size:10.5pt;
font-family:Menlo;
color:black;}
p.p2, li.p2, div.p2
{mso-style-name:p2;
margin:0in;
font-size:10.5pt;
font-family:Menlo;
color:black;}
span.apple-tab-span
{mso-style-name:apple-tab-span;}
span.s1
{mso-style-name:s1;}
span.apple-converted-space
{mso-style-name:apple-converted-space;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;
mso-ligatures:none;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
--></style>
</head>
<body lang="EN-US" link="blue" vlink="purple" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt">Should not take long I think.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="p1"><span style="font-size:11.0pt;font-family:"Aptos",sans-serif">Server:</span>
<span style="font-size:11.5pt">fi_rma_pingpong -s 192.168.1.100 -e msg <o:p></o:p></span></p>
<p class="p1"><span style="font-size:11.0pt;font-family:"Aptos",sans-serif">Client :
</span><span class="s1">fi_rma_pingpong </span><span class="apple-converted-space">
</span><span class="s1">192.168.1.100 -e msg</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="p1"><span class="s1">bytes </span><span class="apple-converted-space">
</span><span class="s1">iters </span><span class="apple-converted-space"> </span>
<span class="s1">total </span><span class="apple-converted-space"> </span><span class="s1">time
</span><span class="apple-converted-space"> </span><span class="s1">MB/sec</span><span class="apple-converted-space">
</span><span class="s1">usec/xfer </span><span class="apple-converted-space"> </span>
<span class="s1">Mxfers/sec</span><o:p></o:p></p>
<p class="p1"><span class="s1">64</span><span class="apple-converted-space">
</span><span class="s1">10k </span><span class="apple-converted-space"> </span>
<span class="s1">1.2m</span><span class="apple-converted-space"> </span><span class="s1">0.03s
</span><span class="apple-converted-space"> </span><span class="s1">42.54 </span>
<span class="apple-converted-space"> </span><span class="s1">1.50 </span><span class="apple-converted-space">
</span><span class="s1">0.66</span><o:p></o:p></p>
<p class="p1"><span class="s1">256 </span><span class="apple-converted-space">
</span><span class="s1">10k </span><span class="apple-converted-space"> </span>
<span class="s1">4.8m</span><span class="apple-converted-space"> </span><span class="s1">0.04s</span><span class="apple-converted-space">
</span><span class="s1">139.01 </span><span class="apple-converted-space"> </span>
<span class="s1">1.84 </span><span class="apple-converted-space"> </span><span class="s1">0.54</span><o:p></o:p></p>
<p class="p1"><span class="s1">1k</span><span class="apple-converted-space">
</span><span class="s1">10k </span><span class="apple-converted-space"> </span>
<span class="s1">19m </span><span class="apple-converted-space"> </span><span class="s1">0.05s</span><span class="apple-converted-space">
</span><span class="s1">424.88 </span><span class="apple-converted-space"> </span>
<span class="s1">2.41 </span><span class="apple-converted-space"> </span><span class="s1">0.41</span><o:p></o:p></p>
<p class="p1"><span class="s1">4k</span><span class="apple-converted-space">
</span><span class="s1">10k </span><span class="apple-converted-space"> </span>
<span class="s1">78m </span><span class="apple-converted-space"> </span><span class="s1">0.05s
</span><span class="apple-converted-space"> </span><span class="s1">1580.58 </span>
<span class="apple-converted-space"> </span><span class="s1">2.59 </span><span class="apple-converted-space">
</span><span class="s1">0.39</span><o:p></o:p></p>
<p class="p1"><span class="s1">64k </span><span class="apple-converted-space">
</span><span class="s1">1k</span><span class="apple-converted-space"> </span>
<span class="s1">125m</span><span class="apple-converted-space"> </span><span class="s1">0.01s
</span><span class="apple-converted-space"> </span><span class="s1">8886.24 </span>
<span class="apple-converted-space"> </span><span class="s1">7.38 </span><span class="apple-converted-space">
</span><span class="s1">0.14</span><o:p></o:p></p>
<p class="p1"><span class="s1">1m</span><span class="apple-converted-space">
</span><span class="s1">100 </span><span class="apple-converted-space"> </span>
<span class="s1">200m</span><span class="apple-converted-space"> </span><span class="s1">0.02s</span><span class="apple-converted-space">
</span><span class="s1">11581.36</span><span class="apple-converted-space"> </span>
<span class="s1">90.54 </span><span class="apple-converted-space"> </span><span class="s1">0.01</span><o:p></o:p></p>
<p class="p2"><o:p> </o:p></p>
<p class="p1"><span class="s1">real</span><span class="apple-tab-span"> </span><span class="s1">0m1.129s</span><o:p></o:p></p>
<p class="p1"><span class="s1">user</span><span class="apple-tab-span"> </span><span class="s1">0m0.225s</span><o:p></o:p></p>
<p class="p1"><span class="s1">sys</span><span class="apple-tab-span"> </span><span class="s1">0m0.478s</span><o:p></o:p></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<div id="mail-editor-reference-message-container">
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">Libfabric-users <libfabric-users-bounces@lists.openfabrics.org> on behalf of Alisa Parashchenko <ge24cuc@mytum.de><br>
<b>Date: </b>Friday, May 3, 2024 at 2:49</span><span style="font-family:"Arial",sans-serif;color:black"> </span><span style="color:black">PM<br>
<b>To: </b>Zegelstein, Seth <szegel@amazon.com>, libfabric-users@lists.openfabrics.org <libfabric-users@lists.openfabrics.org><br>
<b>Subject: </b>Re: [libfabric-users] fi_cq_sread fails with "Resource temporarily unavailable"<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt">Hello,<br>
<br>
I found my mistake. The fi_cq_sread() fails because I set the wrong <br>
timeout (0 instead of -1). I really should have noticed that sooner. <br>
However, with the correct timeout set, fi_inject() returns the same <br>
error. It works eventually if I retry often enough:<br>
<br>
while ((ret = fi_inject(ep, buf, 6, 1)) == -FI_EAGAIN);<br>
<br>
But do I really have to just retry until it works, or is there a better way?<br>
<br>
Running fi_pingpong from utils/ on my setup does work. The <br>
fi_rma_pingpong from fabtests/benchmarks/ keeps running for many <br>
minutes, even if I specify the smallest size and only 10 iterations <br>
(i.e. "fi_rma_pingpong -S l:1 -I 10" for the server, and <br>
"fi_rma_pingpong -S l:1 -I 10 localhost" for the client). Is it supposed <br>
to run this long and I should just wait, or am I doing something wrong?<br>
<br>
Regards,<br>
Alisa<br>
<br>
01.05.2024 17:33, Zegelstein, Seth wrote:<br>
> Hey Alisa,<br>
><br>
> Can you start with trying to run fabtests on your setup? Start with <br>
one of the pinpong tests.<br>
><br>
> Best,<br>
> Seth<br>
><br>
> </span><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"></span><span style="font-size:11.0pt">On 5/1/24, 6:29 AM, "Libfabric-users on behalf of Alisa
<br>
Parashchenko" <libfabric-users-bounces@lists.openfabrics.org <br>
<<a href="mailto:libfabric-users-bounces@lists.openfabrics.org">mailto:libfabric-users-bounces@lists.openfabrics.org</a>> on behalf of
<br>
ge24cuc@mytum.de <<a href="mailto:ge24cuc@mytum.de">mailto:ge24cuc@mytum.de</a>>> wrote:<br>
><br>
><br>
> CAUTION: This email originated from outside of the organization. Do <br>
not click links or open attachments unless you can confirm the sender <br>
and know the content is safe.<br>
><br>
><br>
><br>
><br>
><br>
><br>
> Hello,<br>
><br>
><br>
> I am new to Libfabric and trying to write some code that does RMAs.<br>
> Currently, however, even reading from the completion queue after doing a<br>
> regular fi_recv() is failing with "Resource temporarily unavailable".<br>
><br>
><br>
> Here is a minimal program that gets this error. Could someone tell me<br>
> what I'm doing wrong? Setting FI_LOG_LEVEL=Debug didn't give any helpful<br>
> information. I am on a regular Linux desktop, with Libfabric using its<br>
> TCP provider, if that's relevant.<br>
><br>
><br>
> Regards,<br>
> Alisa<br>
><br>
><br>
> #include <assert.h><br>
> #include <errno.h><br>
> #include <stdlib.h><br>
> #include <stdio.h><br>
> #include <unistd.h><br>
><br>
><br>
> #include <rdma/fabric.h><br>
> #include <rdma/fi_cm.h><br>
> #include <rdma/fi_domain.h><br>
> #include <rdma/fi_endpoint.h><br>
> #include <rdma/fi_rma.h><br>
><br>
><br>
> #define PANIC_NZ(a) if ((ret = a)) panic("" #a "", fi_strerror(ret));<br>
><br>
><br>
> static struct fi_info *info;<br>
> static struct fid_fabric *fabric;<br>
> static struct fid_domain *domain;<br>
> static struct fid_ep *ep;<br>
> static struct fi_av_attr av_attr = { 0 };<br>
> static struct fi_cq_attr cq_attr = { 0 };<br>
> static struct fi_eq_attr eq_attr = { 0 };<br>
> static struct fid_av *av;<br>
> static struct fid_cq *cq;<br>
> static struct fid_eq *eq;<br>
> int ret;<br>
><br>
><br>
> void panic(char *f, const char *msg) {<br>
> fprintf(stderr, "%s failed: %s\n", f, msg);<br>
> exit(1);<br>
> }<br>
><br>
><br>
> void hexdump(int len, void *buf) {<br>
> for (int i = 0; i < len; i++) printf("%02hhx ", ((char*)buf)[i]);<br>
> printf("\n");<br>
> }<br>
><br>
><br>
> int main(int argc, char **argv) {<br>
> char *host = "localhost";<br>
> int is_server = argc <= 1;<br>
> char *port = is_server ? "1234" : "4321" ;<br>
><br>
><br>
> /* Select fabric */<br>
> struct fi_info *hints = fi_allocinfo();<br>
> hints->ep_attr->type = FI_EP_RDM;<br>
> hints->caps = FI_MSG | FI_RMA;<br>
> PANIC_NZ(fi_getinfo(FI_VERSION(1,21), host, port, FI_SOURCE, hints,<br>
> &info));<br>
> printf("Selected fabric \"%s\", domain \"%s\"\n",<br>
> info->fabric_attr->name, info->domain_attr->name);<br>
> fi_freeinfo(hints);<br>
><br>
><br>
> /* Set up address vector */<br>
> PANIC_NZ(fi_fabric(info->fabric_attr, &fabric, NULL));<br>
> PANIC_NZ(fi_domain(fabric, info, &domain, NULL));<br>
> av_attr.type = FI_AV_TABLE;<br>
> av_attr.count = 2;<br>
> PANIC_NZ(fi_av_open(domain, &av_attr, &av, NULL));<br>
><br>
><br>
> /* Open the endpoint, bind it to an EQ, CQ, and AV*/<br>
> PANIC_NZ(fi_endpoint(domain, info, &ep, NULL));<br>
> cq_attr.wait_obj = FI_WAIT_UNSPEC;<br>
> PANIC_NZ(fi_cq_open(domain, &cq_attr, &cq, NULL));<br>
> PANIC_NZ(fi_eq_open(fabric, &eq_attr, &eq, NULL));<br>
> PANIC_NZ(fi_ep_bind(ep, &av->fid, 0));<br>
> PANIC_NZ(fi_ep_bind(ep, &cq->fid, FI_TRANSMIT|FI_RECV));<br>
> PANIC_NZ(fi_ep_bind(ep, &eq->fid, 0));<br>
> PANIC_NZ(fi_enable(ep));<br>
><br>
><br>
> /* Get the address of the endpoint */<br>
> char fi_addr[160];<br>
> size_t fi_addrlen = 160;<br>
> PANIC_NZ(fi_getname(&ep->fid, fi_addr, &fi_addrlen));<br>
> printf("Got libfabric EP addr of length %zu:\n", fi_addrlen);<br>
> hexdump(fi_addrlen, fi_addr);<br>
><br>
><br>
> /* Insert own address and peer's address into AV */<br>
> ret = fi_av_insert(av, fi_addr, 1, NULL, 0, NULL);<br>
> assert(ret == 1);<br>
> /* Obviously not the right way to do this, but the shortest way */<br>
> char *peer_port = is_server ? "\x10\xe1" : "\x04\xd2";<br>
> memcpy(fi_addr + 2, peer_port, 2);<br>
> ret = fi_av_insert(av, fi_addr, 1, NULL, 0, NULL);<br>
> assert(ret == 1);<br>
><br>
><br>
> /* Try to exchange a message */<br>
> if (is_server) {<br>
> char buf[6];<br>
> char cq_buf[128];<br>
> PANIC_NZ(fi_recv(ep, buf, 5, NULL, 1, NULL));<br>
> ret = fi_cq_sread(cq, cq_buf, 1, NULL, 0);<br>
> if (ret < 0) panic("fi_cq_sread", fi_strerror(ret));<br>
> printf("Got message: %s\n", buf);<br>
> } else {<br>
> char buf[6] = "Hello";<br>
> PANIC_NZ(fi_inject(ep, buf, 6, 1));<br>
> }<br>
><br>
><br>
> fi_close((struct fid *) ep);<br>
> fi_close((struct fid *) av);<br>
> fi_close((struct fid *) eq);<br>
> fi_close((struct fid *) cq);<br>
> fi_close((struct fid *) domain);<br>
> fi_close((struct fid *) fabric);<br>
> fi_freeinfo(info);<br>
> return 0;<br>
> }<br>
><br>
><br>
> _______________________________________________<br>
> Libfabric-users mailing list<br>
> Libfabric-users@lists.openfabrics.org <br>
<<a href="mailto:Libfabric-users@lists.openfabrics.org">mailto:Libfabric-users@lists.openfabrics.org</a>><br>
> <a href="https://lists.openfabrics.org/mailman/listinfo/libfabric-users">https://lists.openfabrics.org/mailman/listinfo/libfabric-users</a>
<br>
<<a href="https://lists.openfabrics.org/mailman/listinfo/libfabric-users">https://lists.openfabrics.org/mailman/listinfo/libfabric-users</a>><br>
><br>
><br>
><br>
_______________________________________________<br>
Libfabric-users mailing list<br>
Libfabric-users@lists.openfabrics.org<br>
<a href="https://lists.openfabrics.org/mailman/listinfo/libfabric-users">https://lists.openfabrics.org/mailman/listinfo/libfabric-users</a><o:p></o:p></span></p>
</div>
</div>
</div>
</div>
</body>
</html>