<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Aptos;
panose-1:2 11 0 4 2 2 2 2 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
font-size:12.0pt;
font-family:"Aptos",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
span.EmailStyle19
{mso-style-type:personal-reply;
font-family:"Aptos",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;
mso-ligatures:none;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
--></style>
</head>
<body lang="EN-US" link="blue" vlink="purple" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt">There is an old post on this :
<a href="https://lists.openfabrics.org/pipermail/libfabric-users/2019-January/000438.html">
https://lists.openfabrics.org/pipermail/libfabric-users/2019-January/000438.html</a><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<div id="mail-editor-reference-message-container">
<div>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal" style="margin-bottom:12.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">Libfabric-users <libfabric-users-bounces@lists.openfabrics.org> on behalf of Alisa Parashchenko <ge24cuc@mytum.de><br>
<b>Date: </b>Wednesday, May 1, 2024 at 7:29</span><span style="font-family:"Arial",sans-serif;color:black"> </span><span style="color:black">AM<br>
<b>To: </b>libfabric-users@lists.openfabrics.org <libfabric-users@lists.openfabrics.org><br>
<b>Subject: </b>[libfabric-users] fi_cq_sread fails with "Resource temporarily unavailable"<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><span style="font-size:11.0pt">Hello,<br>
<br>
I am new to Libfabric and trying to write some code that does RMAs. <br>
Currently, however, even reading from the completion queue after doing a <br>
regular fi_recv() is failing with "Resource temporarily unavailable".<br>
<br>
Here is a minimal program that gets this error. Could someone tell me <br>
what I'm doing wrong? Setting FI_LOG_LEVEL=Debug didn't give any helpful <br>
information. I am on a regular Linux desktop, with Libfabric using its <br>
TCP provider, if that's relevant.<br>
<br>
Regards,<br>
Alisa<br>
<br>
#include <assert.h><br>
#include <errno.h><br>
#include <stdlib.h><br>
#include <stdio.h><br>
#include <unistd.h><br>
<br>
#include <rdma/fabric.h><br>
#include <rdma/fi_cm.h><br>
#include <rdma/fi_domain.h><br>
#include <rdma/fi_endpoint.h><br>
#include <rdma/fi_rma.h><br>
<br>
#define PANIC_NZ(a) if ((ret = a)) panic("" #a "", fi_strerror(ret));<br>
<br>
static struct fi_info *info;<br>
static struct fid_fabric *fabric;<br>
static struct fid_domain *domain;<br>
static struct fid_ep *ep;<br>
static struct fi_av_attr av_attr = { 0 };<br>
static struct fi_cq_attr cq_attr = { 0 };<br>
static struct fi_eq_attr eq_attr = { 0 };<br>
static struct fid_av *av;<br>
static struct fid_cq *cq;<br>
static struct fid_eq *eq;<br>
int ret;<br>
<br>
void panic(char *f, const char *msg) {<br>
fprintf(stderr, "%s failed: %s\n", f, msg);<br>
exit(1);<br>
}<br>
<br>
void hexdump(int len, void *buf) {<br>
for (int i = 0; i < len; i++) printf("%02hhx ", ((char*)buf)[i]);<br>
printf("\n");<br>
}<br>
<br>
int main(int argc, char **argv) {<br>
char *host = "localhost";<br>
int is_server = argc <= 1;<br>
char *port = is_server ? "1234" : "4321" ;<br>
<br>
/* Select fabric */<br>
struct fi_info *hints = fi_allocinfo();<br>
hints->ep_attr->type = FI_EP_RDM;<br>
hints->caps = FI_MSG | FI_RMA;<br>
PANIC_NZ(fi_getinfo(FI_VERSION(1,21), host, port, FI_SOURCE, hints, <br>
&info));<br>
printf("Selected fabric \"%s\", domain \"%s\"\n",<br>
info->fabric_attr->name, info->domain_attr->name);<br>
fi_freeinfo(hints);<br>
<br>
/* Set up address vector */<br>
PANIC_NZ(fi_fabric(info->fabric_attr, &fabric, NULL));<br>
PANIC_NZ(fi_domain(fabric, info, &domain, NULL));<br>
av_attr.type = FI_AV_TABLE;<br>
av_attr.count = 2;<br>
PANIC_NZ(fi_av_open(domain, &av_attr, &av, NULL));<br>
<br>
/* Open the endpoint, bind it to an EQ, CQ, and AV*/<br>
PANIC_NZ(fi_endpoint(domain, info, &ep, NULL));<br>
cq_attr.wait_obj = FI_WAIT_UNSPEC;<br>
PANIC_NZ(fi_cq_open(domain, &cq_attr, &cq, NULL));<br>
PANIC_NZ(fi_eq_open(fabric, &eq_attr, &eq, NULL));<br>
PANIC_NZ(fi_ep_bind(ep, &av->fid, 0));<br>
PANIC_NZ(fi_ep_bind(ep, &cq->fid, FI_TRANSMIT|FI_RECV));<br>
PANIC_NZ(fi_ep_bind(ep, &eq->fid, 0));<br>
PANIC_NZ(fi_enable(ep));<br>
<br>
/* Get the address of the endpoint */<br>
char fi_addr[160];<br>
size_t fi_addrlen = 160;<br>
PANIC_NZ(fi_getname(&ep->fid, fi_addr, &fi_addrlen));<br>
printf("Got libfabric EP addr of length %zu:\n", fi_addrlen);<br>
hexdump(fi_addrlen, fi_addr);<br>
<br>
/* Insert own address and peer's address into AV */<br>
ret = fi_av_insert(av, fi_addr, 1, NULL, 0, NULL);<br>
assert(ret == 1);<br>
/* Obviously not the right way to do this, but the shortest way */<br>
char *peer_port = is_server ? "\x10\xe1" : "\x04\xd2";<br>
memcpy(fi_addr + 2, peer_port, 2);<br>
ret = fi_av_insert(av, fi_addr, 1, NULL, 0, NULL);<br>
assert(ret == 1);<br>
<br>
/* Try to exchange a message */<br>
if (is_server) {<br>
char buf[6];<br>
char cq_buf[128];<br>
PANIC_NZ(fi_recv(ep, buf, 5, NULL, 1, NULL));<br>
ret = fi_cq_sread(cq, cq_buf, 1, NULL, 0);<br>
if (ret < 0) panic("fi_cq_sread", fi_strerror(ret));<br>
printf("Got message: %s\n", buf);<br>
} else {<br>
char buf[6] = "Hello";<br>
PANIC_NZ(fi_inject(ep, buf, 6, 1));<br>
}<br>
<br>
fi_close((struct fid *) ep);<br>
fi_close((struct fid *) av);<br>
fi_close((struct fid *) eq);<br>
fi_close((struct fid *) cq);<br>
fi_close((struct fid *) domain);<br>
fi_close((struct fid *) fabric);<br>
fi_freeinfo(info);<br>
return 0;<br>
}<br>
<br>
_______________________________________________<br>
Libfabric-users mailing list<br>
Libfabric-users@lists.openfabrics.org<br>
<a href="https://lists.openfabrics.org/mailman/listinfo/libfabric-users">https://lists.openfabrics.org/mailman/listinfo/libfabric-users</a><o:p></o:p></span></p>
</div>
</div>
</div>
</div>
</body>
</html>