<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style>
</head>
<body dir="ltr">
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
I had a similar issue. Here is the patch I created to get around this problem:<br>
<br>
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
diff --git a/prov/cxi/include/cxip.h b/prov/cxi/include/cxip.h</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
index 53e20f720..71a195447 100644</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
--- a/prov/cxi/include/cxip.h</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+++ b/prov/cxi/include/cxip.h</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
@@ -3020,7 +3020,10 @@ int cxip_cmdq_emit_c_state(struct cxip_cmdq *cmdq,</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
static inline bool cxip_cmdq_empty(struct cxip_cmdq *cmdq)</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
{</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
- return cxi_cq_empty(cmdq->dev_cmdq);</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+ uint64_t wp = cmdq->dev_cmdq->wp32 / 2;</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+ return wp == cmdq->dev_cmdq->status->rd_ptr;</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+ //return cxi_cq_empty(cmdq->dev_cmdq);</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
}</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
static inline bool cxip_cmdq_match(struct cxip_cmdq *cmdq, uint16_t vni,</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
diff --git a/prov/cxi/src/cxip_coll.c b/prov/cxi/src/cxip_coll.c</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
index 40ef8a60f..37dddbca6 100644</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
--- a/prov/cxi/src/cxip_coll.c</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+++ b/prov/cxi/src/cxip_coll.c</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
@@ -421,7 +421,10 @@ static inline int flt_op_to_opcode(int op)</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
{</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
if (op != FI_SUM)</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
return _flt_op_to_opcode[op];</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
-</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+ return (_MM_GET_FLUSH_ZERO_MODE()) ?</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+ COLL_OPCODE_FLT_SUM_FTZ_RND0 :</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+ COLL_OPCODE_FLT_SUM_NOFTZ_RND0;</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+/*</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
switch (fegetround()) {</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
case FE_TONEAREST:</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
return (_MM_GET_FLUSH_ZERO_MODE()) ?</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
@@ -440,6 +443,7 @@ static inline int flt_op_to_opcode(int op)</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
COLL_OPCODE_FLT_SUM_FTZ_RND3 :</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
COLL_OPCODE_FLT_SUM_NOFTZ_RND3;</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
}</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
+*/</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
return -FI_EOPNOTSUPP;</div>
<div style="font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
}</div>
<div id="appendonsend"></div>
<hr style="display:inline-block;width:98%" tabindex="-1">
<div id="divRplyFwdMsg" dir="ltr"><font face="Calibri, sans-serif" style="font-size:11pt" color="#000000"><b>From:</b> Libfabric-users <libfabric-users-bounces@lists.openfabrics.org> on behalf of Raffenetti, Ken via Libfabric-users <libfabric-users@lists.openfabrics.org><br>
<b>Sent:</b> Tuesday, October 8, 2024 11:40 AM<br>
<b>To:</b> Marc Caubet Serrabou <marc.caubet@psi.ch><br>
<b>Cc:</b> libfabric-users@lists.openfabrics.org <libfabric-users@lists.openfabrics.org><br>
<b>Subject:</b> [EXTERNAL] Re: [libfabric-users] CXI libraries present but can not be compiled</font>
<div> </div>
</div>
<style>
<!--
@font-face
{font-family:"Cambria Math"}
@font-face
{font-family:Aptos}
@font-face
{font-family:"Apple Color Emoji"}
@font-face
{font-family:Consolas}
p.x_MsoNormal, li.x_MsoNormal, div.x_MsoNormal
{margin:0in;
font-size:12.0pt;
font-family:"Aptos",sans-serif}
a:link, span.x_MsoHyperlink
{color:blue;
text-decoration:underline}
pre
{margin:0in;
font-size:10.0pt;
font-family:"Courier New"}
span.x_HTMLPreformattedChar
{font-family:Consolas}
span.x_EmailStyle24
{font-family:"Aptos",sans-serif;
color:windowtext}
.x_MsoChpDefault
{font-size:10.0pt}
@page WordSection1
{margin:1.0in 1.0in 1.0in 1.0in}
div.x_WordSection1
{}
-->
</style>
<div lang="EN-US" link="blue" vlink="purple" style="word-wrap:break-word">
<div class="x_WordSection1">
<p class="x_MsoNormal"><span style="font-size:11.0pt">I believe that means that your slingshot host software (i.e. libcxi) is not compatible with the version of the provider being built. Thomas’ branch reverts all the updates that require the upgraded SHS.
When building his branch, you could try increasing the logging to see if there is any indication why the provider isn’t returned from fi_getinfo.</span></p>
<p class="x_MsoNormal"><span style="font-size:11.0pt"> </span></p>
<p class="x_MsoNormal"><span style="font-size:11.0pt">Ken</span></p>
<p class="x_MsoNormal"><span style="font-size:11.0pt"> </span></p>
<div id="x_mail-editor-reference-message-container">
<div>
<div>
<div style="border:none; border-top:solid #B5C4DF 1.0pt; padding:3.0pt 0in 0in 0in">
<p class="x_MsoNormal" style="margin-right:0in; margin-bottom:12.0pt; margin-left:.5in">
<b><span style="color:black">From: </span></b><span style="color:black">Marc Caubet Serrabou <marc.caubet@psi.ch><br>
<b>Date: </b>Tuesday, October 8, 2024 at 3:35</span><span style="font-family:"Arial",sans-serif; color:black"> </span><span style="color:black">AM<br>
<b>To: </b>Raffenetti, Ken <raffenet@anl.gov><br>
<b>Cc: </b>libfabric-users@lists.openfabrics.org <libfabric-users@lists.openfabrics.org><br>
<b>Subject: </b>Re: [libfabric-users] CXI libraries present but can not be compiled</span></p>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:1.0pt; color:white">Hi Ken, Thanks a lot for your answer. I just tried to cherry-pick your commit into the v1.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">22.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">0
tag, but then the compilation crashes for a different reason: copying selected object files to avoid basename conflicts.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">.
CCLD util/fi_strerror CCLD</span></p>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:1.0pt; color:white">ZjQcmQRYFpfptBannerStart</span></p>
</div>
<div id="x_pfptBannergfp6dwz" style="border:none; border-top:solid #90A4AE 3.0pt; padding:0in 0in 0in 0in; display:block!important; text-align:left!important; margin:0px!important; padding:16px!important; border-radius:4px!important; min-width:200px!important; background-color:#D0D8DC!important; border-top:#90a4ae!important">
<div id="x_pfptBannergfp6dwz">
<div id="x_pfptBannergfp6dwz">
<p class="x_MsoNormal" style="margin-left:.5in; line-height:13.5pt; background:#D0D8DC">
<b><span style="font-family:"Arial",sans-serif; color:black">This Message Is From an External Sender
</span></b></p>
</div>
<div id="x_pfptBannergfp6dwz">
<p class="x_MsoNormal" style="margin-left:.5in; line-height:13.5pt; background:#D0D8DC">
<span style="font-family:"Arial",sans-serif; color:black">This message came from outside your organization.
</span></p>
</div>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:.5in; background:#D0D8DC"><span style="color:black"> </span></p>
</div>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:1.0pt; color:white">ZjQcmQRYFpfptBannerEnd</span></p>
</div>
<p style="margin-left:.5in">Hi Ken,</p>
<p style="margin-left:.5in">Thanks a lot for your answer. I just tried to cherry-pick your commit into the v1.22.0 tag, but then the compilation crashes for a different reason:</p>
<p style="margin-left:.5in"><span style="font-family:"Courier New"">copying selected object files to avoid basename conflicts...<br>
CCLD util/fi_strerror<br>
CCLD util/fi_info<br>
CCLD util/fi_pingpong<br>
CCLD prov/cxi/test/multinode/test_frmwk<br>
CCLD prov/cxi/test/multinode/test_zbcoll<br>
CCLD prov/cxi/test/multinode/test_coll<br>
CCLD prov/cxi/test/multinode/test_barrier<br>
/usr/bin/ld: src/.libs/libfabric.so: undefined reference to `cxi_cq_empty'<br>
collect2: error: ld returned 1 exit status<br>
make[1]: *** [Makefile:12746: util/fi_strerror] Error 1<br>
make[1]: *** Waiting for unfinished jobs....<br>
/usr/bin/ld: src/.libs/libfabric.so: undefined reference to `cxi_cq_empty'<br>
collect2: error: ld returned 1 exit status<br>
/usr/bin/ld: src/.libs/libfabric.so: undefined reference to `cxi_cq_empty'<br>
make[1]: *** [Makefile:12740: util/fi_pingpong] Error 1<br>
collect2: error: ld returned 1 exit status<br>
make[1]: *** [Makefile:12734: util/fi_info] Error 1<br>
/usr/bin/ld: src/.libs/libfabric.a(src_libfabric_la-cxip_dom.o): in function `cxip_domain_find_cmdq':<br>
cxip_dom.c:(.text+0x436): undefined reference to `cxi_cq_empty'<br>
/usr/bin/ld: src/.libs/libfabric.a(src_libfabric_la-cxip_dom.o): in function `cxip_domain_find_cmdq':<br>
cxip_dom.c:(.text+0x436): undefined reference to `cxi_cq_empty'<br>
/usr/bin/ld: src/.libs/libfabric.a(src_libfabric_la-cxip_dom.o): in function `cxip_domain_find_cmdq':<br>
cxip_dom.c:(.text+0x436): undefined reference to `cxi_cq_empty'<br>
/usr/bin/ld: src/.libs/libfabric.a(src_libfabric_la-cxip_dom.o): in function `cxip_domain_find_cmdq':<br>
cxip_dom.c:(.text+0x436): undefined reference to `cxi_cq_empty'<br>
collect2: error: ld returned 1 exit status<br>
collect2: error: ld returned 1 exit status<br>
collect2: error: ld returned 1 exit status<br>
make[1]: *** [Makefile:12658: prov/cxi/test/multinode/test_zbcoll] Error 1<br>
make[1]: *** [Makefile:12648: prov/cxi/test/multinode/test_frmwk] Error 1<br>
collect2: error: ld returned 1 exit status<br>
make[1]: *** [Makefile:12638: prov/cxi/test/multinode/test_coll] Error 1<br>
make[1]: *** [Makefile:12628: prov/cxi/test/multinode/test_barrier] Error 1<br>
make[1]: Leaving directory '/var/tmp/caubet_m/libfabric-1.22.0/src'<br>
make: *** [Makefile:6816: all] Error 2<br>
libfabric/1.22.0: compilation failed! </span></p>
<p style="margin-left:.5in">Any ideas?</p>
<p style="margin-left:.5in">On the other hand, I also tried another proposed update from
<a href="https://urldefense.us/v3/__https:/github.com/thomasgillis/libfabric/tree/dev-cxi__;!!G_uCfscf7eWS!ddl49ZeHpPYwGSRVOo04A_tNEaCdZRFp20eq_PyWAAuqOz_KMJMzqnE81f5PQNDWo69E4EoJ0ZurFfPScbpX$">
https://github.com/thomasgillis/libfabric/tree/dev-cxi</a>, and with that one I can compile correctly, but then something is wrong:</p>
<p style="margin-left:.5in"><span style="font-family:"Apple Color Emoji"">🔥</span><span style="font-family:"Courier New""> [caubet_m@login002:~/git/buildblocks/Libraries/libfabric(ofi_1.22.0)]# fi_info -p cxi<br>
fi_getinfo: -61 (No data available)</span></p>
<p style="margin-left:.5in"><span style="font-family:"Apple Color Emoji"">🔥</span><span style="font-family:"Courier New""> [caubet_m@login002:~/git/buildblocks/Libraries/libfabric(ofi_1.22.0)]# ldd $(which fi_info) | grep cxi<br>
libcxi.so.1 => /usr/lib64/libcxi.so.1 (0x00007f122b977000)</span></p>
<p style="margin-left:.5in"><span style="font-family:"Apple Color Emoji"">🔥</span><span style="font-family:"Courier New""> [caubet_m@login002:~/git/buildblocks/Libraries/libfabric(ofi_1.22.0)]# ldd $(which fi_info)<br>
linux-vdso.so.1 (0x00007ffcf52f6000)<br>
libfabric.so.1 => /opt/psi/Libraries/libfabric/1.22.0/lib64/libfabric.so.1 (0x00007f255102a000)<br>
libcxi.so.1 => /usr/lib64/libcxi.so.1 (0x00007f2551004000)<br>
libnl-3.so.200 => /usr/lib64/libnl-3.so.200 (0x00007f2550c00000)<br>
libcurl.so.4 => /usr/lib64/libcurl.so.4 (0x00007f2550f5a000)<br>
libjson-c.so.3 => /usr/lib64/libjson-c.so.3 (0x00007f2550800000)<br>
libm.so.6 => /lib64/libm.so.6 (0x00007f2550ab4000)<br>
libuuid.so.1 => /usr/lib64/libuuid.so.1 (0x00007f2550f28000)<br>
libnuma.so.1 => /usr/lib64/libnuma.so.1 (0x00007f2550400000)<br>
libatomic.so.1 => /usr/lib64/libatomic.so.1 (0x00007f2550f1e000)<br>
librt.so.1 => /lib64/librt.so.1 (0x00007f2550f14000)<br>
libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f2550eee000)<br>
libdl.so.2 => /lib64/libdl.so.2 (0x00007f2550ee9000)<br>
libxpmem.so.0 => /usr/lib64/libxpmem.so.0 (0x00007f2550ee6000)<br>
libc.so.6 => /lib64/libc.so.6 (0x00007f2550209000)<br>
libnghttp2.so.14 => /usr/lib64/libnghttp2.so.14 (0x00007f2550ebd000)<br>
libidn2.so.0 => /usr/lib64/libidn2.so.0 (0x00007f254fe00000)<br>
libssh.so.4 => /usr/lib64/libssh.so.4 (0x00007f2550e4c000)<br>
libpsl.so.5 => /usr/lib64/libpsl.so.5 (0x00007f254fa00000)<br>
libssl.so.1.1 => /usr/lib64/libssl.so.1.1 (0x00007f2550a15000)<br>
libcrypto.so.1.1 => /usr/lib64/libcrypto.so.1.1 (0x00007f254f6c1000)<br>
libgssapi_krb5.so.2 => /usr/lib64/libgssapi_krb5.so.2 (0x00007f25507ae000)<br>
libldap_r-2.4.so.2 => /usr/lib64/libldap_r-2.4.so.2 (0x00007f2550759000)<br>
liblber-2.4.so.2 => /usr/lib64/liblber-2.4.so.2 (0x00007f2550e3a000)<br>
libzstd.so.1 => /usr/lib64/libzstd.so.1 (0x00007f2550628000)<br>
libbrotlidec.so.1 => /usr/lib64/libbrotlidec.so.1 (0x00007f254f400000)<br>
libz.so.1 => /usr/lib64/libz.so.1 (0x00007f255060f000)<br>
/lib64/ld-linux-x86-64.so.2 (0x00007f2551354000)<br>
libunistring.so.2 => /usr/lib64/libunistring.so.2 (0x00007f254f000000)<br>
libjitterentropy.so.3 => /usr/lib64/libjitterentropy.so.3 (0x00007f2550e30000)<br>
libkrb5.so.3 => /usr/lib64/libkrb5.so.3 (0x00007f255012f000)<br>
libk5crypto.so.3 => /usr/lib64/libk5crypto.so.3 (0x00007f2550118000)<br>
libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f2550e2b000)<br>
libkrb5support.so.0 => /usr/lib64/libkrb5support.so.0 (0x00007f2550109000)<br>
libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f25500f1000)<br>
libsasl2.so.3 => /usr/lib64/libsasl2.so.3 (0x00007f25500d3000)<br>
libbrotlicommon.so.1 => /usr/lib64/libbrotlicommon.so.1 (0x00007f254ec00000)<br>
libkeyutils.so.1 => /usr/lib64/libkeyutils.so.1 (0x00007f254e800000)<br>
libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f254e400000)<br>
libpcre.so.1 => /usr/lib64/libpcre.so.1 (0x00007f254e000000)</span></p>
<p style="margin-left:.5in">We are running</p>
<p style="margin-left:.5in"><span style="font-family:"Apple Color Emoji"">🔥</span><span style="font-family:"Courier New""> [caubet_m@login002:~/git/buildblocks/Libraries/libfabric(ofi_1.22.0)]# rpm -qf /usr/lib64/libcxi.so.1<br>
cray-libcxi-0.9-SSHOT2.1.3_20240529150829_3d1dc9246116.x86_64</span></p>
<p style="margin-left:.5in">And we're testing libfabric 1.22.0, I'm open to compiling a newer (or older) version if that makes it work, but I'm not sure how far other Cray systems have been able to get.</p>
<p style="margin-left:.5in">Thanks a lot,</p>
<p style="margin-left:.5in">Marc</p>
<div>
<p class="x_MsoNormal" style="margin-left:.5in">On 07.10.24 17:59, Raffenetti, Ken wrote:</p>
</div>
<blockquote style="margin-top:5.0pt; margin-bottom:5.0pt">
<div>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:11.0pt">Hi Marc,</span></p>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:11.0pt"> </span></p>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:11.0pt">I believe those headers are not necessary for compiling the provider. I proposed removing the checks from configure in
<a href="https://urldefense.us/v3/__https:/github.com/ofiwg/libfabric/pull/9793__;!!G_uCfscf7eWS!ddl49ZeHpPYwGSRVOo04A_tNEaCdZRFp20eq_PyWAAuqOz_KMJMzqnE81f5PQNDWo69E4EoJ0ZurFVSI6UMq$">
https://github.com/ofiwg/libfabric/pull/9793</a>. You could cherry-pick <a href="https://urldefense.us/v3/__https:/github.com/ofiwg/libfabric/pull/9793/commits/5793243aec20c4fee126aa3093ff07bb5889f154__;!!G_uCfscf7eWS!ddl49ZeHpPYwGSRVOo04A_tNEaCdZRFp20eq_PyWAAuqOz_KMJMzqnE81f5PQNDWo69E4EoJ0ZurFZJeP4Dv$">
https://github.com/ofiwg/libfabric/pull/9793/commits/5793243aec20c4fee126aa3093ff07bb5889f154</a> and try again.</span></p>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:11.0pt"> </span></p>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:11.0pt">Ken</span></p>
<p class="x_MsoNormal" style="margin-left:.5in"><span style="font-size:11.0pt"> </span></p>
<div id="x_mail-editor-reference-message-container">
<div>
<div>
<div style="border:none; border-top:solid #B5C4DF 1.0pt; padding:3.0pt 0in 0in 0in">
<p class="x_MsoNormal" style="margin-right:0in; margin-bottom:12.0pt; margin-left:1.0in">
<b><span style="color:black">From: </span></b><span style="color:black">Libfabric-users
<a href="mailto:libfabric-users-bounces@lists.openfabrics.org"><libfabric-users-bounces@lists.openfabrics.org></a> on behalf of Marc Caubet Serrabou
<a href="mailto:marc.caubet@psi.ch"><marc.caubet@psi.ch></a><br>
<b>Date: </b>Monday, October 7, 2024 at 6:03</span><span style="font-family:"Arial",sans-serif; color:black"> </span><span style="color:black">AM<br>
<b>To: </b><a href="mailto:libfabric-users@lists.openfabrics.org">libfabric-users@lists.openfabrics.org</a>
<a href="mailto:libfabric-users@lists.openfabrics.org"><libfabric-users@lists.openfabrics.org></a><br>
<b>Subject: </b>[libfabric-users] CXI libraries present but can not be compiled</span></p>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:1.0in"><span style="font-size:1.0pt; color:white">Hi, I already opened a ticked to ofiwg@</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">lists.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">openfabrics.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">org,
but I also try here in the user list, in case that somebody found a similar issue and has an answer to it. I am trying to compile libfabrics 1.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">22.</span><span style="font-size:1.0pt; font-family:"Arial",sans-serif; color:white"> </span><span style="font-size:1.0pt; color:white">0
with CXI provider support. Despite</span></p>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:1.0in"><span style="font-size:1.0pt; color:white">ZjQcmQRYFpfptBannerStart</span></p>
</div>
<div id="x_pfptBannerblsbz6y" style="border:none; border-top:solid #90A4AE 3.0pt; padding:0in 0in 0in 0in; display:block!important; text-align:left!important; margin:0px!important; padding:16px!important; border-radius:4px!important; min-width:200px!important; background-color:#D0D8DC!important; border-top:#90a4ae!important">
<div id="x_pfptBannerblsbz6y">
<div id="x_pfptBannerblsbz6y">
<p class="x_MsoNormal" style="margin-left:1.0in; line-height:13.5pt; background:#D0D8DC">
<b><span style="font-family:"Arial",sans-serif; color:black">This Message Is From an External Sender
</span></b></p>
</div>
<div id="x_pfptBannerblsbz6y">
<p class="x_MsoNormal" style="margin-left:1.0in; line-height:13.5pt; background:#D0D8DC">
<span style="font-family:"Arial",sans-serif; color:black">This message came from outside your organization.
</span></p>
</div>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:1.0in; background:#D0D8DC"><span style="color:black"> </span></p>
</div>
</div>
<div>
<p class="x_MsoNormal" style="margin-left:1.0in"><span style="font-size:1.0pt; color:white">ZjQcmQRYFpfptBannerEnd</span></p>
</div>
<p style="margin-left:1.0in">Hi,</p>
<p style="margin-left:1.0in">I already opened a ticked to <a href="mailto:ofiwg@lists.openfabrics.org">ofiwg@lists.openfabrics.org</a>, but I also try here in the user list, in case that somebody found a similar issue and has an answer to it.</p>
<p style="margin-left:1.0in">I am trying to compile libfabrics 1.22.0 with CXI provider support. Despite the expected CXI provider header files are present, as well as the CXI library, I get the following errors:</p>
<p style="margin-left:1.0in"><span style="font-family:"Courier New"">configure: WARNING: The EFA provider requires rdma-core v31 or newer.<br>
configure: efa provider: disabled<br>
configure: *** Configuring cxi provider<br>
checking cxi_prov_hw.h usability... no<br>
checking cxi_prov_hw.h presence... yes<br>
configure: WARNING: cxi_prov_hw.h: present but cannot be compiled<br>
configure: WARNING: cxi_prov_hw.h: check for missing prerequisite headers?<br>
configure: WARNING: cxi_prov_hw.h: see the Autoconf documentation<br>
configure: WARNING: cxi_prov_hw.h: section "Present But Cannot Be Compiled"<br>
configure: WARNING: cxi_prov_hw.h: proceeding with the compiler's result<br>
configure: WARNING: ## ------------------------------------------ ##<br>
configure: WARNING: ## Report this to <a href="mailto:ofiwg@lists.openfabrics.org">
ofiwg@lists.openfabrics.org</a> ##<br>
configure: WARNING: ## ------------------------------------------ ##<br>
checking for cxi_prov_hw.h... no<br>
checking uapi/misc/cxi.h usability... no<br>
checking uapi/misc/cxi.h presence... yes<br>
configure: WARNING: uapi/misc/cxi.h: present but cannot be compiled<br>
configure: WARNING: uapi/misc/cxi.h: check for missing prerequisite headers?<br>
configure: WARNING: uapi/misc/cxi.h: see the Autoconf documentation<br>
configure: WARNING: uapi/misc/cxi.h: section "Present But Cannot Be Compiled"<br>
configure: WARNING: uapi/misc/cxi.h: proceeding with the compiler's result<br>
configure: WARNING: ## ------------------------------------------ ##<br>
configure: WARNING: ## Report this to <a href="mailto:ofiwg@lists.openfabrics.org">
ofiwg@lists.openfabrics.org</a> ##<br>
configure: WARNING: ## ------------------------------------------ ##<br>
checking for uapi/misc/cxi.h... no<br>
checking libcxi/libcxi.h usability... yes<br>
checking libcxi/libcxi.h presence... yes<br>
checking for libcxi/libcxi.h... yes<br>
configure: looking for library without search path<br>
checking for cxil_open_device in -lcxi... yes<br>
checking curl/curl.h usability... yes<br>
checking curl/curl.h presence... yes<br>
checking for curl/curl.h... yes<br>
configure: looking for library without search path<br>
checking for curl_global_init in -lcurl... yes<br>
checking json-c/json.h usability... yes<br>
checking json-c/json.h presence... yes<br>
checking for json-c/json.h... yes<br>
configure: looking for library without search path<br>
checking for json_object_get_type in -ljson-c... yes<br>
configure: cxi provider: disabled<br>
configure: WARNING: cxi provider was requested, but cannot be compiled<br>
configure: error: Cannot continue<br>
libfabric/1.22.0: configure failed </span></p>
<p style="margin-left:1.0in">The libraries are the following and come from Cray, and are in the standard directories (<span style="font-family:"Courier New"">/usr</span> for include files,
<span style="font-family:"Courier New"">/usr/lib64</span> for libraries)</p>
<p style="margin-left:1.0in"><span style="font-family:"Apple Color Emoji"">🔥</span><span style="font-family:"Courier New""> [caubet_m@login001:~/git/buildblocks/Libraries/libfabric(ofi_1.22.0)]# rpm -qf /usr/include/uapi/misc/cxi.h /usr/include/cxi_prov_hw.h
/usr/lib64/libcxi.so<br>
warning: Found NDB Packages.db database while attempting bdb backend: using ndb backend.<br>
cray-cxi-driver-devel-0.9-61.9__g3000a93.SSHOT2.1.3.x86_64<br>
cray-cassini-headers-user-1.0-SSHOT2.1.3_20240326210855_321db6bd57af.noarch<br>
cray-libcxi-0.9-SSHOT2.1.3_20240529150829_3d1dc9246116.x86_64</span></p>
<p style="margin-left:1.0in">The configure options are the simplest ones, which should enforce CXI only:</p>
<p style="margin-left:1.0in"><span style="font-family:"Courier New"">/var/tmp/caubet_m/libfabric-1.22.0/src/configure --prefix=/opt/psi/Libraries/libfabric/1.22.0/ --enable-cxi
</span></p>
<p style="margin-left:1.0in">What am I missing, and how shall I proceed? Is the compilation expecting a different set (or version) of CXI libraries?</p>
<p style="margin-left:1.0in">Thanks a lot,</p>
<p style="margin-left:1.0in">Marc</p>
<pre style="margin-left:1.0in">-- </pre>
<pre style="margin-left:1.0in">_________________________________________________________</pre>
<pre style="margin-left:1.0in">Paul Scherrer Institut</pre>
<pre style="margin-left:1.0in">High Performance Computing & Emerging Technologies</pre>
<pre style="margin-left:1.0in">Marc Caubet Serrabou</pre>
<pre style="margin-left:1.0in">Building/Room: OBBA/230</pre>
<pre style="margin-left:1.0in">Forschungsstrasse, 111</pre>
<pre style="margin-left:1.0in">5232 Villigen PSI</pre>
<pre style="margin-left:1.0in">Switzerland</pre>
<pre style="margin-left:1.0in"> </pre>
<pre style="margin-left:1.0in">Telephone: +41 765 42 51 24 // +41 56 310 46 67</pre>
<pre style="margin-left:1.0in">E-Mail: <a href="mailto:marc.caubet@psi.ch">marc.caubet@psi.ch</a></pre>
</div>
</div>
</div>
</div>
</blockquote>
<pre style="margin-left:.5in">-- </pre>
<pre style="margin-left:.5in">_________________________________________________________</pre>
<pre style="margin-left:.5in">Paul Scherrer Institut</pre>
<pre style="margin-left:.5in">High Performance Computing & Emerging Technologies</pre>
<pre style="margin-left:.5in">Marc Caubet Serrabou</pre>
<pre style="margin-left:.5in">Building/Room: OBBA/230</pre>
<pre style="margin-left:.5in">Forschungsstrasse, 111</pre>
<pre style="margin-left:.5in">5232 Villigen PSI</pre>
<pre style="margin-left:.5in">Switzerland</pre>
<pre style="margin-left:.5in"> </pre>
<pre style="margin-left:.5in">Telephone: +41 765 42 51 24 // +41 56 310 46 67</pre>
<pre style="margin-left:.5in">E-Mail: <a href="mailto:marc.caubet@psi.ch">marc.caubet@psi.ch</a></pre>
</div>
</div>
</div>
</div>
</div>
</body>
</html>