[openib-general] umad_recv won't block after first read...
Abhijit Gadgil
Abhijit.Gadgil at pantasys.com
Thu Aug 10 04:21:21 PDT 2006
Hi All,
I am trying to write a simple program using libibumad to 'subscribe' for traps and then receive traps from the SA. Most of the things seem to work fine, however I am facing a small problem where, after first read for the trap, all subsequent reads are not blocking (and return some incorrect length).
Attached is the simple code, can someone tell, what exactly is wrong out here?
Thanks
-abhijit
-------------- next part --------------
#include <stdio.h>
#include <infiniband/verbs.h>
#include <infiniband/umad.h>
#include <infiniband/mad.h>
#define MAX_HCAS (4)
#define MAX_PORTS_PER_CA (2)
#define SA_INFORMINFO_LEN (60)
#define SA_INFORMINFO_OFFSET (56)
#define SA_HEADER_LENGTH (20)
#define SA_CLASS_VERSION (2)
char ca_names[MAX_HCAS][UMAD_CA_NAME_LEN];
umad_ca_t cas[MAX_HCAS];
uint64_t query_tid = 0;
static int set_bit(int nr, void *method_mask)
{
int mask, retval;
long *addr = method_mask;
addr += nr >> 5;
mask = 1 << (nr & 0x1f);
retval = (mask & *addr) != 0;
*addr |= mask;
return retval;
}
static void init_sa_headers(void *mad)
{
mad_set_field(mad, 0, IB_MAD_RESPONSE_F, 0);
mad_set_field(mad, 0, IB_MAD_CLASSVER_F, SA_CLASS_VERSION);
mad_set_field(mad, 0, IB_MAD_MGMTCLASS_F, IB_SA_CLASS);
mad_set_field(mad, 0, IB_MAD_BASEVER_F, 1);
mad_set_field(mad, 0, IB_MAD_STATUS_F, 0);
mad_set_field64(mad, 0, IB_MAD_TRID_F, query_tid++);
mad_set_field(mad, 0, IB_MAD_ATTRID_F, IB_SA_ATTR_INFORMINFO);
mad_set_field(mad, 0, IB_MAD_ATTRMOD_F, 0);
mad_set_field(mad, 0, IB_SA_RMPP_VERS_F, 1);
mad_set_field(mad, 0, IB_SA_RMPP_TYPE_F, IB_RMPP_TYPE_DATA);
mad_set_field(mad, 0, IB_SA_RMPP_RESP_F, 0x1f);
mad_set_field(mad, 0, IB_SA_RMPP_FLAGS_F,
IB_RMPP_FLAG_ACTIVE |
IB_RMPP_FLAG_FIRST |
IB_RMPP_FLAG_LAST );
mad_set_field(mad, 0, IB_SA_RMPP_STATUS_F, 0);
mad_set_field(mad, 0, IB_SA_RMPP_D1_F, 1); /* First packet */
mad_set_field(mad, 0, IB_SA_RMPP_D2_F, SA_INFORMINFO_LEN + SA_HEADER_LENGTH); /* Size of Informinfo */
}
void init_informinfo_set(void *umad_in)
{
union ibv_gid gid;
union ibv_gid mcast_gid;
void *infinfo_set_mad = umad_get_mad(umad_in);
uint8_t char_id = 0x01;
uint8_t char_void = 0x00;
uint32_t vendorid = 0x00000004;
mcast_gid.global.subnet_prefix = 0;
mcast_gid.global.interface_id = 0;
/* hard coded for the time being */
gid.global.subnet_prefix = cas[0].ports[1]->gid_prefix;
gid.global.interface_id = cas[0].ports[1]->port_guid;
init_sa_headers(infinfo_set_mad);
mad_set_field(infinfo_set_mad, 0, IB_MAD_METHOD_F, IB_MAD_METHOD_SET);
/*
mad_set_array(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_SUBGID_F, &gid);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_ENUM_F, 0x0000);
mad_set_field64(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_RESV0_F, 0x00UL);
*/
mad_set_array(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_GID_F, &mcast_gid);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_LID_BEGIN_F, 0xFFFF);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_LID_END_F, 0xFFFF);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_RESV1_F, (uint16_t)0x0000);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_ISGENERIC_F, char_id);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_SUBSCRIBE_F, char_id);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_TYPE_F, 0xFFFF);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_TRAP_DEVID_F, 0xFFFF);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_QPN_F, 0x123456);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_RES2_F, 0x00);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_RESPTIME_F, 0x12);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_RESV3_F, char_void);
mad_set_field(infinfo_set_mad, IB_SA_DATA_OFFS, IB_SA_INFINFO_VENDORID_F, 0x00000004);
}
int main()
{
int num, err = 0, i,j, result ;
struct ibv_device *iter;
int mad_port_hndl, mad_port_found, sa_agent_hndl, trap_umad_hndl, trap_umad_sa_agent;
umad_port_t tmp;
void *umadbuf;
void *recv_umad;
int umad_recv_size;
uint8_t *response;
uint32_t method_mask[4];
err = umad_init();
if(err) {
printf("Error in umad_init %d\n", err);
return -1;
}
umadbuf = umad_alloc(1, umad_size() + IB_MAD_SIZE);
if(!umadbuf) {
printf("Error allocating umad buffer\n");
return umad_done();
}
recv_umad = umad_alloc(1, umad_size() + IB_MAD_SIZE);
if(!recv_umad) {
printf("Error allocating umad buffer\n");
return umad_done();
}
num = umad_get_cas_names(ca_names, MAX_HCAS);
if(num < 0) {
printf("Error in umad_get_cas_names %d\n", err);
return umad_done();
}
/*
** Get the first 'ACTIVE' port of the first 'CA'. If found
** nothing, bailout.
** FIXME: This we'll do later. Apparantly, the libibumad allows to
** do umad_open_port even if port state is not ACTIVE. This is pretty useless.
*/
for(i = 0; i < num; i++) {
err = umad_get_ca(ca_names[i], &cas[i]);
if(err < 0) {
printf("Error in umad_get_ca %d\n", err);
return umad_done();
}
/* LATER : :-P
for(j = 0; j < cas[i].numports; j++) {
printf("ok!\n");
memset(&tmp, 0, sizeof(tmp));
umad_get_port(cas[i].ca_name, (j+1), &tmp);
printf("port %d of HCA %s is in the state %d\n", (j+1), cas[i].ca_name, tmp.state);
mad_port_hndl = umad_open_port(cas[i].ca_name, (j+1), );
if(mad_port_hndl < 0) {
continue;
} else {
printf("opened port %d of HCA %s\n", (j+1), cas[i].ca_name);
mad_port_found = 1;
break;
}
}
if(mad_port_found) {
break;
} */
/*
* following code works only when there is one HCA in the system
*/
mad_port_hndl = umad_open_port(cas[i].ca_name, 2);
if(mad_port_hndl < 0) {
continue;
} else {
printf("opened port %d of HCA %s: port_handle %d\n", 2, cas[i].ca_name, mad_port_hndl);
mad_port_found = 1;
break;
}
}
err = umad_register(mad_port_hndl, IB_SA_CLASS, SA_CLASS_VERSION, 1, 0);
if(err < 0) {
printf("Error in umad_register %d\n", err);
} else {
sa_agent_hndl = err;
}
printf("sa_agent_hndl %d\n", sa_agent_hndl);
memset(umadbuf, 0, umad_size() + IB_MAD_SIZE);
init_informinfo_set(umadbuf);
/*
* (0x12) happens to be the SM_LID on the subnet
*/
umad_set_addr(umadbuf, (0x12), 0x1, 0x0, IB_DEFAULT_QP1_QKEY);
result = umad_send(mad_port_hndl, sa_agent_hndl, umadbuf, (IB_MAD_SIZE), 100, 5);
if(result < 0) {
printf("Error %d in umad_send\n", result);
return -1;
}
umad_recv_size = IB_MAD_SIZE;
result = umad_recv(mad_port_hndl, umadbuf, &umad_recv_size, -1);
if(result >= 0) {
uint32_t val;
printf("umad_status is %x\n", umad_status(umadbuf));
response = umad_get_mad(umadbuf);
sleep(1);
} else {
printf("Error in subscription\n");
return -1;
}
err = umad_unregister(mad_port_hndl, sa_agent_hndl);
if(err < 0) {
printf("Error in unregistering...\n");
}
err = umad_close_port(mad_port_hndl);
if(err) {
printf("Error in umad_close_port %d\n", err);
}
memset(method_mask, 0, sizeof(method_mask));
set_bit(IB_MAD_METHOD_REPORT, &method_mask);
set_bit(IB_MAD_METHOD_TRAP, &method_mask);
set_bit(IB_MAD_METHOD_TRAP_REPRESS, &method_mask);
/*
* now prepare to receive the traps
*/
trap_umad_hndl = umad_open_port(cas[0].ca_name, 2);
if(trap_umad_hndl < 0) {
printf("Error opening port to receive forwarded traps\n");
return -1;
}
err = umad_register(trap_umad_hndl, IB_SA_CLASS, SA_CLASS_VERSION, 0, method_mask);
if(err < 0) {
printf("Error in registering for receiving Traps without RMPP\n");
return -1;
} else {
trap_umad_sa_agent = err;
}
/*
* Wait forever for TRAPS
*/
while(1) {
memset(recv_umad, 0, umad_size() + IB_MAD_SIZE);
umad_recv_size = IB_MAD_SIZE;
result = umad_recv(trap_umad_hndl, recv_umad, &umad_recv_size, -1);
if(result >= 0) {
uint32_t val;
uint64_t val64;
printf("\n\n********** umad dump ************\n");
umad_dump(recv_umad);
printf("********** umad dump end ************\n\n");
printf("result of umad_recv = %d\t", result);
printf("umad_recv_size = %d\t", umad_recv_size);
/*
* dump some fields to indicate that we've received right things
*/
response = umad_get_mad(recv_umad);
val = mad_get_field(response, 0, IB_MAD_METHOD_F);
printf("method %x\t", val);
val = mad_get_field(response, 0, IB_MAD_ATTRID_F);
printf("attribute %x\t", val);
val = mad_get_field(response, IB_SA_DATA_OFFS, IB_NOTICE_TYPE_F);
printf("notice type %x\t", val);
val = mad_get_field(response, IB_SA_DATA_OFFS, IB_NOTICE_TRAP_NUMBER_F);
printf("trap number %x\t", val);
val = mad_get_field(response, IB_SA_DATA_OFFS, IB_NOTICE_ISSUER_LID_F);
printf("from LID %x\t", val);
val64 = mad_get_field64(response, 0, IB_MAD_TRID_F);
printf("Transaction id %lx\n", val64);
mad_set_field(response, 0, IB_MAD_METHOD_F, IB_MAD_METHOD_REPORT_RESPONSE);
mad_set_field(response, 0, IB_MAD_RESPONSE_F, 0x1);
result = umad_send(trap_umad_hndl, trap_umad_sa_agent, recv_umad, IB_MAD_SIZE, 100, 0);
printf("result of umad_send %d\n", result);
} else {
printf("Error %d in umad_recv \n", result);
sleep(1);
}
}
err = umad_close_port(trap_umad_hndl);
if(err) {
printf("Error in umad_close_port %d\n", err);
}
err = umad_release_ca(&cas[0]);
if(err) {
printf("Error in umad_release_ca %d\n", err);
}
err = umad_done();
if(err) {
printf("Error in umad_done %d\n", err);
}
return err;
}
More information about the general
mailing list