libfabric:core:core:fi_param_define_():223<info> registered var perf_cntr
libfabric:core:core:fi_param_get_():272<info> variable perf_cntr=<not set>
libfabric:core:core:fi_param_define_():223<info> registered var hook
libfabric:core:core:fi_param_get_():272<info> variable hook=<not set>
libfabric:core:core:fi_param_define_():223<info> registered var provider
libfabric:core:core:fi_param_define_():223<info> registered var fork_unsafe
libfabric:core:core:fi_param_define_():223<info> registered var universe_size
libfabric:core:core:fi_param_get_():272<info> variable provider=<not set>
libfabric:core:core:fi_param_define_():223<info> registered var provider_path
libfabric:core:core:fi_param_get_():272<info> variable provider_path=<not set>
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:gni:fabric:__gnix_alps_init():275<warn> [8510:1] lli put failed, ret=-1(Bad file descriptor)
libfabric:gni:fabric:_gnix_nics_per_rank():672<warn> [8510:1] __gnix_app_init() failed, ret=-5(Bad file descriptor)
libfabric:gni:fabric:_gnix_nic_init():1407<warn> [8510:1] _gnix_nics_per_rank failed: -5
libfabric:core:core:ofi_register_provider():200<info> registering provider: gni (1.1)
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:ofi_rxm:core:fi_param_define_():223<info> registered var buffer_size
libfabric:ofi_rxm:core:fi_param_define_():223<info> registered var comp_per_progress
libfabric:ofi_rxm:core:fi_param_define_():223<info> registered var sar_limit
libfabric:ofi_rxm:core:fi_param_define_():223<info> registered var use_srx
libfabric:ofi_rxm:core:fi_param_get_():272<info> variable buffer_size=<not set>
libfabric:core:core:ofi_register_provider():200<info> registering provider: ofi_rxm (1.0)
libfabric:core:core:ofi_register_provider():193<warn> no provider structure or name
libfabric:core:core:fi_param_define_():223<info> registered var rxd_enable
libfabric:core:core:fi_param_get_():272<info> variable rxd_enable=<not set>
libfabric:core:core:ofi_register_provider():200<info> registering provider: UDP (1.1)
libfabric:sockets:core:fi_param_define_():223<info> registered var pe_waittime
libfabric:sockets:core:fi_param_define_():223<info> registered var max_conn_retry
libfabric:sockets:core:fi_param_define_():223<info> registered var def_conn_map_sz
libfabric:sockets:core:fi_param_define_():223<info> registered var def_av_sz
libfabric:sockets:core:fi_param_define_():223<info> registered var def_cq_sz
libfabric:sockets:core:fi_param_define_():223<info> registered var def_eq_sz
libfabric:sockets:core:fi_param_define_():223<info> registered var pe_affinity
libfabric:sockets:core:fi_param_define_():223<info> registered var keepalive_enable
libfabric:sockets:core:fi_param_define_():223<info> registered var keepalive_time
libfabric:sockets:core:fi_param_define_():223<info> registered var keepalive_intvl
libfabric:sockets:core:fi_param_define_():223<info> registered var keepalive_probes
libfabric:sockets:core:fi_param_define_():223<info> registered var iface
libfabric:sockets:core:fi_param_get_():272<info> variable iface=<not set>
libfabric:core:core:ofi_register_provider():200<info> registering provider: sockets (2.0)
libfabric:tcp:core:fi_param_define_():223<info> registered var iface
libfabric:core:core:ofi_register_provider():200<info> registering provider: tcp (0.1)
libfabric:gni:fabric:_gnix_ep_getinfo():455<trace> [8510:1]
libfabric:gni:fabric:_gnix_ep_getinfo():455<trace> [8510:1]
libfabric:gni:fabric:_gnix_ep_getinfo():455<trace> [8510:1]
libfabric:gni:fabric:__gnix_getinfo_resolve_node():415<info> [8510:1] node: (null) service: (null)
libfabric:gni:fabric:__gnix_getinfo_resolve_node():420<info> [8510:1] src_pe: 0x315 src_port: 0x0
libfabric:ofi_rxm:core:ofi_check_info():997<info> Unsupported capabilities
libfabric:ofi_rxm:core:ofi_check_info():998<info> Supported: FI_MSG, FI_RMA, FI_TAGGED, FI_READ, FI_WRITE, FI_RECV, FI_SEND, FI_REMOTE_READ, FI_REMOTE_WRITE, FI_MULTI_RECV, FI_LOCAL_COMM, FI_REMOTE_COMM, FI_SOURCE, FI_DIRECTED_RECV
libfabric:ofi_rxm:core:ofi_check_info():998<info> Requested: FI_RMA, FI_ATOMIC, FI_RMA_EVENT
libfabric:core:core:fi_getinfo_():712<warn> fi_getinfo: provider ofi_rxm returned -61 (No data available)
libfabric:UDP:core:ofi_check_ep_type():628<info> Unsupported endpoint type
libfabric:UDP:core:ofi_check_ep_type():629<info> Supported: FI_EP_DGRAM
libfabric:UDP:core:ofi_check_ep_type():629<info> Requested: FI_EP_RDM
libfabric:core:core:fi_getinfo_():712<warn> fi_getinfo: provider UDP returned -61 (No data available)
libfabric:tcp:core:ofi_check_ep_type():628<info> Unsupported endpoint type
libfabric:tcp:core:ofi_check_ep_type():629<info> Supported: FI_EP_MSG
libfabric:tcp:core:ofi_check_ep_type():629<info> Requested: FI_EP_RDM
libfabric:core:core:fi_getinfo_():712<warn> fi_getinfo: provider tcp returned -61 (No data available)
____________________________FI_INFO DETAILS_______________________________
 fi_info:
    caps: [ FI_RMA, FI_ATOMIC, FI_MULTI_RECV, FI_TRIGGER, FI_FENCE, FI_RMA_EVENT ]
    mode: [  ]
    addr_format: FI_ADDR_GNI
    src_addrlen: 48
    dest_addrlen: 48
    src_addr: fi_addr_gni://315
    dest_addr: (null)
    handle: (null)
    fi_tx_attr:
        caps: [ FI_RMA, FI_ATOMIC, FI_MULTI_RECV, FI_TRIGGER, FI_FENCE, FI_RMA_EVENT ]
        mode: [  ]
        op_flags: [ FI_DELIVERY_COMPLETE ]
        msg_order: [ FI_ORDER_SAS, FI_ORDER_STRICT ]
        comp_order: [  ]
        inject_size: 16
        size: 500
        iov_limit: 8
        rma_iov_limit: 1
    fi_rx_attr:
        caps: [ FI_RMA, FI_ATOMIC, FI_MULTI_RECV, FI_TRIGGER, FI_FENCE, FI_RMA_EVENT ]
        mode: [  ]
        op_flags: [  ]
        msg_order: [ FI_ORDER_SAS, FI_ORDER_STRICT ]
        comp_order: [  ]
        total_buffered_recv: 0
        size: 500
        iov_limit: 8
    fi_ep_attr:
        type: FI_EP_RDM
        protocol: FI_PROTO_GNI
        protocol_version: 0
        max_msg_size: 4294967295
        msg_prefix_size: 0
        max_order_raw_size: 0
        max_order_war_size: 0
        max_order_waw_size: 0
        mem_tag_format: 0x0000000000000000
        tx_ctx_cnt: 1
        rx_ctx_cnt: 1
        auth_key_size: 0
    fi_domain_attr:
        domain: 0x0
        name: /sys/class/gni/kgni0
        threading: FI_THREAD_DOMAIN
        control_progress: FI_PROGRESS_AUTO
        data_progress: FI_PROGRESS_AUTO
        resource_mgmt: FI_RM_ENABLED
        av_type: FI_AV_UNSPEC
        mr_mode: [ FI_MR_BASIC ]
        mr_key_size: 8
        cq_data_size: 8
        cq_cnt: 0
        ep_cnt: -1
        tx_ctx_cnt: 3
        rx_ctx_cnt: 4
        max_ep_tx_ctx: 128
        max_ep_rx_ctx: 128
        max_ep_stx_ctx: 0
        max_ep_srx_ctx: 0
        cntr_cnt: 0
        mr_iov_limit: 1
    caps: [ FI_REMOTE_COMM ]
    mode: [  ]
        auth_key_size: 0
        max_err_data: 0
        mr_cnt: 65535
    fi_fabric_attr:
        name: gni
        prov_name: gni
        prov_version: 1.1
        api_version: 1.6
libfabric:core:core:fi_fabric_():896<info> Opened fabric: gni
libfabric:gni:domain:gnix_domain_open():579<trace> [8510:1]
libfabric:gni:fabric:gnix_domain_open():591<info> [8510:1] failed to find authorization key, creating new authorization key
libfabric:gni:fabric:__gnix_alps_init():275<warn> [8510:1] lli put failed, ret=-1(Bad file descriptor)
libfabric:gni:fabric:gnixu_get_rdma_credentials():437<warn> [8510:1] __gnix_app_init() failed, ret=-5(Bad file descriptor)
libfabric:gni:domain:_gnix_auth_key_enable():345<info> [8510:1] pkey=00002aaa ptag=171 key_partition_size=0 key_offset=0 enabled
libfabric:gni:domain:gnix_domain_open():597<info> [8510:1] authorization key=0x63b790 ptag 171 cookie 0x2aaa
libfabric:gni:eq:gnix_eq_open():380<trace> [8510:1]
libfabric:gni:eq:gnix_verify_eq_attr():103<trace> [8510:1]
libfabric:gni:eq:gnix_eq_set_wait():76<trace> [8510:1]
libfabric:gni:eq:gnix_wait_open():536<trace> [8510:1]
libfabric:gni:eq:gnix_verify_wait_attr():367<trace> [8510:1]
libfabric:gni:eq:gnix_init_wait_obj():387<trace> [8510:1]
libfabric:gni:ep_ctrl:__gnix_wait_start_progress():175<trace> [8510:1]
libfabric:gni:ep_ctrl:__gnix_wait_start_progress():179<trace> [8510:1]
libfabric:gni:ep_ctrl:__gnix_wait_nic_prog_thread_fn():72<trace> [8510:2]
libfabric:gni:cq:gnix_cq_open():651<trace> [8510:1]
libfabric:gni:cq:verify_cq_attr():139<trace> [8510:1]
libfabric:gni:cq:gnix_cq_set_wait():192<trace> [8510:1]
libfabric:gni:cq:gnix_cntr_open():433<trace> [8510:1]
libfabric:gni:cq:__verify_cntr_attr():65<trace> [8510:1]
libfabric:gni:eq:gnix_cntr_set_wait():96<trace> [8510:1]
libfabric:gni:eq:gnix_wait_open():536<trace> [8510:1]
libfabric:gni:eq:gnix_verify_wait_attr():367<trace> [8510:1]
libfabric:gni:eq:gnix_init_wait_obj():387<trace> [8510:1]
libfabric:gni:ep_ctrl:__gnix_wait_start_progress():175<trace> [8510:1]
libfabric:gni:cq:gnix_cntr_open():433<trace> [8510:1]
libfabric:gni:cq:__verify_cntr_attr():65<trace> [8510:1]
libfabric:gni:eq:gnix_cntr_set_wait():96<trace> [8510:1]
libfabric:gni:eq:gnix_wait_open():536<trace> [8510:1]
libfabric:gni:eq:gnix_verify_wait_attr():367<trace> [8510:1]
libfabric:gni:eq:gnix_init_wait_obj():387<trace> [8510:1]
libfabric:gni:ep_ctrl:__gnix_wait_start_progress():175<trace> [8510:1]
libfabric:gni:cq:gnix_cntr_open():433<trace> [8510:1]
libfabric:gni:cq:__verify_cntr_attr():65<trace> [8510:1]
libfabric:gni:eq:gnix_cntr_set_wait():96<trace> [8510:1]
libfabric:gni:eq:gnix_wait_open():536<trace> [8510:1]
libfabric:gni:eq:gnix_verify_wait_attr():367<trace> [8510:1]
libfabric:gni:eq:gnix_init_wait_obj():387<trace> [8510:1]
libfabric:gni:ep_ctrl:__gnix_wait_start_progress():175<trace> [8510:1]
libfabric:gni:av:gnix_av_open():838<trace> [8510:1]
libfabric:gni:ep_ctrl:gnix_ep_open():2295<trace> [8510:1]
libfabric:gni:ep_ctrl:__init_tag_storages():2092<info> [8510:1] initializing tag storage, tag_type=1
libfabric:gni:ep_ctrl:_gnix_xpmem_handle_create():238<trace> [8510:1]
libfabric:gni:ep_ctrl:_gnix_cm_nic_alloc():609<trace> [8510:1]
libfabric:gni:fabric:_gnix_resolve_gni_ep_name():118<trace> [8510:1]
libfabric:gni:ep_ctrl:_gnix_cm_nic_alloc():628<info> [8510:1] creating cm_nic for 171/0x2aaa/1302529
libfabric:gni:ep_ctrl:gnix_nic_alloc():947<trace> [8510:1]
libfabric:gni:ep_ctrl:gnix_nic_alloc():1052<warn> [8510:1] GNI_CdmAttach returned GNI_RC_INVALID_PARAM
libfabric:gni:fabric:_gnix_dump_gni_res():729<warn> [8510:1] Device Resources:
dev res:       MDD, avail: 4086 res: 409 held: 0 total: 4095
dev res:        CQ, avail: 2038 res: 64 held: 0 total: 2047
dev res:       FMA, avail: 125 res: 4 held: 0 total: 127
dev res:        CE, avail: 4 res: 0 held: 0 total: 4
dev res:       DLA, avail: 16384 res: 1024 held: 0 total: 16384
dev res:       TCR, avail: 64730 res: 0 held: 0 total: 16
dev res:       DVA, avail: 4398046511104 res: 1099511627776 held: 0 total: 4398046511104
libfabric:gni:fabric:_gnix_dump_gni_res():745<warn> [8510:1] Job Resources:
libfabric:gni:ep_ctrl:_gnix_cm_nic_alloc():649<warn> [8510:1] gnix_nic_alloc returned Invalid argument
libfabric:gni:ep_ctrl:_gnix_ep_nic_init():2188<warn> [8510:1] _gnix_cm_nic_alloc returned Invalid argument
libfabric:gni:ep_ctrl:gnix_ep_open():2404<warn> [8510:1] _gnix_ep_nic_init returned -22
libfabric:gni:ep_ctrl:_gnix_xpmem_handle_destroy():315<trace> [8510:1]
libfabric:gni:ep_ctrl:__xpmem_hndl_destruct():103<trace> [8510:1]
libfabric:gni:ep_ctrl:__destruct_tag_storages():1595<info> [8510:1] destroying tag storage
COMMS ERROR: fi_endpoint creation failed: Invalid argument