[ewg] OFED-4.8, rdma-core, and library paths

[Jason Gunthorpe]

On Wed, Feb 08, 2017 at 08:01:04PM +0200, Leon Romanovsky wrote:
> On Wed, Feb 08, 2017 at 10:33:35AM -0700, Jason Gunthorpe wrote:

> Thanks for the help, my final version which works correctly for build in place, install from
> sources and packages for centos6/centos6 is below:

Looks good

> # Create a special provider with exported symbols in it
> function(rdma_shared_provider DEST VERSION_SCRIPT SOVERSION VERSION)
>   # Installed driver file
>   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${DEST}.driver" "driver ${DEST}\n")
>   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${DEST}.driver" DESTINATION "${CONFIG_DIR}")
> 
>   # Uninstalled driver file
>   file(MAKE_DIRECTORY "${BUILD_ETC}/libibverbs.d/")
>   file(WRITE "${BUILD_ETC}/libibverbs.d/${DEST}.driver" "driver ${BUILD_LIB}/lib${DEST}\n")
> 
>   # Create a static provider library
>   if (ENABLE_STATIC)
>     add_library(${DEST} STATIC ${ARGN})
>     set_target_properties(${DEST} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${BUILD_LIB}")
>     install(TARGETS ${DEST} DESTINATION "${CMAKE_INSTALL_LIBDIR}")
> 
>     list(APPEND RDMA_STATIC_LIBS ${DEST}-rdmav2 ${DEST})
>     set(RDMA_STATIC_LIBS "${RDMA_STATIC_LIBS}" CACHE INTERNAL "")
>   endif()
> 
>   # Create the plugin shared library
>   add_library(${DEST} SHARED ${ARGN})
>   # Even though these are modules we still want to use Wl,--no-undefined
>   set_target_properties(${DEST} PROPERTIES LINK_FLAGS ${CMAKE_SHARED_LINKER_FLAGS})
>   rdma_set_library_map(${DEST} ${VERSION_SCRIPT})
> 
>   target_link_libraries(${DEST} LINK_PRIVATE ${COMMON_LIBS_PIC})
>   target_link_libraries(${DEST} LINK_PRIVATE ibverbs)
>   target_link_libraries(${DEST} LINK_PRIVATE ${CMAKE_THREAD_LIBS_INIT})
>   set_target_properties(${DEST} PROPERTIES
>   	SOVERSION ${SOVERSION}
>   	VERSION ${VERSION}
>   	LIBRARY_OUTPUT_DIRECTORY "${BUILD_LIB}")
>   add_custom_target(share_link ALL DEPENDS "${DEST}"  COMMAND ${CMAKE_COMMAND} -E create_symlink "lib${DEST}.so.${VERSION}"
> 	  "${BUILD_LIB}/lib${DEST}-rdmav2.so")
>   add_dependencies(share_link ${DEST})

Except this really shouldn't be a rule.  The non-rule method is used
everywhere else (eg man pages), so it must work here, it doesn't make
sense that ninja vs make would be any different. When you get
everything working put it back to execute_process..

>   install(TARGETS ${DEST} DESTINATION "${CMAKE_INSTALL_LIBDIR}")
>   execute_process(COMMAND python ${CMAKE_SOURCE_DIR}/buildlib/relpath
>     ${CMAKE_INSTALL_LIBDIR}/lib${DEST}.so.${VERSION} ${VERBS_PROVIDER_DIR}
>     OUTPUT_VARIABLE DEST_LINK_PATH OUTPUT_STRIP_TRAILING_WHITESPACE)
>   rdma_install_symlink("${DEST_LINK_PATH}" "${VERBS_PROVIDER_DIR}/lib${DEST}-rdmav2.so")

> and buildlib/relpath
> import os
> import sys
> 
> print(os.path.relpath(sys.argv[1], sys.argv[2]))

Sure, yes, doing the escaping right would probably be quite hard, that is the
usual nightmare with shell stuff unfortunately.

> But I still have issues with DEB package.
> It creates absolute (and wrong)  symlink instead of relative one.

Hmm. Can you update your github? I'll look for you.

> -rw-r--r-- 1 leonro leonro 46K Feb  8 17:13 libmlx4-rdmav2.so
> lrwxrwxrwx 1 leonro leonro  65 Feb  8 17:13 libmlx5-rdmav2.so -> /home/leonro/src/build-deb/lib/x86_64-linux-gnu/libmlx5.so.1.0.13

Most likely this means that DEST_LINK_PATH passed to
rdma_install_symlink is not correct. Which suggests python relpath is
not working..

relpath in python is done textually, from these args:

>     ${CMAKE_INSTALL_LIBDIR}/lib${DEST}.so.${VERSION} ${VERBS_PROVIDER_DIR}

But

 set(VERBS_PROVIDER_DIR "${CMAKE_INSTALL_FULL_LIBDIR}/libibverbs"

So if CMAKE_INSTALL_FULL_LIBDIR != CMAKE_INSTALL_LIBDIR (which depends
on how the distro build script configures cmake) then it will fail.

Switch to:

     ${CMAKE_INSTALL_FULL_LIBDIR}/lib${DEST}.so.${VERSION} ${VERBS_PROVIDER_DIR}

Jason