prov/psm3: update provider to sync with IEFS 12.0.0.0.36

Updates: - GPU HAL - Removed AVX requirement, now will only warn. - Improved PSM3_RDMA modes 2 & 3's performance. - Improved NIC selection algorithms Signed-off-by: Adam Goldman <adam.goldman@intel.com> (cherry picked from commit f09b96d)
ofiwg · Mar 6, 2025 · 55f288d · 55f288d
1 parent 16223f1
commit 55f288d
Show file tree

Hide file tree

Showing 95 changed files with 8,734 additions and 7,745 deletions.
diff --git a/prov/psm3/COPYING b/prov/psm3/COPYING
@@ -0,0 +1 @@
+../../COPYING
diff --git a/prov/psm3/Makefile.am b/prov/psm3/Makefile.am
@@ -1,6 +1,6 @@
 #
 # Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017-2018 Intel Corporation, Inc. All right reserved.
+# Copyright (c) 2017-2024 Intel Corporation, Inc. All right reserved.
 # Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All rights reserved.
 # (C) Copyright 2020 Hewlett Packard Enterprise Development LP
 #
@@ -97,6 +97,7 @@ common_srcs = \
 	shared/var.c \
 	shared/abi_1_0.c
 
+
 if MACOS
 common_srcs += shared/osx/osd.c
 common_srcs += shared/unix/osd.c
@@ -230,7 +231,7 @@ src_libpsm3_fi_la_LDFLAGS += -lpsm2
 endif !HAVE_PSM3_SRC
 
 if !EMBEDDED
-src_libpsm3_fi_la_LDFLAGS += -version-info 24:0:23
+src_libpsm3_fi_la_LDFLAGS += -version-info 25:0:24
 endif
 
 prov_install_man_pages = man/man7/fi_psm3.7

diff --git a/prov/psm3/Makefile.include b/prov/psm3/Makefile.include
@@ -52,14 +52,11 @@ noinst_LTLIBRARIES += \
 	prov/psm3/psm3/libptl_self.la \
 	prov/psm3/psm3/libhal_verbs.la \
 	prov/psm3/psm3/libhal_sockets.la \
+	prov/psm3/psm3/libgpu.la \
 	prov/psm3/psm3/libpsm3i.la
 
 prov_psm3_psm3_libptl_am_la_SOURCES = \
 	prov/psm3/psm3/ptl_am/am_config.h \
-	prov/psm3/psm3/ptl_am/am_cuda_memhandle_cache.c \
-	prov/psm3/psm3/ptl_am/am_cuda_memhandle_cache.h \
-	prov/psm3/psm3/ptl_am/am_oneapi_memhandle_cache.c \
-	prov/psm3/psm3/ptl_am/am_oneapi_memhandle_cache.h \
 	prov/psm3/psm3/ptl_am/am_reqrep.c \
 	prov/psm3/psm3/ptl_am/am_reqrep_shmem.c \
 	prov/psm3/psm3/ptl_am/cmarw.h \
@@ -191,6 +188,17 @@ prov_psm3_psm3_libhal_sockets_la_CPPFLAGS = \
 prov_psm3_psm3_libhal_sockets_la_CFLAGS = \
 	$(AM_CFLAGS) $(psm3_CFLAGS) $(_psm3_cflags)
 
+prov_psm3_psm3_libgpu_la_SOURCES = \
+	prov/psm3/psm3/gpu/psm_gpu_hal.c \
+	prov/psm3/psm3/gpu/psm_gpu_hal.h \
+	prov/psm3/psm3/gpu/psm_gpu_cuda.c \
+	prov/psm3/psm3/gpu/psm_gpu_oneapi_ze.c
+prov_psm3_psm3_libgpu_la_CPPFLAGS = \
+	-I$(top_srcdir)/prov/psm3/psm3/gpu/ \
+	$(AM_CPPFLAGS) $(psm3_CPPFLAGS) $(_psm3_cppflags)
+prov_psm3_psm3_libgpu_la_CFLAGS = \
+	$(AM_CFLAGS) $(psm3_CFLAGS) $(_psm3_cflags)
+
 prov_psm3_psm3_libpsm3i_la_SOURCES = \
 	prov/psm3/psm3/psm.c \
 	prov/psm3/psm3/psm_am.c \
@@ -218,7 +226,6 @@ prov_psm3_psm3_libpsm3i_la_SOURCES = \
 	prov/psm3/psm3/psm_netutils.h \
 	prov/psm3/psm3/psm_nic_select.c \
 	prov/psm3/psm3/psm_nic_select.h \
-	prov/psm3/psm3/psm_oneapi_ze.c \
 	prov/psm3/psm3/psm_perf.c \
 	prov/psm3/psm3/psm_perf.h \
 	prov/psm3/psm3/psm_rndv_mod.c \
@@ -263,6 +270,7 @@ prov_psm3_psm3_libpsm3i_la_LIBADD = \
 	prov/psm3/psm3/libptl_ips.la \
 	prov/psm3/psm3/libptl_self.la \
 	prov/psm3/psm3/libhal_verbs.la \
+	prov/psm3/psm3/libgpu.la \
 	prov/psm3/psm3/libhal_sockets.la
 
 prov_psm3_psm3_libpsm3i_la_DEPENDENCIES = \
@@ -271,6 +279,7 @@ prov_psm3_psm3_libpsm3i_la_DEPENDENCIES = \
 	prov/psm3/psm3/libptl_ips.la \
 	prov/psm3/psm3/libptl_self.la \
 	prov/psm3/psm3/libhal_verbs.la \
+	prov/psm3/psm3/libgpu.la \
 	prov/psm3/psm3/libhal_sockets.la
 
 # Mirror EXTRA_DIST to end of file
@@ -288,6 +297,7 @@ chksum_srcs += \
 	$(prov_psm3_psm3_libutils_la_SOURCES) \
 	$(prov_psm3_psm3_libhal_verbs_la_SOURCES) \
 	$(prov_psm3_psm3_libhal_sockets_la_SOURCES) \
+	$(prov_psm3_psm3_libgpu_la_SOURCES) \
 	$(prov_psm3_psm3_libpsm3i_la_SOURCES) \
 	$(prov_psm3_extra_dist)
 

diff --git a/prov/psm3/VERSION b/prov/psm3/VERSION
@@ -1 +1 @@
-3_7_0_0
+4_0_0_0
diff --git a/prov/psm3/configure.ac b/prov/psm3/configure.ac
@@ -143,7 +143,7 @@ AS_IF([test "x$enable_psm3_rc" = "xcheck"],
 AS_IF([test "x$enable_psm3_rc" = "xyes"],
       [
        AS_IF([test "x$enable_psm3_verbs" = "xyes"],
-             [CPPFLAGS="$CPPFLAGS -DUSE_RC"],
+             [CPPFLAGS="$CPPFLAGS -DUSE_RC -DUSE_RDMA_READ"],
              [AC_MSG_ERROR([User RC QPs requires Verbs HAL active])])
       ])
 AS_IF([test "x$enable_psm3_src" = "xyes"],
@@ -690,21 +690,25 @@ AS_IF([test "$have_oneapi_ze" = "1"],
         LIBS="$LIBS $ze_LIBS"
 
         dnl - Check for zeMemPutIpcHandle after ZE added to LIBS/*FLAGS
+        save_LDFLAGS="$LDFLAGS"
+        LDFLAGS="$LDFLAGS -lze_loader"
         AC_MSG_CHECKING([for zeMemPutIpcHandle support in level-zero])
         AC_LINK_IFELSE(
             [AC_LANG_PROGRAM([[
                     #include <level_zero/ze_api.h>
                 ]],[[
-                    ze_context_handle_t hContext;
+                    ze_context_handle_t hContext = NULL;
                     ze_ipc_mem_handle_t handle;
                     (void)zeMemPutIpcHandle(hContext, handle);
                 ]])
             ],[
                 AC_MSG_RESULT(yes)
                 have_oneapi_zeMemPutIpcHandle=1
                 CPPFLAGS="$CPPFLAGS -DPSM_HAVE_ONEAPI_ZE_PUT_IPCHANDLE"
+                LDFLAGS="$save_LDFLAGS"
             ],[
                 AC_MSG_RESULT(no)
+                LDFLAGS="$save_LDFLAGS"
             ])
     ])
 
@@ -910,12 +914,9 @@ AS_IF([test ! -z "$CC" && ( test "x${CC%% *}" = "xicc" || test "x${CC%% *}" = "x
       [ dnl ICC/ICX
         CFLAGS="$CFLAGS -Werror -xATOM_SSE4.2 -DPSM_AVX512 -fpic -fPIC -D_GNU_SOURCE -DPACK_STRUCT_STL=packed,"
         LDFLAGS="$LDFLAGS -Wc,-static-intel"
-        PSM3_MARCH="avx2"
       ], [ dnl GCC/other
-        CFLAGS="$CFLAGS -Werror -mavx2 -fpic -fPIC -funwind-tables -Wformat -Wformat-security"
-        PSM3_MARCH="avx2"
+        CFLAGS="$CFLAGS -Werror -msse4.2 -fpic -fPIC -funwind-tables -Wformat -Wformat-security"
       ])
-AC_DEFINE_UNQUOTED([PSM3_MARCH], ["$PSM3_MARCH"], [PSM3 built with instruction set])
 AS_IF([test ! -z "$PSM_CPPFLAGS"], [CPPFLAGS="$CPPFLAGS $PSM_CPPFLAGS"], [])
 AS_IF([test ! -z "$PSM_CFLAGS"], [CFLAGS="$CFLAGS $PSM_CFLAGS"], [])
 
@@ -936,12 +937,10 @@ AC_DEFINE([HAVE_XPMEM], 0, [Ignore HAVE_XPMEM])
 
 dnl Provider-specific checks
 dnl FI_PROVIDER_INIT
-AC_DEFINE([HAVE_BGQ], 0, [Ignore HAVE_BGQ])
-AC_DEFINE([HAVE_BGQ_DL], 0, [Ignore HAVE_BGQ_DL])
+AC_DEFINE([HAVE_CXI], 0, [Ignore HAVE_CXI])
+AC_DEFINE([HAVE_CXI_DL], 0, [Ignore HAVE_CXI_DL])
 AC_DEFINE([HAVE_EFA], 0, [Ignore HAVE_EFA])
 AC_DEFINE([HAVE_EFA_DL], 0, [Ignore HAVE_EFA_DL])
-AC_DEFINE([HAVE_GNI], 0, [Ignore HAVE_GNI])
-AC_DEFINE([HAVE_GNI_DL], 0, [Ignore HAVE_GNI_DL])
 AC_DEFINE([HAVE_MRAIL], 0, [Ignore HAVE_MRAIL])
 AC_DEFINE([HAVE_MRAIL_DL], 0, [Ignore HAVE_MRAIL_DL])
 AC_DEFINE([HAVE_NET], 0, [Ignore HAVE_NET])
@@ -954,8 +953,6 @@ AC_DEFINE([HAVE_PSM2_DL], 0, [Ignore HAVE_PSM2_DL])
 dnl FI_PROVIDER_SETUP([psm3])
 AC_DEFINE([HAVE_OPX], 0, [Ignore HAVE_OPX])
 AC_DEFINE([HAVE_OPX_DL], 0, [Ignore HAVE_OPX_DL])
-AC_DEFINE([HAVE_RSTREAM], 0, [Ignore HAVE_RSTREAM])
-AC_DEFINE([HAVE_RSTREAM_DL], 0, [Ignore HAVE_RSTREAM_DL])
 AC_DEFINE([HAVE_RXD], 0, [Ignore HAVE_RXD])
 AC_DEFINE([HAVE_RXD_DL], 0, [Ignore HAVE_RXD_DL])
 AC_DEFINE([HAVE_RXM], 0, [Ignore HAVE_RXM])
@@ -974,8 +971,6 @@ AC_DEFINE([HAVE_UCX], 0, [Ignore HAVE_UCX])
 AC_DEFINE([HAVE_UCX_DL], 0, [Ignore HAVE_UCX_DL])
 AC_DEFINE([HAVE_UDP], 0, [Ignore HAVE_UDP])
 AC_DEFINE([HAVE_UDP_DL], 0, [Ignore HAVE_UDP_DL])
-AC_DEFINE([HAVE_USNIC], 0, [Ignore HAVE_USNIC])
-AC_DEFINE([HAVE_USNIC_DL], 0, [Ignore HAVE_USNIC_DL])
 AC_DEFINE([HAVE_VERBS], 0, [Ignore HAVE_VERBS])
 AC_DEFINE([HAVE_VERBS_DL], 0, [Ignore HAVE_VERBS_DL])
 dnl FI_PROVIDER_FINI
@@ -991,8 +986,12 @@ AM_COND_IF([HAVE_PSM3_SRC],
         AS_IF([test -z "${PSM3_IEFS_VERSION}"], [PSM3_IEFS_VERSION="${PACKAGE_VERSION}$(whoami)"])
         PSM3_IEFS_VERSION=$(echo "${PSM3_IEFS_VERSION}" | tr '.' '_')
         PSM3_GIT_HASH="$(git rev-parse HEAD)"
-        RPM_RELEASE=$(echo "${PSM3_IEFS_VERSION}" | cut -d'_' -f5)
-        RELEASE_VER=$(echo "${PSM3_IEFS_VERSION}" | cut -d'_' -f1-4 | sed 's/_/./g')
+        RPM_RELEASE=$(echo "${PSM3_IEFS_VERSION}" | tr -s '@<:@A-Z@:>@' '_' | cut -d'_' -f5)
+        RELEASE_VER=$(echo "${PSM3_IEFS_VERSION}" | tr -s '@<:@A-Z@:>@' '_' | cut -d'_' -f1-4 | sed 's/_/./g')
+        char=$(echo "${PSM3_IEFS_VERSION}" | tr -dc '@<:@A-Z@:>@' | tr '@<:@A-Z@:>@' '@<:@a-z@:>@')
+        AS_IF([test -n "$char"], [
+            RPM_RELEASE="0${char}${RPM_RELEASE}"
+        ])
         AS_IF([test x"${RELEASE_VER}" = x"${PACKAGE_VERSION}"], [], [
             AC_MSG_NOTICE([Release Tag does not match VERSION file])
             AC_MSG_NOTICE([${RELEASE_VER} != ${PACKAGE_VERSION}])

diff --git a/prov/psm3/configure.m4 b/prov/psm3/configure.m4
@@ -20,7 +20,6 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
 
      PSM3_HAL_INST=""
      PSM3_HAL_CNT=0
-     PSM3_MARCH=""
 
      psm3_happy=1
      AS_IF([test x"$enable_psm3" != x"no"],
@@ -57,7 +56,7 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
         AS_IF([test "x$enable_psm3_rc" = "xyes"],
               [
             AS_IF([test "x$enable_psm3_verbs" = "xyes"],
-                  [psm3_CPPFLAGS="$psm3_CPPFLAGS -DUSE_RC"],
+                  [psm3_CPPFLAGS="$psm3_CPPFLAGS -DUSE_RC -DUSE_RDMA_READ"],
                   [AC_MSG_ERROR([User RC QPs requires Verbs HAL active])])
               ])
 
@@ -121,39 +120,16 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
             ],[
                 AC_MSG_RESULT([yes])
                 PSM3_ARCH_CFLAGS="-msse4.2"
-                PSM3_MARCH="sse4.2"
             ],[
                 psm3_happy=0
                 AC_MSG_RESULT([no])
-                AC_MSG_NOTICE([psm3 requires minimum of avx instruction set to build])
+                AC_MSG_NOTICE([psm3 requires minimum of sse4.2 instruction set to build])
             ])
         CFLAGS=$save_CFLAGS
 
-        AC_MSG_CHECKING([for -mavx support])
+        AC_MSG_CHECKING([for -mavx2 support (recommended)])
         save_CFLAGS=$CFLAGS
-        CFLAGS="$PSM3_STRIP_OPTFLAGS -mavx -O0"
-        AC_LINK_IFELSE(
-            [AC_LANG_PROGRAM(
-                [[#include <immintrin.h>]],
-                [[unsigned long long _a[4] = {1ULL,2ULL,3ULL,4ULL};
-                  __m256i vA = _mm256_loadu_si256((__m256i *)_a);
-                  __m256i vB;
-                  _mm256_store_si256(&vB, vA);
-                  return 0;]])
-            ],[
-                AC_MSG_RESULT([yes])
-                PSM3_ARCH_CFLAGS="-mavx"
-                PSM3_MARCH="avx"
-            ],[
-                psm3_happy=0
-                AC_MSG_RESULT([no])
-                AC_MSG_NOTICE([psm3 requires minimum of avx instruction set to build])
-            ])
-        CFLAGS=$save_CFLAGS
-
-        AC_MSG_CHECKING([for -mavx2 support])
-        save_CFLAGS=$CFLAGS
-        CFLAGS="$PSM3_STRIP_OPTFLAGS -mavx2 -O0"
+        CFLAGS="$PSM3_STRIP_OPTFLAGS -O0"
         AC_LINK_IFELSE(
             [AC_LANG_PROGRAM(
                 [[#include <immintrin.h>]],
@@ -164,10 +140,9 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
                   return 0;]])
             ],[
                 AC_MSG_RESULT([yes])
-                PSM3_ARCH_CFLAGS="-mavx2"
-                PSM3_MARCH="avx2"
             ],[
                 AC_MSG_RESULT([no])
+                AC_MSG_NOTICE([psm3 recommends minimum of avx2 instruction set for best performance])
             ])
         CFLAGS=$save_CFLAGS
 
@@ -227,20 +202,24 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
 
         AS_IF([test "$have_oneapi_ze" = "1"],
             [
+                save_LDFLAGS="$LDFLAGS"
+                LDFLAGS="$LDFLAGS -lze_loader"
                 AC_MSG_CHECKING([for zeMemPutIpcHandle support in level-zero])
                 AC_LINK_IFELSE(
                     [AC_LANG_PROGRAM([[
                             #include <level_zero/ze_api.h>
                         ]],[[
-                            ze_context_handle_t hContext;
+                            ze_context_handle_t hContext = NULL;
                             ze_ipc_mem_handle_t handle;
                             (void)zeMemPutIpcHandle(hContext, handle);
                         ]])
                     ],[
                         AC_MSG_RESULT(yes)
                         psm3_CPPFLAGS="$psm3_CPPFLAGS -DPSM_HAVE_ONEAPI_ZE_PUT_IPCHANDLE"
+                        LDFLAGS="$save_LDFLAGS"
                     ],[
                         AC_MSG_RESULT(no)
+                        LDFLAGS="$save_LDFLAGS"
                     ])
             ])
 
@@ -413,8 +392,6 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
      AC_SUBST(psm3_LIBS)
      AC_SUBST(PSM3_HAL_CNT)
      AC_SUBST(PSM3_HAL_INST)
-     AC_DEFINE_UNQUOTED([PSM3_MARCH], ["$PSM3_MARCH"], [PSM3 built with instruction set])
-     AC_SUBST(PSM3_MARCH)
 
      PSM3_IEFS_VERSION=m4_normalize(m4_esyscmd([cat prov/psm3/VERSION]))
      AC_SUBST(PSM3_IEFS_VERSION)

diff --git a/prov/psm3/debian/changelog b/prov/psm3/debian/changelog
@@ -1,4 +1,4 @@
-libpsm3-fi (11.7.0.0-110) unstable; urgency=medium
+libpsm3-fi (12.0.0.0-36) unstable; urgency=medium
 
   * Initial release 
 

diff --git a/prov/psm3/libpsm3-fi.spec.in b/prov/psm3/libpsm3-fi.spec.in
@@ -62,7 +62,7 @@ rm -rf %{buildroot}
 %files
 %defattr(-,root,root,-)
 %{_libdir}/libfabric/%{name}*
-%doc README
+%doc README COPYING
 %exclude %{_libdir}/libfabric/*.a
 %exclude %{_libdir}/libfabric/*.la
 %exclude %{_libdir}/pkgconfig

diff --git a/prov/psm3/psm3/Makefile.include b/prov/psm3/psm3/Makefile.include
@@ -17,14 +17,11 @@ noinst_LTLIBRARIES += \
 	psm3/libptl_self.la \
 	psm3/libhal_verbs.la \
 	psm3/libhal_sockets.la \
+	psm3/libgpu.la \
 	psm3/libpsm3i.la
 
 psm3_libptl_am_la_SOURCES = \
 	psm3/ptl_am/am_config.h \
-	psm3/ptl_am/am_cuda_memhandle_cache.c \
-	psm3/ptl_am/am_cuda_memhandle_cache.h \
-	psm3/ptl_am/am_oneapi_memhandle_cache.c \
-	psm3/ptl_am/am_oneapi_memhandle_cache.h \
 	psm3/ptl_am/am_reqrep.c \
 	psm3/ptl_am/am_reqrep_shmem.c \
 	psm3/ptl_am/cmarw.h \
@@ -156,6 +153,17 @@ psm3_libhal_sockets_la_CPPFLAGS = \
 psm3_libhal_sockets_la_CFLAGS = \
 	$(AM_CFLAGS) $(psm3_CFLAGS) $(_psm3_cflags)
 
+psm3_libgpu_la_SOURCES = \
+	psm3/gpu/psm_gpu_hal.c \
+	psm3/gpu/psm_gpu_hal.h \
+	psm3/gpu/psm_gpu_cuda.c \
+	psm3/gpu/psm_gpu_oneapi_ze.c
+psm3_libgpu_la_CPPFLAGS = \
+	-I$(top_srcdir)/psm3/gpu/ \
+	$(AM_CPPFLAGS) $(psm3_CPPFLAGS) $(_psm3_cppflags)
+psm3_libgpu_la_CFLAGS = \
+	$(AM_CFLAGS) $(psm3_CFLAGS) $(_psm3_cflags)
+
 psm3_libpsm3i_la_SOURCES = \
 	psm3/psm.c \
 	psm3/psm_am.c \
@@ -183,7 +191,6 @@ psm3_libpsm3i_la_SOURCES = \
 	psm3/psm_netutils.h \
 	psm3/psm_nic_select.c \
 	psm3/psm_nic_select.h \
-	psm3/psm_oneapi_ze.c \
 	psm3/psm_perf.c \
 	psm3/psm_perf.h \
 	psm3/psm_rndv_mod.c \
@@ -228,6 +235,7 @@ psm3_libpsm3i_la_LIBADD = \
 	psm3/libptl_ips.la \
 	psm3/libptl_self.la \
 	psm3/libhal_verbs.la \
+	psm3/libgpu.la \
 	psm3/libhal_sockets.la
 
 psm3_libpsm3i_la_DEPENDENCIES = \
@@ -236,6 +244,7 @@ psm3_libpsm3i_la_DEPENDENCIES = \
 	psm3/libptl_ips.la \
 	psm3/libptl_self.la \
 	psm3/libhal_verbs.la \
+	psm3/libgpu.la \
 	psm3/libhal_sockets.la
 
 _psm3_extra_dist = \
@@ -252,5 +261,6 @@ chksum_srcs += \
 	$(psm3_libutils_la_SOURCES) \
 	$(psm3_libhal_verbs_la_SOURCES) \
 	$(psm3_libhal_sockets_la_SOURCES) \
+	$(psm3_libgpu_la_SOURCES) \
 	$(psm3_libpsm3i_la_SOURCES) \
 	$(_psm3_extra_dist)