prov/psm3: update provider to sync with IEFS 11.6.0.0.231

Updates: - Full support for Intel oneAPI DPC++/C++ compiler - Improved default tuning for Intel GPUs Signed-off-by: Scott Breyer <scott.breyer@intel.com>
ofiwg · Mar 21, 2024 · acde37d · acde37d
1 parent 530a351
commit acde37d
Show file tree

Hide file tree

Showing 70 changed files with 5,641 additions and 1,851 deletions.
diff --git a/prov/psm3/Makefile.am b/prov/psm3/Makefile.am
@@ -30,9 +30,9 @@ ACLOCAL_AMFLAGS = -I config
 AM_CFLAGS = -Wall
 
 if HAVE_LD_VERSION_SCRIPT
-    libpsm3_fi_version_script = -Wl,--version-script=$(builddir)/libpsm3-fi.map
+	libpsm3_fi_version_script = -Wl,--version-script=$(builddir)/libpsm3-fi.map
 else !HAVE_LD_VERSION_SCRIPT
-    libpsm3_fi_version_script =
+	libpsm3_fi_version_script =
 endif !HAVE_LD_VERSION_SCRIPT
 
 # rdmaincludedir = $(includedir)/rdma
@@ -51,6 +51,8 @@ common_srcs = \
 	shared/hmem_neuron.c \
 	shared/hmem_synapseai.c \
 	shared/hmem_ipc_cache.c \
+	shared/xpmem.c \
+	shared/xpmem_cache.c \
 	shared/common.c \
 	shared/enosys.c \
 	shared/rbtree.c \
@@ -78,13 +80,22 @@ common_srcs = \
 	util/src/util_ns.c \
 	util/src/util_pep.c \
 	util/src/util_poll.c \
+	util/src/util_profile.c \
+	util/src/util_srx.c \
 	util/src/util_wait.c \
 	util/src/rxm_av.c \
 	util/src/cuda_mem_monitor.c \
 	util/src/cuda_ipc_monitor.c \
 	util/src/rocr_mem_monitor.c \
 	util/src/rocr_ipc_monitor.c \
-	util/src/ze_mem_monitor.c
+	util/src/ze_mem_monitor.c \
+	util/src/xpmem_monitor.c \
+	shared/fabric.c \
+	shared/fi_tostr.c \
+	shared/perf.c \
+	shared/log.c \
+	shared/var.c \
+	shared/abi_1_0.c
 
 if MACOS
 common_srcs += shared/osx/osd.c
@@ -103,9 +114,7 @@ if LINUX
 common_srcs += shared/unix/osd.c
 common_srcs += shared/linux/osd.c
 if HAVE_LINUX_PERF_RDPMC
-if !HAVE_PSM3_SRC
-common_srcs += shared/linux/rdpmc.c  #seems to be a copy of psm3/psm_perf.c
-endif
+common_srcs += shared/linux/rdpmc.c
 endif
 common_srcs += inc/linux/rdpmc.h
 common_srcs += inc/linux/osd.h
@@ -120,6 +129,8 @@ bin_SCRIPTS =
 nodist_src_libpsm3_fi_la_SOURCES =
 src_libpsm3_fi_la_SOURCES = \
 	inc/ofi_hmem.h \
+	inc/ofi_cma.h \
+	inc/ofi_xpmem.h \
 	inc/ofi.h \
 	inc/ofi_abi.h \
 	inc/ofi_atom.h \
@@ -137,7 +148,7 @@ src_libpsm3_fi_la_SOURCES = \
 	inc/ofi_proto.h \
 	inc/ofi_recvwin.h \
 	inc/ofi_rbuf.h \
-	inc/ofi_shm.h \
+	inc/ofi_shm_p2p.h \
 	inc/ofi_signal.h \
 	inc/ofi_epoll.h \
 	inc/ofi_tree.h \
@@ -148,10 +159,12 @@ src_libpsm3_fi_la_SOURCES = \
 	inc/ofi_net.h \
 	inc/ofi_perf.h \
 	inc/ofi_coll.h \
+	inc/ofi_mb.h \
 	inc/fasthash.h \
 	inc/rbtree.h \
 	inc/uthash.h \
 	inc/ofi_prov.h \
+	inc/ofi_profile.h \
 	inc/rdma/providers/fi_log.h \
 	inc/rdma/providers/fi_prov.h \
 	inc/rdma/providers/fi_peer.h \
@@ -167,6 +180,7 @@ src_libpsm3_fi_la_SOURCES = \
 	inc/rdma/fi_errno.h \
 	inc/rdma/fi_tagged.h \
 	inc/rdma/fi_trigger.h \
+	inc/rdma/fi_profile.h \
 	src/psmx3.h \
 	src/psmx3_am.c \
 	src/psmx3_atomic.c \
@@ -216,7 +230,7 @@ src_libpsm3_fi_la_LDFLAGS += -lpsm2
 endif !HAVE_PSM3_SRC
 
 if !EMBEDDED
-src_libpsm3_fi_la_LDFLAGS += -version-info 22:0:21
+src_libpsm3_fi_la_LDFLAGS += -version-info 24:0:23
 endif
 
 prov_install_man_pages = man/man7/fi_psm3.7
@@ -249,8 +263,8 @@ src/psm3_src_chksum.h: Makefile $(chksum_srcs)
 
 nroff:
 	@for file in $(prov_install_man_pages); do \
-	    source=`echo $$file | sed -e 's@/man[0-9]@@'`; \
-	    perl $(top_srcdir)/config/md2nroff.pl --source=$(top_srcdir)/$$source.md; \
+		source=`echo $$file | sed -e 's@/man[0-9]@@'`; \
+		perl $(top_srcdir)/config/md2nroff.pl --source=$(top_srcdir)/$$source.md; \
 	done
 
 dist-hook: libpsm3-fi.spec

diff --git a/prov/psm3/Makefile.include b/prov/psm3/Makefile.include
@@ -220,6 +220,8 @@ prov_psm3_psm3_libpsm3i_la_SOURCES = \
 	prov/psm3/psm3/psm_mq_recv.c \
 	prov/psm3/psm3/psm_mq_utils.c \
 	prov/psm3/psm3/psm_netutils.h \
+	prov/psm3/psm3/psm_nic_select.c \
+	prov/psm3/psm3/psm_nic_select.h \
 	prov/psm3/psm3/psm_oneapi_ze.c \
 	prov/psm3/psm3/psm_perf.c \
 	prov/psm3/psm3/psm_perf.h \

diff --git a/prov/psm3/VERSION b/prov/psm3/VERSION
@@ -1 +1 @@
-3_5_1_1
+3_6_0_1
diff --git a/prov/psm3/configure.ac b/prov/psm3/configure.ac
@@ -58,7 +58,7 @@ AC_DEFINE_UNQUOTED([BUILD_ID],["$with_build_id"],
 
 # Override autoconf default CFLAG settings (e.g. "-g -O2") while still
 # allowing the user to explicitly set CFLAGS=""
-: ${CFLAGS="-fvisibility=hidden ${base_c_warn_flags}"}
+: ${CFLAGS="${base_c_warn_flags}"}
 
 # AM_PROG_AS would set CFLAGS="-g -O2" by default if not set already so it
 # should not be called earlier
@@ -242,6 +242,35 @@ AS_IF([test x"$enable_debug" != x"no"],
 AC_DEFINE_UNQUOTED([ENABLE_DEBUG],[$dbg],
                    [defined to 1 if libfabric was configured with --enable-debug, 0 otherwise])
 
+AC_ARG_ENABLE([profile],
+              [AS_HELP_STRING([--enable-profile],
+                              [Enable profiling @<:@default=no@:>@])],
+              [],
+              [enable_profile=no])
+
+AS_IF([test x"$enable_profile" != x"no"],
+      [AC_DEFINE([HAVE_FABRIC_PROFILE], [1],
+                 [defined to 1 if libfabric was configured with --enable-profile, 0 otherwise])
+])
+
+AC_DEFUN([FI_ARG_ENABLE_SANITIZER],[
+        AC_ARG_ENABLE([$1],
+                      [AS_HELP_STRING([--enable-$1],
+                                      [Enable $3Sanitizer @<:@default=no@:>@])
+                      ],
+                      [],
+                      [enable_$1=no])
+        AS_IF([test x"$enable_$1" != x"no"],
+              [CFLAGS="-fsanitize=$2 $CFLAGS"])
+])
+
+m4_map([FI_ARG_ENABLE_SANITIZER],[
+       [asan, address, Address],
+       [lsan, leak, Leak],
+       [tsan, thread, Thread],
+       [ubsan, undefined, UndefinedBehavior]
+])
+
 dnl Checks for header files.
 AC_HEADER_STDC
 
@@ -463,7 +492,9 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[__asm__(".symver main_, main@ABIVER_1.0");]],
 
 ]) dnl AS_IF icc_symver_hack
 
-AC_DEFINE_UNQUOTED([HAVE_SYMVER_SUPPORT], [$ac_asm_symver_support],
+dnl Disable in psm3 to include all symbols without symver
+dnl AC_DEFINE_UNQUOTED([HAVE_SYMVER_SUPPORT], [$ac_asm_symver_support],
+AC_DEFINE_UNQUOTED([HAVE_SYMVER_SUPPORT], [0],
            [Define to 1 if compiler/linker support symbol versioning.])
 
 AC_MSG_CHECKING(for __alias__ attribute support)
@@ -478,8 +509,9 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[
       AC_MSG_RESULT(no)
       ac_prog_cc_alias_symbols=0
    ])
-
-AC_DEFINE_UNQUOTED([HAVE_ALIAS_ATTRIBUTE], [$ac_prog_cc_alias_symbols],
+dnl Disable in psm3 to include all symbols without symver
+dnl AC_DEFINE_UNQUOTED([HAVE_ALIAS_ATTRIBUTE], [$ac_prog_cc_alias_symbols],
+AC_DEFINE_UNQUOTED([HAVE_ALIAS_ATTRIBUTE], [0],
            [Define to 1 if the linker supports alias attribute.])
 AC_CHECK_FUNCS([getifaddrs])
 
@@ -772,6 +804,37 @@ AS_IF([test "x$enable_psm3_umr_cache" != "xno"],
               ])
       ])
 
+dnl ------------- hwloc
+AC_ARG_ENABLE([psm3-hwloc],
+        [AS_HELP_STRING([--enable-psm3-hwloc],
+                        [Enable PSM3 use of hwloc for NIC affinity selections @<:@default=check@:>@])],
+        [], [enable_psm3_hwloc=check])
+psm3_hwloc_happy=0
+AS_IF([test "x$enable_psm3_hwloc" != "xno"],
+      [
+        FI_CHECK_PACKAGE([psm3_hwloc],
+                         [hwloc.h],
+                         [hwloc],
+                         [hwloc_topology_init],
+                         [],
+                         [$psm3_PREFIX],
+                         [$psm3_LIBDIR],
+                         [psm3_hwloc_found=1],
+                         [psm3_hwloc_found=0])
+        AS_IF([test $psm3_hwloc_found -ne 1 && test "x$enable_psm3_hwloc" == "xyes"],
+              [
+                psm3_happy=0
+                AC_MSG_ERROR([hwloc Support requested but hwloc headers and/or library not found.])
+              ])
+        AS_IF([test "$psm3_hwloc_found" -eq 1],
+              [
+                  psm3_hwloc_happy=1
+                  CPPFLAGS="$CPPFLAGS $psm3_hwloc_CPPFLAGS -DPSM_USE_HWLOC"
+                  LDFLAGS="$LDFLAGS $psm3_hwloc_LDFLAGS"
+                  LIBS="$LIBS $psm3_hwloc_LIBS"
+              ])
+      ])
+
 dnl ------------- Driver Modules
 psm3_rv_happy=0
 AC_ARG_WITH([psm3-rv],
@@ -852,6 +915,9 @@ AC_DEFINE_UNQUOTED([PSM3_MARCH], ["$PSM3_MARCH"], [PSM3 built with instruction s
 AS_IF([test ! -z "$PSM_CPPFLAGS"], [CPPFLAGS="$CPPFLAGS $PSM_CPPFLAGS"], [])
 AS_IF([test ! -z "$PSM_CFLAGS"], [CFLAGS="$CFLAGS $PSM_CFLAGS"], [])
 
+dnl Workaround for including fabric.c
+AC_DEFINE([HOOK_NOOP_INIT], NULL, [Ignore HOOK_NOOP_INIT])
+AC_DEFINE([COLL_INIT], NULL, [Ignore COLL_INIT])
 dnl Defines not used in PSM3 provider
 AC_DEFINE([HAVE_DMABUF_PEER_MEM], 0, [Ignore HAVE_DMABUF_PEER_MEM])
 AC_DEFINE([HAVE_GDRCOPY], 0, [Ignore HAVE_GDRCOPY])
@@ -862,10 +928,16 @@ AC_DEFINE([HAVE_NEURON], 0, [Ignore HAVE_NEURON])
 AC_DEFINE([HAVE_ROCR], 0, [Ignore HAVE_ROCR])
 AC_DEFINE([HAVE_SYNAPSEAI], 0, [Ignore HAVE_SYNAPSEAI])
 AC_DEFINE([HAVE_UFFD_MONITOR], 0, [Ignore HAVE_UFFD_MONITOR])
+AC_DEFINE([HAVE_XPMEM], 0, [Ignore HAVE_XPMEM])
+
 dnl Provider-specific checks
 dnl FI_PROVIDER_INIT
+AC_DEFINE([HAVE_BGQ], 0, [Ignore HAVE_BGQ])
+AC_DEFINE([HAVE_BGQ_DL], 0, [Ignore HAVE_BGQ_DL])
 AC_DEFINE([HAVE_EFA], 0, [Ignore HAVE_EFA])
 AC_DEFINE([HAVE_EFA_DL], 0, [Ignore HAVE_EFA_DL])
+AC_DEFINE([HAVE_GNI], 0, [Ignore HAVE_GNI])
+AC_DEFINE([HAVE_GNI_DL], 0, [Ignore HAVE_GNI_DL])
 AC_DEFINE([HAVE_MRAIL], 0, [Ignore HAVE_MRAIL])
 AC_DEFINE([HAVE_MRAIL_DL], 0, [Ignore HAVE_MRAIL_DL])
 AC_DEFINE([HAVE_NET], 0, [Ignore HAVE_NET])
@@ -878,6 +950,8 @@ AC_DEFINE([HAVE_PSM2_DL], 0, [Ignore HAVE_PSM2_DL])
 dnl FI_PROVIDER_SETUP([psm3])
 AC_DEFINE([HAVE_OPX], 0, [Ignore HAVE_OPX])
 AC_DEFINE([HAVE_OPX_DL], 0, [Ignore HAVE_OPX_DL])
+AC_DEFINE([HAVE_RSTREAM], 0, [Ignore HAVE_RSTREAM])
+AC_DEFINE([HAVE_RSTREAM_DL], 0, [Ignore HAVE_RSTREAM_DL])
 AC_DEFINE([HAVE_RXD], 0, [Ignore HAVE_RXD])
 AC_DEFINE([HAVE_RXD_DL], 0, [Ignore HAVE_RXD_DL])
 AC_DEFINE([HAVE_RXM], 0, [Ignore HAVE_RXM])
@@ -980,6 +1054,9 @@ fi
 if test $psm3_dsa_happy -eq 1; then
     afeatures="$afeatures, Intel DSA"
 fi
+if test $psm3_hwloc_happy -eq 1; then
+    afeatures="$afeatures, hwloc"
+fi
 if test "x$enable_psm3_udp" = "xyes"; then
     afeatures="$afeatures, UDP"
 fi

diff --git a/prov/psm3/configure.m4 b/prov/psm3/configure.m4
@@ -371,6 +371,28 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
                   ])
         ])
 
+        AS_IF([test "x$enable_psm3_hwloc" != "xno"],
+              [
+                FI_CHECK_PACKAGE([psm3_hwloc],
+                                 [hwloc.h],
+                                 [hwloc],
+                                 [hwloc_topology_init],
+                                 [],
+                                 [$psm3_PREFIX],
+                                 [$psm3_LIBDIR],
+                                 [psm3_hwloc_found=1],
+                                 [psm3_hwloc_found=0])
+                AS_IF([test $psm3_hwloc_found -ne 1 && test "x$enable_psm3_hwloc" == "xyes"],
+                      [
+                        psm3_happy=0
+                        AC_MSG_ERROR([hwloc Support requested but hwloc headers and/or library not found.])
+                      ])
+                AS_IF([test "$psm3_hwloc_found" -eq 1],
+                      [
+                        psm3_CPPFLAGS="$psm3_CPPFLAGS -DPSM_USE_HWLOC"
+                      ])
+               ])
+
         AS_IF([test $psm3_happy -eq 1], [
             AC_CONFIG_FILES([prov/psm3/psm3/psm2_hal_inlines_i.h \
                  prov/psm3/psm3/psm2_hal_inlines_d.h \
@@ -381,9 +403,9 @@ AC_DEFUN([FI_PSM3_CONFIGURE],[
      AS_IF([test $psm3_happy -eq 1], [$1], [$2])
 
      psm3_ARCH_CFLAGS="$PSM3_ARCH_CFLAGS"
-     psm3_CPPFLAGS="$psm3_CPPFLAGS $psm3_rt_CPPFLAGS $psm3_dl_CPPFLAGS $psm3_numa_CPPFLAGS $psm3_ibv_CPPFLAGS $psm3_uuid_CPPFLAGS"
-     psm3_LDFLAGS="$psm3_LDFLAGS $psm3_rt_LDFLAGS $psm3_dl_LDFLAGS $psm3_numa_LDFLAGS $psm3_ibv_LDFLAGS $psm3_uuid_LDFLAGS"
-     psm3_LIBS="$psm3_LIBS $psm3_rt_LIBS $psm3_dl_LIBS $psm3_numa_LIBS $psm3_ibv_LIBS $psm3_uuid_LIBS"
+     psm3_CPPFLAGS="$psm3_CPPFLAGS $psm3_rt_CPPFLAGS $psm3_dl_CPPFLAGS $psm3_numa_CPPFLAGS $psm3_ibv_CPPFLAGS $psm3_uuid_CPPFLAGS $psm3_hwloc_CPPFLAGS"
+     psm3_LDFLAGS="$psm3_LDFLAGS $psm3_rt_LDFLAGS $psm3_dl_LDFLAGS $psm3_numa_LDFLAGS $psm3_ibv_LDFLAGS $psm3_uuid_LDFLAGS $psm3_hwloc_LDFLAGS"
+     psm3_LIBS="$psm3_LIBS $psm3_rt_LIBS $psm3_dl_LIBS $psm3_numa_LIBS $psm3_ibv_LIBS $psm3_uuid_LIBS $psm3_hwloc_LIBS"
      AC_SUBST(psm3_CFLAGS)
      AC_SUBST(psm3_ARCH_CFLAGS)
      AC_SUBST(psm3_CPPFLAGS)
@@ -448,4 +470,9 @@ AC_ARG_ENABLE([psm3-umr-cache],
         [Enable support for Userspace Memory Region (UMR) Caching @<:@default=check@:>@])],
     [],
     [enable_psm3_umr_cache=check])
+AC_ARG_ENABLE([psm3-hwloc],
+    [AS_HELP_STRING([--enable-psm3-hwloc],
+        [Enable PSM3 use of hwloc for NIC affinity selections @<:@default=check@:>@])],
+    [],
+    [enable_psm3_hwloc=check])
 dnl vim: set ts=4 sw=4 tw=0 et :
diff --git a/prov/psm3/debian/changelog b/prov/psm3/debian/changelog
@@ -1,4 +1,4 @@
-libpsm3-fi (11.5.1.1-1) unstable; urgency=medium
+libpsm3-fi (11.6.0.0-231) unstable; urgency=medium
 
   * Initial release 
 

diff --git a/prov/psm3/debian/control b/prov/psm3/debian/control
@@ -2,7 +2,7 @@ Source: libpsm3-fi
 Section: libs
 Priority: optional
 Maintainer: https://www.intel.com/content/www/us/en/support.html
-Build-Depends: debhelper (>= 12~), uuid-dev, libnuma-dev, libibverbs-dev, librdmacm-dev
+Build-Depends: debhelper (>= 12~), uuid-dev, libnuma-dev, libibverbs-dev, librdmacm-dev, libhwloc-dev
 Standards-Version: 4.5.1
 Rules-Requires-Root: no
 

diff --git a/prov/psm3/libpsm3-fi.spec.in b/prov/psm3/libpsm3-fi.spec.in
@@ -1,6 +1,8 @@
 %{!?configopts: %global configopts LDFLAGS=-Wl,--build-id}
 %{!?provider: %define provider psm3}
 %{!?provider_formal: %define provider_formal PSM3}
+# Disable setting SOURCE_DATE_EPOCH from changelog
+%define source_date_epoch_from_changelog 0
 
 Name: lib%{provider}-fi
 Version: @VERSION@
@@ -18,6 +20,7 @@ Provides: lib${provider}-fi1 = %{version}-%{release}
 
 BuildRequires: libuuid-devel
 BuildRequires: rdma-core-devel
+BuildRequires: hwloc-devel
 %if 0%{?suse_version} >= 1
 BuildRequires: glibc-devel
 BuildRequires: libnuma-devel

diff --git a/prov/psm3/psm3/Makefile.include b/prov/psm3/psm3/Makefile.include
@@ -185,6 +185,8 @@ psm3_libpsm3i_la_SOURCES = \
 	psm3/psm_mq_recv.c \
 	psm3/psm_mq_utils.c \
 	psm3/psm_netutils.h \
+	psm3/psm_nic_select.c \
+	psm3/psm_nic_select.h \
 	psm3/psm_oneapi_ze.c \
 	psm3/psm_perf.c \
 	psm3/psm_perf.h \
@@ -196,13 +198,13 @@ psm3_libpsm3i_la_SOURCES = \
 	psm3/psm_sysbuf.h \
 	psm3/psm_timer.c \
 	psm3/psm_timer.h \
+	psm3/psm_uffd.c \
+	psm3/psm_uffd.h \
 	psm3/psm_user.h \
 	psm3/psm_utils.c \
 	psm3/psm_utils.h \
 	psm3/psm_verbs_mr.c \
 	psm3/psm_verbs_mr.h \
-	psm3/psm_verbs_umrc.c \
-	psm3/psm_verbs_umrc.h \
 	psm3/psmi_wrappers.c \
 	psm3/psmi_wrappers.h \
 	psm3/psm2.h \