prov/efa: Call efa_fork_support_enable_if_requested earlier

efa_hmem_info_initialize calls ibv_reg_mr to test for p2p
support.

On older kernels (without support for IBV_FORK_UNNEEDED),
efa_fork_support_enable_if_requested calls ibv_fork_init,
when fork support is requested.

If ibv_fork_init is called, it must be called before any
ibv_reg_mr calls. Otherwise, ibv_fork_init will return
EINVAL.

Therefore, efa_fork_support_enable_if_requested must be called
before efa_hmem_info_initialize.

Also add a unit test to catch this failure

Signed-off-by: Sai Sunku <sunkusa@amazon.com>

prov/efa/src/efa_prov.c

@@ -201,6 +201,23 @@ EFA_INI
 	 */
 	efa_env_initialize();
 
+	/*
+	 * efa_fork_support_enable_if_requested must be called before
+	 * efa_hmem_info_initialize.
+	 *
+	 * efa_hmem_info_initialize calls ibv_reg_mr to test for p2p support.
+	 *
+	 * On older kernels (without support for IBV_FORK_UNNEEDED),
+	 * efa_fork_support_enable_if_requested calls ibv_fork_init,
+	 * when fork support is requested.
+	 *
+	 * If ibv_fork_init is called, it must be called before any
+	 * ibv_reg_mr calls. Otherwise, ibv_fork_init will return EINVAL.
+	 */
+	err = efa_fork_support_enable_if_requested();
+	if (err)
+		goto err_free;
+
 	err = efa_hmem_info_initialize();
 	if (err)
 		goto err_free;
@@ -213,11 +230,6 @@ EFA_INI
 	if (err)
 		goto err_free;
 
-	err = efa_fork_support_enable_if_requested();
-	if (err) {
-		goto err_free;
-	}
-
 	dlist_init(&g_efa_domain_list);
 
 	return &efa_prov;

prov/efa/test/efa_unit_test_info.c

@@ -70,6 +70,36 @@ void test_info_rdm_attributes()
 	}
 }
 
+/**
+ * @brief Verify that efa rdm path fi_info objects have some expected values
+ * with fork support enabled
+ */
+void test_info_rdm_attributes_fork_support()
+{
+	struct fi_info *hints, *info = NULL, *info_head = NULL;
+	int err;
+
+	hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_FABRIC_NAME);
+	assert_non_null(hints);
+
+	setenv("FI_EFA_FORK_SAFE", "1", 1);
+
+	err = fi_getinfo(FI_VERSION(1,6), NULL, NULL, 0, hints, &info_head);
+	assert_int_equal(err, 0);
+	assert_non_null(info_head);
+
+	for (info = info_head; info; info = info->next) {
+		assert_true(!strcmp(info->fabric_attr->name, EFA_FABRIC_NAME));
+		assert_true(strstr(info->domain_attr->name, "rdm"));
+		assert_int_equal(info->ep_attr->max_msg_size, UINT64_MAX);
+#if HAVE_CUDA || HAVE_NEURON || HAVE_SYNAPSEAI
+		assert_true(info->caps | FI_HMEM);
+#endif
+
+	unsetenv("FI_EFA_FORK_SAFE");
+	}
+}
+
 /**
  * @brief Verify that efa dgram path fi_info objects have some expected values
  */

prov/efa/test/efa_unit_tests.c

@@ -143,6 +143,7 @@ int main(void)
 		/* begin efa_unit_test_info.c */
 		cmocka_unit_test_setup_teardown(test_info_open_ep_with_wrong_info, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_info_rdm_attributes, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_info_rdm_attributes_fork_support, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_info_dgram_attributes, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_info_direct_attributes, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_info_direct_hmem_support_p2p, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),

prov/efa/test/efa_unit_tests.h

@@ -164,6 +164,7 @@ void test_ibv_cq_ex_read_ignore_removed_peer();
 /* begin efa_unit_test_info.c */
 void test_info_open_ep_with_wrong_info();
 void test_info_rdm_attributes();
+void test_info_rdm_attributes_fork_support();
 void test_info_dgram_attributes();
 void test_info_direct_attributes();
 void test_info_direct_hmem_support_p2p();