prov/efa: Rework zcpy recv setting

This patch refactors the efa_rdm_ep_set_use_zcpy_rx function for better readability and logging. It also removes the requirement for FI_MSG_PREFIX Signed-off-by: Shi Jin <sjina@amazon.com>
ofiwg · Jun 25, 2024 · 2a011e2 · 2a011e2
1 parent da62d0f
commit 2a011e2
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 26 deletions.
diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c
@@ -394,16 +394,43 @@ static struct fi_ops efa_rdm_ep_base_ops = {
 static inline
 void efa_rdm_ep_set_use_zcpy_rx(struct efa_rdm_ep *ep)
 {
-	ep->use_zcpy_rx = !(ep->base_ep.util_ep.caps & FI_DIRECTED_RECV) &&
-			  !(ep->base_ep.util_ep.caps & FI_TAGGED) &&
-			  !(ep->base_ep.util_ep.caps & FI_ATOMIC) &&
-			  (ep->max_msg_size <= ep->mtu_size - ep->max_proto_hdr_size) &&
-			  !efa_rdm_ep_need_sas(ep) &&
-			  ep->user_info->mode & FI_MSG_PREFIX &&
-			  efa_env.use_zcpy_rx;
+	uint64_t unsupported_caps = FI_DIRECTED_RECV | FI_TAGGED | FI_ATOMIC;
 
+	ep->use_zcpy_rx = true;
+
+	/* User requests to turn off zcpy recv */
+	if (!efa_env.use_zcpy_rx) {
+		EFA_INFO(FI_LOG_EP_CTRL, "User disables zero-copy receive protocol via environment\n");
+		ep->use_zcpy_rx = false;
+		goto out;
+	}
+
+	/* Unsupported capabilities */
+	if (ep->base_ep.util_ep.caps & unsupported_caps) {
+		EFA_INFO(FI_LOG_EP_CTRL, "Unsupported capabilities, zero-copy receive protocol will be disabled\n");
+		ep->use_zcpy_rx = false;
+		goto out;
+	}
+
+	/* Max msg size is too large, turn off zcpy recv */
+	if (ep->max_msg_size > ep->mtu_size - ep->user_info->ep_attr->msg_prefix_size) {
+		EFA_INFO(FI_LOG_EP_CTRL, "max_msg_size (%zu) is greater than the mtu size limit: %zu. Zero-copy receive protocol will be disabled.\n",
+			ep->max_msg_size, ep->mtu_size - ep->user_info->ep_attr->msg_prefix_size);
+		ep->use_zcpy_rx = false;
+		goto out;
+	}
+
+	/* If app needs sas ordering, turn off zcpy recv */
+	if (efa_rdm_ep_need_sas(ep)) {
+		EFA_INFO(FI_LOG_EP_CTRL, "FI_ORDER_SAS is requested, zero-copy receive protocol will be disabled\n");
+		ep->use_zcpy_rx = false;
+		goto out;
+	}
+
+out:
 	EFA_INFO(FI_LOG_EP_CTRL, "efa_rdm_ep->use_zcpy_rx = %d\n",
 		 ep->use_zcpy_rx);
+	return;
 }
 
 /**

diff --git a/prov/efa/test/efa_unit_test_ep.c b/prov/efa/test/efa_unit_test_ep.c
@@ -688,20 +688,3 @@ void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_sas(struct efa_resource **state
 
 	test_efa_rdm_ep_use_zcpy_rx_impl(resource, false);
 }
-
-/**
- * @brief zcpy will be disabled if app doesn't use FI_MSG_PREFIX mode.
- */
-void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_no_prefix(struct efa_resource **state)
-{
-	struct efa_resource *resource = *state;
-
-	resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM);
-	assert_non_null(resource->hints);
-
-	resource->hints->tx_attr->msg_order = FI_ORDER_NONE;
-	resource->hints->rx_attr->msg_order = FI_ORDER_NONE;
-	resource->hints->caps = FI_MSG;
-
-	test_efa_rdm_ep_use_zcpy_rx_impl(resource, false);
-}
diff --git a/prov/efa/test/efa_unit_tests.c b/prov/efa/test/efa_unit_tests.c
@@ -97,7 +97,6 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_atomic_without_caps, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_zcpy_rx_happy, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_sas, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
-		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_no_prefix, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_dgram_cq_read_empty_cq, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_ibv_cq_ex_read_empty_cq, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_ibv_cq_ex_read_failed_poll, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),

diff --git a/prov/efa/test/efa_unit_tests.h b/prov/efa/test/efa_unit_tests.h
@@ -111,7 +111,6 @@ void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_good();
 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_bad();
 void test_efa_rdm_ep_user_zcpy_rx_happy();
 void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_sas();
-void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_no_prefix();
 void test_dgram_cq_read_empty_cq();
 void test_ibv_cq_ex_read_empty_cq();
 void test_ibv_cq_ex_read_failed_poll();