Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

prov/efa: Fix the max_msg_size reporting for efa-direct #10802

Merged
merged 2 commits into from
Feb 20, 2025
Merged

prov/efa: Fix the max_msg_size reporting for efa-direct #10802

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b36cc24
prov/efa: Fix the max_msg_size reporting for efa-direct
shijin-aws Feb 20, 2025
44c55ac
prov/efa: Add unit test for efa-direct progress model
shijin-aws Feb 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions prov/efa/src/efa_prov_info.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,8 @@ void efa_prov_info_set_ep_attr(struct fi_info *prov_info,
prov_info->ep_attr->max_msg_size = device->ibv_port_attr.max_msg_sz;
prov_info->ep_attr->type = ep_type;

if (ep_type == FI_EP_RDM) {
/* ep_attr->max_msg_size is the maximum of both MSG and RMA operations */
if (prov_info->caps & FI_RMA)
prov_info->ep_attr->max_msg_size = MAX(device->ibv_port_attr.max_msg_sz, device->max_rdma_size);
} else {
assert(ep_type == FI_EP_DGRAM);
if (ep_type == FI_EP_DGRAM)
prov_info->ep_attr->msg_prefix_size = 40;
}
}

/**
Expand Down
5 changes: 5 additions & 0 deletions prov/efa/src/efa_user_info.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,11 @@ int efa_user_info_alter_direct(int version, struct fi_info *info, const struct f
EFA_INFO(FI_LOG_CORE,
"FI_MSG_PREFIX size = %ld\n", info->ep_attr->msg_prefix_size);
}
/* When user requests FI_RMA and it's supported, the max_msg_size should be returned
* as the maximum of both MSG and RMA operations
*/
if (hints->caps & FI_RMA)
info->ep_attr->max_msg_size = MAX(g_device_list[0].ibv_port_attr.max_msg_sz, g_device_list[0].max_rdma_size);
}

/* Print a warning and use FI_AV_TABLE if the app requests FI_AV_MAP */
Expand Down
56 changes: 47 additions & 9 deletions prov/efa/test/efa_unit_test_info.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,24 +94,62 @@ void test_info_dgram_attributes()
/**
* @brief Verify that efa direct path fi_info objects have some expected values
*/
void test_info_direct_attributes()
static void test_info_direct_attributes_impl(struct fi_info *hints,
int expected_return)
{
struct fi_info *hints, *info = NULL, *info_head = NULL;
struct fi_info *info = NULL, *info_head = NULL;
int err;

hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
assert_non_null(hints);
err = fi_getinfo(FI_VERSION(1, 6), NULL, NULL, 0, hints, &info_head);
assert_int_equal(err, expected_return);
if (expected_return)
return;

err = fi_getinfo(FI_VERSION(1,6), NULL, NULL, 0, hints, &info);
assert_int_equal(err, 0);
assert_non_null(info);
assert_non_null(info_head);
for (info = info_head; info; info = info->next) {
assert_true(!strcmp(info->fabric_attr->name, EFA_DIRECT_FABRIC_NAME));
assert_true(!strcmp(info->fabric_attr->name,
EFA_DIRECT_FABRIC_NAME));
assert_true(strstr(info->domain_attr->name, "rdm"));
assert_false(info->caps & (FI_ATOMIC | FI_TAGGED));
assert_false(info->tx_attr->msg_order & FI_ORDER_SAS);
assert_int_equal(info->ep_attr->max_msg_size, g_device_list[0].max_rdma_size);
assert_int_equal(
info->ep_attr->max_msg_size,
(hints->caps & FI_RMA) ?
g_device_list[0].max_rdma_size :
g_device_list[0].ibv_port_attr.max_msg_sz);
}

fi_freeinfo(info_head);
}

void test_info_direct_attributes_no_rma()
{
struct fi_info *hints;

hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
assert_non_null(hints);

/* The default hints for efa-direct only has FI_MSG cap, so it should
* succeed anyway */
test_info_direct_attributes_impl(hints, FI_SUCCESS);
fi_freeinfo(hints);
}

void test_info_direct_attributes_rma()
{
struct fi_info *hints;
bool support_fi_rma = (efa_device_support_rdma_read() &&
efa_device_support_rdma_write() &&
efa_device_support_unsolicited_write_recv());

int expected_return = support_fi_rma ? FI_SUCCESS : -FI_ENODATA;

hints = efa_unit_test_alloc_hints(FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
assert_non_null(hints);

hints->caps |= FI_RMA;
test_info_direct_attributes_impl(hints, expected_return);
fi_freeinfo(hints);
}

/**
Expand Down
3 changes: 2 additions & 1 deletion prov/efa/test/efa_unit_tests.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ int main(void)
cmocka_unit_test_setup_teardown(test_info_open_ep_with_wrong_info, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_rdm_attributes, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_dgram_attributes, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_direct_attributes, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_direct_attributes_no_rma, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_direct_attributes_rma, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_direct_hmem_support_p2p, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_msg_order_rdm_order_none, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_msg_order_rdm_order_sas, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
Expand Down
3 changes: 2 additions & 1 deletion prov/efa/test/efa_unit_tests.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ void test_ibv_cq_ex_read_ignore_removed_peer();
void test_info_open_ep_with_wrong_info();
void test_info_rdm_attributes();
void test_info_dgram_attributes();
void test_info_direct_attributes();
void test_info_direct_attributes_no_rma();
void test_info_direct_attributes_rma();
void test_info_direct_hmem_support_p2p();
void test_info_tx_rx_msg_order_rdm_order_none();
void test_info_tx_rx_msg_order_rdm_order_sas();
Expand Down