Skip to content

Commit

Permalink
prov/efa: Remove efa_av->ep_type in favor of efa_domain->info_type
Browse files Browse the repository at this point in the history 
efa_av->ep_type was used to distinguish between the RDM path and the
DGRAM path. The efa-direct path has FI_EP_RDM ep type but it does not
require a efa_conn. But the efa-direct path still uses unique connid
based on timestamp.

Use efa_domain->info_type to distinguish the three code paths.

Signed-off-by: Sai Sunku <sunkusa@amazon.com>
  • Loading branch information
@sunkuamzn
sunkuamzn committed Feb 26, 2025
1 parent ef15865 commit 10071c7
Show file tree
Hide file tree
Showing 5 changed files with 17 additions and 22 deletions.
22 changes: 9 additions & 13 deletions prov/efa/src/efa_av.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ int efa_conn_rdm_init(struct efa_av *av, struct efa_conn *conn, bool insert_shm_
struct efa_rdm_ep *efa_rdm_ep;
struct efa_rdm_peer *peer;

assert(av->ep_type == FI_EP_RDM);
assert(av->domain->info_type == EFA_INFO_RDM);
assert(conn->ep_addr);

/* currently multiple EP bind to same av is not supported */
Expand Down Expand Up @@ -347,7 +347,7 @@ void efa_conn_rdm_deinit(struct efa_av *av, struct efa_conn *conn)
struct efa_rdm_peer *peer;
struct efa_rdm_ep *ep;

assert(av->ep_type == FI_EP_RDM);
assert(av->domain->info_type == EFA_INFO_RDM);

peer = &conn->rdm_peer;
if (peer->is_local && av->shm_rdm_av) {
Expand Down Expand Up @@ -408,7 +408,7 @@ int efa_av_update_reverse_av(struct efa_av *av, struct efa_ep_addr *raw_addr,
/* We used a static connid for all dgram endpoints, therefore cur_entry should always be NULL,
* and only RDM endpoint can reach here. hence the following assertion
*/
assert(av->ep_type == FI_EP_RDM);
assert(av->domain->info_type == EFA_INFO_RDM);
prv_entry = malloc(sizeof(*prv_entry));
if (!prv_entry) {
EFA_WARN(FI_LOG_AV, "Cannot allocate memory for prv_reverse_av entry\n");
Expand Down Expand Up @@ -478,7 +478,7 @@ struct efa_conn *efa_conn_alloc(struct efa_av *av, struct efa_ep_addr *raw_addr,
if (!conn->ah)
goto err_release;

if (av->ep_type == FI_EP_RDM) {
if (av->domain->info_type == EFA_INFO_RDM) {
err = efa_conn_rdm_init(av, conn, insert_shm_av);
if (err) {
errno = -err;
Expand All @@ -488,7 +488,7 @@ struct efa_conn *efa_conn_alloc(struct efa_av *av, struct efa_ep_addr *raw_addr,

err = efa_av_update_reverse_av(av, raw_addr, conn);
if (err) {
if (av->ep_type == FI_EP_RDM)
if (av->domain->info_type == EFA_INFO_RDM)
efa_conn_rdm_deinit(av, conn);
goto err_release;
}
Expand Down Expand Up @@ -546,7 +546,7 @@ void efa_conn_release(struct efa_av *av, struct efa_conn *conn)
free(prv_reverse_av_entry);
}

if (av->ep_type == FI_EP_RDM)
if (av->domain->info_type == EFA_INFO_RDM)
efa_conn_rdm_deinit(av, conn);

efa_ah_release(av, conn->ah);
Expand Down Expand Up @@ -590,7 +590,7 @@ int efa_av_insert_one(struct efa_av *av, struct efa_ep_addr *addr,
fi_addr_t efa_fiaddr;
int ret = 0;

if (av->ep_type == FI_EP_DGRAM)
if (av->domain->info_type == EFA_INFO_DGRAM)
addr->qkey = EFA_DGRAM_CONNID;

ofi_genlock_lock(&av->util_av.lock);
Expand Down Expand Up @@ -811,7 +811,7 @@ static int efa_av_close(struct fid *fid)
fi_strerror(err));
}

if (av->ep_type == FI_EP_RDM) {
if (av->domain->info_type == EFA_INFO_RDM) {
if (av->shm_rdm_av) {
err = fi_close(&av->shm_rdm_av->fid);
if (OFI_UNLIKELY(err)) {
Expand Down Expand Up @@ -904,9 +904,7 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
if (ret)
goto err;

if (EFA_EP_TYPE_IS_RDM(efa_domain->info)) {
av->ep_type = FI_EP_RDM;

if (efa_domain->info_type == EFA_INFO_RDM) {
av_attr = *attr;
if (efa_domain->fabric && efa_domain->fabric->shm_fabric) {
/*
Expand All @@ -928,8 +926,6 @@ int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
if (ret)
goto err_close_util_av;
}
} else {
av->ep_type = FI_EP_DGRAM;
}

EFA_INFO(FI_LOG_AV, "fi_av_attr:%" PRId64 "\n",
Expand Down
1 change: 0 additions & 1 deletion prov/efa/src/efa_av.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ struct efa_av {
struct efa_prv_reverse_av *prv_reverse_av;
struct efa_ah *ah_map;
struct util_av util_av;
enum fi_ep_type ep_type;
};

int efa_av_open(struct fid_domain *domain_fid, struct fi_av_attr *attr,
Expand Down
8 changes: 4 additions & 4 deletions prov/efa/test/efa_unit_test_av.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,31 @@
*
* @param[in] state struct efa_resource that is managed by the framework
*/
void test_av_ep_type_efa_rdm(struct efa_resource **state)
void test_av_info_type_efa_rdm(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_av *efa_av;

efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_FABRIC_NAME);
g_efa_unit_test_mocks.ibv_create_ah = &efa_mock_ibv_create_ah_check_mock;
efa_av = container_of(resource->av, struct efa_av, util_av.av_fid);
assert(efa_av->ep_type == FI_EP_RDM);
assert(efa_av->domain->info_type == EFA_INFO_RDM);
}

/**
* @brief Verify the ep type in struct efa_av for efa direct path
*
* @param[in] state struct efa_resource that is managed by the framework
*/
void test_av_ep_type_efa_direct(struct efa_resource **state)
void test_av_info_type_efa_direct(struct efa_resource **state)
{
struct efa_resource *resource = *state;
struct efa_av *efa_av;

efa_unit_test_resource_construct(resource, FI_EP_RDM, EFA_DIRECT_FABRIC_NAME);
g_efa_unit_test_mocks.ibv_create_ah = &efa_mock_ibv_create_ah_check_mock;
efa_av = container_of(resource->av, struct efa_av, util_av.av_fid);
assert(efa_av->ep_type == FI_EP_RDM);
assert_false(efa_av->domain->info_type = EFA_INFO_DIRECT);
}

/**
Expand Down
4 changes: 2 additions & 2 deletions prov/efa/test/efa_unit_tests.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ int main(void)
/* Requires an EFA device to work */
const struct CMUnitTest efa_unit_tests[] = {
/* begin efa_unit_test_av.c */
cmocka_unit_test_setup_teardown(test_av_ep_type_efa_rdm, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_av_ep_type_efa_direct, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_av_info_type_efa_rdm, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_av_info_type_efa_direct, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_av_insert_duplicate_raw_addr, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_av_insert_duplicate_gid, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
/* end efa_unit_test_av.c */
Expand Down
4 changes: 2 additions & 2 deletions prov/efa/test/efa_unit_tests.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ struct efa_rdm_ope *efa_unit_test_alloc_rxe(struct efa_resource *resource, uint3
/* test cases */

/* begin efa_unit_test_av.c */
void test_av_ep_type_efa_rdm();
void test_av_ep_type_efa_direct();
void test_av_info_type_efa_rdm();
void test_av_info_type_efa_direct();
void test_av_insert_duplicate_raw_addr();
void test_av_insert_duplicate_gid();
/* end efa_unit_test_av.c */
Expand Down

0 comments on commit 10071c7

Please sign in to comment.