From d85306494497fde3a8c33711a22a0f6e689c8494 Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Fri, 21 Feb 2025 04:25:56 +0000 Subject: [PATCH] prov/efa: Fix the cntr interface for efa-direct 1. efa_ep should support selective_completion after the FI_CONTEXT2 implementation 2. split efa_cntr_open as efa_cntr_open and efa_rdm_cntr_open. efa_rdm_cntr_open is only used for rdm info type. Both of them setup the correct progress function accordingly. We need separate open functions because there is no way to distinguish efa-direct and efa-rdm during the cntr_open call as there is no user_info object Signed-off-by: Shi Jin --- prov/efa/src/efa_cntr.c | 37 ++++++++++++++++++++++++++++++++----- prov/efa/src/efa_cntr.h | 3 +++ prov/efa/src/efa_domain.c | 2 +- prov/efa/src/efa_ep.c | 6 ------ 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/prov/efa/src/efa_cntr.c b/prov/efa/src/efa_cntr.c index c30a3d862d4..7f1c8a56ce7 100644 --- a/prov/efa/src/efa_cntr.c +++ b/prov/efa/src/efa_cntr.c @@ -199,10 +199,40 @@ int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, { int ret; struct efa_cntr *cntr; + + cntr = calloc(1, sizeof(*cntr)); + if (!cntr) + return -FI_ENOMEM; + + dlist_init(&cntr->ibv_cq_poll_list); + cntr->need_to_scan_ep_list = false; + + ret = ofi_cntr_init(&efa_prov, domain, attr, &cntr->util_cntr, + efa_cntr_progress, context); + + if (ret) + goto free; + + *cntr_fid = &cntr->util_cntr.cntr_fid; + cntr->util_cntr.cntr_fid.ops = &efa_cntr_ops; + cntr->util_cntr.cntr_fid.fid.ops = &efa_cntr_fi_ops; + + return FI_SUCCESS; + +free: + free(cntr); + return ret; +} + + +int efa_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, + struct fid_cntr **cntr_fid, void *context) +{ + int ret; + struct efa_cntr *cntr; struct efa_domain *efa_domain; struct fi_cntr_attr shm_cntr_attr = {0}; struct fi_peer_cntr_context peer_cntr_context = {0}; - ofi_cntr_progress_func cntr_progress_func; cntr = calloc(1, sizeof(*cntr)); if (!cntr) @@ -213,11 +243,8 @@ int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, efa_domain = container_of(domain, struct efa_domain, util_domain.domain_fid); - cntr_progress_func = efa_domain->info->ep_attr->type == FI_EP_RDM - ? efa_rdm_cntr_progress - : efa_cntr_progress; ret = ofi_cntr_init(&efa_prov, domain, attr, &cntr->util_cntr, - cntr_progress_func, context); + efa_rdm_cntr_progress, context); if (ret) goto free; diff --git a/prov/efa/src/efa_cntr.h b/prov/efa/src/efa_cntr.h index bcfde8784a2..13d48517536 100644 --- a/prov/efa/src/efa_cntr.h +++ b/prov/efa/src/efa_cntr.h @@ -19,6 +19,9 @@ struct efa_cntr { int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, struct fid_cntr **cntr_fid, void *context); +int efa_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, + struct fid_cntr **cntr_fid, void *context); + void efa_cntr_report_tx_completion(struct util_ep *ep, uint64_t flags); void efa_cntr_report_rx_completion(struct util_ep *ep, uint64_t flags); diff --git a/prov/efa/src/efa_domain.c b/prov/efa/src/efa_domain.c index 1535af56664..441be368b35 100644 --- a/prov/efa/src/efa_domain.c +++ b/prov/efa/src/efa_domain.c @@ -48,7 +48,7 @@ static struct fi_ops_domain efa_domain_ops_rdm = { .cq_open = efa_rdm_cq_open, .endpoint = efa_rdm_ep_open, .scalable_ep = fi_no_scalable_ep, - .cntr_open = efa_cntr_open, + .cntr_open = efa_rdm_cntr_open, .poll_open = fi_poll_create, .stx_ctx = fi_no_stx_context, .srx_ctx = fi_no_srx_context, diff --git a/prov/efa/src/efa_ep.c b/prov/efa/src/efa_ep.c index 8aa3268adf2..91ac038b4b0 100644 --- a/prov/efa/src/efa_ep.c +++ b/prov/efa/src/efa_ep.c @@ -227,12 +227,6 @@ static int efa_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) switch (bfid->fclass) { case FI_CLASS_CQ: - if (flags & FI_SELECTIVE_COMPLETION) { - EFA_WARN(FI_LOG_EP_CTRL, - "Endpoint cannot be bound with selective completion.\n"); - return -FI_EBADFLAGS; - } - /* Must bind a CQ to either RECV or SEND completions */ if (!(flags & (FI_RECV | FI_TRANSMIT))) return -FI_EBADFLAGS;