prov/efa: Fix the cntr interface for efa-direct

1. efa_ep should support selective_completion after
the FI_CONTEXT2 implementation
2. split efa_cntr_open as efa_cntr_open and
efa_rdm_cntr_open. efa_rdm_cntr_open is only
used for rdm info type. Both of them setup
the correct progress function accordingly.
We need separate open functions because
there is no way to distinguish efa-direct
and efa-rdm during the cntr_open call
as there is no user_info object

Signed-off-by: Shi Jin <sjina@amazon.com>

prov/efa/src/efa_cntr.c

@@ -199,10 +199,40 @@ int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
 {
 	int ret;
 	struct efa_cntr *cntr;
+
+	cntr = calloc(1, sizeof(*cntr));
+	if (!cntr)
+		return -FI_ENOMEM;
+
+	dlist_init(&cntr->ibv_cq_poll_list);
+	cntr->need_to_scan_ep_list = false;
+
+	ret = ofi_cntr_init(&efa_prov, domain, attr, &cntr->util_cntr,
+			    efa_cntr_progress, context);
+
+	if (ret)
+		goto free;
+
+	*cntr_fid = &cntr->util_cntr.cntr_fid;
+	cntr->util_cntr.cntr_fid.ops = &efa_cntr_ops;
+	cntr->util_cntr.cntr_fid.fid.ops = &efa_cntr_fi_ops;
+
+	return FI_SUCCESS;
+
+free:
+	free(cntr);
+	return ret;
+}
+
+
+int efa_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
+		       struct fid_cntr **cntr_fid, void *context)
+{
+	int ret;
+	struct efa_cntr *cntr;
 	struct efa_domain *efa_domain;
 	struct fi_cntr_attr shm_cntr_attr = {0};
 	struct fi_peer_cntr_context peer_cntr_context = {0};
-	ofi_cntr_progress_func cntr_progress_func;
 
 	cntr = calloc(1, sizeof(*cntr));
 	if (!cntr)
@@ -213,11 +243,8 @@ int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
 	efa_domain = container_of(domain, struct efa_domain,
 				  util_domain.domain_fid);
 
-	cntr_progress_func = efa_domain->info->ep_attr->type == FI_EP_RDM
-		? efa_rdm_cntr_progress
-		: efa_cntr_progress;
 	ret = ofi_cntr_init(&efa_prov, domain, attr, &cntr->util_cntr,
-			    cntr_progress_func, context);
+			    efa_rdm_cntr_progress, context);
 
 	if (ret)
 		goto free;

prov/efa/src/efa_cntr.h

@@ -19,6 +19,9 @@ struct efa_cntr {
 int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
 		  struct fid_cntr **cntr_fid, void *context);
 
+int efa_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
+		       struct fid_cntr **cntr_fid, void *context);
+
 void efa_cntr_report_tx_completion(struct util_ep *ep, uint64_t flags);
 
 void efa_cntr_report_rx_completion(struct util_ep *ep, uint64_t flags);

prov/efa/src/efa_domain.c

@@ -48,7 +48,7 @@ static struct fi_ops_domain efa_domain_ops_rdm = {
 	.cq_open = efa_rdm_cq_open,
 	.endpoint = efa_rdm_ep_open,
 	.scalable_ep = fi_no_scalable_ep,
-	.cntr_open = efa_cntr_open,
+	.cntr_open = efa_rdm_cntr_open,
 	.poll_open = fi_poll_create,
 	.stx_ctx = fi_no_stx_context,
 	.srx_ctx = fi_no_srx_context,

prov/efa/src/efa_ep.c

@@ -227,12 +227,6 @@ static int efa_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 
 	switch (bfid->fclass) {
 	case FI_CLASS_CQ:
-		if (flags & FI_SELECTIVE_COMPLETION) {
-			EFA_WARN(FI_LOG_EP_CTRL,
-				 "Endpoint cannot be bound with selective completion.\n");
-			return -FI_EBADFLAGS;
-		}
-
 		/* Must bind a CQ to either RECV or SEND completions */
 		if (!(flags & (FI_RECV | FI_TRANSMIT)))
 			return -FI_EBADFLAGS;