prov/efa: Implement the rma interface

Rename efa_dgram_rma.c to efa_rma.c and move it to prov/efa/src as
a common RMA interface for both rdm and dgram ep type.
Implement rdma write and inject.
Support inline rdma write.

Signed-off-by: Jessie Yang <jiaxiyan@amazon.com>

libfabric.vcxproj

@@ -885,9 +885,9 @@
     <ClCompile Include="prov\efa\src\efa_env.c" />
     <ClCompile Include="prov\efa\src\efa_cntr.c" />
     <ClCompile Include="prov\efa\src\efa_msg.c" />
+    <ClCompile Include="prov\efa\src\efa_rma.c" />
     <ClCompile Include="prov\efa\src\dgram\efa_dgram_ep.c" />
     <ClCompile Include="prov\efa\src\dgram\efa_dgram_cq.c" />
-    <ClCompile Include="prov\efa\src\dgram\efa_dgram_rma.c" />
     <ClCompile Include="prov\efa\src\rdm\efa_rdm_ope.c" />
     <ClCompile Include="prov\efa\src\rdm\efa_rdm_rxe_map.c" />
     <ClCompile Include="prov\efa\src\rdm\efa_rdm_atomic.c" />

prov/efa/Makefile.include

@@ -48,9 +48,9 @@ _efa_files = \
 	prov/efa/src/efa_env.c	\
 	prov/efa/src/efa_cntr.c	\
 	prov/efa/src/efa_msg.c \
+	prov/efa/src/efa_rma.c \
 	prov/efa/src/dgram/efa_dgram_ep.c \
 	prov/efa/src/dgram/efa_dgram_cq.c \
-	prov/efa/src/dgram/efa_dgram_rma.c \
 	prov/efa/src/rdm/efa_rdm_peer.c	\
 	prov/efa/src/rdm/efa_rdm_cq.c \
 	prov/efa/src/rdm/efa_rdm_ep_utils.c \

prov/efa/src/efa_base_ep.c

@@ -181,6 +181,12 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
 	init_attr_ex->comp_mask = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
 	init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_SEND | IBV_QP_EX_WITH_SEND_WITH_IMM;
 
+	if (efa_device_support_rdma_read())
+		init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
+	if (efa_device_support_rdma_write()) {
+		init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
+		init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
+	}
 	if (init_attr_ex->qp_type == IBV_QPT_UD) {
 		(*qp)->ibv_qp = ibv_create_qp_ex(init_attr_ex->pd->context,
 					      init_attr_ex);