prov/efa: Use ibv_wr_send_imm to send CQ data

This eliminates the need of extra cq data hdr in the payload. Signed-off-by: Shi Jin <[email protected]>
ofiwg · Jun 26, 2024 · caafe7b · caafe7b
1 parent 7f33058
commit caafe7b
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c
@@ -27,7 +27,7 @@ void efa_rdm_ep_construct_ibv_qp_init_attr_ex(struct efa_rdm_ep *ep,
 	attr_ex->cap.max_inline_data = ep->base_ep.domain->device->efa_attr.inline_buf_size;
 	attr_ex->qp_type = IBV_QPT_DRIVER;
 	attr_ex->comp_mask = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
-	attr_ex->send_ops_flags = IBV_QP_EX_WITH_SEND;
+	attr_ex->send_ops_flags = IBV_QP_EX_WITH_SEND | IBV_QP_EX_WITH_SEND_WITH_IMM;
 	if (efa_device_support_rdma_read())
 		attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
 	if (efa_device_support_rdma_write()) {

diff --git a/prov/efa/src/rdm/efa_rdm_pke.c b/prov/efa/src/rdm/efa_rdm_pke.c
@@ -390,7 +390,14 @@ ssize_t efa_rdm_pke_sendv(struct efa_rdm_pke **pkt_entry_vec,
 		assert(efa_rdm_ep_get_peer(ep, pkt_entry->addr) == peer);
 
 		qp->ibv_qp_ex->wr_id = (uintptr_t)pkt_entry;
-		ibv_wr_send(qp->ibv_qp_ex);
+		if ((pkt_entry->ope->fi_flags & FI_REMOTE_CQ_DATA) &&
+		    (pkt_entry->flags & EFA_RDM_PKE_SEND_TO_USER_RECV_QP)) {
+			/* Currently this is only expected for eager pkts */
+			assert(pkt_entry_cnt == 1);
+			ibv_wr_send_imm(qp->ibv_qp_ex, pkt_entry->ope->cq_entry.data);
+		} else {
+			ibv_wr_send(qp->ibv_qp_ex);
+		}
 		if (pkt_entry->pkt_size <= efa_rdm_ep_domain(ep)->device->efa_attr.inline_buf_size &&
 	            !efa_mr_is_hmem((struct efa_mr *)pkt_entry->payload_mr)) {
 			iov_cnt = 1;