prov/efa: Move efa_rdm_cq_wc_is_unsolicited to efa_cq

Move efa_use_unsolicited_write_recv to efa.h so they can be used outside rdm. Signed-off-by: Jessie Yang <[email protected]>
ofiwg · Dec 20, 2024 · a26ade3 · a26ade3
1 parent 9b7f27c
commit a26ade3
Show file tree

Hide file tree

Showing 6 changed files with 35 additions and 33 deletions.
diff --git a/prov/efa/src/efa.h b/prov/efa/src/efa.h
@@ -221,4 +221,10 @@ static inline void efa_perfset_end(struct efa_rdm_ep *ep, size_t index)
 #define efa_perfset_end(ep, index) do {} while (0)
 #endif
 
+static inline
+bool efa_use_unsolicited_write_recv()
+{
+	return efa_env.use_unsolicited_write_recv && efa_device_support_unsolicited_write_recv();
+}
+
 #endif /* EFA_H */
diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -193,7 +193,7 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
 			init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
 		}
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-		if (efa_rdm_use_unsolicited_write_recv())
+		if (efa_use_unsolicited_write_recv())
 			efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;
 #endif
 		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;

diff --git a/prov/efa/src/efa_cq.h b/prov/efa/src/efa_cq.h
@@ -136,7 +136,7 @@ static inline int efa_cq_ibv_cq_ex_open(struct fi_cq_attr *attr,
 	};
 
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-	if (efa_rdm_use_unsolicited_write_recv())
+	if (efa_use_unsolicited_write_recv())
 		efadv_cq_init_attr.wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED;
 #endif
 
@@ -176,3 +176,27 @@ static inline int efa_cq_ibv_cq_ex_open(struct fi_cq_attr *attr,
 		&init_attr_ex, ibv_ctx, ibv_cq_ex, ibv_cq_ex_type);
 }
 #endif
+
+#if HAVE_CAPS_UNSOLICITED_WRITE_RECV
+/**
+ * @brief Check whether a completion consumes recv buffer
+ *
+ * @param ibv_cq_ex extended ibv cq
+ * @return true the wc consumes a recv buffer
+ * @return false the wc doesn't consume a recv buffer
+ */
+static inline
+bool efa_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
+{
+	return efa_use_unsolicited_write_recv() && efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex));
+}
+
+#else
+
+static inline
+bool efa_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
+{
+	return false;
+}
+
+#endif
diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c
@@ -72,29 +72,6 @@ static struct fi_ops efa_rdm_cq_fi_ops = {
 };
 
 
-#if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-/**
- * @brief Check whether a completion consumes recv buffer
- *
- * @param ibv_cq_ex extended ibv cq
- * @return true the wc consumes a recv buffer
- * @return false the wc doesn't consume a recv buffer
- */
-static inline
-bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
-{
-	return efa_rdm_use_unsolicited_write_recv() && efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex));
-}
-
-#else
-
-static inline
-bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex)
-{
-	return false;
-}
-
-#endif
 /**
  * @brief handle rdma-core CQ completion resulted from IBV_WRITE_WITH_IMM
  *
@@ -148,7 +125,7 @@ void efa_rdm_cq_proc_ibv_recv_rdma_with_imm_completion(
 	 * For unsolicited wc, pkt_entry can be NULL, so we can only
 	 * access it for solicited wc.
 	 */
-	if (!efa_rdm_cq_wc_is_unsolicited(ibv_cq_ex)) {
+	if (!efa_cq_wc_is_unsolicited(ibv_cq_ex)) {
 		/**
 		 * Recv with immediate will consume a pkt_entry, but the pkt is not
 		 * filled, so free the pkt_entry and record we have one less posted
@@ -494,7 +471,7 @@ void efa_rdm_cq_poll_ibv_cq(ssize_t cqe_to_process, struct efa_ibv_cq *ibv_cq)
 				break;
 			case IBV_WC_RECV: /* fall through */
 			case IBV_WC_RECV_RDMA_WITH_IMM:
-				if (efa_rdm_cq_wc_is_unsolicited(ibv_cq->ibv_cq_ex)) {
+				if (efa_cq_wc_is_unsolicited(ibv_cq->ibv_cq_ex)) {
 					EFA_WARN(FI_LOG_CQ, "Receive error %s (%d) for unsolicited write recv",
 						efa_strerror(prov_errno), prov_errno);
 					efa_base_ep_write_eq_error(&ep->base_ep, to_fi_errno(prov_errno), prov_errno);

diff --git a/prov/efa/src/rdm/efa_rdm_util.h b/prov/efa/src/rdm/efa_rdm_util.h
@@ -30,10 +30,5 @@ static inline void efa_rdm_poison_mem_region(void *ptr, size_t size)
 }
 #endif
 
-static inline
-bool efa_rdm_use_unsolicited_write_recv()
-{
-	return efa_env.use_unsolicited_write_recv && efa_device_support_unsolicited_write_recv();
-}
 
 #endif /* _EFA_RDM_UTIL_H */
diff --git a/prov/efa/test/efa_unit_test_cq.c b/prov/efa/test/efa_unit_test_cq.c
@@ -338,7 +338,7 @@ void test_ibv_cq_ex_read_bad_recv_status(struct efa_resource **state)
 	efa_rdm_cq->ibv_cq.ibv_cq_ex->status = IBV_WC_GENERAL_ERR;
 
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
-	if (efa_rdm_use_unsolicited_write_recv()) {
+	if (efa_use_unsolicited_write_recv()) {
 		efadv_cq_from_ibv_cq_ex(efa_rdm_cq->ibv_cq.ibv_cq_ex)->wc_is_unsolicited = &efa_mock_efadv_wc_is_unsolicited;
 		will_return(efa_mock_efadv_wc_is_unsolicited, false);
 	}