prov/efa: Do infinite rnr retry for base ep by default

Currently, efa_base_ep's default rnr_retry is 3 which only does a few retry in the firmware level for RNR. This is due to the efa_rdm_ep supports libfabric level RNR retry. However, the efa-direct ep doesn't support libfabric level RNR retry. Then we should make it do infinite RNR retry (7), which is also the default behavior of SRD QP. Signed-off-by: Shi Jin <[email protected]>
ofiwg · Jan 17, 2025 · 22e941b · 22e941b
1 parent ea082da
commit 22e941b
Show file tree

Hide file tree

Showing 7 changed files with 30 additions and 20 deletions.
diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -350,7 +350,8 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
 		return -FI_ENOMEM;
 	}
 
-	base_ep->rnr_retry = efa_env.rnr_retry;
+	/* This is SRD qp's default behavior */
+	base_ep->rnr_retry = EFA_RNR_INFINITE_RETRY;
 
 	base_ep->efa_recv_wr_vec = calloc(sizeof(struct efa_recv_wr), EFA_RDM_EP_MAX_WR_PER_IBV_POST_RECV);
 	if (!base_ep->efa_recv_wr_vec) {

diff --git a/prov/efa/src/efa_base_ep.h b/prov/efa/src/efa_base_ep.h
@@ -16,6 +16,26 @@
 #define EFA_QP_LOW_LATENCY_SERVICE_LEVEL 8
 #define EFA_ERROR_MSG_BUFFER_LENGTH 1024
 
+/* Default rnr_retry for efa-rdm ep.
+ * If first attempt to send a packet failed,
+ * this value controls how many times firmware
+ * retries the send before it report an RNR error
+ * (via rdma-core error cq entry).
+ * The valid number is from
+ *      0 (no retry)
+ * to
+ *      EFA_RNR_INFINITY_RETRY (retry infinitely)
+ */
+#define EFA_RDM_DEFAULT_RNR_RETRY	(3)
+/**
+ * Infinite retry.
+ * NOTICE: this is the default rnr_retry
+ * mode for SRD qp. So modifying qp_attr.rnr_retry
+ * to this value has the same behavior as
+ * not modifying qp's rnr_retry attribute
+ */
+#define EFA_RNR_INFINITE_RETRY		(7)
+
 #define efa_rx_flags(efa_base_ep) ((efa_base_ep)->util_ep.rx_op_flags)
 #define efa_tx_flags(efa_base_ep) ((efa_base_ep)->util_ep.tx_op_flags)
 

diff --git a/prov/efa/src/efa_env.c b/prov/efa/src/efa_env.c
@@ -34,7 +34,6 @@ struct efa_env efa_env = {
 	.efa_max_gdrcopy_msg_size = 32768,
 	.efa_read_segment_size = 1073741824,
 	.efa_write_segment_size = 1073741824, /* need to confirm this constant. */
-	.rnr_retry = 3, /* Setting this value to EFA_RNR_INFINITE_RETRY makes the firmware retry indefinitey */
 	.host_id_file = "/sys/devices/virtual/dmi/id/board_asset_tag", /* Available on EC2 instances and containers */
 	.use_sm2 = false,
 	.huge_page_setting = EFA_ENV_HUGE_PAGE_UNSPEC,

diff --git a/prov/efa/src/efa_env.h b/prov/efa/src/efa_env.h
@@ -6,12 +6,6 @@
 
 #include "efa_prov.h"
 
-/**
- * Setting ibv_qp_attr.rnr_retry to this number when modifying qp
- * to cause firmware to retry indefinitely.
- */
-#define EFA_RNR_INFINITE_RETRY 7
-
 enum efa_env_huge_page_setting
 {
 	EFA_ENV_HUGE_PAGE_UNSPEC, /**< user did not set FI_EFA_USE_HUGE_PAGE, provider will decide whether to use huge page*/
@@ -48,17 +42,6 @@ struct efa_env {
 	size_t efa_max_gdrcopy_msg_size;
 	size_t efa_read_segment_size;
 	size_t efa_write_segment_size;
-	/* If first attempt to send a packet failed,
-	 * this value controls how many times firmware
-	 * retries the send before it report an RNR error
-	 * (via rdma-core error cq entry).
-	 *
-	 * The valid number is from
-	 *      0 (no retry)
-	 * to
-	 *      EFA_RNR_INFINITY_RETRY (retry infinitely)
-	 */
-	int rnr_retry;
 	/**
 	 * The absolute path to a file that contains an EC2 instance id-like string.
 	 * If host_id_file is provided, the program will attempt to read the

diff --git a/prov/efa/src/rdm/efa_rdm_ep.h b/prov/efa/src/rdm/efa_rdm_ep.h
@@ -274,7 +274,7 @@ int efa_rdm_ep_bulk_post_internal_rx_pkts(struct efa_rdm_ep *ep);
 static inline
 bool efa_rdm_ep_should_write_rnr_completion(struct efa_rdm_ep *ep)
 {
-	return (efa_env.rnr_retry < EFA_RNR_INFINITE_RETRY) &&
+	return (ep->base_ep.rnr_retry < EFA_RNR_INFINITE_RETRY) &&
 		(ep->handle_resource_management == FI_RM_DISABLED);
 }
 

diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c
@@ -487,6 +487,11 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info,
 	efa_rdm_ep->base_ep.max_rma_size = info->ep_attr->max_msg_size;
 	efa_rdm_ep->base_ep.inject_msg_size = info->tx_attr->inject_size;
 	efa_rdm_ep->base_ep.inject_rma_size = info->tx_attr->inject_size;
+	/*
+	 * base ep is configured as infinite retry, use a different default
+	 * for efa_rdm_ep to allow libfabric level retry.
+	 */
+	efa_rdm_ep->base_ep.rnr_retry = EFA_RDM_DEFAULT_RNR_RETRY;
 
 	/* efa_rdm_ep's own fields */
 	efa_rdm_ep->max_tagged_size = info->ep_attr->max_msg_size;

diff --git a/prov/efa/test/efa_unit_test_ep.c b/prov/efa/test/efa_unit_test_ep.c
@@ -1415,6 +1415,7 @@ void test_efa_rdm_ep_default_sizes(struct efa_resource **state)
 	assert_int_equal(efa_rdm_ep->base_ep.max_rma_size, resource->info->ep_attr->max_msg_size);
 	assert_int_equal(efa_rdm_ep->base_ep.inject_msg_size, resource->info->tx_attr->inject_size);
 	assert_int_equal(efa_rdm_ep->base_ep.inject_rma_size, resource->info->tx_attr->inject_size);
+	assert_int_equal(efa_rdm_ep->base_ep.rnr_retry, EFA_RDM_DEFAULT_RNR_RETRY);
 
 	/* efa_rdm_ep's own fields */
 	assert_int_equal(efa_rdm_ep->max_tagged_size, resource->info->ep_attr->max_msg_size);
@@ -1448,6 +1449,7 @@ void test_efa_ep_open(struct efa_resource **state)
 	/* TODO: update inject_rma_size to inline size after firmware
 	 * supports inline rdma write */
 	assert_true(efa_ep->inject_rma_size == 0);
+	assert_int_equal(efa_ep->rnr_retry, EFA_RNR_INFINITE_RETRY);
 }
 
 /**