fabtests/common: Add dmabuf support to ft_reg_mr

Signed-off-by: Shi Jin <[email protected]>
shijin-aws · Oct 31, 2023 · f53453b · f53453b
1 parent b548cad
commit f53453b
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 13 deletions.
diff --git a/fabtests/common/shared.c b/fabtests/common/shared.c
@@ -375,11 +375,15 @@ void ft_free_bit_combo(uint64_t *combo)
 	free(combo);
 }
 
-void ft_fill_mr_attr(struct iovec *iov, int iov_count, uint64_t access,
+void ft_fill_mr_attr(struct iovec *iov, struct fi_mr_dmabuf *dmabuf,
+		     int iov_count, uint64_t access,
 		     uint64_t key, enum fi_hmem_iface iface, uint64_t device,
-		     struct fi_mr_attr *attr)
+		     struct fi_mr_attr *attr, uint64_t flags)
 {
-	attr->mr_iov = iov;
+	if (flags & FI_MR_DMABUF)
+		attr->dmabuf = dmabuf;
+	else
+		attr->mr_iov = iov;
 	attr->iov_count = iov_count;
 	attr->access = access;
 	attr->offset = 0;
@@ -418,15 +422,32 @@ int ft_reg_mr(struct fi_info *fi, void *buf, size_t size, uint64_t access,
 	struct iovec iov = {0};
 	int ret;
 	uint64_t flags;
+	int dmabuf_fd;
+	uint64_t dmabuf_offset;
+	struct fi_mr_dmabuf dmabuf = {0};
 
 	if (!ft_need_mr_reg(fi))
 		return 0;
 
 	iov.iov_base = buf;
 	iov.iov_len = size;
-	ft_fill_mr_attr(&iov, 1, access, key, iface, device, &attr);
 
 	flags = (iface) ? FI_HMEM_DEVICE_ONLY : 0;
+
+	if (opts.options & FT_OPT_REG_DMABUF_MR) {
+		ret = ft_hmem_get_dmabuf_fd(iface, buf, size,
+					    &dmabuf_fd, &dmabuf_offset);
+		if (ret)
+			return ret;
+
+		dmabuf.fd = dmabuf_fd;
+		dmabuf.offset = dmabuf_offset;
+		dmabuf.len = size;
+		dmabuf.base_addr = (void *)((uintptr_t) buf - dmabuf_offset);
+		flags |= FI_MR_DMABUF;
+	}
+
+	ft_fill_mr_attr(&iov, &dmabuf, 1, access, key, iface, device, &attr, flags);
 	ret = fi_mr_regattr(domain, &attr, flags, mr);
 	if (ret)
 		return ret;
@@ -3115,6 +3136,7 @@ void ft_hmem_usage()
 			    "Automatically enables FI_HMEM (-H)");
 	FT_PRINT_OPTS_USAGE("-i <device_id>", "Specify which device to use (default: 0)");
 	FT_PRINT_OPTS_USAGE("-H", "Enable provider FI_HMEM support");
+	FT_PRINT_OPTS_USAGE("-R", "Register HMEM memory with fi_mr_dmabuf API");
 }
 
 void ft_mcusage(char *name, char *desc)
@@ -3282,6 +3304,9 @@ void ft_parse_hmem_opts(int op, char *optarg, struct ft_opts *opts)
 	case 'H':
 		opts->options |= FT_OPT_ENABLE_HMEM;
 		break;
+	case 'R':
+		opts->options |= FT_OPT_REG_DMABUF_MR;
+		break;
 	default:
 		/* Let getopt handle unknown opts*/
 		break;

diff --git a/fabtests/functional/unexpected_msg.c b/fabtests/functional/unexpected_msg.c
@@ -89,8 +89,8 @@ static int alloc_bufs(void)
 		iov.iov_base = buf;
 		iov.iov_len = buf_size;
 
-		ft_fill_mr_attr(&iov, 1, ft_info_to_mr_access(fi), FT_MR_KEY,
-				opts.iface, opts.device, &mr_attr);
+		ft_fill_mr_attr(&iov, NULL, 1, ft_info_to_mr_access(fi), FT_MR_KEY,
+				opts.iface, opts.device, &mr_attr, 0);
 
 		flags = (opts.iface) ? FI_HMEM_DEVICE_ONLY : 0;
 		ret = fi_mr_regattr(domain, &mr_attr, flags, &mr);

diff --git a/fabtests/include/shared.h b/fabtests/include/shared.h
@@ -134,6 +134,7 @@ enum {
 	FT_OPT_PERF			= 1 << 24,
 	FT_OPT_DISABLE_TAG_VALIDATION	= 1 << 25,
 	FT_OPT_ADDR_IS_OOB		= 1 << 26,
+	FT_OPT_REG_DMABUF_MR		= 1 << 27,
 	FT_OPT_OOB_CTRL			= FT_OPT_OOB_SYNC | FT_OPT_OOB_ADDR_EXCH,
 };
 
@@ -278,7 +279,7 @@ extern int sock, oob_sock;
 extern int listen_sock;
 #define ADDR_OPTS "B:P:s:a:b::E::C:F:O:"
 #define FAB_OPTS "f:d:p:K"
-#define HMEM_OPTS "D:i:H"
+#define HMEM_OPTS "D:i:HR"
 #define INFO_OPTS FAB_OPTS HMEM_OPTS "e:M:"
 #define CS_OPTS ADDR_OPTS "I:QS:mc:t:w:l"
 #define API_OPTS "o:"
@@ -452,9 +453,10 @@ int ft_init_av_dst_addr(struct fid_av *av_ptr, struct fid_ep *ep_ptr,
 int ft_init_av_addr(struct fid_av *av, struct fid_ep *ep,
 		fi_addr_t *addr);
 int ft_exchange_keys(struct fi_rma_iov *peer_iov);
-void ft_fill_mr_attr(struct iovec *iov, int iov_count, uint64_t access,
+void ft_fill_mr_attr(struct iovec *iov, struct fi_mr_dmabuf *dmabuf,
+		     int iov_count, uint64_t access,
 		     uint64_t key, enum fi_hmem_iface iface, uint64_t device,
-		     struct fi_mr_attr *attr);
+		     struct fi_mr_attr *attr, uint64_t flags);
 bool ft_need_mr_reg(struct fi_info *fi);
 int ft_reg_mr(struct fi_info *info, void *buf, size_t size, uint64_t access,
 	      uint64_t key, enum fi_hmem_iface iface, uint64_t device,

diff --git a/fabtests/unit/mr_test.c b/fabtests/unit/mr_test.c
@@ -178,8 +178,8 @@ static int mr_regattr()
 			base += iov[j].iov_len;
 		}
 
-		ft_fill_mr_attr(&iov[0], n, ft_info_to_mr_access(fi),
-				FT_MR_KEY, opts.iface, opts.device, &attr);
+		ft_fill_mr_attr(&iov[0], NULL, n, ft_info_to_mr_access(fi),
+				FT_MR_KEY, opts.iface, opts.device, &attr, 0);
 		ret = fi_mr_regattr(domain, &attr, 0, &mr);
 		if (ret) {
 			FT_UNIT_STRERR(err_buf, "fi_mr_regattr failed", ret);
@@ -213,8 +213,8 @@ static int mr_reg_free_then_alloc()
 	for (i = 0; i < numtry; ++i) {
 		iov.iov_base = buf;
 		iov.iov_len = buf_size;
-		ft_fill_mr_attr(&iov, 1, ft_info_to_mr_access(fi),
-				FT_MR_KEY, opts.iface, opts.device, &attr);
+		ft_fill_mr_attr(&iov, NULL, 1, ft_info_to_mr_access(fi),
+				FT_MR_KEY, opts.iface, opts.device, &attr, 0);
 
 		ret = fi_mr_regattr(domain, &attr, 0, &mr);
 		if (ret) {