prov/efa: Introduce efa specific domain operations

Make efa onboard the fi_open_ops API, which allows
user to access efa specific domain ops. The usage of
this API is documented in fi_efa.7.md.

Also added a unit test.

Signed-off-by: Shi Jin <[email protected]>

man/fi_efa.7.md

@@ -134,6 +134,43 @@ provider for AWS Neuron or Habana SynapseAI.
   delivered to the target buffer only once. If endpoint is not able to support
   this feature, it will return -FI_EOPNOTSUPP for the call to fi_setopt().
 
+# PROVIDER SPECIFIC DOMAIN OPS
+The efa provider exports extensions for additional operations
+that are not provided by the standard libfabric interface. These extensions
+are available via the "`fi_ext_efa.h`" header file.
+
+## Domain Operation Extension
+
+Domain operation extension is obtained by calling `fi_open_ops`
+(see [`fi_domain`(3)](fi_domain.3.html))
+```c
+int fi_open_ops(struct fid *domain, const char *name, uint64_t flags,
+    void **ops, void *context);
+```
+and requesting `FI_EFA_DOMAIN_OPS_1` in `name`. `fi_open_ops` returns `ops` as
+the pointer to the function table `fi_efa_ops_domain` defined as follows:
+
+```c
+struct fi_efa_ops_domain {
+	int (*query_mr)(struct fid_mr *mr, struct fi_efa_mr_attr *mr_attr);
+};
+```
+
+It contains the following operations
+
+### query_mr
+This op query an existing memory registration (mr), and returns the efa
+specific mr attribute which is defined as follows
+
+```c
+struct fi_efa_mr_attr {
+    uint32_t pci_bus_id;
+};
+```
+. The `pci_bus_id` represents the pci bus id of the rx (receive, remote-write etc.)
+performed into the memory registration.
+
+
 # RUNTIME PARAMETERS
 
 *FI_EFA_TX_SIZE*

prov/efa/Makefile.include

@@ -94,6 +94,7 @@ _efa_headers = \
 	prov/efa/src/efa_tp.h \
 	prov/efa/src/efa_prov.h \
 	prov/efa/src/efa_env.h \
+	prov/efa/src/fi_ext_efa.h \
 	prov/efa/src/dgram/efa_dgram_ep.h \
 	prov/efa/src/dgram/efa_dgram_cq.h \
 	prov/efa/src/rdm/efa_rdm_peer.h	\
@@ -132,6 +133,7 @@ nodist_prov_efa_test_efa_unit_test_SOURCES = \
 	prov/efa/test/efa_unit_tests.c \
 	prov/efa/test/efa_unit_test_mocks.c \
 	prov/efa/test/efa_unit_test_common.c \
+	prov/efa/test/efa_unit_test_domain.c \
 	prov/efa/test/efa_unit_test_ep.c \
 	prov/efa/test/efa_unit_test_av.c \
 	prov/efa/test/efa_unit_test_cq.c \
@@ -171,6 +173,9 @@ efa_CPPFLAGS += \
 	-I$(top_srcdir)/prov/efa/src/dgram/ \
 	-I$(top_srcdir)/prov/efa/src/rdm/
 
+rdmainclude_HEADERS += \
+	prov/efa/src/fi_ext_efa.h
+
 if HAVE_EFA_DL
 pkglib_LTLIBRARIES += libefa-fi.la
 libefa_fi_la_SOURCES = $(_efa_files) $(_efa_headers) $(common_srcs)

prov/efa/src/efa.h

@@ -81,6 +81,7 @@
 #include "rdm/efa_rdm_pke.h"
 #include "rdm/efa_rdm_peer.h"
 #include "rdm/efa_rdm_util.h"
+#include "fi_ext_efa.h"
 
 #define EFA_ABI_VER_MAX_LEN 8
 

prov/efa/src/efa_domain.c

@@ -47,12 +47,15 @@ struct dlist_entry g_efa_domain_list;
 
 static int efa_domain_close(fid_t fid);
 
+static int efa_domain_ops_open(struct fid *fid, const char *ops_name,
+				uint64_t flags, void **ops, void *context);
+
 static struct fi_ops efa_ops_domain_fid = {
 	.size = sizeof(struct fi_ops),
 	.close = efa_domain_close,
 	.bind = fi_no_bind,
 	.control = fi_no_control,
-	.ops_open = fi_no_ops_open,
+	.ops_open = efa_domain_ops_open,
 };
 
 static struct fi_ops_domain efa_ops_domain_dgram = {
@@ -380,3 +383,36 @@ static int efa_domain_close(fid_t fid)
 	return 0;
 }
 
+/**
+ * @brief Get efa specific mr attr
+ *
+ * @param mr ptr to fid_mr
+ * @param mr_attr  ptr to fi_efa_mr_attr
+ * @return int 0 on success, negative integer on failure
+ */
+static int
+efa_domain_query_mr(struct fid_mr *mr, struct fi_efa_mr_attr *mr_attr)
+{
+	return -FI_ENOSYS;
+}
+
+static struct fi_efa_ops_domain efa_ops_domain = {
+	.query_mr = efa_domain_query_mr,
+};
+
+static int
+efa_domain_ops_open(struct fid *fid, const char *ops_name, uint64_t flags,
+		     void **ops, void *context)
+{
+	int ret = FI_SUCCESS;
+
+	if (strcmp(ops_name, FI_EFA_DOMAIN_OPS_1) == 0) {
+		*ops = &efa_ops_domain;
+	} else {
+		EFA_WARN(FI_LOG_DOMAIN,
+			"Unknown ops name: %s\n", ops_name);
+		ret = -FI_EINVAL;
+	}
+
+	return ret;
+}

prov/efa/src/fi_ext_efa.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) Amazon.com, Inc. or its affiliates. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _FI_EXT_EFA_H_
+#define _FI_EXT_EFA_H_
+
+#define FI_EFA_DOMAIN_OPS_1 "efa domain ops 1"
+
+struct fi_efa_mr_attr {
+    /* The pci bus id of the rx performed into the mr */
+    uint32_t pci_bus_id;
+};
+
+struct fi_efa_ops_domain {
+	int (*query_mr)(struct fid_mr *mr, struct fi_efa_mr_attr *mr_attr);
+};
+
+#endif /* _FI_EXT_EFA_H_ */

prov/efa/test/efa_unit_test_domain.c

@@ -0,0 +1,15 @@
+#include "efa_unit_tests.h"
+
+void test_efa_domain_open_ops(struct efa_resource **state)
+{
+    struct efa_resource *resource = *state;
+    int ret;
+    struct fi_efa_ops_domain *efa_domain_ops;
+    struct fi_efa_mr_attr efa_mr_attr = {0};
+
+    ret = fi_open_ops(resource->domain->fid, FI_EFA_DOMAIN_OPS_1, &efa_domain_ops);
+    assert_int_equal(ret, 0);
+
+    ret = efa_domain_ops->query_mr(NULL, &efa_mr_attr);
+    assert_int_equal(ret, -FI_ENOSYS);
+}

prov/efa/test/efa_unit_tests.c

@@ -133,6 +133,7 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_rdm_msg_send_to_local_peer_with_null_desc, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_fork_support_request_initialize_when_ibv_fork_support_is_needed, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_fork_support_request_initialize_when_ibv_fork_support_is_unneeded, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_domain_open_ops, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 	};
 
 	cmocka_set_message_output(CM_OUTPUT_XML);

prov/efa/test/efa_unit_tests.h

@@ -141,5 +141,6 @@ void test_efa_rdm_ope_post_write_0_byte();
 void test_efa_rdm_msg_send_to_local_peer_with_null_desc();
 void test_efa_fork_support_request_initialize_when_ibv_fork_support_is_needed();
 void test_efa_fork_support_request_initialize_when_ibv_fork_support_is_unneeded();
+void test_efa_domain_open_ops();
 
 #endif