From b33531039511bb4db4d602136760a4e7af79d15c Mon Sep 17 00:00:00 2001 From: Steve Welch Date: Thu, 8 Aug 2024 11:11:52 -0500 Subject: [PATCH] prov/util: Allow providers to update cache MR IOV A provider may update the memory region that is added to accommodate for instance alignment of the region to a larger page boundary. In such cases, the MR cache info used to search the cache should use the updated region. This allows the provider to avoid walking /proc/pid/smaps if the underlying kernel component may more efficiently determine the backing page size. Signed-off-by: Steve Welch Signed-off-by: Ian Ziemba --- include/ofi_mr.h | 9 +++++---- prov/util/src/util_mr_cache.c | 10 ++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/ofi_mr.h b/include/ofi_mr.h index 1ebb07a8e11..64bae5f0755 100644 --- a/include/ofi_mr.h +++ b/include/ofi_mr.h @@ -417,14 +417,15 @@ bool ofi_mr_cache_flush(struct ofi_mr_cache *cache, bool flush_lru); * a new ofi_mr_entry and assign it to entry. * * @param[in] cache The cache the entry belongs to - * @param[in] info Information about the mr entry to search + * @param[in out] info Information about the mr entry to search. Info IOV may + * be updated by providers to reflect region registered by + * the provider. * @param[out] entry The registered entry corresponding to the * region described in info. * @returns On success, returns 0. On failure, returns a negative error code. */ -int ofi_mr_cache_search(struct ofi_mr_cache *cache, - const struct ofi_mr_info *info, - struct ofi_mr_entry **entry); +int ofi_mr_cache_search(struct ofi_mr_cache *cache, struct ofi_mr_info *info, + struct ofi_mr_entry **entry); /** * Given an attr (with an iov range), if the iov range is already registered, diff --git a/prov/util/src/util_mr_cache.c b/prov/util/src/util_mr_cache.c index f2148e56267..ea8bf15570f 100644 --- a/prov/util/src/util_mr_cache.c +++ b/prov/util/src/util_mr_cache.c @@ -267,7 +267,7 @@ void ofi_mr_cache_delete(struct ofi_mr_cache *cache, struct ofi_mr_entry *entry) * restart the entire operation. */ static int -util_mr_cache_create(struct ofi_mr_cache *cache, const struct ofi_mr_info *info, +util_mr_cache_create(struct ofi_mr_cache *cache, struct ofi_mr_info *info, struct ofi_mr_entry **entry) { struct ofi_mr_entry *cur; @@ -291,6 +291,12 @@ util_mr_cache_create(struct ofi_mr_cache *cache, const struct ofi_mr_info *info, if (ret) goto free; + /* Providers may have expanded the MR. Update MR info input + * accordingly. + */ + assert(ofi_iov_within(&(*info).iov, &(*entry)->info.iov)); + *info = (*entry)->info; + pthread_mutex_lock(&mm_lock); cur = ofi_mr_rbt_find(&cache->tree, info); if (cur) { @@ -329,7 +335,7 @@ util_mr_cache_create(struct ofi_mr_cache *cache, const struct ofi_mr_info *info, return ret; } -int ofi_mr_cache_search(struct ofi_mr_cache *cache, const struct ofi_mr_info *info, +int ofi_mr_cache_search(struct ofi_mr_cache *cache, struct ofi_mr_info *info, struct ofi_mr_entry **entry) { struct ofi_mem_monitor *monitor;