prov/util: support specific placement of addr into the av

there situations where being able to insert addresses into
specific indexes of the av would be convienient

example: enable the tcp provider to duplicate established
avs into other domains

Co-authored-by: Alexia Ingerson <[email protected]>
Signed-off-by: Stephen Oost <[email protected]>
(cherry picked from commit c55adcf)

include/ofi_mem.h

@@ -574,6 +574,33 @@ static inline void *ofi_ibuf_alloc(struct ofi_bufpool *pool)
 	return ofi_buf_data(buf_hdr);
 }
 
+static inline void *ofi_ibuf_alloc_at(struct ofi_bufpool *pool, size_t index)
+{
+	void *buf;
+	struct ofi_bufpool_hdr *buf_hdr;
+	struct ofi_bufpool_region *buf_region;
+	size_t region_index = index / pool->attr.chunk_cnt;
+
+	assert(pool->attr.flags & OFI_BUFPOOL_INDEXED);
+	while (region_index >= pool->region_cnt) {
+		if (ofi_bufpool_grow(pool))
+			return NULL;
+	}
+	buf_region = pool->region_table[region_index];
+	buf = buf_region->mem_region +
+		(index % pool->attr.chunk_cnt) * pool->entry_size;
+	buf_hdr = ofi_buf_hdr(buf);
+	assert(ofi_atomic_inc32(&buf_hdr->region->use_cnt));
+	assert(!ofi_buf_is_valid(buf));
+
+	dlist_remove(&buf_hdr->entry.dlist);
+	buf_hdr->entry.dlist.next = &buf_hdr->entry.dlist;
+
+	if (dlist_empty(&buf_region->free_list))
+		dlist_remove_init(&buf_region->entry);
+
+	return buf;
+}
 
 /*
  * Persistent memory support

include/ofi_util.h

@@ -976,6 +976,7 @@ int ofi_av_close(struct util_av *av);
 int ofi_av_close_lightweight(struct util_av *av);
 
 size_t ofi_av_size(struct util_av *av);
+int ofi_av_insert_addr_at(struct util_av *av, const void *addr, fi_addr_t fi_addr);
 int ofi_av_insert_addr(struct util_av *av, const void *addr, fi_addr_t *fi_addr);
 int ofi_av_remove_addr(struct util_av *av, fi_addr_t fi_addr);
 fi_addr_t ofi_av_lookup_fi_addr_unsafe(struct util_av *av, const void *addr);

prov/util/src/util_av.c

@@ -283,6 +283,34 @@ int ofi_verify_av_insert(struct util_av *av, uint64_t flags, void *context)
 	return 0;
 }
 
+int ofi_av_insert_addr_at(struct util_av *av, const void *addr, fi_addr_t fi_addr)
+{
+	struct util_av_entry *entry = NULL;
+
+	assert(ofi_mutex_held(&av->lock));
+	ofi_straddr_log(av->prov, FI_LOG_INFO, FI_LOG_AV, "inserting addr", addr);
+	HASH_FIND(hh, av->hash, addr, av->addrlen, entry);
+	if (entry) {
+		if (fi_addr == ofi_buf_index(entry))
+			return FI_SUCCESS;
+
+		ofi_straddr_log(av->prov, FI_LOG_WARN, FI_LOG_AV,
+						"addr already in AV", addr);
+		return -FI_EALREADY;
+	}
+
+	entry = ofi_ibuf_alloc_at(av->av_entry_pool, fi_addr);
+	if (!entry)
+		return -FI_ENOMEM;
+
+	memcpy(entry->data, addr, av->addrlen);
+	ofi_atomic_initialize32(&entry->use_cnt, 1);
+	HASH_ADD(hh, av->hash, data, av->addrlen, entry);
+	FI_INFO(av->prov, FI_LOG_AV, "fi_addr: %" PRIu64 "\n",
+		ofi_buf_index(entry));
+	return 0;
+}
+
 int ofi_av_insert_addr(struct util_av *av, const void *addr, fi_addr_t *fi_addr)
 {
 	struct util_av_entry *entry = NULL;