util/bufpool: Add new flag to skip zeroing new memory allocations

When a buf pool allocates a new chunk of memory, the entire memory
chunk is zeroed out via a call to memset. This can be quite costly,
occur at inconvenient times, and end up being wasteful when buf pool
users initialize their own entries.

This change adds a new OFI_BUFPOOL_NO_ZERO flag and will skip
the memset when that flag is set.

Signed-off-by: Ben Lynam <Ben.Lynam@cornelisnetworks.com>

include/ofi_mem.h

@@ -327,6 +327,7 @@ enum {
 	OFI_BUFPOOL_NO_TRACK		= 1 << 2,
 	OFI_BUFPOOL_HUGEPAGES		= 1 << 3,
 	OFI_BUFPOOL_NONSHARED		= 1 << 4,
+	OFI_BUFPOOL_NO_ZERO		= 1 << 5,
 };
 
 struct ofi_bufpool_region;

prov/util/src/util_buf.c

@@ -142,7 +142,7 @@ int ofi_bufpool_grow(struct ofi_bufpool *pool)
 	size_t mem_allocated = 0;
 
 	FI_DBG(&core_prov, FI_LOG_CORE, "%s pool %p  size %ld region_size %zu "
-	       "entry_cnt %d chuck_cnt %d\n",
+	       "entry_cnt %d chunk_cnt %d\n",
 		__func__, pool, pool->alloc_size, pool->region_size,
 		(int)(pool->entry_cnt+pool->attr.chunk_cnt),
 		(int)pool->attr.chunk_cnt);
@@ -167,7 +167,8 @@ int ofi_bufpool_grow(struct ofi_bufpool *pool)
 
 	mem_allocated += buf_region->pool->alloc_size;
 
-	memset(buf_region->alloc_region, 0, pool->alloc_size);
+	if (!(pool->attr.flags & OFI_BUFPOOL_NO_ZERO))
+		memset(buf_region->alloc_region, 0, pool->alloc_size);
 	buf_region->mem_region = buf_region->alloc_region + pool->entry_size;
 	if (pool->attr.alloc_fn) {
 		ret = pool->attr.alloc_fn(buf_region);
@@ -270,7 +271,7 @@ int ofi_bufpool_create_attr(struct ofi_bufpool_attr *attr,
 
 	FI_DBG(&core_prov, FI_LOG_CORE,
 		"%s alloc_size %zu region_size %zu align_entry %zu "
-		"entry_size %zu chuck_cnt %ld pool %p  (%p)\n",
+		"entry_size %zu chunk_cnt %ld pool %p  (%p)\n",
 		__func__, pool->alloc_size, pool->region_size, pool->entry_size,
 		attr->size, pool->attr.chunk_cnt, pool, buf_pool);