Merge pull request #7181 from hzhou/2410_req_handles

request: increase the maximum number of requests

Approved-by: Ken Raffenetti

src/include/mpir_handlemem.h

@@ -172,6 +172,7 @@ static inline void *MPIR_Handle_indirect_init(MPIR_Object_alloc_t * objmem,
             return 0;
         }
         objmem->indirect_size = 0;
+        objmem->indirect_block_size = indirect_num_indices;
     }
 
     /* See if we can allocate another block */

src/include/mpir_objects.h

@@ -517,6 +517,7 @@ typedef struct MPIR_Object_alloc_t {
     int initialized;            /* */
     void **indirect;            /* Pointer to indirect object blocks */
     int indirect_size;          /* Number of allocated indirect blocks */
+    int indirect_block_size;    /* Number of objects in each indirect block */
     int num_allocated;          /* Total capacity of this allocator including both
                                  * direct and indirect */
     int num_avail;              /* Number of available objects including both direct and indirect */

src/include/mpir_request.h

@@ -243,24 +243,32 @@ void MPIR_Persist_coll_free_cb(MPIR_Request * request);
  * MPIR_Request_create_from_pool is used to create request objects from a specific pool.
  * MPIR_Request_create is a wrapper to create request from pool 0.
  */
-/* Handle Bits - 2+4+6+8+12 - Type, Kind, Pool_idx, Block_idx, Object_idx */
-#define REQUEST_POOL_MASK    0x03f00000
-#define REQUEST_POOL_SHIFT   20
+/* Handle Bits - 2+4+1+6+7+12 - Type, Kind, NonZeroVCI, Pool_idx, Block_idx, Object_idx */
+/* UPDATE: If the 6th bit (NonZeroVCI) is 0, it allocated from Pool 0, Block_idx extends to 13 bits (taking over Pool_idx).
+ *         If the 6th bit is 1, BLOCK_idx shrinks to 7 bits.
+ */
+#define REQUEST_POOL_FLAG    0x02000000
+#define REQUEST_POOL_MASK    0x01f80000
+#define REQUEST_POOL_SHIFT   19
 #define REQUEST_POOL_MAX     64
-#define REQUEST_BLOCK_MASK   0x000ff000
+#define REQUEST_BLOCK_MASK0  0x01fff000
+#define REQUEST_BLOCK_MASK   0x0007f000
 #define REQUEST_BLOCK_SHIFT  12
-#define REQUEST_BLOCK_MAX    256
+#define REQUEST_BLOCK_MAX    128
 #define REQUEST_OBJECT_MASK  0x00000fff
 #define REQUEST_OBJECT_SHIFT 0
 #define REQUEST_OBJECT_MAX   4096
 
-#define REQUEST_NUM_BLOCKS   256
-#define REQUEST_NUM_INDICES  1024
+#define REQUEST_NUM_BLOCKS0  8192
+#define REQUEST_NUM_BLOCKS   128
+#define REQUEST_NUM_INDICES  4096
 
 #define MPIR_MAX_VCIS REQUEST_POOL_MAX
 #define MPIR_REQUEST_NUM_POOLS REQUEST_POOL_MAX
 
-#define MPIR_REQUEST_POOL(req_) (((req_)->handle & REQUEST_POOL_MASK) >> REQUEST_POOL_SHIFT)
+#define MPIR_REQUEST_POOL_FROM_HANDLE(handle) (((handle) & REQUEST_POOL_FLAG) ? \
+    (((handle) & REQUEST_POOL_MASK) >> REQUEST_POOL_SHIFT) : 0)
+#define MPIR_REQUEST_POOL(req_) MPIR_REQUEST_POOL_FROM_HANDLE((req_)->handle)
 
 extern MPIR_Request MPIR_Request_builtin[MPIR_REQUEST_N_BUILTIN];
 extern MPIR_Object_alloc_t MPIR_Request_mem[MPIR_REQUEST_NUM_POOLS];
@@ -269,7 +277,7 @@ extern MPIR_Request MPIR_Request_direct[MPIR_REQUEST_PREALLOC];
 #define MPIR_Request_get_ptr(a, ptr) \
     do { \
         int pool, blk, idx; \
-        pool = ((a) & REQUEST_POOL_MASK) >> REQUEST_POOL_SHIFT; \
+        pool = MPIR_REQUEST_POOL_FROM_HANDLE(a); \
         switch (HANDLE_GET_KIND(a)) { \
         case HANDLE_KIND_BUILTIN: \
             if (a == MPI_MESSAGE_NO_PROC) { \
@@ -284,7 +292,11 @@ extern MPIR_Request MPIR_Request_direct[MPIR_REQUEST_PREALLOC];
             ptr = MPIR_Request_direct + HANDLE_INDEX(a); \
             break; \
         case HANDLE_KIND_INDIRECT: \
-            blk = ((a) & REQUEST_BLOCK_MASK) >> REQUEST_BLOCK_SHIFT; \
+            if (pool == 0) { \
+                blk = ((a) & REQUEST_BLOCK_MASK0) >> REQUEST_BLOCK_SHIFT; \
+            } else { \
+                blk = ((a) & REQUEST_BLOCK_MASK) >> REQUEST_BLOCK_SHIFT; \
+            } \
             idx = ((a) & REQUEST_OBJECT_MASK) >> REQUEST_OBJECT_SHIFT; \
             ptr = ((MPIR_Request *) MPIR_Request_mem[pool].indirect[blk]) + idx; \
             break; \
@@ -419,13 +431,15 @@ static inline MPIR_Request *MPIR_Request_create_from_pool(MPIR_Request_kind_t ki
 #ifdef MPICH_DEBUG_MUTEX
     MPID_THREAD_ASSERT_IN_CS(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
 #endif
-    req = MPIR_Handle_obj_alloc_unsafe(&MPIR_Request_mem[pool],
-                                       REQUEST_NUM_BLOCKS, REQUEST_NUM_INDICES);
+    int max_blocks = (pool == 0) ? REQUEST_NUM_BLOCKS0 : REQUEST_NUM_BLOCKS;
+    req = MPIR_Handle_obj_alloc_unsafe(&MPIR_Request_mem[pool], max_blocks, REQUEST_NUM_INDICES);
     if (req == NULL)
         goto fn_fail;
 
     /* Patch the handle for pool index. */
-    req->handle |= (pool << REQUEST_POOL_SHIFT);
+    if (pool > 0) {
+        req->handle |= (REQUEST_POOL_FLAG | (pool << REQUEST_POOL_SHIFT));
+    }
 
     MPL_DBG_MSG_P(MPIR_DBG_REQUEST, VERBOSE, "allocated request, handle=0x%08x", req->handle);
 #ifdef MPICH_DBG_OUTPUT

src/mpi/attr/attrutil.c

@@ -21,7 +21,7 @@
 /* Preallocated keyval objects */
 MPII_Keyval MPII_Keyval_direct[MPID_KEYVAL_PREALLOC];
 
-MPIR_Object_alloc_t MPII_Keyval_mem = { 0, 0, 0, 0, 0, 0, MPIR_KEYVAL,
+MPIR_Object_alloc_t MPII_Keyval_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_KEYVAL,
     sizeof(MPII_Keyval),
     MPII_Keyval_direct,
     MPID_KEYVAL_PREALLOC,
@@ -31,7 +31,7 @@ MPIR_Object_alloc_t MPII_Keyval_mem = { 0, 0, 0, 0, 0, 0, MPIR_KEYVAL,
 /* Preallocated keyval objects */
 MPIR_Attribute MPID_Attr_direct[MPIR_ATTR_PREALLOC];
 
-MPIR_Object_alloc_t MPID_Attr_mem = { 0, 0, 0, 0, 0, 0, MPIR_ATTR,
+MPIR_Object_alloc_t MPID_Attr_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_ATTR,
     sizeof(MPIR_Attribute),
     MPID_Attr_direct,
     MPIR_ATTR_PREALLOC,
@@ -227,10 +227,10 @@ int MPIR_Attr_delete_list(int handle, MPIR_Attribute ** attr)
         /* For this attribute, find the delete function for the
          * corresponding keyval. If the delete function fails,
          * we record the last failure */
-         rc = MPIR_Call_attr_delete(handle, p);
-         if (rc != 0) {
-             mpi_errno = rc;
-         }
+        rc = MPIR_Call_attr_delete(handle, p);
+        if (rc != 0) {
+            mpi_errno = rc;
+        }
 
         /* We must also remove the keyval reference.  If the keyval
          * was freed earlier (reducing the refcount), the actual

src/mpi/coll/op/op_impl.c

@@ -9,7 +9,7 @@
 MPIR_Op MPIR_Op_builtin[MPIR_OP_N_BUILTIN];
 MPIR_Op MPIR_Op_direct[MPIR_OP_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Op_mem = { 0, 0, 0, 0, 0, 0, MPIR_OP,
+MPIR_Object_alloc_t MPIR_Op_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_OP,
     sizeof(MPIR_Op),
     MPIR_Op_direct,
     MPIR_OP_PREALLOC,

src/mpi/comm/commutil.c

@@ -18,14 +18,7 @@
 MPIR_Comm MPIR_Comm_builtin[MPIR_COMM_N_BUILTIN];
 MPIR_Comm MPIR_Comm_direct[MPIR_COMM_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Comm_mem = {
-    0,
-    0,
-    0,
-    0,
-    0,
-    0,
-    MPIR_COMM,
+MPIR_Object_alloc_t MPIR_Comm_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_COMM,
     sizeof(MPIR_Comm),
     MPIR_Comm_direct,
     MPIR_COMM_PREALLOC,

src/mpi/datatype/typeutil.c

@@ -13,7 +13,7 @@
 MPIR_Datatype MPIR_Datatype_builtin[MPIR_DATATYPE_N_BUILTIN];
 MPIR_Datatype MPIR_Datatype_direct[MPIR_DATATYPE_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Datatype_mem = { 0, 0, 0, 0, 0, 0, MPIR_DATATYPE,
+MPIR_Object_alloc_t MPIR_Datatype_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_DATATYPE,
     sizeof(MPIR_Datatype), MPIR_Datatype_direct,
     MPIR_DATATYPE_PREALLOC,
     NULL, {0}

src/mpi/errhan/errutil.c

@@ -116,7 +116,7 @@ static int checkForUserErrcode(int);
 MPIR_Errhandler MPIR_Errhandler_builtin[MPIR_ERRHANDLER_N_BUILTIN];
 MPIR_Errhandler MPIR_Errhandler_direct[MPIR_ERRHANDLER_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Errhandler_mem = { 0, 0, 0, 0, 0, 0, MPIR_ERRHANDLER,
+MPIR_Object_alloc_t MPIR_Errhandler_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_ERRHANDLER,
     sizeof(MPIR_Errhandler),
     MPIR_Errhandler_direct,
     MPIR_ERRHANDLER_PREALLOC,

src/mpi/group/grouputil.c

@@ -10,7 +10,7 @@
 MPIR_Group MPIR_Group_builtin[MPIR_GROUP_N_BUILTIN];
 MPIR_Group MPIR_Group_direct[MPIR_GROUP_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Group_mem = { 0, 0, 0, 0, 0, 0, MPIR_GROUP,
+MPIR_Object_alloc_t MPIR_Group_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_GROUP,
     sizeof(MPIR_Group), MPIR_Group_direct,
     MPIR_GROUP_PREALLOC,
     NULL, {0}

src/mpi/info/infoutil.c

@@ -14,7 +14,7 @@
 MPIR_Info MPIR_Info_builtin[MPIR_INFO_N_BUILTIN];
 MPIR_Info MPIR_Info_direct[MPIR_INFO_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Info_mem = { 0, 0, 0, 0, 0, 0, MPIR_INFO,
+MPIR_Object_alloc_t MPIR_Info_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_INFO,
     sizeof(MPIR_Info), MPIR_Info_direct,
     MPIR_INFO_PREALLOC,
     NULL, {0}

src/mpi/request/mpir_greq.c

@@ -8,7 +8,7 @@
 /* preallocated grequest classes */
 MPIR_Grequest_class MPIR_Grequest_class_direct[MPIR_GREQ_CLASS_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Grequest_class_mem = { 0, 0, 0, 0, 0, 0, MPIR_GREQ_CLASS,
+MPIR_Object_alloc_t MPIR_Grequest_class_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_GREQ_CLASS,
     sizeof(MPIR_Grequest_class),
     MPIR_Grequest_class_direct,
     MPIR_GREQ_CLASS_PREALLOC,

src/mpi/request/mpir_request.c

@@ -37,9 +37,9 @@ void MPII_init_request(void)
 #endif
 
     /* *INDENT-OFF* */
-    MPIR_Request_mem[0] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), MPIR_Request_direct, MPIR_REQUEST_PREALLOC, lock_ptr, {0}};
+    MPIR_Request_mem[0] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), MPIR_Request_direct, MPIR_REQUEST_PREALLOC, lock_ptr, {0}};
     for (int i = 1; i < MPIR_REQUEST_NUM_POOLS; i++) {
-        MPIR_Request_mem[i] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), NULL, 0, lock_ptr, {0}};
+        MPIR_Request_mem[i] = (MPIR_Object_alloc_t) { 0, 0, 0, 0, 0, 0, 0, MPIR_REQUEST, sizeof(MPIR_Request), NULL, 0, lock_ptr, {0}};
     }
     /* *INDENT-ON* */
 

src/mpi/rma/winutil.c

@@ -11,7 +11,7 @@
 /* Preallocated window objects */
 MPIR_Win MPIR_Win_direct[MPIR_WIN_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Win_mem = { 0, 0, 0, 0, 0, 0, MPIR_WIN,
+MPIR_Object_alloc_t MPIR_Win_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_WIN,
     sizeof(MPIR_Win), MPIR_Win_direct,
     MPIR_WIN_PREALLOC,
     NULL, {0}

src/mpi/session/session_util.c

@@ -13,7 +13,7 @@
 /* Preallocated session object */
 MPIR_Session MPIR_Session_direct[MPIR_SESSION_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Session_mem = { 0, 0, 0, 0, 0, 0,
+MPIR_Object_alloc_t MPIR_Session_mem = { 0, 0, 0, 0, 0, 0, 0,
     MPIR_SESSION, sizeof(MPIR_Session),
     MPIR_Session_direct, MPIR_SESSION_PREALLOC,
     NULL, {0}

src/mpi/stream/stream_impl.c

@@ -57,7 +57,7 @@ static int deallocate_vci(int *vci)
 
 MPIR_Stream MPIR_Stream_direct[MPIR_STREAM_PREALLOC];
 
-MPIR_Object_alloc_t MPIR_Stream_mem = { 0, 0, 0, 0, 0, 0, MPIR_STREAM,
+MPIR_Object_alloc_t MPIR_Stream_mem = { 0, 0, 0, 0, 0, 0, 0, MPIR_STREAM,
     sizeof(MPIR_Stream), MPIR_Stream_direct,
     MPIR_STREAM_PREALLOC,
     NULL, {0}

src/util/mpir_handlemem.c

@@ -92,7 +92,7 @@ int MPIR_check_handles_on_finalize(void *objmem_ptr)
             void **indirect = (void **) objmem->indirect;
             for (i = 0; i < objmem->indirect_size; i++) {
                 char *start = indirect[i];
-                char *end = start + HANDLE_NUM_INDICES * objmem->size;
+                char *end = start + objmem->indirect_block_size * objmem->size;
                 if ((char *) ptr >= start && (char *) ptr < end) {
                     nIndirect[i]++;
                     break;
@@ -106,7 +106,7 @@ int MPIR_check_handles_on_finalize(void *objmem_ptr)
                 printf("direct block is [%p,%p]\n", direct, directEnd);
                 if (objmem->indirect_size) {
                     printf("indirect block is [%p,%p]\n", indirect[0],
-                           (char *) indirect[0] + HANDLE_NUM_INDICES * objmem->size);
+                           (char *) indirect[0] + objmem->indirect_block_size * objmem->size);
                 }
             }
         }
@@ -124,11 +124,12 @@ int MPIR_check_handles_on_finalize(void *objmem_ptr)
                MPIR_Handle_get_kind_str(objmem->kind), directSize - nDirect);
     }
     for (i = 0; i < objmem->indirect_size; i++) {
-        if (nIndirect[i] != HANDLE_NUM_INDICES) {
+        if (nIndirect[i] != objmem->indirect_block_size) {
             leaked_handles = TRUE;
             printf
                 ("In indirect memory block %d for handle type %s, %d handles are still allocated\n",
-                 i, MPIR_Handle_get_kind_str(objmem->kind), HANDLE_NUM_INDICES - nIndirect[i]);
+                 i, MPIR_Handle_get_kind_str(objmem->kind),
+                 objmem->indirect_block_size - nIndirect[i]);
         }
     }