Hi, I'm encountering an unexpected deadlock in this Fortran test program, compiled using Parallel Studio XE 2017 Update 4 on an Amazon EC2 cluster (Linux system).
$ mpiifort -traceback nbtest.f90 -o test.x
On one node, the program runs just fine, but any more and it deadlocks, leading me to suspect a internode comm failure, but my knowledge in this area is lacking. FYI, the test code is hardcoded to be run on 16 cores.
Any help or insight is appreciated!
Danny
Code
program nbtest
use mpi
implicit none
!***____________________ Definitions _______________
integer, parameter :: r4 = SELECTED_REAL_KIND(6,37)
integer :: irank
integer, allocatable :: gstart1(:)
integer, allocatable :: gend1(:)
integer, allocatable :: gstartz(:)
integer, allocatable :: gendz(:)
integer, allocatable :: ind_fl(:)
integer, allocatable :: blen(:),disp(:)
integer, allocatable :: ddt_recv(:),ddt_send(:)
real(kind=r4), allocatable :: tmp_array(:,:,:)
real(kind=r4), allocatable :: tmp_in(:,:,:)
integer :: cnt, i, j
integer :: count_send, count_recv
integer :: ssend
integer :: srecv
integer :: esend
integer :: erecv
integer :: erecv2, srecv2
integer :: mpierr, ierr, old, typesize, typesize2,typesize3
integer :: mpi_requests(2*16)
integer :: mpi_status_arr(MPI_STATUS_SIZE,2*16)
character(MPI_MAX_ERROR_STRING) :: string
integer :: resultlen
integer :: errorcode
!***________Code___________________________
!*_________initialize MPI__________________
call MPI_INIT(ierr)
call MPI_COMM_RANK(MPI_COMM_WORLD,irank,ierr)
call MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN,ierr)
allocate(gstart1(0:15), &
gend1(0:15), &
gstartz(0:15), &
gendz(0:15))
gstart1(0) = 1
gend1(0) = 40
gstartz(0) = 1
gendz(0) = 27
do i = 2, 16
gstart1(i-1) = gend1(i-2) + 1
gend1(i-1) = gend1(i-2) + 40
gstartz(i-1) = gendz(i-2) + 1
gendz(i-1) = gendz(i-2) + 27
end do
allocate(ind_fl(15))
cnt = 1
do i = 1, 16
if ( (i-1) == irank ) cycle
ind_fl(cnt) = (i - 1)
cnt = cnt + 1
end do
cnt = 1
do i = 1, 16
if ( (i-1) == irank ) cycle
ind_fl(cnt) = (i - 1)
cnt = cnt + 1
end do
!*_________new datatype__________________
allocate(ddt_recv(16),ddt_send(16))
allocate(blen(60), disp(60))
call mpi_type_size(MPI_REAL,typesize,ierr)
do i = 1, 15
call mpi_type_contiguous(3240,MPI_REAL, &
ddt_send(i),ierr)
call mpi_type_commit(ddt_send(i),ierr)
srecv2 = (gstartz(ind_fl(i))-1)*2+1
erecv2 = gendz(ind_fl(i))*2
blen(:) = erecv2 - srecv2 + 1
do j = 1, 60
disp(j) = (j-1)*(852) + srecv2 - 1
end do
call mpi_type_indexed(60,blen,disp,MPI_REAL, &
ddt_recv(i),ierr)
call mpi_type_commit(ddt_recv(i),ierr)
old = ddt_recv(i)
call mpi_type_create_resized(old,int(0,kind=MPI_ADDRESS_KIND),&
int(51120*typesize,kind=MPI_ADDRESS_KIND),&
ddt_recv(i),ierr)
call mpi_type_free(old,ierr)
call mpi_type_commit(ddt_recv(i),ierr)
end do
allocate(tmp_array(852,60,40))
allocate(tmp_in(54,60,640))
tmp_array = 0.0_r4
tmp_in = 0.0_r4
ssend = gstart1(irank)
esend = gend1(irank)
cnt = 0
do i = 1, 15
srecv = gstart1(ind_fl(i))
erecv = gend1(ind_fl(i))
! Calculate the number of bytes to send (for MPI_SEND)
count_send = erecv - srecv + 1
count_recv = esend - ssend + 1
cnt = cnt + 1
call mpi_irecv(tmp_array,count_recv,ddt_recv(i), &
ind_fl(i),ind_fl(i),MPI_COMM_WORLD,mpi_requests(cnt),ierr)
cnt = cnt + 1
call mpi_isend(tmp_in(:,:,srecv:erecv), &
count_send,ddt_send(i),ind_fl(i), &
irank,MPI_COMM_WORLD,mpi_requests(cnt),ierr)
end do
call mpi_waitall(cnt,mpi_requests(1:cnt),mpi_status_arr(:,1:cnt),ierr)
if (ierr /= MPI_SUCCESS) then
do i = 1,cnt
errorcode = mpi_status_arr(MPI_ERROR,i)
if (errorcode /= 0 .AND. errorcode /= MPI_ERR_PENDING) then
call MPI_Error_string(errorcode,string,resultlen,mpierr)
print *, "rank: ",irank, string
!call MPI_Abort(MPI_COMM_WORLD,errorcode,ierr)
end if
end do
end if
deallocate(tmp_array)
deallocate(tmp_in)
print *, "great success"
call MPI_FINALIZE(ierr)
end program nbtest
Running gdb on one of the processors during the deadlock:
(gdb) bt
#0 0x00002acb4c6bf733 in __select_nocancel () from /lib64/libc.so.6
#1 0x00002acb4b496a2e in MPID_nem_tcp_connpoll () from /opt/intel/psxe_runtime_2017.4.196/linux/mpi/intel64/lib/libmpi.so.12
#2 0x00002acb4b496048 in MPID_nem_tcp_poll () from /opt/intel/psxe_runtime_2017.4.196/linux/mpi/intel64/lib/libmpi.so.12
#3 0x00002acb4b350020 in MPID_nem_network_poll () from /opt/intel/psxe_runtime_2017.4.196/linux/mpi/intel64/lib/libmpi.so.12
#4 0x00002acb4b0cc5f2 in PMPIDI_CH3I_Progress () from /opt/intel/psxe_runtime_2017.4.196/linux/mpi/intel64/lib/libmpi.so.12
#5 0x00002acb4b50328f in PMPI_Waitall () from /opt/intel/psxe_runtime_2017.4.196/linux/mpi/intel64/lib/libmpi.so.12
#6 0x00002acb4ad1d53f in pmpi_waitall_ (v1=0x1e, v2=0xb0c320, v3=0x0, ierr=0x2acb4c6bf733 <__select_nocancel+10>) at ../../src/binding/fortran/mpif_h/waitallf.c:275
#7 0x00000000004064b0 in MAIN__ ()
#8 0x000000000040331e in main ()
Output log after I kill the job:
$ mpirun -n 16 ./test.x
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
test.x 000000000040C12A Unknown Unknown Unknown
libpthread-2.17.s 00002BA8B42F95A0 Unknown Unknown Unknown
libmpi.so.12 00002BA8B3303EBF PMPIDI_CH3I_Progr Unknown Unknown
libmpi.so.12 00002BA8B373B28F PMPI_Waitall Unknown Unknown
libmpifort.so.12. 00002BA8B2F5553F pmpi_waitall Unknown Unknown
test.x 00000000004064B0 MAIN__ 129 nbtest.f90
test.x 000000000040331E Unknown Unknown Unknown
libc-2.17.so 00002BA8B4829C05 __libc_start_main Unknown Unknown
test.x 0000000000403229 Unknown Unknown Unknown
(repeated 15 times, once for each processor)
Output with I_MPI_DEBUG = 6
[0] MPI startup(): Intel(R) MPI Library, Version 2017 Update 3 Build 20170405 (id: 17193)
[0] MPI startup(): Copyright (C) 2003-2017 Intel Corporation. All rights reserved.
[0] MPI startup(): Multi-threaded optimized library
[12] MPI startup(): cannot open dynamic library libdat2.so.2
[7] MPI startup(): cannot open dynamic library libdat2.so.2
[10] MPI startup(): cannot open dynamic library libdat2.so.2
[13] MPI startup(): cannot open dynamic library libdat2.so.2
[4] MPI startup(): cannot open dynamic library libdat2.so.2
[9] MPI startup(): cannot open dynamic library libdat2.so.2
[14] MPI startup(): cannot open dynamic library libdat2.so.2
[5] MPI startup(): cannot open dynamic library libdat2.so.2
[11] MPI startup(): cannot open dynamic library libdat2.so.2
[15] MPI startup(): cannot open dynamic library libdat2.so.2
[6] MPI startup(): cannot open dynamic library libdat2.so.2
[8] MPI startup(): cannot open dynamic library libdat2.so.2
[0] MPI startup(): cannot open dynamic library libdat2.so.2
[3] MPI startup(): cannot open dynamic library libdat2.so.2
[2] MPI startup(): cannot open dynamic library libdat2.so.2
[4] MPI startup(): cannot open dynamic library libdat2.so
[7] MPI startup(): cannot open dynamic library libdat2.so
[8] MPI startup(): cannot open dynamic library libdat2.so
[9] MPI startup(): cannot open dynamic library libdat2.so
[6] MPI startup(): cannot open dynamic library libdat2.so
[10] MPI startup(): cannot open dynamic library libdat2.so
[13] MPI startup(): cannot open dynamic library libdat2.so
[0] MPI startup(): cannot open dynamic library libdat2.so
[15] MPI startup(): cannot open dynamic library libdat2.so
[3] MPI startup(): cannot open dynamic library libdat2.so
[12] MPI startup(): cannot open dynamic library libdat2.so
[4] MPI startup(): cannot open dynamic library libdat.so.1
[14] MPI startup(): cannot open dynamic library libdat2.so
[7] MPI startup(): cannot open dynamic library libdat.so.1
[5] MPI startup(): cannot open dynamic library libdat2.so
[8] MPI startup(): cannot open dynamic library libdat.so.1
[1] MPI startup(): cannot open dynamic library libdat2.so.2
[6] MPI startup(): cannot open dynamic library libdat.so.1
[9] MPI startup(): cannot open dynamic library libdat.so.1
[10] MPI startup(): cannot open dynamic library libdat.so.1
[0] MPI startup(): cannot open dynamic library libdat.so.1
[12] MPI startup(): cannot open dynamic library libdat.so.1
[4] MPI startup(): cannot open dynamic library libdat.so
[11] MPI startup(): cannot open dynamic library libdat2.so
[3] MPI startup(): cannot open dynamic library libdat.so.1
[13] MPI startup(): cannot open dynamic library libdat.so.1
[5] MPI startup(): cannot open dynamic library libdat.so.1
[15] MPI startup(): cannot open dynamic library libdat.so.1
[5] MPI startup(): cannot open dynamic library libdat.so
[7] MPI startup(): cannot open dynamic library libdat.so
[1] MPI startup(): cannot open dynamic library libdat2.so
[9] MPI startup(): cannot open dynamic library libdat.so
[8] MPI startup(): cannot open dynamic library libdat.so
[11] MPI startup(): cannot open dynamic library libdat.so.1
[6] MPI startup(): cannot open dynamic library libdat.so
[10] MPI startup(): cannot open dynamic library libdat.so
[14] MPI startup(): cannot open dynamic library libdat.so.1
[11] MPI startup(): cannot open dynamic library libdat.so
[13] MPI startup(): cannot open dynamic library libdat.so
[15] MPI startup(): cannot open dynamic library libdat.so
[12] MPI startup(): cannot open dynamic library libdat.so
[0] MPI startup(): cannot open dynamic library libdat.so
[14] MPI startup(): cannot open dynamic library libdat.so
[1] MPI startup(): cannot open dynamic library libdat.so.1
[3] MPI startup(): cannot open dynamic library libdat.so
[1] MPI startup(): cannot open dynamic library libdat.so
[2] MPI startup(): cannot open dynamic library libdat2.so
[2] MPI startup(): cannot open dynamic library libdat.so.1
[2] MPI startup(): cannot open dynamic library libdat.so
[4] MPI startup(): cannot load default tmi provider
[7] MPI startup(): cannot load default tmi provider
[5] MPI startup(): cannot load default tmi provider
[9] MPI startup(): cannot load default tmi provider
[0] MPI startup(): cannot load default tmi provider
[6] MPI startup(): cannot load default tmi provider
[10] MPI startup(): cannot load default tmi provider
[3] MPI startup(): cannot load default tmi provider
[15] MPI startup(): cannot load default tmi provider
[8] MPI startup(): cannot load default tmi provider
[1] MPI startup(): cannot load default tmi provider
[14] MPI startup(): cannot load default tmi provider
[11] MPI startup(): cannot load default tmi provider
[2] MPI startup(): cannot load default tmi provider
[12] MPI startup(): cannot load default tmi provider
[13] MPI startup(): cannot load default tmi provider
[12] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[4] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[9] ERROR - load_iblibrary(): [15] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[5] ERROR - load_iblibrary(): [0] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[10] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[1] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[3] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[13] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[7] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[2] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[6] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[8] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[11] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[14] ERROR - load_iblibrary(): Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
Can't open IB verbs library: libibverbs.so.1: cannot open shared object file: No such file or directory
[0] MPI startup(): shm and tcp data transfer modes
[1] MPI startup(): shm and tcp data transfer modes
[2] MPI startup(): shm and tcp data transfer modes
[3] MPI startup(): shm and tcp data transfer modes
[4] MPI startup(): shm and tcp data transfer modes
[5] MPI startup(): shm and tcp data transfer modes
[7] MPI startup(): shm and tcp data transfer modes
[9] MPI startup(): shm and tcp data transfer modes
[8] MPI startup(): shm and tcp data transfer modes
[6] MPI startup(): shm and tcp data transfer modes
[10] MPI startup(): shm and tcp data transfer modes
[11] MPI startup(): shm and tcp data transfer modes
[12] MPI startup(): shm and tcp data transfer modes
[13] MPI startup(): shm and tcp data transfer modes
[14] MPI startup(): shm and tcp data transfer modes
[15] MPI startup(): shm and tcp data transfer modes
[0] MPI startup(): Device_reset_idx=1
[0] MPI startup(): Allgather: 4: 1-4 & 0-4
[0] MPI startup(): Allgather: 1: 5-11 & 0-4
[0] MPI startup(): Allgather: 4: 12-28 & 0-4
[0] MPI startup(): Allgather: 1: 29-1694 & 0-4
p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 11.0px Menlo; color: #000000; background-color: #ffffff}
span.s1 {font-variant-ligatures: no-common-ligatures}
[0] MPI startup(): Allgather: 4: 1695-3413 & 0-4
[0] MPI startup(): Allgather: 1: 3414-513494 & 0-4
[0] MPI startup(): Allgather: 3: 513495-1244544 & 0-4
[0] MPI startup(): Allgather: 4: 0-2147483647 & 0-4
[0] MPI startup(): Allgather: 4: 1-16 & 5-16
[0] MPI startup(): Allgather: 1: 17-38 & 5-16
[0] MPI startup(): Allgather: 3: 0-2147483647 & 5-16
[0] MPI startup(): Allgather: 4: 1-8 & 17-2147483647
[0] MPI startup(): Allgather: 1: 9-23 & 17-2147483647
[0] MPI startup(): Allgather: 4: 24-35 & 17-2147483647
[0] MPI startup(): Allgather: 3: 0-2147483647 & 17-2147483647
[0] MPI startup(): Allgatherv: 1: 0-3669 & 0-4
[0] MPI startup(): Allgatherv: 4: 3669-4949 & 0-4
[0] MPI startup(): Allgatherv: 1: 4949-17255 & 0-4
[0] MPI startup(): Allgatherv: 4: 17255-46775 & 0-4
[0] MPI startup(): Allgatherv: 3: 46775-836844 & 0-4
[0] MPI startup(): Allgatherv: 4: 0-2147483647 & 0-4
[0] MPI startup(): Allgatherv: 4: 0-10 & 5-16
[0] MPI startup(): Allgatherv: 1: 10-38 & 5-16
[0] MPI startup(): Allgatherv: 3: 0-2147483647 & 5-16
[0] MPI startup(): Allgatherv: 4: 0-8 & 17-2147483647
[0] MPI startup(): Allgatherv: 1: 8-21 & 17-2147483647
[0] MPI startup(): Allgatherv: 3: 0-2147483647 & 17-2147483647
[0] MPI startup(): Allreduce: 5: 0-6 & 0-8
[0] MPI startup(): Allreduce: 7: 6-11 & 0-8
[0] MPI startup(): Allreduce: 5: 11-26 & 0-8
[0] MPI startup(): Allreduce: 4: 26-43 & 0-8
[0] MPI startup(): Allreduce: 5: 43-99 & 0-8
[0] MPI startup(): Allreduce: 1: 99-176 & 0-8
[0] MPI startup(): Allreduce: 6: 176-380 & 0-8
[0] MPI startup(): Allreduce: 2: 380-2967 & 0-8
[0] MPI startup(): Allreduce: 1: 2967-9460 & 0-8
[0] MPI startup(): Allreduce: 2: 0-2147483647 & 0-8
[0] MPI startup(): Allreduce: 5: 0-95 & 9-16
[0] MPI startup(): Allreduce: 1: 95-301 & 9-16
[0] MPI startup(): Allreduce: 2: 301-2577 & 9-16
[0] MPI startup(): Allreduce: 6: 2577-5427 & 9-16
[0] MPI startup(): Allreduce: 1: 5427-10288 & 9-16
[0] MPI startup(): Allreduce: 2: 0-2147483647 & 9-16
[0] MPI startup(): Allreduce: 6: 0-6 & 17-2147483647
[0] MPI startup(): Allreduce: 5: 6-11 & 17-2147483647
[0] MPI startup(): Allreduce: 6: 11-452 & 17-2147483647
[0] MPI startup(): Allreduce: 2: 452-2639 & 17-2147483647
[0] MPI startup(): Allreduce: 6: 2639-5627 & 17-2147483647
[0] MPI startup(): Allreduce: 1: 5627-9956 & 17-2147483647
[0] MPI startup(): Allreduce: 2: 9956-2587177 & 17-2147483647
[0] MPI startup(): Allreduce: 3: 0-2147483647 & 17-2147483647
p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 11.0px Menlo; color: #000000; background-color: #ffffff}
span.s1 {font-variant-ligatures: no-common-ligatures}
[0] MPI startup(): Alltoall: 4: 1-16 & 0-8
[0] MPI startup(): Alltoall: 1: 17-69 & 0-8
[0] MPI startup(): Alltoall: 2: 70-1024 & 0-8
[0] MPI startup(): Alltoall: 2: 1024-52228 & 0-8
[0] MPI startup(): Alltoall: 4: 52229-74973 & 0-8
[0] MPI startup(): Alltoall: 2: 74974-131148 & 0-8
[0] MPI startup(): Alltoall: 3: 131149-335487 & 0-8
[0] MPI startup(): Alltoall: 4: 0-2147483647 & 0-8
[0] MPI startup(): Alltoall: 4: 1-16 & 9-16
[0] MPI startup(): Alltoall: 1: 17-40 & 9-16
[0] MPI startup(): Alltoall: 2: 41-497 & 9-16
[0] MPI startup(): Alltoall: 1: 498-547 & 9-16
[0] MPI startup(): Alltoall: 2: 548-1024 & 9-16
[0] MPI startup(): Alltoall: 2: 1024-69348 & 9-16
[0] MPI startup(): Alltoall: 4: 0-2147483647 & 9-16
[0] MPI startup(): Alltoall: 4: 0-1 & 17-2147483647
[0] MPI startup(): Alltoall: 1: 2-4 & 17-2147483647
[0] MPI startup(): Alltoall: 4: 5-24 & 17-2147483647
[0] MPI startup(): Alltoall: 2: 25-1024 & 17-2147483647
[0] MPI startup(): Alltoall: 2: 1024-20700 & 17-2147483647
[0] MPI startup(): Alltoall: 4: 20701-57414 & 17-2147483647
[0] MPI startup(): Alltoall: 3: 57415-66078 & 17-2147483647
[0] MPI startup(): Alltoall: 4: 0-2147483647 & 17-2147483647
[0] MPI startup(): Alltoallv: 2: 0-2147483647 & 0-2147483647
[0] MPI startup(): Alltoallw: 0: 0-2147483647 & 0-2147483647
[0] MPI startup(): Barrier: 0: 0-2147483647 & 0-2147483647
[0] MPI startup(): Bcast: 4: 1-29 & 0-8
[0] MPI startup(): Bcast: 7: 30-37 & 0-8
[0] MPI startup(): Bcast: 4: 38-543 & 0-8
[0] MPI startup(): Bcast: 6: 544-1682 & 0-8
[0] MPI startup(): Bcast: 4: 1683-2521 & 0-8
[0] MPI startup(): Bcast: 6: 2522-30075 & 0-8
[0] MPI startup(): Bcast: 7: 30076-34889 & 0-8
[0] MPI startup(): Bcast: 4: 34890-131072 & 0-8
[0] MPI startup(): Bcast: 6: 131072-409051 & 0-8
[0] MPI startup(): Bcast: 7: 0-2147483647 & 0-8
[0] MPI startup(): Bcast: 4: 1-13 & 9-2147483647
[0] MPI startup(): Bcast: 1: 14-25 & 9-2147483647
[0] MPI startup(): Bcast: 4: 26-691 & 9-2147483647
[0] MPI startup(): Bcast: 6: 692-2367 & 9-2147483647
[0] MPI startup(): Bcast: 4: 2368-7952 & 9-2147483647
[0] MPI startup(): Bcast: 6: 7953-10407 & 9-2147483647
[0] MPI startup(): Bcast: 4: 10408-17900 & 9-2147483647
[0] MPI startup(): Bcast: 6: 17901-36385 & 9-2147483647
[0] MPI startup(): Bcast: 7: 36386-131072 & 9-2147483647
[0] MPI startup(): Bcast: 7: 0-2147483647 & 9-2147483647
[0] MPI startup(): Exscan: 0: 0-2147483647 & 0-2147483647
p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 11.0px Menlo; color: #000000; background-color: #ffffff}
span.s1 {font-variant-ligatures: no-common-ligatures}
[0] MPI startup(): Gather: 2: 1-3 & 0-8
[0] MPI startup(): Gather: 3: 4-4 & 0-8
[0] MPI startup(): Gather: 2: 5-66 & 0-8
[0] MPI startup(): Gather: 3: 67-174 & 0-8
[0] MPI startup(): Gather: 2: 175-478 & 0-8
[0] MPI startup(): Gather: 3: 479-531 & 0-8
[0] MPI startup(): Gather: 2: 532-2299 & 0-8
[0] MPI startup(): Gather: 3: 0-2147483647 & 0-8
[0] MPI startup(): Gather: 2: 1-141 & 9-16
[0] MPI startup(): Gather: 3: 142-456 & 9-16
[0] MPI startup(): Gather: 2: 457-785 & 9-16
[0] MPI startup(): Gather: 3: 786-70794 & 9-16
[0] MPI startup(): Gather: 2: 70795-254351 & 9-16
[0] MPI startup(): Gather: 3: 0-2147483647 & 9-16
[0] MPI startup(): Gather: 2: 1-89 & 17-2147483647
[0] MPI startup(): Gather: 3: 90-472 & 17-2147483647
[0] MPI startup(): Gather: 2: 473-718 & 17-2147483647
[0] MPI startup(): Gather: 3: 719-16460 & 17-2147483647
[0] MPI startup(): Gather: 2: 0-2147483647 & 17-2147483647
[0] MPI startup(): Gatherv: 2: 0-2147483647 & 0-16
[0] MPI startup(): Gatherv: 2: 0-2147483647 & 17-2147483647
p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 11.0px Menlo; color: #000000; background-color: #ffffff}
span.s1 {font-variant-ligatures: no-common-ligatures}
[0] MPI startup(): Reduce_scatter: 5: 0-5 & 0-4
[0] MPI startup(): Reduce_scatter: 1: 5-192 & 0-4
[0] MPI startup(): Reduce_scatter: 3: 192-349 & 0-4
[0] MPI startup(): Reduce_scatter: 1: 349-3268 & 0-4
[0] MPI startup(): Reduce_scatter: 3: 3268-71356 & 0-4
[0] MPI startup(): Reduce_scatter: 2: 71356-513868 & 0-4
[0] MPI startup(): Reduce_scatter: 5: 513868-731452 & 0-4
[0] MPI startup(): Reduce_scatter: 2: 731452-1746615 & 0-4
[0] MPI startup(): Reduce_scatter: 5: 1746615-2485015 & 0-4
[0] MPI startup(): Reduce_scatter: 2: 0-2147483647 & 0-4
[0] MPI startup(): Reduce_scatter: 5: 0-5 & 5-16
[0] MPI startup(): Reduce_scatter: 1: 5-59 & 5-16
[0] MPI startup(): Reduce_scatter: 5: 59-99 & 5-16
[0] MPI startup(): Reduce_scatter: 3: 99-198 & 5-16
[0] MPI startup(): Reduce_scatter: 1: 198-360 & 5-16
[0] MPI startup(): Reduce_scatter: 3: 360-3606 & 5-16
[0] MPI startup(): Reduce_scatter: 2: 3606-4631 & 5-16
[0] MPI startup(): Reduce_scatter: 3: 0-2147483647 & 5-16
[0] MPI startup(): Reduce_scatter: 5: 0-22 & 17-2147483647
[0] MPI startup(): Reduce_scatter: 1: 22-44 & 17-2147483647
[0] MPI startup(): Reduce_scatter: 5: 44-278 & 17-2147483647
[0] MPI startup(): Reduce_scatter: 3: 278-3517 & 17-2147483647
[0] MPI startup(): Reduce_scatter: 5: 3517-4408 & 17-2147483647
[0] MPI startup(): Reduce_scatter: 3: 0-2147483647 & 17-2147483647
[0] MPI startup(): Reduce: 4: 4-5 & 0-4
[0] MPI startup(): Reduce: 1: 6-59 & 0-4
[0] MPI startup(): Reduce: 2: 60-188 & 0-4
[0] MPI startup(): Reduce: 6: 189-362 & 0-4
[0] MPI startup(): Reduce: 2: 363-7776 & 0-4
[0] MPI startup(): Reduce: 5: 7777-151371 & 0-4
[0] MPI startup(): Reduce: 1: 0-2147483647 & 0-4
[0] MPI startup(): Reduce: 4: 4-60 & 5-16
[0] MPI startup(): Reduce: 3: 61-88 & 5-16
[0] MPI startup(): Reduce: 4: 89-245 & 5-16
[0] MPI startup(): Reduce: 3: 246-256 & 5-16
[0] MPI startup(): Reduce: 4: 257-8192 & 5-16
[0] MPI startup(): Reduce: 3: 8192-1048576 & 5-16
[0] MPI startup(): Reduce: 3: 0-2147483647 & 5-16
[0] MPI startup(): Reduce: 4: 4-8192 & 17-2147483647
[0] MPI startup(): Reduce: 3: 8192-1048576 & 17-2147483647
[0] MPI startup(): Reduce: 3: 0-2147483647 & 17-2147483647
[0] MPI startup(): Scan: 0: 0-2147483647 & 0-2147483647
[0] MPI startup(): Scatter: 2: 1-7 & 0-16
[0] MPI startup(): Scatter: 3: 8-9 & 0-16
[0] MPI startup(): Scatter: 2: 10-64 & 0-16
[0] MPI startup(): Scatter: 3: 65-372 & 0-16
[0] MPI startup(): Scatter: 2: 373-811 & 0-16
[0] MPI startup(): Scatter: 3: 812-115993 & 0-16
[0] MPI startup(): Scatter: 2: 115994-173348 & 0-16
[0] MPI startup(): Scatter: 3: 0-2147483647 & 0-16
[0] MPI startup(): Scatter: 1: 1-1 & 17-2147483647
[0] MPI startup(): Scatter: 2: 2-76 & 17-2147483647
[0] MPI startup(): Scatter: 3: 77-435 & 17-2147483647
[0] MPI startup(): Scatter: 2: 436-608 & 17-2147483647
[0] MPI startup(): Scatter: 3: 0-2147483647 & 17-2147483647
[0] MPI startup(): Scatterv: 1: 0-2147483647 & 0-2147483647
[5] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[1] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[7] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[2] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[6] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[3] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[13] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[4] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[9] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[14] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[11] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[15] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[8] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[12] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[10] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[0] MPI startup(): Rank Pid Node name Pin cpu
[0] MPI startup(): 0 10691 ip-10-0-0-189 0
[0] MPI startup(): 1 10692 ip-10-0-0-189 1
[0] MPI startup(): 2 10693 ip-10-0-0-189 2
[0] MPI startup(): 3 10694 ip-10-0-0-189 3
[0] MPI startup(): 4 10320 ip-10-0-0-174 0
[0] MPI startup(): 5 10321 ip-10-0-0-174 1
[0] MPI startup(): 6 10322 ip-10-0-0-174 2
[0] MPI startup(): 7 10323 ip-10-0-0-174 3
[0] MPI startup(): 8 10273 ip-10-0-0-104 0
[0] MPI startup(): 9 10274 ip-10-0-0-104 1
[0] MPI startup(): 10 10275 ip-10-0-0-104 2
[0] MPI startup(): 11 10276 ip-10-0-0-104 3
[0] MPI startup(): 12 10312 ip-10-0-0-158 0
[0] MPI startup(): 13 10313 ip-10-0-0-158 1
[0] MPI startup(): 14 10314 ip-10-0-0-158 2
[0] MPI startup(): 15 10315 ip-10-0-0-158 3
[0] MPI startup(): Recognition=2 Platform(code=32 ippn=2 dev=5) Fabric(intra=1 inter=6 flags=0x0)
[0] MPI startup(): I_MPI_DEBUG=6
[0] MPI startup(): I_MPI_HYDRA_UUID=bb290000-2b37-e5b2-065d-050000bd0a00
[0] MPI startup(): I_MPI_INFO_NUMA_NODE_NUM=1
[0] MPI startup(): I_MPI_PIN_MAPPING=4:0 0,1 1,2 2,3 3