//  run with MPI:  ff-mpirun -np 4 script.edp
// NBPROC 4

verbosity=2;
cout << " rank = " << mpirank << " " << " size " << mpisize << endl;
cout << " rank = " << mpiRank(mpiCommWorld) << " " << " size " << mpiSize(mpiCommWorld) << endl;
if(mpisize>4)
  { 
    int[int] procs=[1,4];
    mpiGroup gpr(procs);
    cout << " before  ...  " << mpirank << endl;
    mpiComm comm(gpr);
    cout << " after ...  " << mpirank << "  in : " << bool(comm)   
	 << " rank " << mpiRank(comm) << " / " << mpiSize(comm) <<endl;
    mpiComm  n0comm(mpiCommWorld,mpirank%3,0);// MPI_Comm_split
    mpiComm  n1comm(mpiCommWorld,mpirank%3,1);// MPI_Comm_split
    mpiComm  n2comm(mpiCommWorld,mpirank%3,2);// MPI_Comm_split
    if( ! n1comm)  // not in n1comm 
      {
	// MPI_Intercomm_create constructor 
	mpiComm  nicomm(processor(n1comm,1),processor(n2comm,2),1000); 
	mpiComm  n12comm(nicomm,true);  // MPI_Intercomm_merge cosntructor 
      }
  }

if ( mpirank<2 && mpisize>=2)
  { 
    
    if( mpirank==0) 
      {
	cout << mpirank << " send to 1 " << endl;
	processor(1) << 123456; //  unblock send
      }
    else if (mpirank==1) 
      {
	int k;
	processor(0) >> k; // block recv 
	cout << " recived " << k << endl;
      }
    {
      // UnBlock communication 
      mpiRequest rq;
      if( mpirank==0) 
	{
	  Isend(processor(1,rq),16.);
	  mpiWait(rq);// Send
	}
      else if (mpirank==1) 
	{
	  real  k;
	  Irecv(processor(0,rq),k);
	  mpiWait(rq);// Recv 
	  assert(k==16.);// verif..
	}
    }


    {
    //  Block Communication
      if( mpirank==0) 
	  Send(processor(1),16.);
      else if (mpirank==1) 
	{
	  real  k;
	  Recv(processor(0),k);
	  assert(k==16.);// verif..
	}
    }

  }
int l= mpirank+100;
cout << " l == " << l << " broadcast l from 0   mpirank = " << mpirank << endl;
broadcast(processor(0),l);
cout << " l == " << l << "  mpirank ="<< mpirank << endl;

{
matrix<complex> A;
matrix<complex>[int]  B(10);
if( mpirank==0)
   A=[[1+1i,1i],[0,2-1i],[0,2-1i]];
broadcast(processor(0),A);


if (mpirank==1)
  { A=A+A;
    processor(0) << A;
  }
 else if (mpirank==0)
   processor(1) >>  B[3];

cout << " mpirank = " << mpirank << " A =  " << endl;
if (mpirank==0)
cout <<  " **** " <<  B[3] << endl;

cout << " " <<  norm(A(1,1) - 2-1i) << endl;
mpiRequest rr;
if (mpirank==1) Isend(processor(0,mpiCommWorld,rr),A);
else if (mpirank==0) Irecv(processor(1,mpiCommWorld,rr),B[3]);
mpiWait(rr);
if (mpirank==0)  cout << " B3= " << B[3] << endl;
if (mpirank==1)  cout << " A= " << A << endl;

}

// asyncronous  send/recv  messages....
mpiRequest rr;
real[int] vv(mpirank*mpisize:mpirank*mpisize+mpisize-1),ww(mpisize);
processor(0,rr) << vv;
if(mpirank==0)
  {
    mpiRequest[int] rq(mpisize);
    cout << " --- in ++ " <<mpisize << endl;
    for (int i=0;i<mpisize;++i)
       processor(i,rq[i]) >> ww;
    cout << " --- out ++ " <<mpisize << endl;

    int k=0;
    for (int i=0;i<mpisize;++i)
      {
	int k= mpiWaitAny(rq);
	cout << k << "  mpiWaitAny " << ww << endl; 
      }
  }
cout << " mpiWait " << mpiWait(rr) << " " << mpirank << endl;
//  a way to get data from any proc ... 
processor(0,rr) << vv;
if(mpirank==0)
  {
    for (int i=0;i<mpisize;++i)
      {
	processor(mpiAnySource) >> ww;  // correction jan 2024  mpiAnySource==-2 in mpich 
	cout << " case " << i << "   " <<  ww[0] << endl ;
      }
  }


mpiAlltoall(vv,ww);
cout << " all2all " << mpirank << " :  "  ;
for(int i=0;i<mpisize;i++) 
  cout << vv[i] << " ";
cout << " -> "  ;
for(int i=0;i<mpisize;i++) 
  cout << ww[i] << " ";
cout << endl; 


real s0=mpiWtime();
real[int,int] aa(10,mpisize),bb(10,mpisize);
real [int,int] ss(10,mpisize);
for(int i=0;i<10;i++)
  for(int j=0;j<mpisize;j++) 
    aa(i,j)=100*i+j;
mpiAlltoall(aa,bb);
mpiAlltoall(aa,bb,mpiCommWorld);
mpiAllgather(aa(:,mpirank),ss);
if(mpirank == 1) // just printing the result of one process ..
    cout << " AllGather " << ss << endl;
mpiGather (aa(:,mpirank),ss,processor(0,mpiCommWorld));
if(mpirank == 0)
    cout << " Gather " << ss << endl;

mpiScatter(aa,bb(:,mpirank),processor(0,mpiCommWorld));

cout << " all2all " << mpirank << " " << mpiWtime()-s0 << "s :  "  ;
cout << mpirank << " aa = " << aa << endl;
cout << mpirank << " bb = " << bb << endl;


mpiBarrier(mpiCommWorld);
