geo_mpi_partition.cc
Go to the documentation of this file.
// call to parmetis
#include "rheolef/config.h"
#ifdef _RHEOLEF_HAVE_MPI
#include "rheolef/geo_element.h"
#include "rheolef/hack_array.h"
#if defined(_RHEOLEF_HAVE_SCOTCH)
#include "geo_partition_scotch.h"
#elif defined(_RHEOLEF_HAVE_PARMETIS)
#include <parmetis.h>
typedef idx_t idxtype; // change from previous parmetis versions
#endif // _RHEOLEF_HAVE_PARMETIS
namespace rheolef {
disarray<size_t>
geo_mpi_partition (
const std::array<hack_array<geo_element_hack>,reference_element::max_variant>&
ios_geo_element,
const distributor& ios_ownership, // by_dimension
size_t map_dim,
size_t dis_nv)
{
typedef disarray<int>::size_type size_type;
typedef hack_array<geo_element_hack>::const_iterator const_iterator_by_variant;
communicator comm = ios_ownership.comm();
size_type nproc = comm.size();
size_type my_proc = comm.rank();
disarray<size_type> partition (ios_ownership);
size_type iproc = 0;
if (partition.dis_size() <= nproc || dis_nv <= nproc) {
// mesh is very small (nelts <= nproc): parmetis does not support it !
for (size_type ie = 0, nie = ios_ownership.size(); ie < nie; ie++) {
partition [ie] = my_proc;
}
return partition;
}
// here, mesh is not so small: sufficient for parmetis
disarray<idxtype> part (ios_ownership);
std::vector<idxtype> elmdist (nproc+1);
std::copy (
ios_ownership.begin(),
ios_ownership.end(),
elmdist.begin());
std::vector<idxtype> eptr (ios_ownership.dis_size()+1);
{
size_type ie = 0;
eptr [0] = 0;
for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
iter != last; iter++, ie++) {
const geo_element& K = *iter;
eptr [ie+1] = eptr[ie] + K.size();
}
}
}
std::vector<idxtype> eind (eptr [ios_ownership.size()]);
std::vector<idxtype>::iterator iter_eind = eind.begin();
{
for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
iter != last; iter++) {
const geo_element& K = *iter;
for (size_type iloc = 0; iloc < K.size(); iloc++, iter_eind++) {
*iter_eind = K[iloc];
}
}
}
}
idxtype wgtflag = 0;
idxtype numflag = 0;
idxtype mgcnum = map_dim;
idxtype ncon = 1;
idxtype nparts = nproc;
std::vector<float> tpwgts (nparts*ncon);
std::fill (tpwgts.begin(), tpwgts.end(), 1./nparts);
float ubvec [12];
std::fill (ubvec, ubvec+ncon, 1.05);
idxtype options [10];
options[0] = 1;
const int pvm3_option_dbglvl = 1;
const int pvm3_option_seed = 2;
options[pvm3_option_dbglvl] = 0; // otherwise: timming print to stdout...
options[pvm3_option_seed] = 0;
idxtype edgecut;
MPI_Comm raw_comm = comm;
idxtype *elmwgt = 0;
#if defined(_RHEOLEF_HAVE_SCOTCH)
geo_partition_scotch (
elmdist.begin().operator->(),
eptr.begin().operator->(),
eind,
elmwgt,
&ncon,
&mgcnum,
&nparts,
tpwgts.begin().operator->(),
ubvec,
&edgecut,
part.begin().operator->(),
comm);
#elif defined(_RHEOLEF_HAVE_PARMETIS)
ParMETIS_V3_PartMeshKway(
elmdist.begin().operator->(),
eptr.begin().operator->(),
eind.begin().operator->(),
NULL,
&wgtflag,
&numflag,
&ncon,
&mgcnum,
&nparts,
tpwgts.begin().operator->(),
ubvec,
options,
&edgecut,
part.begin().operator->(),
&raw_comm);
#else // _RHEOLEF_HAVE_PARMETIS
# error either parmetis nor scotch partitioner founded
#endif // _RHEOLEF_HAVE_PARMETIS
// convert int to size_type (64 bits : not the same sizeof...)
std::copy (part.begin(), part.end(), partition.begin());
return partition;
}
} // namespace rheolef
#endif // _RHEOLEF_HAVE_MPI
size_type
field::size_type size_type
Definition: branch.cc:425
rheolef::disarray< size_type >
rheolef::disarray::size_type
rep::base::size_type size_type
Definition: disarray.h:463
rheolef::distributor
see the distributor page for the full documentation
Definition: distributor.h:62
rheolef::distributor::dis_size
size_type dis_size() const
global and local sizes
Definition: distributor.h:207
rheolef::distributor::size
size_type size(size_type iproc) const
Definition: distributor.h:163
rheolef::distributor::comm
const communicator_type & comm() const
Definition: distributor.h:145
rheolef::geo_element
see the geo_element page for the full documentation
Definition: geo_element.h:102
rheolef::geo_element::size
size_type size() const
Definition: geo_element.h:168
rheolef::hack_array
Definition: hack_array.h:346
rheolef::hack_array::const_iterator
std::vector< T, A >::const_iterator const_iterator
Definition: hack_array.h:351
rheolef::reference_element::max_variant
static const variant_type max_variant
Definition: reference_element.h:82
rheolef::reference_element::last_variant_by_dimension
static variant_type last_variant_by_dimension(size_type dim)
Definition: reference_element.h:150
rheolef::reference_element::first_variant_by_dimension
static variant_type first_variant_by_dimension(size_type dim)
Definition: reference_element.h:148
rheolef::size_type
size_t size_type
Definition: basis_get.cc:76
rheolef::idxtype
my_idxtype idxtype
Definition: geo_partition_scotch.h:30
geo_partition_scotch.h
mkgeo_ball.variant
int variant
Definition: mkgeo_ball.sh:149
mkgeo_ball.map_dim
map_dim
Definition: mkgeo_ball.sh:337
rheolef
This file is part of Rheolef.
Definition: compiler_eigen.h:37
rheolef::geo_mpi_partition
disarray< size_t > geo_mpi_partition(const std::array< hack_array< geo_element_hack >, reference_element::max_variant > &ios_geo_element, const distributor &ownership_by_dimension, size_t map_dim, size_t dis_nv)
Definition: geo_mpi_partition.cc:38
rheolef::geo_partition_scotch
void geo_partition_scotch(my_idxtype *elmdist, my_idxtype *eptr, vector< my_idxtype > &eind, my_idxtype *elmwgt, int *ncon, int *ncommonnodes, int *nparts, float *tpwgts, float *ubvec, int *edgecut, my_idxtype *part, const mpi::communicator &comm)
Definition: geo_mpi_partition_scotch.cc:48
rheolef::Vector::end
iterator end()
Definition: Vector.h:225
rheolef::Vector::begin
iterator begin()
Definition: Vector.h:217