EsysMPI.h
Go to the documentation of this file.
/*****************************************************************************
*
* Copyright (c) 2003-2017 by The University of Queensland
* http://www.uq.edu.au
*
* Primary Business: Queensland, Australia
* Licensed under the Apache License, version 2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Development until 2012 by Earth Systems Science Computational Center (ESSCC)
* Development 2012-2013 by School of Earth Sciences
* Development from 2014 by Centre for Geoscience Computing (GeoComp)
*
*****************************************************************************/
#ifndef __ESCRIPT_ESYSMPI_H__
#define __ESCRIPT_ESYSMPI_H__
#include <escript/DataTypes.h>
#include <ctime>
#include <sstream>
#include <boost/shared_ptr.hpp>
#ifdef _OPENMP
#include <omp.h>
#endif
#ifdef ESYS_MPI
#include <mpi.h>
#ifdef ESYS_INDEXTYPE_LONG
#define MPI_DIM_T MPI_LONG
#else
#define MPI_DIM_T MPI_INT
#endif
#else
typedef int MPI_Comm;
typedef int MPI_Request;
typedef int MPI_Op;
typedef int MPI_Status;
#define MPI_INT 6
#define MPI_DOUBLE 11
#define MPI_COMM_WORLD 91
#define MPI_COMM_NULL 0
// MPI_Op replacements for non-MPI - these values are arbitrary
#define MPI_SUM 100
#define MPI_MIN 101
#define MPI_MAX 102
#define MPI_OP_NULL 17
// end MPI_op
#endif // ESYS_MPI
namespace escript {
inline int getSubWorldTag()
{
return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
}
class JMPI_;
typedef boost::shared_ptr<JMPI_> JMPI;
JMPI makeInfo(MPI_Comm comm, bool owncom=false);
class JMPI_
{
public:
~JMPI_();
DataTypes::dim_t setDistribution(DataTypes::index_t min_id,
DataTypes::index_t max_id,
DataTypes::index_t* distribution);
void split(DataTypes::dim_t N, DataTypes::dim_t* local_N,
DataTypes::index_t* offset);
inline int mod_rank(int k) const
{
int out=0;
#ifdef ESYS_MPI
if (size > 1) {
const int q = k/size;
if (k > 0) {
out=k-size*q;
} else if (k < 0) {
out=k-size*(q-1);
}
}
#endif
return out;
}
inline std::string appendRankToFileName(const std::string& fileName) const
{
#ifdef ESYS_MPI
if (size > 1) {
std::stringstream ss;
ss << fileName << '.';
ss.fill('0');
ss.width(4);
ss << rank;
return ss.str();
}
#endif
return fileName;
}
inline int counter() const
{
return msg_tag_counter;
}
inline void incCounter(int i=1)
{
msg_tag_counter+=i;
// there is no particular significance here other than being 7 digits
// and prime (because why not). It just needs to be big.
msg_tag_counter %= 1010201;
}
inline void setCounter(int value)
{
msg_tag_counter = value%1010201;
}
inline bool isValid() const
{
return comm!=MPI_COMM_NULL;
}
int size;
int rank;
MPI_Comm comm;
private:
JMPI_(MPI_Comm comm, bool owncomm);
friend JMPI makeInfo(MPI_Comm comm, bool owncom);
bool ownscomm;
int msg_tag_counter;
};
// Does not cope with nested calls
class NoCOMM_WORLD
{
public:
NoCOMM_WORLD();
~NoCOMM_WORLD();
static bool active();
};
bool checkResult(int input, int& output, const JMPI& comm);
bool shipString(const char* src, char** dest, MPI_Comm& comm);
inline double gettime()
{
double out;
#ifdef ESYS_MPI
out = MPI_Wtime();
#else
#ifdef _OPENMP
out=omp_get_wtime();
#else
out=((double) clock())/CLOCKS_PER_SEC;
#endif
#endif
return out;
}
} // namespace escript
#endif // __ESCRIPT_ESYSMPI_H__
escript::JMPI_::ownscomm
bool ownscomm
Definition: EsysMPI.h:162
escript::shipString
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition: EsysMPI.cpp:163
escript::getSubWorldTag
int getSubWorldTag()
tag reserved for use by SubWorld code This value should be higher than the modulus used in JMPI_::set...
Definition: EsysMPI.h:66
escript::JMPI_
Definition: EsysMPI.h:79
MPI_Op
int MPI_Op
Definition: EsysMPI.h:43
escript::JMPI_::isValid
bool isValid() const
returns true if this has a valid MPI communicator
Definition: EsysMPI.h:149
MPI_Status
int MPI_Status
Definition: EsysMPI.h:44
escript::makeInfo
JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:28
escript
Definition: AbstractContinuousDomain.cpp:22
escript::JMPI_::appendRankToFileName
std::string appendRankToFileName(const std::string &fileName) const
appends MPI rank to a file name if MPI size > 1
Definition: EsysMPI.h:112
escript::NoCOMM_WORLD
Definition: EsysMPI.h:167
escript::JMPI
boost::shared_ptr< JMPI_ > JMPI
Definition: EsysMPI.h:71
escript::JMPI_::JMPI_
JMPI_(MPI_Comm comm, bool owncomm)
Definition: EsysMPI.cpp:37
escript::JMPI_::counter
int counter() const
returns the current value of the message tag counter
Definition: EsysMPI.h:128
escript::JMPI_::mod_rank
int mod_rank(int k) const
Definition: EsysMPI.h:95
escript::JMPI_::makeInfo
friend JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:28
escript::JMPI_::rank
int rank
Definition: EsysMPI.h:155
escript::JMPI_::comm
MPI_Comm comm
Definition: EsysMPI.h:156
escript::checkResult
bool checkResult(int res, int &mres, const JMPI &info)
Everyone puts in their error code and everyone gets the largest one.
Definition: EsysMPI.cpp:109
escript::JMPI_::setDistribution
DataTypes::dim_t setDistribution(DataTypes::index_t min_id, DataTypes::index_t max_id, DataTypes::index_t *distribution)
Definition: EsysMPI.cpp:64
escript::DataTypes::index_t
int index_t
type for array/matrix indices used both globally and on each rank
Definition: DataTypes.h:59
escript::JMPI_::size
int size
Definition: EsysMPI.h:154
escript::JMPI_::~JMPI_
~JMPI_()
Definition: EsysMPI.cpp:56
escript::JMPI_::setCounter
void setCounter(int value)
sets the message tag counter to value
Definition: EsysMPI.h:143
MPI_COMM_NULL
#define MPI_COMM_NULL
Definition: EsysMPI.h:48
paso::N
static dim_t N
Definition: SparseMatrix_saveHB.cpp:37
MPI_Request
int MPI_Request
Definition: EsysMPI.h:42
escript::JMPI_::msg_tag_counter
int msg_tag_counter
Definition: EsysMPI.h:163
MPI_Comm
int MPI_Comm
Definition: EsysMPI.h:41
escript::JMPI_::incCounter
void incCounter(int i=1)
increments the message tag counter by i
Definition: EsysMPI.h:134
escript::JMPI_::split
void split(DataTypes::dim_t N, DataTypes::dim_t *local_N, DataTypes::index_t *offset)
Definition: EsysMPI.cpp:91
escript::gettime
double gettime()
returns the current ticks for timing
Definition: EsysMPI.h:184
escript::DataTypes::dim_t
index_t dim_t
Definition: DataTypes.h:64