Loading...
Searching...
No Matches
globalindices.hh
Go to the documentation of this file.
// -*- mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
// vi: set et ts=4 sw=4 sts=4:
/*
This file is part of the Open Porous Media project (OPM).
OPM is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
OPM is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with OPM. If not, see <http://www.gnu.org/licenses/>.
Consult the COPYING file in the top-level source directory of this
module for the precise wording of the license and the list of
copyright holders.
*/
#ifndef EWOMS_GLOBAL_INDICES_HH
#define EWOMS_GLOBAL_INDICES_HH
#include <dune/grid/common/datahandleif.hh>
#include <dune/istl/bcrsmatrix.hh>
#include <dune/istl/scalarproducts.hh>
#include <dune/istl/operators.hh>
#include <algorithm>
#include <set>
#include <map>
#include <iostream>
#include <tuple>
#if HAVE_MPI
#include <mpi.h>
#endif
#include "overlaptypes.hh"
namespace Opm {
namespace Linear {
template <class ForeignOverlap>
class GlobalIndices
{
GlobalIndices(const GlobalIndices& ) = delete;
using GlobalToDomesticMap = std::map<Index, Index>;
using DomesticToGlobalMap = std::map<Index, Index>;
public:
explicit GlobalIndices(const ForeignOverlap& foreignOverlap)
: foreignOverlap_(foreignOverlap)
{
myRank_ = 0;
mpiSize_ = 1;
#if HAVE_MPI
{
int tmp;
MPI_Comm_rank(MPI_COMM_WORLD, &tmp);
myRank_ = static_cast<ProcessRank>(tmp);
MPI_Comm_size(MPI_COMM_WORLD, &tmp);
mpiSize_ = static_cast<size_t>(tmp);
}
#endif
// calculate the domestic overlap (i.e. all overlap indices in
// foreign processes which the current process overlaps.)
// This requires communication via MPI.
buildGlobalIndices_();
}
Index domesticToGlobal(Index domesticIdx) const
{
assert(domesticToGlobal_.find(domesticIdx) != domesticToGlobal_.end());
return domesticToGlobal_.find(domesticIdx)->second;
}
Index globalToDomestic(Index globalIdx) const
{
const auto& tmp = globalToDomestic_.find(globalIdx);
if (tmp == globalToDomestic_.end())
return -1;
return tmp->second;
}
size_t numLocal() const
{ return foreignOverlap_.numLocal(); }
size_t numDomestic() const
{ return numDomestic_; }
void addIndex(Index domesticIdx, Index globalIdx)
{
domesticToGlobal_[domesticIdx] = globalIdx;
globalToDomestic_[globalIdx] = domesticIdx;
numDomestic_ = domesticToGlobal_.size();
assert(domesticToGlobal_.size() == globalToDomestic_.size());
}
void sendBorderIndex([[maybe_unused]] ProcessRank peerRank,
[[maybe_unused]] Index domesticIdx,
[[maybe_unused]] Index peerLocalIdx)
{
#if HAVE_MPI
PeerIndexGlobalIndex sendBuf;
sendBuf.peerIdx = peerLocalIdx;
sendBuf.globalIdx = domesticToGlobal(domesticIdx);
MPI_Send(&sendBuf, // buff
sizeof(PeerIndexGlobalIndex), // count
MPI_BYTE, // data type
static_cast<int>(peerRank), // peer process
0, // tag
MPI_COMM_WORLD); // communicator
#endif
}
void receiveBorderIndex([[maybe_unused]] ProcessRank peerRank)
{
#if HAVE_MPI
PeerIndexGlobalIndex recvBuf;
MPI_Recv(&recvBuf, // buff
sizeof(PeerIndexGlobalIndex), // count
MPI_BYTE, // data type
static_cast<int>(peerRank), // peer process
0, // tag
MPI_COMM_WORLD, // communicator
MPI_STATUS_IGNORE); // status
Index domesticIdx = foreignOverlap_.nativeToLocal(recvBuf.peerIdx);
if (domesticIdx >= 0) {
Index globalIdx = recvBuf.globalIdx;
addIndex(domesticIdx, globalIdx);
}
#endif // HAVE_MPI
}
bool hasGlobalIndex(Index globalIdx) const
{ return globalToDomestic_.find(globalIdx) != globalToDomestic_.end(); }
void print() const
{
std::cout << "(domestic index, global index, domestic->global->domestic)"
<< " list for rank " << myRank_ << "\n";
for (size_t domIdx = 0; domIdx < domesticToGlobal_.size(); ++domIdx)
std::cout << "(" << domIdx << ", " << domesticToGlobal(domIdx)
<< ", " << globalToDomestic(domesticToGlobal(domIdx)) << ") ";
std::cout << "\n" << std::flush;
}
protected:
// retrieve the offset for the indices where we are master in the
// global index list
void buildGlobalIndices_()
{
#if HAVE_MPI
numDomestic_ = 0;
#else
numDomestic_ = foreignOverlap_.numLocal();
#endif
#if HAVE_MPI
if (myRank_ == 0) {
// the first rank starts at index zero
domesticOffset_ = 0;
}
else {
// all other ranks retrieve their offset from the next
// lower rank
MPI_Recv(&domesticOffset_, // buffer
1, // count
MPI_INT, // data type
static_cast<int>(myRank_ - 1), // peer rank
0, // tag
MPI_COMM_WORLD, // communicator
MPI_STATUS_IGNORE);
}
// create maps for all indices for which the current process
// is the master
int numMaster = 0;
for (unsigned i = 0; i < foreignOverlap_.numLocal(); ++i) {
if (!foreignOverlap_.iAmMasterOf(static_cast<Index>(i)))
continue;
addIndex(static_cast<Index>(i),
static_cast<Index>(domesticOffset_ + numMaster));
++numMaster;
}
if (myRank_ < mpiSize_ - 1) {
// send the domestic offset plus the number of master
// indices to the process which is one rank higher
int tmp = domesticOffset_ + numMaster;
MPI_Send(&tmp, // buff
1, // count
MPI_INT, // data type
static_cast<int>(myRank_ + 1), // peer rank
0, // tag
MPI_COMM_WORLD); // communicator
}
typename PeerSet::const_iterator peerIt;
typename PeerSet::const_iterator peerEndIt = peerSet_().end();
// receive the border indices from the lower ranks
peerIt = peerSet_().begin();
for (; peerIt != peerEndIt; ++peerIt) {
if (*peerIt < myRank_)
receiveBorderFrom_(*peerIt);
}
// send the border indices to the higher ranks
peerIt = peerSet_().begin();
for (; peerIt != peerEndIt; ++peerIt) {
if (*peerIt > myRank_)
sendBorderTo_(*peerIt);
}
// receive the border indices from the higher ranks
peerIt = peerSet_().begin();
for (; peerIt != peerEndIt; ++peerIt) {
if (*peerIt > myRank_)
receiveBorderFrom_(*peerIt);
}
// send the border indices to the lower ranks
peerIt = peerSet_().begin();
for (; peerIt != peerEndIt; ++peerIt) {
if (*peerIt < myRank_)
sendBorderTo_(*peerIt);
}
#endif // HAVE_MPI
}
void sendBorderTo_([[maybe_unused]] ProcessRank peerRank)
{
#if HAVE_MPI
// send (local index on myRank, global index) pairs to the
// peers
BorderList::const_iterator borderIt = borderList_().begin();
BorderList::const_iterator borderEndIt = borderList_().end();
for (; borderIt != borderEndIt; ++borderIt) {
ProcessRank borderPeer = borderIt->peerRank;
BorderDistance borderDistance = borderIt->borderDistance;
if (borderPeer != peerRank || borderDistance != 0)
continue;
Index localIdx = foreignOverlap_.nativeToLocal(borderIt->localIdx);
Index peerIdx = borderIt->peerIdx;
assert(localIdx >= 0);
if (foreignOverlap_.iAmMasterOf(localIdx)) {
sendBorderIndex(borderPeer, localIdx, peerIdx);
}
}
#endif // HAVE_MPI
}
void receiveBorderFrom_([[maybe_unused]] ProcessRank peerRank)
{
#if HAVE_MPI
// retrieve the global indices for which we are not master
// from the processes with lower rank
BorderList::const_iterator borderIt = borderList_().begin();
BorderList::const_iterator borderEndIt = borderList_().end();
for (; borderIt != borderEndIt; ++borderIt) {
ProcessRank borderPeer = borderIt->peerRank;
BorderDistance borderDistance = borderIt->borderDistance;
if (borderPeer != peerRank || borderDistance != 0)
continue;
Index nativeIdx = borderIt->localIdx;
Index localIdx = foreignOverlap_.nativeToLocal(nativeIdx);
if (localIdx >= 0 && foreignOverlap_.masterRank(localIdx) == borderPeer)
receiveBorderIndex(borderPeer);
}
#endif // HAVE_MPI
}
const PeerSet& peerSet_() const
{ return foreignOverlap_.peerSet(); }
const BorderList& borderList_() const
{ return foreignOverlap_.borderList(); }
ProcessRank myRank_;
size_t mpiSize_;
int domesticOffset_;
size_t numDomestic_;
const ForeignOverlap& foreignOverlap_;
GlobalToDomesticMap globalToDomestic_;
DomesticToGlobalMap domesticToGlobal_;
};
} // namespace Linear
} // namespace Opm
#endif
Opm::Linear::GlobalIndices::print
void print() const
Prints the global indices of all domestic indices for debugging purposes.
Definition globalindices.hh:190
Opm::Linear::GlobalIndices::addIndex
void addIndex(Index domesticIdx, Index globalIdx)
Add an index to the domestic<->global mapping.
Definition globalindices.hh:127
Opm::Linear::GlobalIndices::numDomestic
size_t numDomestic() const
Returns the number domestic indices.
Definition globalindices.hh:121
Opm::Linear::GlobalIndices::domesticToGlobal
Index domesticToGlobal(Index domesticIdx) const
Converts a domestic index to a global one.
Definition globalindices.hh:88
Opm::Linear::GlobalIndices::sendBorderIndex
void sendBorderIndex(ProcessRank peerRank, Index domesticIdx, Index peerLocalIdx)
Send a border index to a remote process.
Definition globalindices.hh:139
Opm::Linear::GlobalIndices::receiveBorderIndex
void receiveBorderIndex(ProcessRank peerRank)
Receive an index on the border from a remote process and add it the translation maps.
Definition globalindices.hh:160
Opm::Linear::GlobalIndices::globalToDomestic
Index globalToDomestic(Index globalIdx) const
Converts a global index to a domestic one.
Definition globalindices.hh:98
Opm::Linear::GlobalIndices::hasGlobalIndex
bool hasGlobalIndex(Index globalIdx) const
Return true iff a given global index already exists.
Definition globalindices.hh:183
Opm::Linear::GlobalIndices::numLocal
size_t numLocal() const
Returns the number of indices which are in the interior or on the border of the current rank.
Definition globalindices.hh:112
Opm
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition blackoilbioeffectsmodules.hh:43
overlaptypes.hh
This files provides several data structures for storing tuples of indices of remote and/or local proc...
Opm::Linear::ProcessRank
unsigned ProcessRank
The type of the rank of a process.
Definition overlaptypes.hh:49
Opm::Linear::BorderDistance
unsigned BorderDistance
The type representing the distance of an index to the border.
Definition overlaptypes.hh:54
Opm::Linear::Index
int Index
The type of an index of a degree of freedom.
Definition overlaptypes.hh:44
Opm::Linear::BorderList
std::list< BorderIndex > BorderList
This class managages a list of indices which are on the border of a process' partition of the grid.
Definition overlaptypes.hh:120
Opm::Linear::PeerIndexGlobalIndex
This structure stores a local index on a peer process and a global index.
Definition overlaptypes.hh:70