Loading...
Searching...
No Matches
indicessyncer.hh
Go to the documentation of this file.
// -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
// vi: set et ts=4 sw=2 sts=2:
// SPDX-FileCopyrightInfo: Copyright © DUNE Project contributors, see file LICENSE.md in module root
// SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
#ifndef DUNE_COMMON_PARALLEL_INDICESSYNCER_HH
#define DUNE_COMMON_PARALLEL_INDICESSYNCER_HH
#if HAVE_MPI
#include <algorithm>
#include <cassert>
#include <cmath>
#include <functional>
#include <limits>
#include <map>
#include <tuple>
#include <mpi.h>
#include <dune/common/stdstreams.hh>
#include <dune/common/sllist.hh>
#include <dune/common/parallel/indexset.hh>
#include <dune/common/parallel/remoteindices.hh>
namespace Dune
{
template<typename T>
class IndicesSyncer
{
public:
typedef T ParallelIndexSet;
typedef typename ParallelIndexSet::IndexPair IndexPair;
typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
typedef typename ParallelIndexSet::LocalIndex::Attribute Attribute;
typedef Dune::RemoteIndices<ParallelIndexSet> RemoteIndices;
IndicesSyncer(ParallelIndexSet& indexSet,
RemoteIndices& remoteIndices);
void sync();
template<typename T1>
void sync(T1& numberer, bool useFixedOrder = false);
private:
ParallelIndexSet& indexSet_;
RemoteIndices& remoteIndices_;
char** sendBuffers_;
char* receiveBuffer_;
std::size_t* sendBufferSizes_;
int receiveBufferSize_; // int because of MPI
struct MessageInformation
{
MessageInformation()
: publish(), pairs()
{}
int publish;
int pairs;
};
class DefaultNumberer
{
public:
std::size_t operator()([[maybe_unused]] const GlobalIndex& global)
{
return std::numeric_limits<size_t>::max();
}
};
MPI_Datatype datatype_;
int rank_;
typedef SLList<std::pair<GlobalIndex,Attribute>, typename RemoteIndices::Allocator> GlobalIndexList;
typedef typename GlobalIndexList::ModifyIterator GlobalIndexModifier;
typedef typename SLList<GlobalIndex, typename RemoteIndices::Allocator>::iterator
GlobalIndexIterator;
typedef std::map<int, GlobalIndexList> GlobalIndicesMap;
GlobalIndicesMap globalMap_;
typedef SLList<bool, typename RemoteIndices::Allocator> BoolList;
typedef typename BoolList::iterator BoolIterator;
typedef typename BoolList::ModifyIterator BoolListModifier;
typedef std::map<int,BoolList> BoolMap;
BoolMap oldMap_;
std::map<int,MessageInformation> infoSend_;
typedef typename RemoteIndices::RemoteIndexList RemoteIndexList;
typedef typename RemoteIndexList::ModifyIterator RemoteIndexModifier;
typedef Dune::RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
typedef typename RemoteIndexList::iterator RemoteIndexIterator;
typedef typename RemoteIndexList::const_iterator ConstRemoteIndexIterator;
typedef std::tuple<RemoteIndexModifier,GlobalIndexModifier,BoolListModifier,
const ConstRemoteIndexIterator> IteratorTuple;
class Iterators
{
friend class IndicesSyncer<T>;
public:
Iterators(RemoteIndexList& remoteIndices, GlobalIndexList& globalIndices,
BoolList& booleans);
Iterators();
Iterators& operator++();
void insert(const RemoteIndex& index,
const std::pair<GlobalIndex,Attribute>& global);
RemoteIndex& remoteIndex() const;
std::pair<GlobalIndex,Attribute>& globalIndexPair() const;
Attribute& attribute() const;
bool isOld() const;
void reset(RemoteIndexList& remoteIndices, GlobalIndexList& globalIndices,
BoolList& booleans);
bool isNotAtEnd() const;
bool isAtEnd() const;
private:
IteratorTuple iterators_;
};
typedef std::map<int,Iterators> IteratorsMap;
IteratorsMap iteratorsMap_;
void calculateMessageSizes();
void packAndSend(int destination, char* buffer, std::size_t bufferSize, MPI_Request& req);
template<typename T1>
void recvAndUnpack(T1& numberer, int hardSource, bool useHardSource);
void registerMessageDatatype();
void insertIntoRemoteIndexList(int process,
const std::pair<GlobalIndex,Attribute>& global,
char attribute);
void resetIteratorsMap();
bool checkReset();
bool checkReset(const Iterators& iterators, RemoteIndexList& rlist, GlobalIndexList& gList,
BoolList& bList);
};
template<typename TG, typename TA>
bool operator<(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
const std::pair<TG,TA>& i2)
{
return i1.global() < i2.first ||
(i1.global() == i2.first && i1.local().attribute()<i2.second);
}
template<typename TG, typename TA>
bool operator<(const std::pair<TG,TA>& i1,
const IndexPair<TG,ParallelLocalIndex<TA> >& i2)
{
return i1.first < i2.global() ||
(i1.first == i2.global() && i1.second<i2.local().attribute());
}
template<typename TG, typename TA>
bool operator==(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
const std::pair<TG,TA>& i2)
{
return (i1.global() == i2.first && i1.local().attribute()==i2.second);
}
template<typename TG, typename TA>
bool operator!=(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
const std::pair<TG,TA>& i2)
{
return (i1.global() != i2.first || i1.local().attribute()!=i2.second);
}
template<typename TG, typename TA>
bool operator==(const std::pair<TG,TA>& i2,
const IndexPair<TG,ParallelLocalIndex<TA> >& i1)
{
return (i1.global() == i2.first && i1.local().attribute()==i2.second);
}
template<typename TG, typename TA>
bool operator!=(const std::pair<TG,TA>& i2,
const IndexPair<TG,ParallelLocalIndex<TA> >& i1)
{
return (i1.global() != i2.first || i1.local().attribute()!=i2.second);
}
template<typename T, typename A, typename A1>
void storeGlobalIndicesOfRemoteIndices(std::map<int,SLList<std::pair<typename T::GlobalIndex, typename T::LocalIndex::Attribute>,A> >& globalMap,
const RemoteIndices<T,A1>& remoteIndices)
{
for(auto remote = remoteIndices.begin(), end =remoteIndices.end(); remote != end; ++remote) {
typedef typename RemoteIndices<T,A1>::RemoteIndexList RemoteIndexList;
typedef SLList<std::pair<typename T::GlobalIndex, typename T::LocalIndex::Attribute>,A> GlobalIndexList;
GlobalIndexList& global = globalMap[remote->first];
RemoteIndexList& rList = *(remote->second.first);
for(auto index = rList.begin(), riEnd = rList.end();
index != riEnd; ++index) {
global.push_back(std::make_pair(index->localIndexPair().global(),
index->localIndexPair().local().attribute()));
}
}
}
template<typename T, typename A, typename A1>
inline void repairLocalIndexPointers(std::map<int,
SLList<std::pair<typename T::GlobalIndex,
typename T::LocalIndex::Attribute>,A> >& globalMap,
RemoteIndices<T,A1>& remoteIndices,
const T& indexSet)
{
assert(globalMap.size()==static_cast<std::size_t>(remoteIndices.neighbours()));
// Repair pointers to index set in remote indices.
auto global = globalMap.begin();
auto end = remoteIndices.remoteIndices_.end();
for(auto remote = remoteIndices.remoteIndices_.begin(); remote != end; ++remote, ++global) {
assert(remote->first==global->first);
assert(remote->second.first->size() == global->second.size());
auto riEnd = remote->second.first->end();
auto rIndex = remote->second.first->begin();
auto gIndex = global->second.begin();
auto index = indexSet.begin();
assert(rIndex==riEnd || gIndex != global->second.end());
while(rIndex != riEnd) {
// Search for the index in the set.
assert(gIndex != global->second.end());
while(!(index->global() == gIndex->first
&& index->local().attribute() == gIndex->second)) {
++index;
// this is only needed for ALU, where there may exist
// more entries with the same global index in the remote index set
// than in the index set
if (index->global() > gIndex->first) {
index=indexSet.begin();
}
}
assert(index != indexSet.end() && *index == *gIndex);
rIndex->localIndex_ = &(*index);
++index;
++rIndex;
++gIndex;
}
}
remoteIndices.sourceSeqNo_ = remoteIndices.source_->seqNo();
remoteIndices.destSeqNo_ = remoteIndices.target_->seqNo();
}
template<typename T>
IndicesSyncer<T>::IndicesSyncer(ParallelIndexSet& indexSet,
RemoteIndices& remoteIndices)
: indexSet_(indexSet), remoteIndices_(remoteIndices)
{
// index sets must match.
assert(remoteIndices.source_ == remoteIndices.target_);
assert(remoteIndices.source_ == &indexSet);
MPI_Comm_rank(remoteIndices_.communicator(), &rank_);
}
template<typename T>
IndicesSyncer<T>::Iterators::Iterators(RemoteIndexList& remoteIndices,
GlobalIndexList& globalIndices,
BoolList& booleans)
: iterators_(remoteIndices.beginModify(), globalIndices.beginModify(),
booleans.beginModify(), remoteIndices.end())
{ }
template<typename T>
IndicesSyncer<T>::Iterators::Iterators()
: iterators_()
{}
template<typename T>
inline typename IndicesSyncer<T>::Iterators& IndicesSyncer<T>::Iterators::operator++()
{
++(std::get<0>(iterators_));
++(std::get<1>(iterators_));
++(std::get<2>(iterators_));
return *this;
}
template<typename T>
inline void IndicesSyncer<T>::Iterators::insert(const RemoteIndex & index,
const std::pair<GlobalIndex,Attribute>& global)
{
std::get<0>(iterators_).insert(index);
std::get<1>(iterators_).insert(global);
std::get<2>(iterators_).insert(false);
}
template<typename T>
inline typename IndicesSyncer<T>::RemoteIndex&
IndicesSyncer<T>::Iterators::remoteIndex() const
{
return *(std::get<0>(iterators_));
}
template<typename T>
inline std::pair<typename IndicesSyncer<T>::GlobalIndex,typename IndicesSyncer<T>::Attribute>&
IndicesSyncer<T>::Iterators::globalIndexPair() const
{
return *(std::get<1>(iterators_));
}
template<typename T>
inline bool IndicesSyncer<T>::Iterators::isOld() const
{
return *(std::get<2>(iterators_));
}
template<typename T>
inline void IndicesSyncer<T>::Iterators::reset(RemoteIndexList& remoteIndices,
GlobalIndexList& globalIndices,
BoolList& booleans)
{
std::get<0>(iterators_) = remoteIndices.beginModify();
std::get<1>(iterators_) = globalIndices.beginModify();
std::get<2>(iterators_) = booleans.beginModify();
}
template<typename T>
inline bool IndicesSyncer<T>::Iterators::isNotAtEnd() const
{
return std::get<0>(iterators_) != std::get<3>(iterators_);
}
template<typename T>
inline bool IndicesSyncer<T>::Iterators::isAtEnd() const
{
return std::get<0>(iterators_) == std::get<3>(iterators_);
}
template<typename T>
void IndicesSyncer<T>::registerMessageDatatype()
{
MPI_Datatype type[2] = {MPI_INT, MPI_INT};
int blocklength[2] = {1,1};
MPI_Aint displacement[2];
MPI_Aint base;
// Compute displacement
MessageInformation message;
MPI_Get_address( &(message.publish), displacement);
MPI_Get_address( &(message.pairs), displacement+1);
// Make the displacement relative
MPI_Get_address(&message, &base);
displacement[0] -= base;
displacement[1] -= base;
MPI_Type_create_struct( 2, blocklength, displacement, type, &datatype_);
MPI_Type_commit(&datatype_);
}
template<typename T>
void IndicesSyncer<T>::calculateMessageSizes()
{
auto iEnd = indexSet_.end();
auto collIter = remoteIndices_.template iterator<true>();
for(auto index = indexSet_.begin(); index != iEnd; ++index) {
collIter.advance(index->global(), index->local().attribute());
if(collIter.empty())
break;
int knownRemote=0;
auto end = collIter.end();
// Count the remote indices we know.
for(auto valid = collIter.begin(); valid != end; ++valid) {
++knownRemote;
}
if(knownRemote>0) {
Dune::dverb<<rank_<<": publishing "<<knownRemote<<" for index "<<index->global()<< " for processes ";
// Update MessageInformation
for(auto valid = collIter.begin(); valid != end; ++valid) {
++(infoSend_[valid.process()].publish);
(infoSend_[valid.process()].pairs) += knownRemote;
Dune::dverb<<valid.process()<<" ";
Dune::dverb<<"(publish="<<infoSend_[valid.process()].publish<<", pairs="<<infoSend_[valid.process()].pairs
<<") ";
}
Dune::dverb<<std::endl;
}
}
const auto end = infoSend_.end();
// Now determine the buffersizes needed for each neighbour using MPI_Pack_size
MessageInformation dummy;
auto messageIter= infoSend_.begin();
const auto rend = remoteIndices_.end();
int neighbour=0;
for(auto remote = remoteIndices_.begin(); remote != rend; ++remote, ++neighbour) {
MessageInformation* message;
MessageInformation recv;
if(messageIter != end && messageIter->first==remote->first) {
// We want to send message information to that process
message = const_cast<MessageInformation*>(&(messageIter->second));
++messageIter;
}else
// We do not want to send information but the other process might.
message = &dummy;
sendBufferSizes_[neighbour]=0;
int tsize;
// The number of indices published
MPI_Pack_size(1, MPI_INT,remoteIndices_.communicator(), &tsize);
sendBufferSizes_[neighbour] += tsize;
for(int i=0; i < message->publish; ++i) {
// The global index
MPI_Pack_size(1, MPITraits<GlobalIndex>::getType(), remoteIndices_.communicator(), &tsize);
sendBufferSizes_[neighbour] += tsize;
// The attribute in the local index
MPI_Pack_size(1, MPI_CHAR, remoteIndices_.communicator(), &tsize);
sendBufferSizes_[neighbour] += tsize;
// The number of corresponding remote indices
MPI_Pack_size(1, MPI_INT, remoteIndices_.communicator(), &tsize);
sendBufferSizes_[neighbour] += tsize;
}
for(int i=0; i < message->pairs; ++i) {
// The process of the remote index
MPI_Pack_size(1, MPI_INT, remoteIndices_.communicator(), &tsize);
sendBufferSizes_[neighbour] += tsize;
// The attribute of the remote index
MPI_Pack_size(1, MPI_CHAR, remoteIndices_.communicator(), &tsize);
sendBufferSizes_[neighbour] += tsize;
}
Dune::dverb<<rank_<<": Buffer (neighbour="<<remote->first<<") size is "<< sendBufferSizes_[neighbour]<<" for publish="<<message->publish<<" pairs="<<message->pairs<<std::endl;
}
}
template<typename T>
inline void IndicesSyncer<T>::sync()
{
DefaultNumberer numberer;
sync(numberer);
}
template<typename T>
template<typename T1>
void IndicesSyncer<T>::sync(T1& numberer, bool useFixedOrder)
{
// The pointers to the local indices in the remote indices
// will become invalid due to the resorting of the index set.
// Therefore store the corresponding global indices.
// Mark all indices as not added
const auto end = remoteIndices_.end();
// Number of neighbours might change during the syncing.
// save the old neighbours
std::size_t noOldNeighbours = remoteIndices_.neighbours();
int* oldNeighbours = new int[noOldNeighbours];
sendBufferSizes_ = new std::size_t[noOldNeighbours];
std::size_t neighbourI = 0;
for(auto remote = remoteIndices_.begin(); remote != end; ++remote, ++neighbourI) {
oldNeighbours[neighbourI] = remote->first;
// Make sure we only have one remote index list.
assert(remote->second.first==remote->second.second);
RemoteIndexList& rList = *(remote->second.first);
// Store the corresponding global indices.
GlobalIndexList& global = globalMap_[remote->first];
BoolList& added = oldMap_[remote->first];
auto riEnd = rList.end();
for(auto index = rList.begin();
index != riEnd; ++index) {
global.push_back(std::make_pair(index->localIndexPair().global(),
index->localIndexPair().local().attribute()));
added.push_back(true);
}
Iterators iterators(rList, global, added);
iteratorsMap_.insert(std::make_pair(remote->first, iterators));
assert(checkReset(iteratorsMap_[remote->first], rList,global,added));
}
// Exchange indices with each neighbour
calculateMessageSizes();
// Allocate the buffers
receiveBufferSize_=1;
sendBuffers_ = new char*[noOldNeighbours];
for(std::size_t i=0; i<noOldNeighbours; ++i) {
sendBuffers_[i] = new char[sendBufferSizes_[i]];
receiveBufferSize_ = std::max<int>(receiveBufferSize_, sendBufferSizes_[i]);
}
receiveBuffer_=new char[receiveBufferSize_];
indexSet_.beginResize();
Dune::dverb<<rank_<<": Neighbours: ";
for(std::size_t i = 0; i<noOldNeighbours; ++i)
Dune::dverb<<oldNeighbours[i]<<" ";
Dune::dverb<<std::endl;
MPI_Request* requests = new MPI_Request[noOldNeighbours];
MPI_Status* statuses = new MPI_Status[noOldNeighbours];
// Pack Message data and start the sends
for(std::size_t i = 0; i<noOldNeighbours; ++i)
packAndSend(oldNeighbours[i], sendBuffers_[i], sendBufferSizes_[i], requests[i]);
// Probe for incoming messages, receive and unpack them
for(std::size_t i = 0; i<noOldNeighbours; ++i)
recvAndUnpack(numberer, oldNeighbours[i], useFixedOrder);
// }else{
// recvAndUnpack(oldNeighbours[i], numberer);
// packAndSend(oldNeighbours[i]);
// }
// }
delete[] receiveBuffer_;
// Wait for the completion of the sends
// Wait for completion of sends
if(MPI_SUCCESS!=MPI_Waitall(noOldNeighbours, requests, statuses)) {
std::cerr<<": MPI_Error occurred while sending message"<<std::endl;
for(std::size_t i=0; i< noOldNeighbours; i++)
if(MPI_SUCCESS!=statuses[i].MPI_ERROR)
std::cerr<<"Destination "<<statuses[i].MPI_SOURCE<<" error code: "<<statuses[i].MPI_ERROR<<std::endl;
}
delete[] statuses;
delete[] requests;
for(std::size_t i=0; i<noOldNeighbours; ++i)
delete[] sendBuffers_[i];
delete[] sendBuffers_;
delete[] sendBufferSizes_;
// No need for the iterator tuples any more
iteratorsMap_.clear();
indexSet_.endResize();
delete[] oldNeighbours;
repairLocalIndexPointers(globalMap_, remoteIndices_, indexSet_);
oldMap_.clear();
globalMap_.clear();
// update the sequence number
remoteIndices_.sourceSeqNo_ = remoteIndices_.destSeqNo_ = indexSet_.seqNo();
}
template<typename T>
void IndicesSyncer<T>::packAndSend(int destination, char* buffer, std::size_t bufferSize, MPI_Request& request)
{
auto iEnd = indexSet_.end();
int bpos = 0;
int published = 0;
int pairs = 0;
assert(checkReset());
// Pack the number of indices we publish
MPI_Pack(&(infoSend_[destination].publish), 1, MPI_INT, buffer, bufferSize, &bpos,
remoteIndices_.communicator());
for(auto index = indexSet_.begin(); index != iEnd; ++index) {
// Search for corresponding remote indices in all iterator tuples
auto iteratorsEnd = iteratorsMap_.end();
// advance all iterators to a position with global index >= index->global()
for(auto iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators) {
while(iterators->second.isNotAtEnd() &&
iterators->second.globalIndexPair().first < index->global())
++(iterators->second);
assert(!iterators->second.isNotAtEnd() || iterators->second.globalIndexPair().first >= index->global());
}
// Add all remote indices positioned at global which were already present before calling sync
// to the message.
// Count how many remote indices we will send
int indices = 0;
bool knownRemote = false; // Is the remote process supposed to know this index?
for(auto iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators)
{
std::pair<GlobalIndex,Attribute> p;
if (iterators->second.isNotAtEnd())
{
p = iterators->second.globalIndexPair();
}
if(iterators->second.isNotAtEnd() && iterators->second.isOld()
&& iterators->second.globalIndexPair().first == index->global()) {
indices++;
if(destination == iterators->first)
knownRemote = true;
}
}
if(!knownRemote)
// We do not need to send any indices
continue;
Dune::dverb<<rank_<<": sending "<<indices<<" for index "<<index->global()<<" to "<<destination<<std::endl;
// Pack the global index, the attribute and the number
MPI_Pack(const_cast<GlobalIndex*>(&(index->global())), 1, MPITraits<GlobalIndex>::getType(), buffer, bufferSize, &bpos,
remoteIndices_.communicator());
char attr = index->local().attribute();
MPI_Pack(&attr, 1, MPI_CHAR, buffer, bufferSize, &bpos,
remoteIndices_.communicator());
// Pack the number of remote indices we send.
MPI_Pack(&indices, 1, MPI_INT, buffer, bufferSize, &bpos,
remoteIndices_.communicator());
// Pack the information about the remote indices
for(auto iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators)
if(iterators->second.isNotAtEnd() && iterators->second.isOld()
&& iterators->second.globalIndexPair().first == index->global()) {
int process = iterators->first;
++pairs;
assert(pairs <= infoSend_[destination].pairs);
MPI_Pack(&process, 1, MPI_INT, buffer, bufferSize, &bpos,
remoteIndices_.communicator());
char attr2 = iterators->second.remoteIndex().attribute();
MPI_Pack(&attr2, 1, MPI_CHAR, buffer, bufferSize, &bpos,
remoteIndices_.communicator());
--indices;
}
assert(indices==0);
++published;
Dune::dvverb<<" (publish="<<published<<", pairs="<<pairs<<")"<<std::endl;
assert(published <= infoSend_[destination].publish);
}
// Make sure we send all expected entries
assert(published == infoSend_[destination].publish);
assert(pairs == infoSend_[destination].pairs);
resetIteratorsMap();
Dune::dverb << rank_<<": Sending message of "<<bpos<<" bytes to "<<destination<<std::endl;
MPI_Issend(buffer, bpos, MPI_PACKED, destination, 345, remoteIndices_.communicator(),&request);
}
template<typename T>
inline void IndicesSyncer<T>::insertIntoRemoteIndexList(int process,
const std::pair<GlobalIndex,Attribute>& globalPair,
char attribute)
{
Dune::dverb<<"Inserting from "<<process<<" "<<globalPair.first<<", "<<
globalPair.second<<" "<<attribute<<std::endl;
resetIteratorsMap();
// There might be cases where there no remote indices for that process yet
typename IteratorsMap::iterator found = iteratorsMap_.find(process);
if( found == iteratorsMap_.end() ) {
Dune::dverb<<"Discovered new neighbour "<<process<<std::endl;
RemoteIndexList* rlist = new RemoteIndexList();
remoteIndices_.remoteIndices_.insert(std::make_pair(process,std::make_pair(rlist,rlist)));
Iterators iterators = Iterators(*rlist, globalMap_[process], oldMap_[process]);
found = iteratorsMap_.insert(std::make_pair(process, iterators)).first;
}
Iterators& iterators = found->second;
// Search for the remote index
while(iterators.isNotAtEnd() && iterators.globalIndexPair() < globalPair) {
// Increment all iterators
++iterators;
}
if(iterators.isAtEnd() || iterators.globalIndexPair() != globalPair) {
// The entry is not yet known
// Insert in the list and do not change the first iterator.
iterators.insert(RemoteIndex(Attribute(attribute)),globalPair);
return;
}
// Global indices match
bool indexIsThere=false;
for(Iterators tmpIterators = iterators;
!tmpIterators.isAtEnd() && tmpIterators.globalIndexPair() == globalPair;
++tmpIterators)
//entry already exists with the same attribute
if(tmpIterators.globalIndexPair().second == attribute) {
indexIsThere=true;
break;
}
if(!indexIsThere)
// The entry is not yet known
// Insert in the list and do not change the first iterator.
iterators.insert(RemoteIndex(Attribute(attribute)),globalPair);
}
template<typename T>
template<typename T1>
void IndicesSyncer<T>::recvAndUnpack(T1& numberer, int hardSource, bool useHardSource)
{
const ParallelIndexSet& constIndexSet = indexSet_;
auto iEnd = constIndexSet.end();
auto index = constIndexSet.begin();
int bpos = 0;
int publish;
assert(checkReset());
MPI_Status status;
// We have to determine the message size and source before the receive
MPI_Probe(useHardSource ? hardSource : MPI_ANY_SOURCE, 345, remoteIndices_.communicator(), &status);
int source=status.MPI_SOURCE;
int count;
MPI_Get_count(&status, MPI_PACKED, &count);
Dune::dvverb<<rank_<<": Receiving message from "<< source<<" with "<<count<<" bytes"<<std::endl;
if(count>receiveBufferSize_) {
receiveBufferSize_=count;
delete[] receiveBuffer_;
receiveBuffer_ = new char[receiveBufferSize_];
}
MPI_Recv(receiveBuffer_, count, MPI_PACKED, source, 345, remoteIndices_.communicator(), &status);
// How many global entries were published?
MPI_Unpack(receiveBuffer_, count, &bpos, &publish, 1, MPI_INT, remoteIndices_.communicator());
// Now unpack the remote indices and add them.
while(publish>0) {
// Unpack information about the local index on the source process
GlobalIndex global; // global index of the current entry
char sourceAttribute; // Attribute on the source process
int pairs;
MPI_Unpack(receiveBuffer_, count, &bpos, &global, 1, MPITraits<GlobalIndex>::getType(),
remoteIndices_.communicator());
MPI_Unpack(receiveBuffer_, count, &bpos, &sourceAttribute, 1, MPI_CHAR,
remoteIndices_.communicator());
MPI_Unpack(receiveBuffer_, count, &bpos, &pairs, 1, MPI_INT,
remoteIndices_.communicator());
// Insert the entry on the remote process to our
// remote index list
SLList<std::pair<int,Attribute> > sourceAttributeList;
sourceAttributeList.push_back(std::make_pair(source,Attribute(sourceAttribute)));
#ifndef NDEBUG
bool foundSelf = false;
#endif
Attribute myAttribute=Attribute();
// Unpack the remote indices
for(; pairs>0; --pairs) {
// Unpack the process id that knows the index
int process;
char attribute;
MPI_Unpack(receiveBuffer_, count, &bpos, &process, 1, MPI_INT,
remoteIndices_.communicator());
// Unpack the attribute
MPI_Unpack(receiveBuffer_, count, &bpos, &attribute, 1, MPI_CHAR,
remoteIndices_.communicator());
if(process==rank_) {
#ifndef NDEBUG
foundSelf=true;
#endif
myAttribute=Attribute(attribute);
// Now we know the local attribute of the global index
//Only add the index if it is unknown.
// Do we know that global index already?
auto pos = std::lower_bound(index, iEnd, IndexPair(global));
if(pos == iEnd || pos->global() != global) {
// no entry with this global index
indexSet_.add(global,
ParallelLocalIndex<Attribute>(numberer(global),
myAttribute, true));
Dune::dvverb << "Adding "<<global<<" "<<myAttribute<<std::endl;
continue;
}
// because of above the global indices match. Add only if the attribute is different
bool indexIsThere = false;
index=pos;
for(; pos->global()==global; ++pos)
if(pos->local().attribute() == myAttribute) {
Dune::dvverb<<"found "<<global<<" "<<myAttribute<<std::endl;
indexIsThere = true;
break;
}
if(!indexIsThere) {
indexSet_.add(global,
ParallelLocalIndex<Attribute>(numberer(global),
myAttribute, true));
Dune::dvverb << "Adding "<<global<<" "<<myAttribute<<std::endl;
}
}else{
sourceAttributeList.push_back(std::make_pair(process,Attribute(attribute)));
}
}
assert(foundSelf);
// Insert remote indices
typedef typename SLList<std::pair<int,Attribute> >::const_iterator Iter;
for(Iter i=sourceAttributeList.begin(), end=sourceAttributeList.end();
i!=end; ++i)
insertIntoRemoteIndexList(i->first, std::make_pair(global, myAttribute),
i->second);
--publish;
}
resetIteratorsMap();
}
template<typename T>
void IndicesSyncer<T>::resetIteratorsMap(){
// Reset iterators in all tuples.
const auto remoteEnd = remoteIndices_.remoteIndices_.end();
auto iterators = iteratorsMap_.begin();
auto global = globalMap_.begin();
auto added = oldMap_.begin();
for(auto remote = remoteIndices_.remoteIndices_.begin();
remote != remoteEnd; ++remote, ++global, ++added, ++iterators) {
iterators->second.reset(*(remote->second.first), global->second, added->second);
}
}
template<typename T>
bool IndicesSyncer<T>::checkReset(const Iterators& iterators, RemoteIndexList& rList, GlobalIndexList& gList,
BoolList& bList){
if(std::get<0>(iterators.iterators_) != rList.begin())
return false;
if(std::get<1>(iterators.iterators_) != gList.begin())
return false;
if(std::get<2>(iterators.iterators_) != bList.begin())
return false;
return true;
}
template<typename T>
bool IndicesSyncer<T>::checkReset(){
// Reset iterators in all tuples.
const auto remoteEnd = remoteIndices_.remoteIndices_.end();
auto iterators = iteratorsMap_.begin();
auto global = globalMap_.begin();
auto added = oldMap_.begin();
bool ret = true;
for(auto remote = remoteIndices_.remoteIndices_.begin();
remote != remoteEnd; ++remote, ++global, ++added, ++iterators) {
if(!checkReset(iterators->second, *(remote->second.first), global->second,
added->second))
ret=false;
}
return ret;
}
}
#endif // HAVE_MPI
#endif // DUNE_COMMON_PARALLEL_INDICESSYNCER_HH
remoteindices.hh
Classes describing a distributed indexset.
indexset.hh
Provides a map between global and local indices.
sllist.hh
Implements a singly linked list together with the necessary iterators.
stdstreams.hh
Standard Dune debug streams.
Dune::SLList::push_back
void push_back(const MemberType &item)
Add a new entry to the end of the list.
Definition sllist.hh:643
Dune::SLList::iterator
SLListIterator< T, A > iterator
The mutable iterator of the list.
Definition sllist.hh:69
Dune::SLList::end
iterator end()
Get an iterator pointing to the end of the list.
Definition sllist.hh:774
Dune::SLList::beginModify
ModifyIterator beginModify()
Get an iterator capable of deleting and inserting elements.
Definition sllist.hh:787
Dune::SLList< RemoteIndex, Allocator >::const_iterator
SLListConstIterator< RemoteIndex, Allocator > const_iterator
Definition sllist.hh:74
Dune::SLList< std::pair< GlobalIndex, Attribute >, typename RemoteIndices::Allocator >::ModifyIterator
SLListModifyIterator< std::pair< GlobalIndex, Attribute >, typename RemoteIndices::Allocator > ModifyIterator
Definition sllist.hh:103
Dune::SLList::begin
iterator begin()
Get an iterator pointing to the first element in the list.
Definition sllist.hh:762
Dune::operator<
constexpr EnableIfInterOperable< T1, T2, bool >::type operator<(const RandomAccessIteratorFacade< T1, V1, R1, D > &lhs, const RandomAccessIteratorFacade< T2, V2, R2, D > &rhs)
Comparison operator.
Definition iteratorfacades.hh:638
Dune::IndicesSyncer::Iterators::Iterators
Iterators()
Default constructor.
Definition indicessyncer.hh:563
Dune::IndicesSyncer::RemoteIndices
Dune::RemoteIndices< ParallelIndexSet > RemoteIndices
Type of the remote indices.
Definition indicessyncer.hh:63
Dune::IndicesSyncer::Iterators::isOld
bool isOld() const
Was this entry already in the remote index list before the sync process?
Definition indicessyncer.hh:600
Dune::IndicesSyncer::MessageInformation::publish
int publish
The number of indices we publish for the other process.
Definition indicessyncer.hh:132
Dune::repairLocalIndexPointers
void repairLocalIndexPointers(std::map< int, SLList< std::pair< typename T::GlobalIndex, typename T::LocalIndex::Attribute >, A > > &globalMap, RemoteIndices< T, A1 > &remoteIndices, const T &indexSet)
Repair the pointers to the local indices in the remote indices.
Definition indicessyncer.hh:495
Dune::IndicesSyncer::Iterators::isAtEnd
bool isAtEnd() const
Are we at the end of the list?
Definition indicessyncer.hh:622
Dune::RemoteIndices::communicator
MPI_Comm communicator() const
Get the mpi communicator used.
Definition remoteindices.hh:1695
Dune::IndicesSyncer::GlobalIndex
ParallelIndexSet::GlobalIndex GlobalIndex
Type of the global index used in the index set.
Definition indicessyncer.hh:55
Dune::IndicesSyncer::MessageInformation::pairs
int pairs
The number of pairs (attribute and process number) we publish to the neighbour process.
Definition indicessyncer.hh:137
Dune::RemoteIndices::end
const_iterator end() const
Get an iterator over all remote index lists.
Definition remoteindices.hh:1528
Dune::IndicesSyncer::Iterators::attribute
Attribute & attribute() const
Dune::IndicesSyncer::Iterators::operator++
Iterators & operator++()
Increment all iterators.
Definition indicessyncer.hh:568
Dune::IndicesSyncer::Attribute
ParallelIndexSet::LocalIndex::Attribute Attribute
Type of the attribute used in the index set.
Definition indicessyncer.hh:58
Dune::IndicesSyncer::Iterators::globalIndexPair
std::pair< GlobalIndex, Attribute > & globalIndexPair() const
Get the global index of the remote index at current position.
Definition indicessyncer.hh:594
Dune::IndicesSyncer::IndicesSyncer
IndicesSyncer(ParallelIndexSet &indexSet, RemoteIndices &remoteIndices)
Constructor.
Definition indicessyncer.hh:544
Dune::IndicesSyncer::Iterators::reset
void reset(RemoteIndexList &remoteIndices, GlobalIndexList &globalIndices, BoolList &booleans)
Reset all the underlying iterators.
Definition indicessyncer.hh:606
Dune::IndicesSyncer::DefaultNumberer::operator()
std::size_t operator()(const GlobalIndex &global)
Provide the local index, always std::numeric_limits<size_t>::max().
Definition indicessyncer.hh:151
Dune::IndicesSyncer::ParallelIndexSet
T ParallelIndexSet
The type of the index set.
Definition indicessyncer.hh:49
Dune::IndicesSyncer::Iterators::isNotAtEnd
bool isNotAtEnd() const
Are we not at the end of the list?
Definition indicessyncer.hh:616
Dune::type
MPI_Datatype MPITraits< ParallelLocalIndex< T > >::type
Definition plocalindex.hh:317
Dune::RemoteIndices::neighbours
int neighbours() const
Get the number of processors we share indices with.
Definition remoteindices.hh:1445
Dune::ParallelIndexSet::GlobalIndex
TG GlobalIndex
the type of the global index. This type has to provide at least a operator< for sorting.
Definition indexset.hh:226
Dune::storeGlobalIndicesOfRemoteIndices
void storeGlobalIndicesOfRemoteIndices(std::map< int, SLList< std::pair< typename T::GlobalIndex, typename T::LocalIndex::Attribute >, A > > &globalMap, const RemoteIndices< T, A1 > &remoteIndices)
Stores the corresponding global indices of the remote index information.
Definition indicessyncer.hh:469
Dune::IndicesSyncer::sync
void sync()
Sync the index set.
Definition indicessyncer.hh:736
Dune::ParallelIndexSet::seqNo
int seqNo() const
Get the internal sequence number.
Dune::IndicesSyncer::IndexPair
ParallelIndexSet::IndexPair IndexPair
The type of the index pair.
Definition indicessyncer.hh:52
Dune::RemoteIndices::begin
const_iterator begin() const
Get an iterator over all remote index lists.
Definition remoteindices.hh:1521
Dune::IndicesSyncer::Iterators::remoteIndex
RemoteIndex & remoteIndex() const
Get the remote index at current position.
Definition indicessyncer.hh:587
Dune::IndicesSyncer::MessageInformation::MessageInformation
MessageInformation()
Definition indicessyncer.hh:128
Dune::ParallelIndexSet::IndexPair
Dune::IndexPair< GlobalIndex, LocalIndex > IndexPair
The type of the pair stored.
Definition indexset.hh:244
Dune::IndicesSyncer::Iterators::insert
void insert(const RemoteIndex &index, const std::pair< GlobalIndex, Attribute > &global)
Insert a new remote index to the underlying remote index list.
Definition indicessyncer.hh:577
Dune::dvverb
DVVerbType dvverb(std::cout)
stream for very verbose output.
Definition stdstreams.hh:96
Dune::dverb
DVerbType dverb(std::cout)
Singleton of verbose debug stream.
Definition stdstreams.hh:117
Dune
Dune namespace
Definition alignedallocator.hh:13
Dune::operator!=
constexpr auto operator!=(const HybridMultiIndex< S... > &lhs, const HybridMultiIndex< T... > &rhs)
Compare two HybridMultiIndexs for inequality.
Definition hybridmultiindex.hh:441
Dune::operator==
constexpr bool operator==(const HybridMultiIndex< S... > &lhs, const HybridMultiIndex< T... > &rhs)
Compare two HybridMultiIndexs for value equality.
Definition hybridmultiindex.hh:404
Dune::MPITraits::getType
static MPI_Datatype getType()
Definition mpitraits.hh:48
Dune::IndexPair
A pair consisting of a global and local index.
Definition indexset.hh:85
Dune::IndicesSyncer
Class for recomputing missing indices of a distributed index set.
Definition indicessyncer.hh:45
Dune::ParallelLocalIndex
An index present on the local process with an additional attribute flag.
Definition plocalindex.hh:52
Dune::RemoteIndices
The indices present on remote processes.
Definition remoteindices.hh:190
Dune::RemoteIndices::RemoteIndexList
Dune::SLList< RemoteIndex, Allocator > RemoteIndexList
The type of the remote index list.
Definition remoteindices.hh:242
Dune::RemoteIndices::Allocator
typename std::allocator_traits< A >::template rebind_alloc< RemoteIndex > Allocator
The type of the allocator for the remote index list.
Definition remoteindices.hh:238
Dune::RemoteIndex
Information about an index residing on another processor.
Definition remoteindices.hh:74
Dune::SLList
A single linked list.
Definition sllist.hh:44
Dune::type