blocktools.h
Go to the documentation of this file.
/*****************************************************************************
*
* Copyright (c) 2014-2018 by The University of Queensland
* http://www.uq.edu.au
*
* Primary Business: Queensland, Australia
* Licensed under the Apache License, version 2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Development until 2012 by Earth Systems Science Computational Center (ESSCC)
* Development 2012-2013 by School of Earth Sciences
* Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
* Development from 2019 by School of Earth and Environmental Sciences
**
*****************************************************************************/
#ifndef __RIPLEY_BLOCKTOOLS_H__
#define __RIPLEY_BLOCKTOOLS_H__
/* This file contains two main classes for dealing with a large 3D region which
* has been divided into a 3D Grid of Blocks (usually to be distributed).
* Each block is divided into 27 subblocks. The first and last subblocks in
* each dimension are cubes.
*
* class Block manages a single block. It has methods for copying between a
* flat array (storing all the values in the 3D block) and buffers storing
* individual subblocks. These buffers can be used to transfer individual
* subblocks to other blocks (using some external means).
*
* class BlockGrid deals with the position of a given block in relation to the
* rest of the blocks in the grid. (In an MPI setting, there would be one block
* per rank.)
* It also requires transfers of subblocks into and out of the block in order
* to make the whole (global) array consistent.
*
* Each block has a region "inset" wide in from each edge which is shared with
* neighbouring blocks. Where these regions overlap with another block, the
* block closest to block 0,0,0 is used.
* Or more precisely, values move left->right then lowy->highy and finally
* lowz -> highz.
*
* Please don't mix external calls into this file, it may be useful to separate
* it for debugging purposes.
*
* Types required:
* neighbourID_t - Stores the label of a neighbouring block.
* In an MPI setting, this will be the type used refer to
* ranks
* coord_t - Stores a position of a block in the grid of blocks
* (it could be within one dimension or overall in a flat
* structure. It is not (necessarily) the same as
* neighbourID_t because coord_t should be _unsigned_ and
* there is no guarantee that neighbourID_t will even be
* an integral type.
*/
#include <escript/EsysMPI.h>
#include <vector>
typedef int neighbourID_t; // This should be the MPI_rank type
typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
typedef std::pair<neighbourID_t, int> neighpair;
typedef std::vector<neighpair> neighbourvector;
typedef struct
{
public:
neighbourID_t sourceID; // ranks involved in communication
neighbourID_t destID;
int tag;
unsigned char srcbuffid; // number of buffer to use for coms
unsigned char destbuffid;
} message;
typedef std::vector<message> messvec;
class BlockGrid
{
public:
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const;
// generate all incoming com messages for this block.
// for each subblock (27 of them), there may be an x, y, z direction to search in
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
// generate all outgoing com messages for this block
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
private:
coord_t xmax;
coord_t ymax;
coord_t zmax;
};
/* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
They do not exist, such buffers would be:
1) big
2) unnecessary since the centre sub-block is not sent anywhere
Note that this class does not deal with data transfer between blocks
Sub-blocks are copied to and from buffers. Other code is required to
actually move the data.
"dpp" == doubles per point and gives the number of doubles that make up each "point"
This is required to calculate offsets and buffer sizes.
*/
class Block
{
public:
// s? specifiy the [local] size (in points) of each dimension
Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen,
size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
~Block();
// Out buffers are loaded with the contents of the flat array and are
// to be sent to other blocks
double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
double* getOutBuffer(unsigned char bid);
// In buffers are populated from external communications
// and copied back to the flat array
double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
double* getInBuffer(unsigned char bid);
// return number of doubles in the given block
size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
size_t getBuffSize(unsigned char bid);
// where does the subblock specified start in a source array
size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
// debug only
void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
// Copy a 3d region from a flat array into a buffer
void copyToBuffer(unsigned char buffid, double* src);
// Copy a 3d region from a buffer into a flat array
void copyFromBuffer(unsigned char buffid, double* dest);
void copyAllToBuffer(double* src);
void copyUsedFromBuffer(double* dest);
void setUsed(unsigned char buffid);
private:
// determines the dimensions of each subblock
void populateDimsTable();
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
double* inbuff;
double* outbuff;
size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
size_t flatoffsets[27]; // starting point of each block within a flat array
bool used[27];
size_t dims[27][3]; // dimension of each subblock
size_t sx;
size_t sy;
size_t sz;
size_t inset;
size_t xmidlen;
size_t ymidlen;
size_t zmidlen;
double* inbuffptr[27];
double* outbuffptr[27];
const unsigned int dpsize; // number of doubles which make up a point
};
// Returns the MPI message tag to use for a transfer between the two subblocks
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez,
unsigned char targetx, unsigned char targety, unsigned char targetz);
// computes the tag based on the destination and the direction it comes from
// the booleans indicate whether a negative shift in that direction is required
int getTag(unsigned char destx, unsigned char desty, unsigned char destz,
bool deltax, bool deltay, bool deltaz);
// the booleans indicate whether a negative shift in that direction is required
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty,
unsigned char destz, bool deltax, bool deltay,
bool deltaz);
/* Now the 2D versions */
// 2D version
class BlockGrid2
{
public:
BlockGrid2(coord_t maxx, coord_t maxy);
neighbourID_t getNID(coord_t x, coord_t y) const;
// generate all incoming com messages for this block.
// for each subblock (9 of them), there may be an x, y direction to search in
void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
// generate all outgoing com messages for this block
void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
private:
coord_t xmax;
coord_t ymax;
};
// The 2D version - there is no block 4
class Block2
{
public:
// s? specifiy the [local] size (in points) of each dimension
Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen,
size_t ymidlen, unsigned int dpp=1);
~Block2();
// Out buffers are loaded with the contents of the flat array and are
// to be sent to other blocks
double* getOutBuffer(unsigned char subx, unsigned char suby);
double* getOutBuffer(unsigned char bid);
// In buffers are populated from external communications
// and copied back to the flat array
double* getInBuffer(unsigned char subx, unsigned char suby);
double* getInBuffer(unsigned char bid);
// return number of doubles in the given block
size_t getBuffSize(unsigned char subx, unsigned char suby);
size_t getBuffSize(unsigned char bid);
// where does the subblock specified start in a source array
size_t startOffset(unsigned char subx, unsigned char suby);
// debug only
void displayBlock(unsigned char subx, unsigned char suby, bool out);
// Copy a 3d region from a flat array into a buffer
void copyToBuffer(unsigned char buffid, double* src);
// Copy a 3d region from a buffer into a flat array
void copyFromBuffer(unsigned char buffid, double* dest);
void copyAllToBuffer(double* src);
void copyUsedFromBuffer(double* dest);
void setUsed(unsigned char buffid);
private:
// determines the dimensions of each subblock
void populateDimsTable();
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);
double* inbuff;
double* outbuff;
size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays
size_t flatoffsets[9]; // starting point of each block within a flat array
bool used[9];
size_t dims[9][2]; // dimension of each subblock
size_t sx;
size_t sy;
size_t inset;
size_t xmidlen;
size_t ymidlen;
double* inbuffptr[9];
double* outbuffptr[9];
const unsigned int dpsize; // number of doubles which make up a point
};
// Returns the MPI message tag to use for a transfer between the two subblocks
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
// computes the tag based on the destination and the direction it comes from
// the booleans indicate whether a negative shift in that direction is required
int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
// the booleans indicate whether a negative shift in that direction is required
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
#endif // __RIPLEY_BLOCKTOOLS_H__
Block::getInBuffer
double * getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:133
Block::Block
Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen, unsigned int dpp=1)
Definition: blocktools.cpp:278
Block::displayBlock
void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out)
Definition: blocktools.cpp:324
Block::getOutBuffer
double * getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:123
message::sourceID
neighbourID_t sourceID
Definition: blocktools.h:72
Block2::populateOffsetTable
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen)
Definition: blocktools2.cpp:184
Block2::startOffset
size_t startOffset(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:277
Block::sx
size_t sx
Definition: blocktools.h:175
message
Definition: blocktools.h:69
getTag
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition: blocktools.cpp:413
Block2::createBuffArrays
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen)
Definition: blocktools2.cpp:206
BlockGrid::BlockGrid
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz)
Definition: blocktools.cpp:25
message::destID
neighbourID_t destID
Definition: blocktools.h:73
Block2::dpsize
const unsigned int dpsize
Definition: blocktools.h:289
Block::buffoffsets
size_t buffoffsets[27]
Definition: blocktools.h:171
Block2::dims
size_t dims[9][2]
Definition: blocktools.h:281
neighpair
std::pair< neighbourID_t, int > neighpair
Definition: blocktools.h:65
Block::startOffset
size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:310
Block::inset
size_t inset
Definition: blocktools.h:178
Block2::sx
size_t sx
Definition: blocktools.h:282
Block::copyFromBuffer
void copyFromBuffer(unsigned char buffid, double *dest)
Definition: blocktools.cpp:387
Block::inbuff
double * inbuff
Definition: blocktools.h:169
neighbourvector
std::vector< neighpair > neighbourvector
Definition: blocktools.h:66
Block2::outbuff
double * outbuff
Definition: blocktools.h:277
Block2::populateDimsTable
void populateDimsTable()
Definition: blocktools2.cpp:166
getTag
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition: blocktools.cpp:413
message::tag
int tag
Definition: blocktools.h:74
getSrcBuffID
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition: blocktools.cpp:431
Block::populateOffsetTable
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition: blocktools.cpp:215
Block2::inbuff
double * inbuff
Definition: blocktools.h:276
Block::dims
size_t dims[27][3]
Definition: blocktools.h:174
getTag2
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety)
Definition: blocktools2.cpp:365
generateInNeighbours
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Block2::getOutBuffer
double * getOutBuffer(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:101
Block::dpsize
const unsigned int dpsize
Definition: blocktools.h:184
getSrcBuffID
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition: blocktools.cpp:431
Block2::outbuffptr
double * outbuffptr[9]
Definition: blocktools.h:288
Block2
Definition: blocktools.h:226
Block::ymidlen
size_t ymidlen
Definition: blocktools.h:180
Block::outbuffptr
double * outbuffptr[27]
Definition: blocktools.h:183
Block::createBuffArrays
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition: blocktools.cpp:237
Block::copyAllToBuffer
void copyAllToBuffer(double *src)
Definition: blocktools.cpp:252
BlockGrid::generateOutNeighbours
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:83
Block::outbuff
double * outbuff
Definition: blocktools.h:170
BlockGrid::zmax
coord_t zmax
Definition: blocktools.h:101
BlockGrid
Definition: blocktools.h:82
BlockGrid::getNID
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const
Definition: blocktools.cpp:29
Block2::copyFromBuffer
void copyFromBuffer(unsigned char buffid, double *dest)
Definition: blocktools2.cpp:346
Block2::sy
size_t sy
Definition: blocktools.h:283
BlockGrid::generateInNeighbours
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:44
Block2::copyToBuffer
void copyToBuffer(unsigned char buffid, double *src)
Definition: blocktools2.cpp:318
BlockGrid2::getNID
neighbourID_t getNID(coord_t x, coord_t y) const
Definition: blocktools2.cpp:28
Block2::getInBuffer
double * getInBuffer(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:111
Block::xmidlen
size_t xmidlen
Definition: blocktools.h:179
Block2::flatoffsets
size_t flatoffsets[9]
Definition: blocktools.h:279
messvec
std::vector< message > messvec
Definition: blocktools.h:79
Block2::Block2
Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen, size_t ymidlen, unsigned int dpp=1)
Definition: blocktools2.cpp:247
Block::flatoffsets
size_t flatoffsets[27]
Definition: blocktools.h:172
Block2::buffoffsets
size_t buffoffsets[9]
Definition: blocktools.h:278
Block2::xmidlen
size_t xmidlen
Definition: blocktools.h:285
Block::populateDimsTable
void populateDimsTable()
Definition: blocktools.cpp:188
getSrcBuffID2
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay)
Definition: blocktools2.cpp:381
Block2::getBuffSize
size_t getBuffSize(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:121
Block::setUsed
void setUsed(unsigned char buffid)
Definition: blocktools.cpp:247
Block2::used
bool used[9]
Definition: blocktools.h:280
BlockGrid::ymax
coord_t ymax
Definition: blocktools.h:100
Block::copyToBuffer
void copyToBuffer(unsigned char buffid, double *src)
Definition: blocktools.cpp:358
generateOutNeighbours
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Block2::setUsed
void setUsed(unsigned char buffid)
Definition: blocktools2.cpp:216
Block::zmidlen
size_t zmidlen
Definition: blocktools.h:181
Block::used
bool used[27]
Definition: blocktools.h:173
BlockGrid::xmax
coord_t xmax
Definition: blocktools.h:99
BlockGrid2
Definition: blocktools.h:206
Block::getBuffSize
size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:143
Block
Definition: blocktools.h:119
blocktools.h
Block::sy
size_t sy
Definition: blocktools.h:176
Block2::displayBlock
void displayBlock(unsigned char subx, unsigned char suby, bool out)
Definition: blocktools2.cpp:289
BlockGrid2::BlockGrid2
BlockGrid2(coord_t maxx, coord_t maxy)
Definition: blocktools2.cpp:24
Block::copyUsedFromBuffer
void copyUsedFromBuffer(double *dest)
Definition: blocktools.cpp:265
neighbourID_t
int neighbourID_t
Definition: blocktools.h:62
BlockGrid2::xmax
coord_t xmax
Definition: blocktools.h:221
Block::sz
size_t sz
Definition: blocktools.h:177
message::destbuffid
unsigned char destbuffid
Definition: blocktools.h:76
message::srcbuffid
unsigned char srcbuffid
Definition: blocktools.h:75
Block::inbuffptr
double * inbuffptr[27]
Definition: blocktools.h:182
Block2::ymidlen
size_t ymidlen
Definition: blocktools.h:286
Block2::~Block2
~Block2()
Definition: blocktools2.cpp:160
Block2::copyAllToBuffer
void copyAllToBuffer(double *src)
Definition: blocktools2.cpp:221
Block::~Block
~Block()
Definition: blocktools.cpp:182
Block2::inset
size_t inset
Definition: blocktools.h:284
Block2::inbuffptr
double * inbuffptr[9]
Definition: blocktools.h:287
BlockGrid2::ymax
coord_t ymax
Definition: blocktools.h:222
BlockGrid2::generateInNeighbours
void generateInNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition: blocktools2.cpp:35
Block2::copyUsedFromBuffer
void copyUsedFromBuffer(double *dest)
Definition: blocktools2.cpp:234
coord_t
unsigned coord_t
Definition: blocktools.h:63
BlockGrid2::generateOutNeighbours
void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition: blocktools2.cpp:68