Loading...
Searching...
No Matches
dyscostman.h
Go to the documentation of this file.
#ifndef DYSCO_STORAGE_MANAGER_H
#define DYSCO_STORAGE_MANAGER_H
#include <casacore/tables/DataMan/DataManager.h>
#include <casacore/casa/Containers/Record.h>
#include <cstdint>
#include <fstream>
#include <memory>
#include <mutex>
#include <vector>
#include "dyscodistribution.h"
#include "dysconormalization.h"
#include "threadgroup.h"
#include "uvector.h"
#ifndef DOXYGEN_SHOULD_SKIP_THIS
extern "C" {
#endif
void register_dyscostman();
#ifndef DOXYGEN_SHOULD_SKIP_THIS
}
#endif
namespace dyscostman {
class DyscoStManColumn;
class DyscoStMan : public casacore::DataManager {
public:
DyscoStMan(unsigned dataBitRate, unsigned weightBitRate,
const casacore::String &name = "DyscoStMan");
void SetGaussianDistribution() { _distribution = GaussianDistribution; }
void SetUniformDistribution() { _distribution = UniformDistribution; }
void SetStudentsTDistribution(double nu) {
_distribution = StudentsTDistribution;
_studentTNu = nu;
}
void SetTruncatedGaussianDistribution(double truncationSigma) {
_distribution = TruncatedGaussianDistribution;
_distributionTruncation = truncationSigma;
}
void SetNormalization(Normalization normalization) {
_normalization = normalization;
}
void SetStaticSeed(bool staticSeed) { _staticSeed = staticSeed; }
DyscoStMan(const casacore::String &name, const casacore::Record &spec);
DyscoStMan(const DyscoStMan &source);
~DyscoStMan();
DyscoStMan &operator=(const DyscoStMan &source) = delete;
virtual casacore::DataManager *clone() const final override {
return new DyscoStMan(*this);
}
virtual casacore::String dataManagerType() const final override {
return "DyscoStMan";
}
virtual casacore::String dataManagerName() const final override {
return _name;
}
virtual casacore::Record dataManagerSpec() const final override;
uint getNRow() const { return _nRow; }
virtual casacore::Bool canAddRow() const final override { return true; }
virtual casacore::Bool canRemoveRow() const final override { return true; }
virtual casacore::Bool canAddColumn() const final override { return true; }
virtual casacore::Bool canRemoveColumn() const final override { return true; }
static casacore::DataManager *makeObject(const casacore::String &name,
const casacore::Record &spec) {
return new DyscoStMan(name, spec);
}
static void registerClass();
protected:
uint64_t nBlocksInFile() const {
std::lock_guard<std::mutex> lock(_mutex);
return _nBlocksInFile;
}
size_t nRowsInBlock() const { return _rowsPerBlock; }
size_t nAntennae() const { return _antennaCount; }
size_t getBlockIndex(uint64_t row) const { return row / _rowsPerBlock; }
size_t getRowWithinBlock(uint64_t row) const { return row % _rowsPerBlock; }
uint64_t getRowIndex(size_t block) const {
return uint64_t(block) * uint64_t(_rowsPerBlock);
}
bool areOffsetsInitialized() const { return _rowsPerBlock != 0; }
void initializeRowsPerBlock(size_t rowsPerBlock, size_t antennaCount,
bool writeToHeader);
private:
friend class DyscoStManColumn;
const static unsigned short VERSION_MAJOR, VERSION_MINOR;
void readCompressedData(size_t blockIndex, const DyscoStManColumn *column,
unsigned char *dest, size_t size);
void writeCompressedData(size_t blockIndex, const DyscoStManColumn *column,
const unsigned char *data, size_t size);
void readHeader();
void writeHeader();
void makeEmpty();
void setFromSpec(const casacore::Record &spec);
size_t getFileOffset(size_t blockIndex) const {
return _blockSize * blockIndex + _headerSize;
}
// Flush and optionally fsync the data.
// The AipsIO stream represents the main table file and can be
// used by virtual column engines to store SMALL amounts of data.
virtual casacore::Bool flush(casacore::AipsIO &,
casacore::Bool doFsync) final override;
// Let the storage manager create files as needed for a new table.
// This allows a column with an indirect array to create its file.
virtual void create64(casacore::rownr_t nRow) final override;
// Open the storage manager file for an existing table.
// Return the number of rows in the data file.
virtual casacore::rownr_t open64(casacore::rownr_t nRow, casacore::AipsIO &) final override;
// Create a column in the storage manager on behalf of a table column.
// The caller will NOT delete the newly created object.
// Create a scalar column.
virtual casacore::DataManagerColumn *makeScalarColumn(
const casacore::String &name, int dataType,
const casacore::String &dataTypeID) final override;
// Create a direct array column.
virtual casacore::DataManagerColumn *makeDirArrColumn(
const casacore::String &name, int dataType,
const casacore::String &dataTypeID) final override;
// Create an indirect array column.
virtual casacore::DataManagerColumn *makeIndArrColumn(
const casacore::String &name, int dataType,
const casacore::String &dataTypeID) final override;
virtual casacore::rownr_t resync64(casacore::rownr_t nRow) final override;
virtual void deleteManager() final override;
// Prepare the columns, let the data manager initialize itself further.
// Prepare is called after create/open has been called for all
// columns. In this way one can be sure that referenced columns
// are read back and partly initialized.
virtual void prepare() final override;
// Reopen the storage manager files for read/write.
virtual void reopenRW() final override;
// Add rows to the storage manager.
virtual void addRow64(casacore::rownr_t nrrow) final override;
// Delete a row from all columns.
virtual void removeRow64(casacore::rownr_t rowNr) final override;
// Do the final addition of a column.
virtual void addColumn(casacore::DataManagerColumn *) final override;
// Remove a column from the data file.
virtual void removeColumn(casacore::DataManagerColumn *) final override;
uint64_t _nRow;
uint64_t _nBlocksInFile;
uint32_t _rowsPerBlock;
uint32_t _antennaCount;
uint32_t _blockSize;
unsigned _headerSize;
mutable std::mutex _mutex;
std::unique_ptr<std::fstream> _fStream;
std::string _name;
unsigned _dataBitCount;
unsigned _weightBitCount;
DyscoDistribution _distribution;
Normalization _normalization;
double _studentTNu, _distributionTruncation;
bool _staticSeed;
std::vector<std::unique_ptr<DyscoStManColumn>> _columns;
};
} // namespace dyscostman
#endif
casacore::AipsIO
Definition AipsIO.h:167
casacore::DataManagerColumn
Definition DataManagerColumn.h:149
casacore::DataManager
Abstract base class for a data manager.
Definition DataManager.h:219
casacore::String
String: the storage and methods of handling collections of characters.
Definition String.h:223
dyscostman::DyscoStManColumn
Base class for columns of the DyscoStMan.
Definition dyscostmancol.h:22
dyscostman::DyscoStMan::canRemoveColumn
virtual casacore::Bool canRemoveColumn() const final override
Whether columns can be removed.
Definition dyscostman.h:209
dyscostman::DyscoStMan::_studentTNu
double _studentTNu
Definition dyscostman.h:401
dyscostman::DyscoStMan::open64
virtual casacore::rownr_t open64(casacore::rownr_t nRow, casacore::AipsIO &) final override
Open the storage manager file for an existing table.
dyscostman::DyscoStMan::deleteManager
virtual void deleteManager() final override
The data manager will be deleted (because all its columns are requested to be deleted).
dyscostman::DyscoStMan::dataManagerType
virtual casacore::String dataManagerType() const final override
Type of manager.
Definition dyscostman.h:164
dyscostman::DyscoStMan::_rowsPerBlock
uint32_t _rowsPerBlock
Definition dyscostman.h:388
dyscostman::DyscoStMan::registerClass
static void registerClass()
This function makes the DyscoStMan known to casacore.
dyscostman::DyscoStMan::_fStream
std::unique_ptr< std::fstream > _fStream
Definition dyscostman.h:394
dyscostman::DyscoStMan::_normalization
Normalization _normalization
Definition dyscostman.h:400
dyscostman::DyscoStMan::dataManagerName
virtual casacore::String dataManagerName() const final override
Returns the name of this manager as specified during construction.
Definition dyscostman.h:169
dyscostman::DyscoStMan::writeCompressedData
void writeCompressedData(size_t blockIndex, const DyscoStManColumn *column, const unsigned char *data, size_t size)
dyscostman::DyscoStMan::VERSION_MAJOR
static const unsigned short VERSION_MAJOR
Definition dyscostman.h:310
dyscostman::DyscoStMan::_dataBitCount
unsigned _dataBitCount
Definition dyscostman.h:397
dyscostman::DyscoStMan::reopenRW
virtual void reopenRW() final override
Reopen the storage manager files for read/write.
dyscostman::DyscoStMan::SetGaussianDistribution
void SetGaussianDistribution()
Initialize the storage manager to use a Gaussian distribution for the quantization.
Definition dyscostman.h:72
dyscostman::DyscoStMan::makeScalarColumn
virtual casacore::DataManagerColumn * makeScalarColumn(const casacore::String &name, int dataType, const casacore::String &dataTypeID) final override
Create a column in the storage manager on behalf of a table column.
dyscostman::DyscoStMan::SetStudentsTDistribution
void SetStudentsTDistribution(double nu)
Initialize the storage manager to use a Student T distribution for the quantization (i....
Definition dyscostman.h:95
dyscostman::DyscoStMan::SetTruncatedGaussianDistribution
void SetTruncatedGaussianDistribution(double truncationSigma)
Initialize the storage manager to use a Uniform distribution for the quantization (i....
Definition dyscostman.h:111
dyscostman::DyscoStMan::addRow64
virtual void addRow64(casacore::rownr_t nrrow) final override
Add rows to the storage manager.
dyscostman::DyscoStMan::readCompressedData
void readCompressedData(size_t blockIndex, const DyscoStManColumn *column, unsigned char *dest, size_t size)
dyscostman::DyscoStMan::prepare
virtual void prepare() final override
Prepare the columns, let the data manager initialize itself further.
dyscostman::DyscoStMan::SetStaticSeed
void SetStaticSeed(bool staticSeed)
Definition dyscostman.h:125
dyscostman::DyscoStMan::_distributionTruncation
double _distributionTruncation
Definition dyscostman.h:401
dyscostman::DyscoStMan::nAntennae
size_t nAntennae() const
Number of antennae used in a time block.
Definition dyscostman.h:260
dyscostman::DyscoStMan::flush
virtual casacore::Bool flush(casacore::AipsIO &, casacore::Bool doFsync) final override
Flush and optionally fsync the data.
dyscostman::DyscoStMan::_blockSize
uint32_t _blockSize
Definition dyscostman.h:390
dyscostman::DyscoStMan::addColumn
virtual void addColumn(casacore::DataManagerColumn *) final override
Do the final addition of a column.
dyscostman::DyscoStMan::makeEmpty
void makeEmpty()
dyscostman::DyscoStMan::_antennaCount
uint32_t _antennaCount
Definition dyscostman.h:389
dyscostman::DyscoStMan::create64
virtual void create64(casacore::rownr_t nRow) final override
Let the storage manager create files as needed for a new table.
dyscostman::DyscoStMan::clone
virtual casacore::DataManager * clone() const final override
Polymorphical copy constructor, equal to DyscoStMan(const DyscoStMan&).
Definition dyscostman.h:158
dyscostman::DyscoStMan::getFileOffset
size_t getFileOffset(size_t blockIndex) const
Definition dyscostman.h:326
dyscostman::DyscoStMan::setFromSpec
void setFromSpec(const casacore::Record &spec)
dyscostman::DyscoStMan::~DyscoStMan
~DyscoStMan()
Destructor.
dyscostman::DyscoStMan::_nBlocksInFile
uint64_t _nBlocksInFile
Definition dyscostman.h:387
dyscostman::DyscoStMan::_staticSeed
bool _staticSeed
Definition dyscostman.h:402
dyscostman::DyscoStMan::makeDirArrColumn
virtual casacore::DataManagerColumn * makeDirArrColumn(const casacore::String &name, int dataType, const casacore::String &dataTypeID) final override
Create a direct array column.
dyscostman::DyscoStMan::canAddColumn
virtual casacore::Bool canAddColumn() const final override
Whether columns can be added.
Definition dyscostman.h:202
dyscostman::DyscoStMan::initializeRowsPerBlock
void initializeRowsPerBlock(size_t rowsPerBlock, size_t antennaCount, bool writeToHeader)
To be called by a column once it determines rowsPerBlock and antennaCount.
dyscostman::DyscoStMan::nBlocksInFile
uint64_t nBlocksInFile() const
The number of rows that are actually stored in the file.
Definition dyscostman.h:239
dyscostman::DyscoStMan::_nRow
uint64_t _nRow
Definition dyscostman.h:386
dyscostman::DyscoStMan::VERSION_MINOR
static const unsigned short VERSION_MINOR
Definition dyscostman.h:310
dyscostman::DyscoStMan::DyscoStManColumn
friend class DyscoStManColumn
Definition dyscostman.h:308
dyscostman::DyscoStMan::getNRow
uint getNRow() const
Get the number of rows in the measurement set.
Definition dyscostman.h:183
dyscostman::DyscoStMan::_distribution
DyscoDistribution _distribution
Definition dyscostman.h:399
dyscostman::DyscoStMan::SetUniformDistribution
void SetUniformDistribution()
Initialize the storage manager to use a Uniform distribution for the quantization (i....
Definition dyscostman.h:84
dyscostman::DyscoStMan::nRowsInBlock
size_t nRowsInBlock() const
Number of rows in one "time-block", i.e.
Definition dyscostman.h:251
dyscostman::DyscoStMan::_columns
std::vector< std::unique_ptr< DyscoStManColumn > > _columns
Definition dyscostman.h:404
dyscostman::DyscoStMan::operator=
DyscoStMan & operator=(const DyscoStMan &source)=delete
Assignment – new dyscostman takes the settings of the source (but not the columns and/or data).
dyscostman::DyscoStMan::makeIndArrColumn
virtual casacore::DataManagerColumn * makeIndArrColumn(const casacore::String &name, int dataType, const casacore::String &dataTypeID) final override
Create an indirect array column.
dyscostman::DyscoStMan::getRowIndex
uint64_t getRowIndex(size_t block) const
Calculate first measurement set row index of a given block index.
Definition dyscostman.h:286
dyscostman::DyscoStMan::_name
std::string _name
Definition dyscostman.h:396
dyscostman::DyscoStMan::DyscoStMan
DyscoStMan(const DyscoStMan &source)
Copy constructor that initializes a storage manager with similar specs.
dyscostman::DyscoStMan::canAddRow
virtual casacore::Bool canAddRow() const final override
Whether rows can be added.
Definition dyscostman.h:189
dyscostman::DyscoStMan::canRemoveRow
virtual casacore::Bool canRemoveRow() const final override
Whether rows can be removed.
Definition dyscostman.h:195
dyscostman::DyscoStMan::getRowWithinBlock
size_t getRowWithinBlock(uint64_t row) const
Return the offset of the row within the block.
Definition dyscostman.h:279
dyscostman::DyscoStMan::resync64
virtual casacore::rownr_t resync64(casacore::rownr_t nRow) final override
dyscostman::DyscoStMan::SetNormalization
void SetNormalization(Normalization normalization)
Set the type of normalization.
Definition dyscostman.h:121
dyscostman::DyscoStMan::_headerSize
unsigned _headerSize
Definition dyscostman.h:392
dyscostman::DyscoStMan::getBlockIndex
size_t getBlockIndex(uint64_t row) const
Return index of block that contains the given measurement set row.
Definition dyscostman.h:269
dyscostman::DyscoStMan::removeColumn
virtual void removeColumn(casacore::DataManagerColumn *) final override
Remove a column from the data file.
dyscostman::DyscoStMan::dataManagerSpec
virtual casacore::Record dataManagerSpec() const final override
Get manager specifications.
dyscostman::DyscoStMan::makeObject
static casacore::DataManager * makeObject(const casacore::String &name, const casacore::Record &spec)
Create an object with given name and spec.
Definition dyscostman.h:219
dyscostman::DyscoStMan::_weightBitCount
unsigned _weightBitCount
Definition dyscostman.h:398
dyscostman::DyscoStMan::_mutex
std::mutex _mutex
Definition dyscostman.h:393
dyscostman::DyscoStMan::areOffsetsInitialized
bool areOffsetsInitialized() const
This method returns true when the number of rows per block and the number of antennae per block are k...
Definition dyscostman.h:296
dyscostman::DyscoStMan::DyscoStMan
DyscoStMan(const casacore::String &name, const casacore::Record &spec)
This constructor is called by Casa when it needs to create a DyscoStMan.
dyscostman::DyscoStMan::removeRow64
virtual void removeRow64(casacore::rownr_t rowNr) final override
Delete a row from all columns.
dyscostman::DyscoStMan::readHeader
void readHeader()
dyscostman::DyscoStMan::writeHeader
void writeHeader()
dyscostman::DyscoStMan::DyscoStMan
DyscoStMan(unsigned dataBitRate, unsigned weightBitRate, const casacore::String &name="DyscoStMan")
Convenience constructor to create a new storage manager with some settings without having to fill a '...
dyscodistribution.h
dysconormalization.h
register_dyscostman
void register_dyscostman()
casacore::Record
Record()
casacore
this file contains all the compiler specific defines
Definition mainpage.dox:28
casacore::Bool
bool Bool
Define the standard types used by Casacore.
Definition aipstype.h:40
casacore::rownr_t
uInt64 rownr_t
Define the type of a row number in a table.
Definition aipsxtype.h:44
dyscostman
Definition bytepacker.h:7
dyscostman::Normalization
Normalization
Definition dysconormalization.h:5
dyscostman::DyscoDistribution
DyscoDistribution
Definition dyscodistribution.h:5
dyscostman::GaussianDistribution
@ GaussianDistribution
Definition dyscodistribution.h:6
dyscostman::StudentsTDistribution
@ StudentsTDistribution
Definition dyscodistribution.h:8
dyscostman::UniformDistribution
@ UniformDistribution
Definition dyscodistribution.h:7
dyscostman::TruncatedGaussianDistribution
@ TruncatedGaussianDistribution
Definition dyscodistribution.h:9
std
Define real & complex conjugation for non-complex types and put comparisons into std namespace.
Definition Complex.h:350
threadgroup.h
uvector.h
Header file for uvector and its relational and swap functions.