Loading...
Searching...
No Matches
MultiHDF5.h
Go to the documentation of this file.
//# MultiHDF5.h: Class to combine multiple files in a single HDF5 file
//# Copyright (C) 2015
//# Associated Universities, Inc. Washington DC, USA.
//#
//# This library is free software; you can redistribute it and/or modify it
//# under the terms of the GNU Library General Public License as published by
//# the Free Software Foundation; either version 2 of the License, or (at your
//# option) any later version.
//#
//# This library is distributed in the hope that it will be useful, but WITHOUT
//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
//# License for more details.
//#
//# You should have received a copy of the GNU Library General Public License
//# along with this library; if not, write to the Free Software Foundation,
//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
//#
//# Correspondence concerning AIPS++ should be addressed as follows:
//# Internet email: casa-feedback@nrao.edu.
//# Postal address: AIPS++ Project Office
//# National Radio Astronomy Observatory
//# 520 Edgemont Road
//# Charlottesville, VA 22903-2475 USA
#ifndef CASA_MULTIHDF5_H
#define CASA_MULTIHDF5_H
//# Includes
#include <casacore/casa/aips.h>
#include <casacore/casa/IO/MultiFile.h>
#include <casacore/casa/HDF5/HDF5File.h>
#include <casacore/casa/HDF5/HDF5Group.h>
#include <memory>
namespace casacore { //# NAMESPACE CASACORE - BEGIN
// <summary>
// Class to combine multiple files in a single HDF5 file.
// </summary>
// <use visibility=export>
// <reviewed reviewer="" date="" tests="tMultiHDF5" demos="">
// </reviewed>
// <synopsis>
// This class is a container file holding multiple virtual files. It is
// primarily meant as a container file for the storage manager files of a
// table to reduce the number of files used (especially for Lustre) and to
// reduce the number of open files (especially when concatenating tables).
// <br>A secondary goal is offering the ability to use an IO buffer size
// that matches the file system well (large buffer size for e.g. ZFS).
//
// The SetupNewTable constructor has a StorageOption argument to define
// if a MultiFile has to be used and if so, the buffer size to use.
// It is also possible to specify that through aipsrc variables.
//
// A virtual file is spread over multiple (fixed size) data blocks in the
// MultiFile. A data block is never shared by multiple files.
// For each virtual file MultiFile keeps a MultiFileInfo object telling
// the file size and the blocks numbers used for the file. When flushing
// the MultiFile, this meta info is written into a header block and,
// if needed, continuation blocks. On open and resync, it is read back.
// <br>
//
// A virtual file is represented by an MFFileIO object, which is derived
// from ByteIO and as such part of the casacore IO framework. It makes it
// possible for applications to access a virtual file in the same way as
// a regular file.
//
// It is possible to delete a virtual file. Its blocks will be added to
// the free block list (which is also stored in the meta info).
// </synopsis>
// <example>
// In principle it is possible to use the MultiFile functions directly.
// However, in general it is much easier to use an MFFileIO object
// per virtual file as shown below.
// <srcblock>
// // Create a new MultiFile using a block size of 1 MB.
// MultiFile mfile("file.mf', ByteIO::New, 1048576);
// // Create a virtual file in it.
// MFFileIO mf1(mfile, "mf1", ByteIO::New);
// // Use it (for example) as the sink of AipsIO.
// AipsIO stream (&mf1);
// // Write values.
// stream << (Int)10;
// stream << True;
// // Seek to beginning of file and read data in.
// stream.setpos (0);
// Int vali;
// Bool valb;
// stream >> vali >> valb;
// </srcblock>
// </example>
class MultiHDF5 : public MultiFileBase
{
public:
// Open or create a MultiHDF5 with the given name.
// Upon creation the block size can be given. If 0, it uses the block size
// of the file system the file is on.
explicit MultiHDF5 (const String& name, ByteIO::OpenOption, Int blockSize=0);
// Open or create a MultiHDF5 which is nested in the given parent.
// The data are read/written in a group with the given name in the parent.
// Upon creation the block size can be given. If 0, it uses the block size
// of the parent.
explicit MultiHDF5 (const String& name,
const std::shared_ptr<MultiFileBase>& parent,
ByteIO::OpenOption, Int blockSize=0);
// The destructor flushes and closes the file.
~MultiHDF5() override;
// Copy constructor and assignment not possible.
MultiHDF5 (const MultiHDF5&) = delete;
MultiHDF5& operator= (const MultiHDF5&) = delete;
// Make the correct MultiFileBase object for a nested file.
// It creates a new group under which the virtual files are created.
std::shared_ptr<MultiFileBase> makeNested
(const std::shared_ptr<MultiFileBase>& parent, const String& name,
ByteIO::OpenOption, Int blockSize) const override;
// Open the given logical file and return its file id.
// If the name is unknown, an exception is thrown.
// It creates the HDF5 group and dataset opbject.
// Reopen the underlying file for read/write access.
// Nothing will be done if the file is writable already.
// Otherwise it will be reopened and an exception will be thrown
// if it is not possible to reopen it for read/write access.
void reopenRW() override;
// Fsync the file (i.e., force the data to be physically written).
void fsync() override;
// Get the HDF5File object.
const std::shared_ptr<HDF5File>& getHDF5File() const
{ return itsFile; }
const HDF5Object& getHDF5Object() const
{ return *itsHDF5; }
private:
// Initialize the MultiFile object.
void init (ByteIO::OpenOption option);
// Do the class-specific actions on opening a file.
void doOpenFile (MultiFileInfo&) override;
// Do the class-specific actions on closing a file.
void doCloseFile (MultiFileInfo&) override;
// Do the class-specific actions on adding a file.
void doAddFile (MultiFileInfo&) override;
// Do the class-specific actions on deleting a file.
void doDeleteFile (MultiFileInfo&) override;
// Truncate the file to <src>nrblk</src> blocks (does nothing).
void doTruncateFile (MultiFileInfo& info, uInt64 nrblk) override;
// Flush the file itself.
void doFlushFile() override;
// Flush and close the file.
void close() override;
// Write the header info.
void writeHeader() override;
// Read the header info. If always==False, the info is only read if the
// header counter has changed.
void readHeader (Bool always=True) override;
// Extend the virtual file to fit lastblk.
void extend (MultiFileInfo& info, Int64 lastblk) override;
// Read a data block.
void readBlock (MultiFileInfo& info, Int64 blknr,
void* buffer) override;
// Write a data block.
void writeBlock (MultiFileInfo& info, Int64 blknr,
const void* buffer) override;
//# Data members
std::shared_ptr<HDF5File> itsFile;
std::shared_ptr<HDF5Group> itsGroup;
// This points to the HDF5Group if present, otherwise to the HDF5File.
// It tells where the HDF5 data are accessed.
HDF5Object* itsHDF5;
};
} //# NAMESPACE CASACORE - END
#endif
casacore::ByteIO::OpenOption
OpenOption
Define the possible ByteIO open options.
Definition ByteIO.h:63
casacore::HDF5Object
Definition HDF5Object.h:69
casacore::MultiFileBase::blockSize
Int64 blockSize() const
Get the block size used.
Definition MultiFileBase.h:212
casacore::MultiFileBase::MultiFileBase
MultiFileBase(const String &name, Int blockSize, Bool useODirect)
Create a MultiFileBase object with the given name.
casacore::MultiFileBase::info
const std::vector< MultiFileInfo > & info() const
Get the info object (for test purposes mainly).
Definition MultiFileBase.h:223
casacore::MultiHDF5::~MultiHDF5
~MultiHDF5() override
The destructor flushes and closes the file.
casacore::MultiHDF5::itsGroup
std::shared_ptr< HDF5Group > itsGroup
Definition MultiHDF5.h:180
casacore::MultiHDF5::getHDF5File
const std::shared_ptr< HDF5File > & getHDF5File() const
Get the HDF5File object.
Definition MultiHDF5.h:142
casacore::MultiHDF5::doOpenFile
void doOpenFile(MultiFileInfo &) override
Do the class-specific actions on opening a file.
casacore::MultiHDF5::makeNested
std::shared_ptr< MultiFileBase > makeNested(const std::shared_ptr< MultiFileBase > &parent, const String &name, ByteIO::OpenOption, Int blockSize) const override
Make the correct MultiFileBase object for a nested file.
casacore::MultiHDF5::fsync
void fsync() override
Fsync the file (i.e., force the data to be physically written).
casacore::MultiHDF5::operator=
MultiHDF5 & operator=(const MultiHDF5 &)=delete
casacore::MultiHDF5::reopenRW
void reopenRW() override
Open the given logical file and return its file id.
casacore::MultiHDF5::getHDF5Object
const HDF5Object & getHDF5Object() const
Definition MultiHDF5.h:144
casacore::MultiHDF5::readHeader
void readHeader(Bool always=True) override
Read the header info.
casacore::MultiHDF5::doCloseFile
void doCloseFile(MultiFileInfo &) override
Do the class-specific actions on closing a file.
casacore::MultiHDF5::doFlushFile
void doFlushFile() override
Flush the file itself.
casacore::MultiHDF5::itsFile
std::shared_ptr< HDF5File > itsFile
Definition MultiHDF5.h:179
casacore::MultiHDF5::doTruncateFile
void doTruncateFile(MultiFileInfo &info, uInt64 nrblk) override
Truncate the file to nrblk blocks (does nothing).
casacore::MultiHDF5::close
void close() override
Flush and close the file.
casacore::MultiHDF5::doAddFile
void doAddFile(MultiFileInfo &) override
Do the class-specific actions on adding a file.
casacore::MultiHDF5::itsHDF5
HDF5Object * itsHDF5
This points to the HDF5Group if present, otherwise to the HDF5File.
Definition MultiHDF5.h:183
casacore::MultiHDF5::writeHeader
void writeHeader() override
Write the header info.
casacore::MultiHDF5::readBlock
void readBlock(MultiFileInfo &info, Int64 blknr, void *buffer) override
Read a data block.
casacore::MultiHDF5::extend
void extend(MultiFileInfo &info, Int64 lastblk) override
Extend the virtual file to fit lastblk.
casacore::MultiHDF5::writeBlock
void writeBlock(MultiFileInfo &info, Int64 blknr, const void *buffer) override
Write a data block.
casacore::MultiHDF5::doDeleteFile
void doDeleteFile(MultiFileInfo &) override
Do the class-specific actions on deleting a file.
casacore::MultiHDF5::MultiHDF5
MultiHDF5(const String &name, ByteIO::OpenOption, Int blockSize=0)
Open or create a MultiHDF5 with the given name.
casacore::String
String: the storage and methods of handling collections of characters.
Definition String.h:223
casacore
this file contains all the compiler specific defines
Definition mainpage.dox:28
casacore::Int64
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size).
Definition aipsxtype.h:36
casacore::name
String name() const
Return the name of the field.
Definition RecordField.h:163
casacore::init
void init()
Other internal helper function(s).
casacore::Int
int Int
Definition aipstype.h:48
casacore::Bool
bool Bool
Define the standard types used by Casacore.
Definition aipstype.h:40
casacore::True
const Bool True
Definition aipstype.h:41
casacore::uInt64
unsigned long long uInt64
Definition aipsxtype.h:37
casacore::MultiFileInfo
Helper class for MultiFileBase containing info per logical file.
Definition MultiFileBase.h:77