Loading...
Searching...
No Matches
MultiFileBase.h
Go to the documentation of this file.
//# MultiFileBase.h: Abstract base class to combine multiple files in a single one
//# Copyright (C) 2014
//# Associated Universities, Inc. Washington DC, USA.
//#
//# This library is free software; you can redistribute it and/or modify it
//# under the terms of the GNU Library General Public License as published by
//# the Free Software Foundation; either version 2 of the License, or (at your
//# option) any later version.
//#
//# This library is distributed in the hope that it will be useful, but WITHOUT
//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
//# License for more details.
//#
//# You should have received a copy of the GNU Library General Public License
//# along with this library; if not, write to the Free Software Foundation,
//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
//#
//# Correspondence concerning AIPS++ should be addressed as follows:
//# Internet email: casa-feedback@nrao.edu.
//# Postal address: AIPS++ Project Office
//# National Radio Astronomy Observatory
//# 520 Edgemont Road
//# Charlottesville, VA 22903-2475 USA
#ifndef CASA_MULTIFILEBASE_H
#define CASA_MULTIFILEBASE_H
//# Includes
#include <casacore/casa/aips.h>
#include <casacore/casa/IO/ByteIO.h>
#include <casacore/casa/BasicSL/String.h>
#include <casacore/casa/ostream.h>
#include <vector>
#include <memory>
namespace casacore { //# NAMESPACE CASACORE - BEGIN
//# Forward declaration.
class AipsIO;
class HDF5Group;
class HDF5DataSet;
// <summary>
// Helper class for MultiFileInfo holding a data buffer
// </summary>
// <synopsis>
// The buffer can be allocated with posix_memalign (for O_DIRECT support).
// Hence the memory must be freed using free, which makes it impossible
// to use a shared_ptr to that memory. Hence it is encapsulated in this class.
// </synopsis>
class MultiFileBuffer {
public:
MultiFileBuffer (size_t bufSize, Bool useODirect);
~MultiFileBuffer()
{ if (itsData) free(itsData); }
// Forbid copy constructor.
MultiFileBuffer (const MultiFileBuffer&) = delete;
// Forbid assignment.
MultiFileBuffer& operator= (const MultiFileBuffer&) = delete;
char* data()
{ return itsData; }
private:
// Data members
char* itsData;
};
// <summary>
// Helper class for MultiFileBase containing info per logical file.
// </summary>
// <synopsis>
// This struct defines the basic fields describing a logical file in a
// class derived from MultiFileBase (such as MultiFile or MultiHDF5).
// </synopsis>
// <use visibility=local>
struct MultiFileInfo {
// Initialize the object. The buffer is created when the file is opened.
explicit MultiFileInfo();
// Allocate the buffer.
void allocBuffer (size_t bufSize, Bool useODirect)
{ buffer = std::make_shared<MultiFileBuffer>(bufSize, useODirect); }
//# Data members.
std::vector<Int64> blockNrs; // physical blocknrs for this logical file
Int64 curBlock; // the data block held in buffer (<0 is none)
Int64 fsize; // file size (in bytes)
String name; // the virtual file name
Bool nested; // is the file a nested MultiFile?
Bool dirty; // has data in buffer been changed?
std::shared_ptr<MultiFileBuffer> buffer; // buffer holding a data block
std::shared_ptr<HDF5Group> group;
std::shared_ptr<HDF5DataSet> dataSet;
};
ostream& operator<< (ostream&, const MultiFileInfo&);
AipsIO& operator<< (AipsIO&, const MultiFileInfo&);
AipsIO& operator>> (AipsIO&, MultiFileInfo&);
void getInfoVersion1 (AipsIO&, std::vector<MultiFileInfo>&);
// <summary>
// Abstract base class to combine multiple logical files in a single one.
// </summary>
// <use visibility=export>
// <reviewed reviewer="" date="" tests="tMultiFile" demos="">
// </reviewed>
// <synopsis>
// This class is the abstract base class for classes defining a container
// file holding multiple logical files. These classes are meant as a
// container files for the storage manager files of a table to reduce the
// number of files used (especially for Lustre) and to reduce the number
// of open files (especially when concatenating tables).
// The derived classes MultiFile and MultiHDF5 implement such container
// files using a regular file and HDF5, resp.
//
// MultiFileBase implements several functions with common functionality
// for the derived classes.
//
// A logical file is represented by an MFFileIO object, which is derived
// from ByteIO and as such part of the casacore IO framework. It makes it
// possible for applications to access a logical file in the same way as
// a regular file.
// </synopsis>
class MultiFileBase
{
public:
// Create a MultiFileBase object with the given name.
// <br>Upon creation of the container file the block size can be given.
// If <=0, it uses the block size of the file system the file is on,
// but it will not be less than the absolute value of the given block size.
// <br>If useODIrect=True, it means that O_DIRECT is used. If the OS does not
// support it (as determined at configure time), the flag will always be
// set to False. If True, the data buffers will have a proper alignment
// and size (as needed by O_DIRECT).
MultiFileBase (const String& name, Int blockSize, Bool useODirect);
// The destructor flushes dirty blocks and closes the container file.
virtual ~MultiFileBase();
// Forbid copy constructor.
MultiFileBase (const MultiFileBase&) = delete;
// Forbid assignment.
MultiFileBase& operator= (const MultiFileBase&) = delete;
// Open the correct MultiFileBase (as plain or HDF5).
static std::shared_ptr<MultiFileBase> openMF (const String& fileName);
// Make the correct MultiFileBase object for a nested file.
virtual std::shared_ptr<MultiFileBase> makeNested
(const std::shared_ptr<MultiFileBase>& parent, const String& name,
ByteIO::OpenOption, Int blockSize) const = 0;
// Get the file name of the MultiFileBase container file.
String fileName() const
{ return itsName; }
// Is the container file writable?
Bool isWritable() const
{ return itsWritable; }
// Open the given logical file and return its file id.
// If the name is unknown, an exception is thrown.
// It allocates the internal buffer of the logical file.
Int openFile (const String& name);
// Create a new logical file and return its file id.
// Only the base name of the given file name is used. In this way the
// MultiFileBase container file can be moved.
// If the logical file already exists, it is deleted if ByteIO::New is
// given. Otherwise an exception is thrown.
// It allocates the internal buffer of the logical file.
Int createFile (const String& name, ByteIO::OpenOption = ByteIO::New);
// Flush the possible dirty buffer of the given logical file.
void flushFile (Int fileId);
// Close a logical file.
// It flushes and deallocates its buffer.
void closeFile (Int fileId);
// Delete a logical file. It adds its blocks to the free block list.
void deleteFile (Int fileId);
// Get the size of a logical file.
Int64 fileSize (Int fileId) const;
// Read a block at the given offset in the logical file.
// It returns the actual size read.
Int64 read (Int fileId, void* buffer, Int64 size, Int64 offset);
// Write a block at the given offset in the logical file.
// It returns the actual size written.
Int64 write (Int fileId, const void* buffer, Int64 size, Int64 offset);
// Truncate the logical file to the given size.
void truncate (Int fileId, Int64 size);
// Reopen the underlying file for read/write access.
// Nothing will be done if the file is writable already.
// Otherwise it will be reopened and an exception will be thrown
// if it is not possible to reopen it for read/write access.
virtual void reopenRW() = 0;
// Flush the file by writing all dirty data and all header info.
void flush();
// Get the block size used.
Int64 blockSize() const
{ return itsBlockSize; }
// Get the nr of logical files.
uInt nfile() const;
// Get the total nr of data blocks used.
Int64 nblock() const
{ return itsNrBlock; }
// Get the info object (for test purposes mainly).
const std::vector<MultiFileInfo>& info() const
{ return itsInfo; }
// Get the free blocks (for test purposes mainly).
const std::vector<Int64>& freeBlocks() const
{ return itsFreeBlocks; }
// Return the file id of a file in the MultiFileBase object.
// If the name is unknown, an exception is thrown if throwExcp is set.
// Otherwise it returns -1.
Int fileId (const String& name, Bool throwExcp=True) const;
// Is O_DIRECT used?
Bool useODirect() const
{ return itsUseODirect; }
protected:
// Resync with another process by clearing the buffers and rereading
// the header. The header is only read if its counter has changed.
void resync();
// Fsync the file (i.e., force the data to be physically written).
virtual void fsync() = 0;
private:
// Write the dirty block and clear dirty flag.
void writeDirty (MultiFileInfo& info)
{
writeBlock (info, info.curBlock, info.buffer->data());
info.dirty = False;
}
// Add a file to the MultiFileBase object. It returns the file id.
// Only the base name of the given file name is used. In this way the
// MultiFileBase container file can be moved.
Int addFile (const String& name);
// Do the class-specific actions on opening a logical file.
virtual void doOpenFile (MultiFileInfo&) = 0;
// Do the class-specific actions on closing a logical file.
virtual void doCloseFile (MultiFileInfo&) = 0;
// Do the class-specific actions on adding a logical file.
virtual void doAddFile (MultiFileInfo&) = 0;
// Do the class-specific actions on deleting a logical file.
virtual void doDeleteFile (MultiFileInfo&) = 0;
// Truncate the container file to <src>nrblk</src> blocks.
virtual void doTruncateFile (MultiFileInfo& info, uInt64 nrblk) = 0;
// Flush the container file.
virtual void doFlushFile() = 0;
// Flush and close the container file.
virtual void close() = 0;
// Write the header info.
virtual void writeHeader() = 0;
// Read the header info. If always==False, the info is only read if the
// header counter has changed.
virtual void readHeader (Bool always=True) = 0;
// Extend a logical file to fit lastblk.
virtual void extend (MultiFileInfo& info, Int64 lastblk) = 0;
// Write a data block of a logical file into the container file.
virtual void writeBlock (MultiFileInfo& info, Int64 blknr,
const void* buffer) = 0;
// Read a data block of a logical file from the container file.
virtual void readBlock (MultiFileInfo& info, Int64 blknr,
void* buffer) = 0;
protected:
// Set the flags and blockSize for a new MultiFile/HDF5.
void setNewFile();
//# Data members
String itsName;
Int64 itsBlockSize; // The blocksize used
Int64 itsNrBlock; // The total nr of blocks actually used
Int64 itsHdrCounter; // Counter of header changes
std::vector<MultiFileInfo> itsInfo;
std::shared_ptr<MultiFileBuffer> itsBuffer;
Bool itsUseODirect; // use O_DIRECT?
Bool itsWritable; // Is the file writable?
Bool itsChanged; // Has header info changed since last flush?
std::vector<Int64> itsFreeBlocks;
};
} //# NAMESPACE CASACORE - END
#endif
casacore::AipsIO
Definition AipsIO.h:167
casacore::ByteIO::OpenOption
OpenOption
Define the possible ByteIO open options.
Definition ByteIO.h:63
casacore::ByteIO::New
@ New
read/write; create file if not exist.
Definition ByteIO.h:70
casacore::HDF5DataSet
Definition HDF5DataSet.h:91
casacore::HDF5Group
Definition HDF5Group.h:53
casacore::MultiFileBase::itsName
String itsName
Definition MultiFileBase.h:293
casacore::MultiFileBase::flushFile
void flushFile(Int fileId)
Flush the possible dirty buffer of the given logical file.
casacore::MultiFileBase::doCloseFile
virtual void doCloseFile(MultiFileInfo &)=0
Do the class-specific actions on closing a logical file.
casacore::MultiFileBase::nblock
Int64 nblock() const
Get the total nr of data blocks used.
Definition MultiFileBase.h:219
casacore::MultiFileBase::blockSize
Int64 blockSize() const
Get the block size used.
Definition MultiFileBase.h:212
casacore::MultiFileBase::openMF
static std::shared_ptr< MultiFileBase > openMF(const String &fileName)
Open the correct MultiFileBase (as plain or HDF5).
casacore::MultiFileBase::operator=
MultiFileBase & operator=(const MultiFileBase &)=delete
Forbid assignment.
casacore::MultiFileBase::extend
virtual void extend(MultiFileInfo &info, Int64 lastblk)=0
Extend a logical file to fit lastblk.
casacore::MultiFileBase::doFlushFile
virtual void doFlushFile()=0
Flush the container file.
casacore::MultiFileBase::writeHeader
virtual void writeHeader()=0
Write the header info.
casacore::MultiFileBase::itsFreeBlocks
std::vector< Int64 > itsFreeBlocks
Definition MultiFileBase.h:302
casacore::MultiFileBase::MultiFileBase
MultiFileBase(const String &name, Int blockSize, Bool useODirect)
Create a MultiFileBase object with the given name.
casacore::MultiFileBase::MultiFileBase
MultiFileBase(const MultiFileBase &)=delete
Forbid copy constructor.
casacore::MultiFileBase::itsWritable
Bool itsWritable
Definition MultiFileBase.h:300
casacore::MultiFileBase::itsHdrCounter
Int64 itsHdrCounter
Definition MultiFileBase.h:296
casacore::MultiFileBase::fsync
virtual void fsync()=0
Fsync the file (i.e., force the data to be physically written).
casacore::MultiFileBase::itsBlockSize
Int64 itsBlockSize
Definition MultiFileBase.h:294
casacore::MultiFileBase::readHeader
virtual void readHeader(Bool always=True)=0
Read the header info.
casacore::MultiFileBase::write
Int64 write(Int fileId, const void *buffer, Int64 size, Int64 offset)
Write a block at the given offset in the logical file.
casacore::MultiFileBase::resync
void resync()
Resync with another process by clearing the buffers and rereading the header.
casacore::MultiFileBase::makeNested
virtual std::shared_ptr< MultiFileBase > makeNested(const std::shared_ptr< MultiFileBase > &parent, const String &name, ByteIO::OpenOption, Int blockSize) const =0
Make the correct MultiFileBase object for a nested file.
casacore::MultiFileBase::createFile
Int createFile(const String &name, ByteIO::OpenOption=ByteIO::New)
Create a new logical file and return its file id.
casacore::MultiFileBase::addFile
Int addFile(const String &name)
Add a file to the MultiFileBase object.
casacore::MultiFileBase::openFile
Int openFile(const String &name)
Open the given logical file and return its file id.
casacore::MultiFileBase::fileId
Int fileId(const String &name, Bool throwExcp=True) const
Return the file id of a file in the MultiFileBase object.
casacore::MultiFileBase::doTruncateFile
virtual void doTruncateFile(MultiFileInfo &info, uInt64 nrblk)=0
Truncate the container file to nrblk blocks.
casacore::MultiFileBase::info
const std::vector< MultiFileInfo > & info() const
Get the info object (for test purposes mainly).
Definition MultiFileBase.h:223
casacore::MultiFileBase::freeBlocks
const std::vector< Int64 > & freeBlocks() const
Get the free blocks (for test purposes mainly).
Definition MultiFileBase.h:227
casacore::MultiFileBase::fileSize
Int64 fileSize(Int fileId) const
Get the size of a logical file.
casacore::MultiFileBase::deleteFile
void deleteFile(Int fileId)
Delete a logical file.
casacore::MultiFileBase::itsNrBlock
Int64 itsNrBlock
Definition MultiFileBase.h:295
casacore::MultiFileBase::doOpenFile
virtual void doOpenFile(MultiFileInfo &)=0
Do the class-specific actions on opening a logical file.
casacore::MultiFileBase::read
Int64 read(Int fileId, void *buffer, Int64 size, Int64 offset)
Read a block at the given offset in the logical file.
casacore::MultiFileBase::itsUseODirect
Bool itsUseODirect
Definition MultiFileBase.h:299
casacore::MultiFileBase::fileName
String fileName() const
Get the file name of the MultiFileBase container file.
Definition MultiFileBase.h:158
casacore::MultiFileBase::useODirect
Bool useODirect() const
Is O_DIRECT used?
Definition MultiFileBase.h:236
casacore::MultiFileBase::truncate
void truncate(Int fileId, Int64 size)
Truncate the logical file to the given size.
casacore::MultiFileBase::~MultiFileBase
virtual ~MultiFileBase()
The destructor flushes dirty blocks and closes the container file.
casacore::MultiFileBase::isWritable
Bool isWritable() const
Is the container file writable?
Definition MultiFileBase.h:162
casacore::MultiFileBase::doDeleteFile
virtual void doDeleteFile(MultiFileInfo &)=0
Do the class-specific actions on deleting a logical file.
casacore::MultiFileBase::writeBlock
virtual void writeBlock(MultiFileInfo &info, Int64 blknr, const void *buffer)=0
Write a data block of a logical file into the container file.
casacore::MultiFileBase::reopenRW
virtual void reopenRW()=0
Reopen the underlying file for read/write access.
casacore::MultiFileBase::itsChanged
Bool itsChanged
Definition MultiFileBase.h:301
casacore::MultiFileBase::itsInfo
std::vector< MultiFileInfo > itsInfo
Definition MultiFileBase.h:297
casacore::MultiFileBase::readBlock
virtual void readBlock(MultiFileInfo &info, Int64 blknr, void *buffer)=0
Read a data block of a logical file from the container file.
casacore::MultiFileBase::close
virtual void close()=0
Flush and close the container file.
casacore::MultiFileBase::setNewFile
void setNewFile()
Set the flags and blockSize for a new MultiFile/HDF5.
casacore::MultiFileBase::doAddFile
virtual void doAddFile(MultiFileInfo &)=0
Do the class-specific actions on adding a logical file.
casacore::MultiFileBase::flush
void flush()
Flush the file by writing all dirty data and all header info.
casacore::MultiFileBase::nfile
uInt nfile() const
Get the nr of logical files.
casacore::MultiFileBase::closeFile
void closeFile(Int fileId)
Close a logical file.
casacore::MultiFileBase::itsBuffer
std::shared_ptr< MultiFileBuffer > itsBuffer
Definition MultiFileBase.h:298
casacore::MultiFileBase::writeDirty
void writeDirty(MultiFileInfo &info)
Write the dirty block and clear dirty flag.
Definition MultiFileBase.h:249
casacore::MultiFileBuffer::data
char * data()
Definition MultiFileBase.h:62
casacore::MultiFileBuffer::~MultiFileBuffer
~MultiFileBuffer()
Definition MultiFileBase.h:56
casacore::MultiFileBuffer::itsData
char * itsData
Data members.
Definition MultiFileBase.h:66
casacore::MultiFileBuffer::MultiFileBuffer
MultiFileBuffer(size_t bufSize, Bool useODirect)
casacore::MultiFileBuffer::MultiFileBuffer
MultiFileBuffer(const MultiFileBuffer &)=delete
Forbid copy constructor.
casacore::MultiFileBuffer::operator=
MultiFileBuffer & operator=(const MultiFileBuffer &)=delete
Forbid assignment.
casacore::String
String: the storage and methods of handling collections of characters.
Definition String.h:223
free
free(pool)
casacore
this file contains all the compiler specific defines
Definition mainpage.dox:28
casacore::False
const Bool False
Definition aipstype.h:42
casacore::operator>>
friend AipsIO & operator>>(AipsIO &os, Record &rec)
Read the Record from an input stream.
Definition Record.h:462
casacore::operator<<
ostream & operator<<(ostream &os, const IComplex &)
Show on ostream.
casacore::offset
int offset(int, int) const
compute a linear offset from array indicies
casacore::uInt
unsigned int uInt
Definition aipstype.h:49
casacore::Int64
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size).
Definition aipsxtype.h:36
casacore::name
String name() const
Return the name of the field.
Definition RecordField.h:163
casacore::Int
int Int
Definition aipstype.h:48
casacore::Bool
bool Bool
Define the standard types used by Casacore.
Definition aipstype.h:40
casacore::size
uInt size() const
Definition RecordInterface.h:224
casacore::getInfoVersion1
void getInfoVersion1(AipsIO &, std::vector< MultiFileInfo > &)
casacore::True
const Bool True
Definition aipstype.h:41
casacore::uInt64
unsigned long long uInt64
Definition aipsxtype.h:37
casacore::MultiFileInfo
Helper class for MultiFileBase containing info per logical file.
Definition MultiFileBase.h:77
casacore::MultiFileInfo::name
String name
Definition MultiFileBase.h:87
casacore::MultiFileInfo::allocBuffer
void allocBuffer(size_t bufSize, Bool useODirect)
Allocate the buffer.
Definition MultiFileBase.h:81
casacore::MultiFileInfo::fsize
Int64 fsize
Definition MultiFileBase.h:86
casacore::MultiFileInfo::buffer
std::shared_ptr< MultiFileBuffer > buffer
Definition MultiFileBase.h:90
casacore::MultiFileInfo::curBlock
Int64 curBlock
Definition MultiFileBase.h:85
casacore::MultiFileInfo::blockNrs
std::vector< Int64 > blockNrs
Definition MultiFileBase.h:84
casacore::MultiFileInfo::dirty
Bool dirty
Definition MultiFileBase.h:89
casacore::MultiFileInfo::group
std::shared_ptr< HDF5Group > group
Definition MultiFileBase.h:91
casacore::MultiFileInfo::nested
Bool nested
Definition MultiFileBase.h:88
casacore::MultiFileInfo::MultiFileInfo
MultiFileInfo()
Initialize the object.
casacore::MultiFileInfo::dataSet
std::shared_ptr< HDF5DataSet > dataSet
Definition MultiFileBase.h:92