Loading...
Searching...
No Matches
casacore::VarBufferedColumnarFile< BufferSize > Class Template Reference

Class that provides binary table I/O. More...

`#include <BufferedColumnarFile.h>`

## Public Member Functions

| VarBufferedColumnarFile () noexcept=default
| VarBufferedColumnarFile (const VarBufferedColumnarFile &rhs)=delete
| VarBufferedColumnarFile (VarBufferedColumnarFile &&rhs) noexcept
| ~VarBufferedColumnarFile () noexcept
VarBufferedColumnarFile & | operator= (VarBufferedColumnarFile &&rhs)
void | Close ()
| Close the file.
void | Read (uint64_t row, uint64_t column_offset, float *data, uint64_t n)
| Read one cell containing an array of floats.
void | Read (uint64_t row, uint64_t column_offset, double *data, uint64_t n)
| Read array of doubles.
void | Read (uint64_t row, uint64_t column_offset, int32_t *data, uint64_t n)
| Read array of int32_t.
void | Read (uint64_t row, uint64_t column_offset, std::complex< float > *data, uint64_t n)
| Read array of complex floats.
void | Read (uint64_t row, uint64_t column_offset, bool *data, uint64_t n)
| Read an array of bools.
void | Write (uint64_t row, uint64_t column_offset, const float *data, uint64_t n)
| Write one cell containing an array of floats.
void | Write (uint64_t row, uint64_t column_offset, const double *data, uint64_t n)
| Write an array of doubles.
void | Write (uint64_t row, uint64_t column_offset, const int32_t *data, uint64_t n)
| Write an array of int32_t.
void | Write (uint64_t row, uint64_t column_offset, const std::complex< float > *data, uint64_t n)
| Write an array of complex floats.
void | Write (uint64_t row, uint64_t column_offset, const std::complex< double > *data, uint64_t n)
| Write an array of complex doubles.
void | Write (uint64_t row, uint64_t column_offset, const bool *data, uint64_t n)
| Write an array of bools.
void | SetStride (uint64_t new_stride)
| Set the number of bytes per row for this file.
void | AddRows (uint64_t n_rows)
| Adds a given number of rows to the back of the file.
void | DeleteRow ()
| Deletes the last row.
const std::string & | Filename () const
bool | IsOpen () const
uint64_t | NRows () const
| Total number of rows stored in this file.
void | ReadHeader (unsigned char *data)
| Read an optional extra header to the file.
uint64_t | Stride () const
| Total number of bytes in one row.
void | WriteHeader (const unsigned char *data)
| Write an optional extra header to the file.

## Static Public Member Functions

static VarBufferedColumnarFile | CreateNew (const std::string &filename, uint64_t header_size, uint64_t stride)
| Create a new file on disk.
static VarBufferedColumnarFile | OpenExisting (const std::string &filename, size_t header_size)
| Open an existing file from disk.

## Private Member Functions

| VarBufferedColumnarFile (const std::string &filename, uint64_t header_size, uint64_t stride)
| Create or overwrite a new columnar file on disk.
| VarBufferedColumnarFile (const std::string &filename, size_t header_size)
| Open an existing columnar file.
void | ActivateBlock (uint64_t row)
template<typename ValueType>
void | ReadImplementation (uint64_t row, uint64_t column_offset, ValueType *data, uint64_t n)
template<typename ValueType>
void | WriteImplementation (uint64_t row, uint64_t column_offset, const ValueType *data, uint64_t n)
void | WriteActiveBlock ()
Private Member Functions inherited from casacore::RowBasedFile
| RowBasedFile ()=default
| RowBasedFile (const RowBasedFile &rhs)=delete
| RowBasedFile (RowBasedFile &&rhs) noexcept
| RowBasedFile (const std::string &filename, uint64_t header_size, uint64_t stride)
| Create or overwrite a new columnar file on disk.
| RowBasedFile (const std::string &filename, size_t header_size)
| Open an existing columnar file.
| ~RowBasedFile () noexcept
RowBasedFile & | operator= (RowBasedFile &&rhs)
void | Close ()
| Close the file.
void | Truncate (uint64_t n_rows)
void | Seek (off_t pos, int seek_direction)
void | ReadData (unsigned char *data, uint64_t size)
void | WriteData (const unsigned char *data, uint64_t size)
bool | IsOpen () const
uint64_t | DataLocation () const
| Offset of the first row in the file.
const std::string & | Filename () const
uint64_t | HeaderSize () const
| Number of bytes reserved for an optional header.
void | WriteHeader (const unsigned char *data)
| Write an optional extra header to the file.
void | ReadHeader (unsigned char *data)
| Read an optional extra header to the file.
uint64_t | NRows () const
| Total number of rows stored in this file.
void | SetNRows (uint64_t new_n_rows)
uint64_t | Stride () const
| Total number of bytes in one row.
void | SetStride (uint64_t new_stride)
| Set the number of bytes per row for this file.
void | AddRows (uint64_t n_rows)
| Adds a given number of rows to the back of the file.
void | DeleteRow ()
| Deletes the last row.

## Private Attributes

std::vector< unsigned char > | packed_buffer_
| This buffer is used temporarily for (un)packing booleans.
bool | block_changed_
uint64_t | active_block_
uint64_t | rows_per_block_
std::vector< unsigned char > | block_buffer_

## Detailed Description

template<uint64_t BufferSize = 100 * 1024>
class casacore::VarBufferedColumnarFile< BufferSize >

Class that provides binary table I/O.

It is rather low-level, and requires the user to keep track of the columns and their datatype. It uses a simple binary format where the columns are interleaved into the output file. Booleans are written with bit-packing.

This class limits calls to the read/write functions by caching a block of data around the last accessed row. The size of the buffer is templated to make testing of the caching easier. Production code can normally use the alias BufferedColumnarFile, which uses a reasonable buffer size.

This class aims to be as fast as possible for large data files that are read from or written to consecutively. It does not try to optimize random access, partial data access and access is not transactional.

The class uses exceptions to handle any I/O errors.

Definition at line 33 of file BufferedColumnarFile.h.

## Constructor & Destructor Documentation

## VarBufferedColumnarFile() [1/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile | ( |  | ) |

default noexcept

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::CreateNew(), and casacore::VarBufferedColumnarFile< 100 *1024 >::OpenExisting().

## VarBufferedColumnarFile() [2/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile | ( | const VarBufferedColumnarFile< BufferSize > & | rhs | ) |

delete

## VarBufferedColumnarFile() [3/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile | ( | VarBufferedColumnarFile< BufferSize > && | rhs | ) |

inline noexcept

Definition at line 47 of file BufferedColumnarFile.h.

## ~VarBufferedColumnarFile()

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile | ( |  | ) |

inline noexcept

Definition at line 58 of file BufferedColumnarFile.h.

## VarBufferedColumnarFile() [4/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile | ( | const std::string & | filename,
|  | uint64_t | header_size,
|  | uint64_t | stride )

inline private

Create or overwrite a new columnar file on disk.

Definition at line 248 of file BufferedColumnarFile.h.

## VarBufferedColumnarFile() [5/5]

template<uint64_t BufferSize = 100 * 1024>

casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile | ( | const std::string & | filename,
|  | size_t | header_size )

inline private

Open an existing columnar file.

Definition at line 257 of file BufferedColumnarFile.h.

## Member Function Documentation

## ActivateBlock()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock | ( | uint64_t | row | ) |

inline private

Fill the remainder of block_buffer_ with zeroes. Doing it here makes the code robust and avoids the need for inserting zeroes when adding rows out-of-order, e.g., when adding row 5 while NRows() is 2."

Definition at line 267 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::ReadImplementation(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), and casacore::VarBufferedColumnarFile< 100 *1024 >::WriteImplementation().

## AddRows()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::AddRows | ( | uint64_t | n_rows | ) |

inline

Adds a given number of rows to the back of the file.

Definition at line 232 of file RowBasedFile.h.

## Close()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Close | ( |  | ) |

inline

Close the file.

After closing, all calls to I/O functions cause undefined behaviour, untill the class is assigned to a new instance.

Even if writing fails, the file should still be closed to prevent the file to remain open after destruction.

Definition at line 87 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::operator=().

## CreateNew()

template<uint64_t BufferSize = 100 * 1024>

VarBufferedColumnarFile casacore::VarBufferedColumnarFile< BufferSize >::CreateNew | ( | const std::string & | filename,
|  | uint64_t | header_size,
|  | uint64_t | stride )

inline static

Create a new file on disk.

If the file exists, it is overwritten.

Parameters

  header | Optional header.
  stride | The number of bytes in one row (total over all columns).

Definition at line 108 of file BufferedColumnarFile.h.

## DeleteRow()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::DeleteRow | ( |  | ) |

inline

Deletes the last row.

Definition at line 237 of file RowBasedFile.h.

## Filename()

template<uint64_t BufferSize = 100 * 1024>

const std::string & casacore::RowBasedFile::Filename | ( |  | ) | const

inline

Definition at line 178 of file RowBasedFile.h.

## IsOpen()

template<uint64_t BufferSize = 100 * 1024>

bool casacore::RowBasedFile::IsOpen | ( |  | ) | const

inline

Definition at line 172 of file RowBasedFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Close(), and casacore::VarBufferedColumnarFile< 100 *1024 >::~VarBufferedColumnarFile().

## NRows()

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::RowBasedFile::NRows | ( |  | ) | const

inline

Total number of rows stored in this file.

Definition at line 203 of file RowBasedFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::ActivateBlock(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::ReadImplementation(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< 100 *1024 >::WriteImplementation(), and casacore::VarBufferedColumnarFile< 100 *1024 >::~VarBufferedColumnarFile().

## OpenExisting()

template<uint64_t BufferSize = 100 * 1024>

VarBufferedColumnarFile casacore::VarBufferedColumnarFile< BufferSize >::OpenExisting | ( | const std::string & | filename,
|  | size_t | header_size )

inline static

Open an existing file from disk.

If the file does not exists, an exception is thrown.

Parameters

  header | Optional header.
  stride | The number of bytes in one row (total over all columns).

Definition at line 120 of file BufferedColumnarFile.h.

## operator=()

template<uint64_t BufferSize = 100 * 1024>

VarBufferedColumnarFile & casacore::VarBufferedColumnarFile< BufferSize >::operator= | ( | VarBufferedColumnarFile< BufferSize > && | rhs | ) |

inline

Definition at line 71 of file BufferedColumnarFile.h.

## Read() [1/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | bool * | data,
|  | uint64_t | n )

inline

Read an array of bools.

See float version for documentation. Booleans are stored with bit-packing.

Definition at line 164 of file BufferedColumnarFile.h.

## Read() [2/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | double * | data,
|  | uint64_t | n )

inline

Read array of doubles.

See float version for documentation.

Definition at line 141 of file BufferedColumnarFile.h.

## Read() [3/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | float * | data,
|  | uint64_t | n )

inline

Read one cell containing an array of floats.

If the row was not written yet, zeros are returned.

Parameters

  row | The cell's row index.
  column_offset | The position of this column counted from the start of the row, in bytes.
  data | Buffer in which the data will be stored.
  n | Size of the column in number of elements (NOT in bytes!).

Definition at line 134 of file BufferedColumnarFile.h.

## Read() [4/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | int32_t * | data,
|  | uint64_t | n )

inline

Read array of int32_t.

See float version for documentation.

Definition at line 148 of file BufferedColumnarFile.h.

## Read() [5/5]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Read | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | std::complex< float > * | data,
|  | uint64_t | n )

inline

Read array of complex floats.

See float version for documentation.

Definition at line 155 of file BufferedColumnarFile.h.

## ReadHeader()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::ReadHeader | ( | unsigned char * | data | ) |

inline

Read an optional extra header to the file.

See also
  WriteHeader().

Definition at line 196 of file RowBasedFile.h.

## ReadImplementation()

template<uint64_t BufferSize = 100 * 1024>
template<typename ValueType>

void casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | ValueType * | data,
|  | uint64_t | n )

inline private

Definition at line 290 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), and casacore::VarBufferedColumnarFile< 100 *1024 >::Read().

## SetStride()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::SetStride | ( | uint64_t | new_stride | ) |

inline

Set the number of bytes per row for this file.

This changes the format of the file, and because of this the file is emptied.

Definition at line 236 of file BufferedColumnarFile.h.

## Stride()

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::RowBasedFile::Stride | ( |  | ) | const

inline

Total number of bytes in one row.

This value is also stored in the file, and is read from the file in OpenExisting().

Definition at line 214 of file RowBasedFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::ActivateBlock(), casacore::VarBufferedColumnarFile< 100 *1024 >::Read(), casacore::VarBufferedColumnarFile< 100 *1024 >::ReadImplementation(), casacore::VarBufferedColumnarFile< 100 *1024 >::VarBufferedColumnarFile(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< 100 *1024 >::WriteImplementation(), and casacore::VarBufferedColumnarFile< 100 *1024 >::~VarBufferedColumnarFile().

## Write() [1/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const bool * | data,
|  | uint64_t | n )

inline

Write an array of bools.

Bools are stored with bit-packing. See float version for documentation.

Definition at line 221 of file BufferedColumnarFile.h.

## Write() [2/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const double * | data,
|  | uint64_t | n )

inline

Write an array of doubles.

See float version for documentation.

Definition at line 188 of file BufferedColumnarFile.h.

## Write() [3/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const float * | data,
|  | uint64_t | n )

inline

Write one cell containing an array of floats.

If the row is past the end of the file, the file is enlarged (making NRows() = row + 1).

Definition at line 180 of file BufferedColumnarFile.h.

## Write() [4/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const int32_t * | data,
|  | uint64_t | n )

inline

Write an array of int32_t.

See float version for documentation.

Definition at line 196 of file BufferedColumnarFile.h.

## Write() [5/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const std::complex< double > * | data,
|  | uint64_t | n )

inline

Write an array of complex doubles.

See float version for documentation.

Definition at line 212 of file BufferedColumnarFile.h.

## Write() [6/6]

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::Write | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const std::complex< float > * | data,
|  | uint64_t | n )

inline

Write an array of complex floats.

See float version for documentation.

Definition at line 204 of file BufferedColumnarFile.h.

## WriteActiveBlock()

template<uint64_t BufferSize = 100 * 1024>

void casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock | ( |  | ) |

inline private

Definition at line 319 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::ActivateBlock(), and casacore::VarBufferedColumnarFile< 100 *1024 >::Close().

## WriteHeader()

template<uint64_t BufferSize = 100 * 1024>

void casacore::RowBasedFile::WriteHeader | ( | const unsigned char * | data | ) |

inline

Write an optional extra header to the file.

When creating the file, the requested space is saved to store this header.

Parameters

  data | An array equal to the size of the header given in the CreateNew() and OpenExisting() calls.

Definition at line 189 of file RowBasedFile.h.

## WriteImplementation()

template<uint64_t BufferSize = 100 * 1024>
template<typename ValueType>

void casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation | ( | uint64_t | row,
|  | uint64_t | column_offset,
|  | const ValueType * | data,
|  | uint64_t | n )

inline private

Definition at line 306 of file BufferedColumnarFile.h.

Referenced by casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), casacore::VarBufferedColumnarFile< 100 *1024 >::Write(), and casacore::VarBufferedColumnarFile< 100 *1024 >::Write().

## Member Data Documentation

## active_block_

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::VarBufferedColumnarFile< BufferSize >::active_block_

private

Definition at line 335 of file BufferedColumnarFile.h.

## block_buffer_

template<uint64_t BufferSize = 100 * 1024>

std::vector<unsigned char> casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_

private

Definition at line 337 of file BufferedColumnarFile.h.

## block_changed_

template<uint64_t BufferSize = 100 * 1024>

bool casacore::VarBufferedColumnarFile< BufferSize >::block_changed_

private

Definition at line 334 of file BufferedColumnarFile.h.

## packed_buffer_

template<uint64_t BufferSize = 100 * 1024>

std::vector<unsigned char> casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_

private

This buffer is used temporarily for (un)packing booleans.

Storing it as a member avoids memory allocations.

Definition at line 332 of file BufferedColumnarFile.h.

## rows_per_block_

template<uint64_t BufferSize = 100 * 1024>

uint64_t casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_

private

Definition at line 336 of file BufferedColumnarFile.h.

The documentation for this class was generated from the following file:

- tables/AlternateMans/BufferedColumnarFile.h