Loading...
Searching...
No Matches
Compression.h
Go to the documentation of this file.
// Copyright Contributors to the OpenVDB Project
// SPDX-License-Identifier: MPL-2.0
#ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
#define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
#include <openvdb/Types.h>
#include <openvdb/MetaMap.h>
#include <openvdb/math/Math.h> // for negative()
#include "io.h" // for getDataCompression(), etc.
#include "DelayedLoadMetadata.h"
#include <algorithm>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
namespace openvdb {
OPENVDB_USE_VERSION_NAMESPACE
namespace OPENVDB_VERSION_NAME {
namespace io {
/// @brief OR-able bit flags for compression options on input and output streams
/// @details
/// <dl>
/// <dt><tt>COMPRESS_NONE</tt>
/// <dd>On write, don't compress data.<br>
/// On read, the input stream contains uncompressed data.
///
/// <dt><tt>COMPRESS_ZIP</tt>
/// <dd>When writing grids other than level sets or fog volumes, apply
/// ZLIB compression to internal and leaf node value buffers.<br>
/// When reading grids other than level sets or fog volumes, indicate that
/// the value buffers of internal and leaf nodes are ZLIB-compressed.<br>
/// ZLIB compresses well but is slow.
///
/// <dt><tt>COMPRESS_ACTIVE_MASK</tt>
/// <dd>When writing a grid of any class, don't output a node's inactive values
/// if it has two or fewer distinct values. Instead, output minimal information
/// to permit the lossless reconstruction of inactive values.<br>
/// On read, nodes might have been stored without inactive values.
/// Where necessary, reconstruct inactive values from available information.
///
/// <dt><tt>COMPRESS_BLOSC</tt>
/// <dd>When writing grids other than level sets or fog volumes, apply
/// Blosc compression to internal and leaf node value buffers.<br>
/// When reading grids other than level sets or fog volumes, indicate that
/// the value buffers of internal and leaf nodes are Blosc-compressed.<br>
/// Blosc is much faster than ZLIB and produces comparable file sizes.
/// </dl>
enum {
COMPRESS_NONE = 0,
COMPRESS_ZIP = 0x1,
COMPRESS_ACTIVE_MASK = 0x2,
COMPRESS_BLOSC = 0x4
};
/// Return a string describing the given compression flags.
OPENVDB_API std::string compressionToString(uint32_t flags);
////////////////////////////////////////
/// @internal Per-node indicator byte that specifies what additional metadata
/// is stored to permit reconstruction of inactive values
enum {
/*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
/*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
/*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
/*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
/*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
/*5*/ MASK_AND_TWO_INACTIVE_VALS, // mask selects between two non-background inactive vals
/*6*/ NO_MASK_AND_ALL_VALS // > 2 inactive vals, so no mask compression at all
};
template <typename ValueT, typename MaskT>
struct MaskCompress
{
// Comparison function for values
static inline bool eq(const ValueT& a, const ValueT& b) {
return math::isExactlyEqual(a, b);
}
MaskCompress(
const MaskT& valueMask, const MaskT& childMask,
const ValueT* srcBuf, const ValueT& background)
{
/// @todo Consider all values, not just inactive values?
inactiveVal[0] = inactiveVal[1] = background;
int numUniqueInactiveVals = 0;
for (typename MaskT::OffIterator it = valueMask.beginOff();
numUniqueInactiveVals < 3 && it; ++it)
{
const Index32 idx = it.pos();
// Skip inactive values that are actually child node pointers.
if (childMask.isOn(idx)) continue;
const ValueT& val = srcBuf[idx];
const bool unique = !(
(numUniqueInactiveVals > 0 && MaskCompress::eq(val, inactiveVal[0])) ||
(numUniqueInactiveVals > 1 && MaskCompress::eq(val, inactiveVal[1]))
);
if (unique) {
if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
++numUniqueInactiveVals;
}
}
metadata = NO_MASK_OR_INACTIVE_VALS;
if (numUniqueInactiveVals == 1) {
if (!MaskCompress::eq(inactiveVal[0], background)) {
if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
metadata = NO_MASK_AND_MINUS_BG;
} else {
metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
}
}
} else if (numUniqueInactiveVals == 2) {
metadata = NO_MASK_OR_INACTIVE_VALS;
if (!MaskCompress::eq(inactiveVal[0], background) && !MaskCompress::eq(inactiveVal[1], background)) {
// If neither inactive value is equal to the background, both values
// need to be saved, along with a mask that selects between them.
metadata = MASK_AND_TWO_INACTIVE_VALS;
} else if (MaskCompress::eq(inactiveVal[1], background)) {
if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
// If the second inactive value is equal to the background and
// the first is equal to -background, neither value needs to be saved,
// but save a mask that selects between -background and +background.
metadata = MASK_AND_NO_INACTIVE_VALS;
} else {
// If the second inactive value is equal to the background, only
// the first value needs to be saved, along with a mask that selects
// between it and the background.
metadata = MASK_AND_ONE_INACTIVE_VAL;
}
} else if (MaskCompress::eq(inactiveVal[0], background)) {
if (MaskCompress::eq(inactiveVal[1], math::negative(background))) {
// If the first inactive value is equal to the background and
// the second is equal to -background, neither value needs to be saved,
// but save a mask that selects between -background and +background.
metadata = MASK_AND_NO_INACTIVE_VALS;
std::swap(inactiveVal[0], inactiveVal[1]);
} else {
// If the first inactive value is equal to the background, swap it
// with the second value and save only that value, along with a mask
// that selects between it and the background.
std::swap(inactiveVal[0], inactiveVal[1]);
metadata = MASK_AND_ONE_INACTIVE_VAL;
}
}
} else if (numUniqueInactiveVals > 2) {
metadata = NO_MASK_AND_ALL_VALS;
}
}
int8_t metadata = NO_MASK_AND_ALL_VALS;
ValueT inactiveVal[2];
};
////////////////////////////////////////
/// @brief RealToHalf and its specializations define a mapping from
/// floating-point data types to analogous half float types.
template<typename T>
struct RealToHalf {
enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
using HalfT = T; // type T's half float analogue is T itself
static HalfT convert(const T& val) { return val; }
};
template<> struct RealToHalf<float> {
enum { isReal = true };
using HalfT = math::half;
static HalfT convert(float val) { return HalfT(val); }
};
template<> struct RealToHalf<double> {
enum { isReal = true };
using HalfT = math::half;
// A half can only be constructed from a float, so cast the value to a float first.
static HalfT convert(double val) { return HalfT(float(val)); }
};
template<> struct RealToHalf<Vec2s> {
enum { isReal = true };
using HalfT = Vec2H;
static HalfT convert(const Vec2s& val) { return HalfT(val); }
};
template<> struct RealToHalf<Vec2d> {
enum { isReal = true };
using HalfT = Vec2H;
// A half can only be constructed from a float, so cast the vector's elements to floats first.
static HalfT convert(const Vec2d& val) { return HalfT(Vec2s(val)); }
};
template<> struct RealToHalf<Vec3s> {
enum { isReal = true };
using HalfT = Vec3H;
static HalfT convert(const Vec3s& val) { return HalfT(val); }
};
template<> struct RealToHalf<Vec3d> {
enum { isReal = true };
using HalfT = Vec3H;
// A half can only be constructed from a float, so cast the vector's elements to floats first.
static HalfT convert(const Vec3d& val) { return HalfT(Vec3s(val)); }
};
/// Return the given value truncated to 16-bit float precision.
template<typename T>
inline T
truncateRealToHalf(const T& val)
{
return T(RealToHalf<T>::convert(val));
}
////////////////////////////////////////
OPENVDB_API size_t zipToStreamSize(const char* data, size_t numBytes);
OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
OPENVDB_API size_t bloscToStreamSize(const char* data, size_t valSize, size_t numVals);
OPENVDB_API void bloscToStream(std::ostream&, const char* data, size_t valSize, size_t numVals);
OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes);
/// @brief Read data from a stream.
/// @param is the input stream
/// @param data the contiguous array of data to read in
/// @param count the number of elements to read in
/// @param compression whether and how the data is compressed (either COMPRESS_NONE,
/// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
/// @param metadata optional pointer to a DelayedLoadMetadata object that stores
/// the size of the compressed buffer
/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null
/// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
/// without Blosc support.
/// @details This default implementation is instantiated only for types
/// whose size can be determined by the sizeof() operator.
template<typename T>
inline void
readData(std::istream& is, T* data, Index count, uint32_t compression,
DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0))
{
const bool seek = data == nullptr;
if (seek) {
assert(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable());
}
const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP);
if (metadata && seek && hasCompression) {
size_t compressedSize = metadata->getCompressedSize(metadataOffset);
is.seekg(compressedSize, std::ios_base::cur);
} else if (compression & COMPRESS_BLOSC) {
bloscFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
} else if (compression & COMPRESS_ZIP) {
unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
} else if (seek) {
is.seekg(sizeof(T) * count, std::ios_base::cur);
} else {
is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
}
}
/// Specialization for std::string input
template<>
inline void
readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/,
DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/)
{
for (Index i = 0; i < count; ++i) {
size_t len = 0;
is >> len;
//data[i].resize(len);
//is.read(&(data[i][0]), len);
std::string buffer(len+1, ' ');
is.read(&buffer[0], len+1);
if (data != nullptr) data[i].assign(buffer, 0, len);
}
}
/// HalfReader wraps a static function, read(), that is analogous to readData(), above,
/// except that it is partially specialized for floating-point types in order to promote
/// 16-bit half float values to full float. A wrapper class is required because
/// only classes, not functions, can be partially specialized.
template<bool IsReal, typename T> struct HalfReader;
/// Partial specialization for non-floating-point types (no half to float promotion)
template<typename T>
struct HalfReader</*IsReal=*/false, T> {
static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
readData(is, data, count, compression, metadata, metadataOffset);
}
};
/// Partial specialization for floating-point types
template<typename T>
struct HalfReader</*IsReal=*/true, T> {
using HalfT = typename RealToHalf<T>::HalfT;
static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
if (count < 1) return;
if (data == nullptr) {
// seek mode - pass through null pointer
readData<HalfT>(is, nullptr, count, compression, metadata, metadataOffset);
} else {
std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression,
metadata, metadataOffset);
// Copy half float values from the temporary buffer to the full float output array.
std::copy(halfData.begin(), halfData.end(), data);
}
}
};
template<typename T>
inline size_t
writeDataSize(const T *data, Index count, uint32_t compression)
{
if (compression & COMPRESS_BLOSC) {
return bloscToStreamSize(reinterpret_cast<const char*>(data), sizeof(T), count);
} else if (compression & COMPRESS_ZIP) {
return zipToStreamSize(reinterpret_cast<const char*>(data), sizeof(T) * count);
} else {
return sizeof(T) * count;
}
}
/// Specialization for std::string output
template<>
inline size_t
writeDataSize<std::string>(const std::string* data, Index count,
uint32_t /*compression*/) ///< @todo add compression
{
size_t size(0);
for (Index i = 0; i < count; ++i) {
const size_t len = data[i].size();
size += sizeof(size_t) + (len+1);
}
return size;
}
/// Write data to a stream.
/// @param os the output stream
/// @param data the contiguous array of data to write
/// @param count the number of elements to write out
/// @param compression whether and how to compress the data (either COMPRESS_NONE,
/// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
/// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
/// without Blosc support.
/// @details This default implementation is instantiated only for types
/// whose size can be determined by the sizeof() operator.
template<typename T>
inline void
writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
{
if (compression & COMPRESS_BLOSC) {
bloscToStream(os, reinterpret_cast<const char*>(data), sizeof(T), count);
} else if (compression & COMPRESS_ZIP) {
zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
} else {
os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
}
}
/// Specialization for std::string output
template<>
inline void
writeData<std::string>(std::ostream& os, const std::string* data, Index count,
uint32_t /*compression*/) ///< @todo add compression
{
for (Index i = 0; i < count; ++i) {
const size_t len = data[i].size();
os << len;
os.write(data[i].c_str(), len+1);
//os.write(&(data[i][0]), len );
}
}
/// HalfWriter wraps a static function, write(), that is analogous to writeData(), above,
/// except that it is partially specialized for floating-point types in order to quantize
/// floating-point values to 16-bit half float. A wrapper class is required because
/// only classes, not functions, can be partially specialized.
template<bool IsReal, typename T> struct HalfWriter;
/// Partial specialization for non-floating-point types (no float to half quantization)
template<typename T>
struct HalfWriter</*IsReal=*/false, T> {
static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
return writeDataSize(data, count, compression);
}
static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
writeData(os, data, count, compression);
}
};
/// Partial specialization for floating-point types
template<typename T>
struct HalfWriter</*IsReal=*/true, T> {
using HalfT = typename RealToHalf<T>::HalfT;
static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
if (count < 1) return size_t(0);
// Convert full float values to half float, then output the half float array.
std::vector<HalfT> halfData(count);
for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
}
static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
if (count < 1) return;
// Convert full float values to half float, then output the half float array.
std::vector<HalfT> halfData(count);
for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
}
};
#ifdef _WIN32
/// Specialization to avoid double to float warnings in MSVC
template<>
struct HalfWriter</*IsReal=*/true, double> {
using HalfT = RealToHalf<double>::HalfT;
static inline size_t writeSize(const double* data, Index count, uint32_t compression)
{
if (count < 1) return size_t(0);
// Convert full float values to half float, then output the half float array.
std::vector<HalfT> halfData(count);
for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
}
static inline void write(std::ostream& os, const double* data, Index count,
uint32_t compression)
{
if (count < 1) return;
// Convert full float values to half float, then output the half float array.
std::vector<HalfT> halfData(count);
for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
}
};
#endif // _WIN32
////////////////////////////////////////
/// Populate the given buffer with @a destCount values of type @c ValueT
/// read from the given stream, taking into account that the stream might
/// have been compressed via one of several supported schemes.
/// [Mainly for internal use]
/// @param is a stream from which to read data (possibly compressed,
/// depending on the stream's compression settings)
/// @param destBuf a buffer into which to read values of type @c ValueT
/// @param destCount the number of values to be stored in the buffer
/// @param valueMask a bitmask (typically, a node's value mask) indicating
/// which positions in the buffer correspond to active values
/// @param fromHalf if true, read 16-bit half floats from the input stream
/// and convert them to full floats
template<typename ValueT, typename MaskT>
inline void
readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
const MaskT& valueMask, bool fromHalf)
{
// Get the stream's compression settings.
auto meta = getStreamMetadataPtr(is);
const uint32_t compression = getDataCompression(is);
const bool maskCompressed = compression & COMPRESS_ACTIVE_MASK;
const bool seek = (destBuf == nullptr);
assert(!seek || (!meta || meta->seekable()));
// Get delayed load metadata if it exists
DelayedLoadMetadata::Ptr delayLoadMeta;
uint64_t leafIndex(0);
if (seek && meta && meta->delayedLoadMeta()) {
delayLoadMeta =
meta->gridMetadata().getMetadata<DelayedLoadMetadata>("file_delayed_load");
leafIndex = meta->leaf();
}
int8_t metadata = NO_MASK_AND_ALL_VALS;
if (getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
// Read the flag that specifies what, if any, additional metadata
// (selection mask and/or inactive value(s)) is saved.
if (seek && !maskCompressed) {
is.seekg(/*bytes=*/1, std::ios_base::cur);
} else if (seek && delayLoadMeta) {
metadata = delayLoadMeta->getMask(leafIndex);
is.seekg(/*bytes=*/1, std::ios_base::cur);
} else {
is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
}
}
ValueT background = zeroVal<ValueT>();
if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
background = *static_cast<const ValueT*>(bgPtr);
}
ValueT inactiveVal1 = background;
ValueT inactiveVal0 =
((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background));
if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
metadata == MASK_AND_ONE_INACTIVE_VAL ||
metadata == MASK_AND_TWO_INACTIVE_VALS)
{
// Read one of at most two distinct inactive values.
if (seek) {
is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
} else {
is.read(reinterpret_cast<char*>(&inactiveVal0), /*bytes=*/sizeof(ValueT));
}
if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
// Read the second of two distinct inactive values.
if (seek) {
is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
} else {
is.read(reinterpret_cast<char*>(&inactiveVal1), /*bytes=*/sizeof(ValueT));
}
}
}
MaskT selectionMask;
if (metadata == MASK_AND_NO_INACTIVE_VALS ||
metadata == MASK_AND_ONE_INACTIVE_VAL ||
metadata == MASK_AND_TWO_INACTIVE_VALS)
{
// For use in mask compression (only), read the bitmask that selects
// between two distinct inactive values.
if (seek) {
is.seekg(/*bytes=*/selectionMask.memUsage(), std::ios_base::cur);
} else {
selectionMask.load(is);
}
}
ValueT* tempBuf = destBuf;
std::unique_ptr<ValueT[]> scopedTempBuf;
Index tempCount = destCount;
if (maskCompressed && metadata != NO_MASK_AND_ALL_VALS
&& getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION)
{
tempCount = valueMask.countOn();
if (!seek && tempCount != destCount) {
// If this node has inactive voxels, allocate a temporary buffer
// into which to read just the active values.
scopedTempBuf.reset(new ValueT[tempCount]);
tempBuf = scopedTempBuf.get();
}
}
// Read in the buffer.
if (fromHalf) {
HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(
is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
} else {
readData<ValueT>(
is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
}
// If mask compression is enabled and the number of active values read into
// the temp buffer is smaller than the size of the destination buffer,
// then there are missing (inactive) values.
if (!seek && maskCompressed && tempCount != destCount) {
// Restore inactive values, using the background value and, if available,
// the inside/outside mask. (For fog volumes, the destination buffer is assumed
// to be initialized to background value zero, so inactive values can be ignored.)
for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
if (valueMask.isOn(destIdx)) {
// Copy a saved active value into this node's buffer.
destBuf[destIdx] = tempBuf[tempIdx];
++tempIdx;
} else {
// Reconstruct an unsaved inactive value and copy it into this node's buffer.
destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
}
}
}
}
template<typename ValueT, typename MaskT>
inline size_t
writeCompressedValuesSize(ValueT* srcBuf, Index srcCount,
const MaskT& valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
{
using NonConstValueT = typename std::remove_const<ValueT>::type;
const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
Index tempCount = srcCount;
ValueT* tempBuf = srcBuf;
std::unique_ptr<NonConstValueT[]> scopedTempBuf;
if (maskCompress && maskMetadata != NO_MASK_AND_ALL_VALS) {
tempCount = 0;
Index64 onVoxels = valueMask.countOn();
if (onVoxels > Index64(0)) {
// Create a new array to hold just the active values.
scopedTempBuf.reset(new NonConstValueT[onVoxels]);
NonConstValueT* localTempBuf = scopedTempBuf.get();
// Copy active values to a new, contiguous array.
for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
localTempBuf[tempCount] = srcBuf[it.pos()];
}
tempBuf = scopedTempBuf.get();
}
}
// Return the buffer size.
if (toHalf) {
return HalfWriter<RealToHalf<NonConstValueT>::isReal, NonConstValueT>::writeSize(
tempBuf, tempCount, compress);
} else {
return writeDataSize<NonConstValueT>(tempBuf, tempCount, compress);
}
}
/// Write @a srcCount values of type @c ValueT to the given stream, optionally
/// after compressing the values via one of several supported schemes.
/// [Mainly for internal use]
/// @param os a stream to which to write data (possibly compressed, depending
/// on the stream's compression settings)
/// @param srcBuf a buffer containing values of type @c ValueT to be written
/// @param srcCount the number of values stored in the buffer
/// @param valueMask a bitmask (typically, a node's value mask) indicating
/// which positions in the buffer correspond to active values
/// @param childMask a bitmask (typically, a node's child mask) indicating
/// which positions in the buffer correspond to child node pointers
/// @param toHalf if true, convert floating-point values to 16-bit half floats
template<typename ValueT, typename MaskT>
inline void
writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
const MaskT& valueMask, const MaskT& childMask, bool toHalf)
{
// Get the stream's compression settings.
const uint32_t compress = getDataCompression(os);
const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
Index tempCount = srcCount;
ValueT* tempBuf = srcBuf;
std::unique_ptr<ValueT[]> scopedTempBuf;
int8_t metadata = NO_MASK_AND_ALL_VALS;
if (!maskCompress) {
os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
} else {
// A valid level set's inactive values are either +background (outside)
// or -background (inside), and a fog volume's inactive values are all zero.
// Rather than write out all of these values, we can store just the active values
// (given that the value mask specifies their positions) and, if necessary,
// an inside/outside bitmask.
const ValueT zero = zeroVal<ValueT>();
ValueT background = zero;
if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
background = *static_cast<const ValueT*>(bgPtr);
}
MaskCompress<ValueT, MaskT> maskCompressData(valueMask, childMask, srcBuf, background);
metadata = maskCompressData.metadata;
os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
metadata == MASK_AND_ONE_INACTIVE_VAL ||
metadata == MASK_AND_TWO_INACTIVE_VALS)
{
if (!toHalf) {
// Write one of at most two distinct inactive values.
os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[0]), sizeof(ValueT));
if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
// Write the second of two distinct inactive values.
os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[1]), sizeof(ValueT));
}
} else {
// Write one of at most two distinct inactive values.
ValueT truncatedVal = static_cast<ValueT>(truncateRealToHalf(maskCompressData.inactiveVal[0]));
os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
// Write the second of two distinct inactive values.
truncatedVal = truncateRealToHalf(maskCompressData.inactiveVal[1]);
os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
}
}
}
if (metadata == NO_MASK_AND_ALL_VALS) {
// If there are more than two unique inactive values, the entire input buffer
// needs to be saved (both active and inactive values).
/// @todo Save the selection mask as long as most of the inactive values
/// are one of two values?
} else {
// Create a new array to hold just the active values.
scopedTempBuf.reset(new ValueT[srcCount]);
tempBuf = scopedTempBuf.get();
if (metadata == NO_MASK_OR_INACTIVE_VALS ||
metadata == NO_MASK_AND_MINUS_BG ||
metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
{
// Copy active values to the contiguous array.
tempCount = 0;
for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
tempBuf[tempCount] = srcBuf[it.pos()];
}
} else {
// Copy active values to a new, contiguous array and populate a bitmask
// that selects between two distinct inactive values.
MaskT selectionMask;
tempCount = 0;
for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
if (valueMask.isOn(srcIdx)) { // active value
tempBuf[tempCount] = srcBuf[srcIdx];
++tempCount;
} else { // inactive value
if (MaskCompress<ValueT, MaskT>::eq(srcBuf[srcIdx], maskCompressData.inactiveVal[1])) {
selectionMask.setOn(srcIdx); // inactive value 1
} // else inactive value 0
}
}
assert(tempCount == valueMask.countOn());
// Write out the mask that selects between two inactive values.
selectionMask.save(os);
}
}
}
// Write out the buffer.
if (toHalf) {
HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, compress);
} else {
writeData(os, tempBuf, tempCount, compress);
}
}
} // namespace io
} // namespace OPENVDB_VERSION_NAME
} // namespace openvdb
#endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
DelayedLoadMetadata.h
Math.h
General-purpose arithmetic and comparison routines, most of which accept arbitrary value types (or at...
MetaMap.h
OPENVDB_API
#define OPENVDB_API
Definition Platform.h:251
openvdb::v10_0::io::DelayedLoadMetadata
Store a buffer of data that can be optionally used during reading for faster delayed-load I/O perform...
Definition DelayedLoadMetadata.h:22
openvdb::v10_0::io::DelayedLoadMetadata::Ptr
SharedPtr< DelayedLoadMetadata > Ptr
Definition DelayedLoadMetadata.h:24
io.h
openvdb::v10_0::compression
Definition StreamCompression.h:33
openvdb::v10_0::io::getDataCompression
OPENVDB_API uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK,...
openvdb::v10_0::io::zipToStreamSize
OPENVDB_API size_t zipToStreamSize(const char *data, size_t numBytes)
openvdb::v10_0::io::writeCompressedValues
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition Compression.h:645
openvdb::v10_0::io::getFormatVersion
OPENVDB_API uint32_t getFormatVersion(std::ios_base &)
Return the file format version number associated with the given input stream.
openvdb::v10_0::io::zipToStream
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
openvdb::v10_0::io::COMPRESS_BLOSC
@ COMPRESS_BLOSC
Definition Compression.h:56
openvdb::v10_0::io::COMPRESS_NONE
@ COMPRESS_NONE
Definition Compression.h:53
openvdb::v10_0::io::COMPRESS_ACTIVE_MASK
@ COMPRESS_ACTIVE_MASK
Definition Compression.h:55
openvdb::v10_0::io::COMPRESS_ZIP
@ COMPRESS_ZIP
Definition Compression.h:54
openvdb::v10_0::io::truncateRealToHalf
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition Compression.h:216
openvdb::v10_0::io::unzipFromStream
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
openvdb::v10_0::io::bloscToStreamSize
OPENVDB_API size_t bloscToStreamSize(const char *data, size_t valSize, size_t numVals)
openvdb::v10_0::io::bloscToStream
OPENVDB_API void bloscToStream(std::ostream &, const char *data, size_t valSize, size_t numVals)
openvdb::v10_0::io::readCompressedValues
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition Compression.h:465
openvdb::v10_0::io::writeData
void writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition Compression.h:363
openvdb::v10_0::io::getGridBackgroundValuePtr
OPENVDB_API const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
openvdb::v10_0::io::compressionToString
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
openvdb::v10_0::io::getStreamMetadataPtr
OPENVDB_API SharedPtr< StreamMetadata > getStreamMetadataPtr(std::ios_base &)
Return a shared pointer to an object that stores metadata (file format, compression scheme,...
openvdb::v10_0::io::bloscFromStream
OPENVDB_API void bloscFromStream(std::istream &, char *data, size_t numBytes)
openvdb::v10_0::io::MASK_AND_NO_INACTIVE_VALS
@ MASK_AND_NO_INACTIVE_VALS
Definition Compression.h:72
openvdb::v10_0::io::MASK_AND_TWO_INACTIVE_VALS
@ MASK_AND_TWO_INACTIVE_VALS
Definition Compression.h:74
openvdb::v10_0::io::NO_MASK_AND_MINUS_BG
@ NO_MASK_AND_MINUS_BG
Definition Compression.h:70
openvdb::v10_0::io::NO_MASK_AND_ONE_INACTIVE_VAL
@ NO_MASK_AND_ONE_INACTIVE_VAL
Definition Compression.h:71
openvdb::v10_0::io::MASK_AND_ONE_INACTIVE_VAL
@ MASK_AND_ONE_INACTIVE_VAL
Definition Compression.h:73
openvdb::v10_0::io::NO_MASK_AND_ALL_VALS
@ NO_MASK_AND_ALL_VALS
Definition Compression.h:75
openvdb::v10_0::io::NO_MASK_OR_INACTIVE_VALS
@ NO_MASK_OR_INACTIVE_VALS
Definition Compression.h:69
openvdb::v10_0::io::readData
void readData(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Read data from a stream.
Definition Compression.h:247
openvdb::v10_0::io::writeCompressedValuesSize
size_t writeCompressedValuesSize(ValueT *srcBuf, Index srcCount, const MaskT &valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
Definition Compression.h:591
openvdb::v10_0::io::writeDataSize
size_t writeDataSize(const T *data, Index count, uint32_t compression)
Definition Compression.h:324
openvdb::v10_0::math::Vec2d
Vec2< double > Vec2d
Definition Vec2.h:533
openvdb::v10_0::math::Vec2s
Vec2< float > Vec2s
Definition Vec2.h:532
openvdb::v10_0::math::Vec3d
Vec3< double > Vec3d
Definition Vec3.h:664
openvdb::v10_0::math::negative
T negative(const T &val)
Return the unary negation of the given value.
Definition Math.h:128
openvdb::v10_0::math::isExactlyEqual
bool isExactlyEqual(const T0 &a, const T1 &b)
Return true if a is exactly equal to b.
Definition Math.h:443
openvdb::v10_0::math::half
internal::half half
Definition Types.h:29
openvdb::v10_0::math::Vec3s
Vec3< float > Vec3s
Definition Vec3.h:663
openvdb::v10_0::Vec2H
math::Vec2< math::half > Vec2H
Definition Types.h:66
openvdb::v10_0::Index
Index32 Index
Definition Types.h:54
openvdb::v10_0::Vec3H
math::Vec3< math::half > Vec3H
Definition Types.h:75
openvdb::v10_0::OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
@ OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
Definition version.h.in:256
openvdb::v10_0::Index32
uint32_t Index32
Definition Types.h:52
openvdb::v10_0::zeroVal
T zeroVal()
Return the value of type T that corresponds to zero.
Definition Math.h:70
openvdb::v10_0::Index64
uint64_t Index64
Definition Types.h:53
openvdb
Definition Exceptions.h:13
Types.h
openvdb::v10_0::io::HalfReader< false, T >::read
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition Compression.h:296
openvdb::v10_0::io::HalfReader< true, T >::HalfT
typename RealToHalf< T >::HalfT HalfT
Definition Compression.h:304
openvdb::v10_0::io::HalfReader< true, T >::read
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition Compression.h:305
openvdb::v10_0::io::HalfReader
Definition Compression.h:292
openvdb::v10_0::io::HalfWriter< false, T >::writeSize
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition Compression.h:396
openvdb::v10_0::io::HalfWriter< false, T >::write
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition Compression.h:399
openvdb::v10_0::io::HalfWriter< true, T >::writeSize
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition Compression.h:407
openvdb::v10_0::io::HalfWriter< true, T >::HalfT
typename RealToHalf< T >::HalfT HalfT
Definition Compression.h:406
openvdb::v10_0::io::HalfWriter< true, T >::write
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition Compression.h:414
openvdb::v10_0::io::HalfWriter
Definition Compression.h:392
openvdb::v10_0::io::MaskCompress
Definition Compression.h:81
openvdb::v10_0::io::MaskCompress::metadata
int8_t metadata
Definition Compression.h:162
openvdb::v10_0::io::MaskCompress::MaskCompress
MaskCompress(const MaskT &valueMask, const MaskT &childMask, const ValueT *srcBuf, const ValueT &background)
Definition Compression.h:87
openvdb::v10_0::io::MaskCompress::eq
static bool eq(const ValueT &a, const ValueT &b)
Definition Compression.h:83
openvdb::v10_0::io::MaskCompress::inactiveVal
ValueT inactiveVal[2]
Definition Compression.h:163
openvdb::v10_0::io::RealToHalf< Vec2d >::HalfT
Vec2H HalfT
Definition Compression.h:196
openvdb::v10_0::io::RealToHalf< Vec2d >::isReal
@ isReal
Definition Compression.h:195
openvdb::v10_0::io::RealToHalf< Vec2d >::convert
static HalfT convert(const Vec2d &val)
Definition Compression.h:198
openvdb::v10_0::io::RealToHalf< Vec2s >::HalfT
Vec2H HalfT
Definition Compression.h:191
openvdb::v10_0::io::RealToHalf< Vec2s >::isReal
@ isReal
Definition Compression.h:190
openvdb::v10_0::io::RealToHalf< Vec2s >::convert
static HalfT convert(const Vec2s &val)
Definition Compression.h:192
openvdb::v10_0::io::RealToHalf< Vec3d >::isReal
@ isReal
Definition Compression.h:206
openvdb::v10_0::io::RealToHalf< Vec3d >::HalfT
Vec3H HalfT
Definition Compression.h:207
openvdb::v10_0::io::RealToHalf< Vec3d >::convert
static HalfT convert(const Vec3d &val)
Definition Compression.h:209
openvdb::v10_0::io::RealToHalf< Vec3s >::HalfT
Vec3H HalfT
Definition Compression.h:202
openvdb::v10_0::io::RealToHalf< Vec3s >::isReal
@ isReal
Definition Compression.h:201
openvdb::v10_0::io::RealToHalf< Vec3s >::convert
static HalfT convert(const Vec3s &val)
Definition Compression.h:203
openvdb::v10_0::io::RealToHalf< double >::HalfT
math::half HalfT
Definition Compression.h:185
openvdb::v10_0::io::RealToHalf< double >::isReal
@ isReal
Definition Compression.h:184
openvdb::v10_0::io::RealToHalf< double >::convert
static HalfT convert(double val)
Definition Compression.h:187
openvdb::v10_0::io::RealToHalf< float >::convert
static HalfT convert(float val)
Definition Compression.h:181
openvdb::v10_0::io::RealToHalf< float >::isReal
@ isReal
Definition Compression.h:179
openvdb::v10_0::io::RealToHalf< float >::HalfT
math::half HalfT
Definition Compression.h:180
openvdb::v10_0::io::RealToHalf
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition Compression.h:173
openvdb::v10_0::io::RealToHalf::isReal
@ isReal
Definition Compression.h:174
openvdb::v10_0::io::RealToHalf::HalfT
T HalfT
Definition Compression.h:175
openvdb::v10_0::io::RealToHalf::convert
static HalfT convert(const T &val)
Definition Compression.h:176
OPENVDB_VERSION_NAME
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition version.h.in:121
OPENVDB_USE_VERSION_NAMESPACE
#define OPENVDB_USE_VERSION_NAMESPACE
Definition version.h.in:212