Loading...
Searching...
No Matches
IO.h
Go to the documentation of this file.
// Copyright Contributors to the OpenVDB Project
// SPDX-License-Identifier: MPL-2.0
/*!
\file IO.h
\author Ken Museth
\date May 1, 2020
\brief Implements I/O for NanoVDB grids. Features optional BLOSC and ZIP
file compression, support for multiple grids per file as well as
multiple grid types.
\note This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC
*/
#ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
#define NANOVDB_IO_H_HAS_BEEN_INCLUDED
#include "../NanoVDB.h"
#include "GridHandle.h"
#include <fstream> // for std::ifstream
#include <iostream> // for std::cerr/cout
#include <string> // for std::string
#include <sstream> // for std::stringstream
#include <cstring> // for std::strcmp
#include <memory> // for std::unique_ptr
#include <vector> // for std::vector
#ifdef NANOVDB_USE_ZIP
#include <zlib.h> // for ZIP compression
#endif
#ifdef NANOVDB_USE_BLOSC
#include <blosc.h> // for BLOSC compression
#endif
// Due to a bug in older versions of gcc, including fstream might
// define "major" and "minor" which are used as member data below.
// See https://bugzilla.redhat.com/show_bug.cgi?id=130601
#if defined(major) || defined(minor)
#undef major
#undef minor
#endif
namespace nanovdb {
namespace io {
/// We fix a specific size for counting bytes in files so that they
/// are saved the same regardless of machine precision. (Note there are
/// still little/bigendian issues, however)
using fileSize_t = uint64_t;
/// @brief Optional compression codecs
///
/// @note NONE is the default, ZIP is slow but compact and BLOSC offers a great balance.
///
/// @warning NanoVDB optionally supports ZIP and BLOSC compression and will throw an exception
/// if it support is required but missing.
enum class Codec : uint16_t { NONE = 0,
ZIP = 1,
BLOSC = 2,
END = 3 };
inline __hostdev__ const char* toStr(Codec codec)
{
static const char * LUT[] = { "NONE", "ZIP", "BLOSC" , "END" };
return LUT[static_cast<int>(codec)];
}
/// @brief Internal functions for compressed read/write of a NanoVDB GridHandle into a stream
///
/// @warning These functions should never be called directly by client code
namespace Internal {
static constexpr fileSize_t MAX_SIZE = 1UL << 30; // size is 1 GB
template<typename BufferT>
static fileSize_t write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec);
template<typename BufferT>
static void read(std::istream& is, GridHandle<BufferT>& handle, Codec codec);
}; // namespace Internal
/// @brief Standard hash function to use on strings; std::hash may vary by
/// platform/implementation and is know to produce frequent collisions.
uint64_t stringHash(const char* cstr);
/// @brief Return a uint64_t hash key of a std::string
inline uint64_t stringHash(const std::string& str)
{
return stringHash(str.c_str());
}
/// @brief Return a uint64_t with its bytes reversed so we can check for endianness
inline uint64_t reverseEndianness(uint64_t val)
{
return (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) |
(((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) |
(((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) |
(((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000);
}
/// @brief Data encoded at the head of each segment of a file or stream.
///
/// @note A file or stream is composed of one or more segments that each contain
// one or more grids.
// Magic number of NanoVDB files (uint64_t) |
// Version numbers of this file (uint32_t) | one header for each segment
// Number of grids in this segment (uint16_t) |
// Compression mode (uint16_t) |
struct Header
{// 16 bytes
uint64_t magic; // 8 bytes
Version version;// 4 bytes version numbers
uint16_t gridCount; // 2 bytes
Codec codec; // 2 bytes
Header(Codec c = Codec::NONE)
: magic(NANOVDB_MAGIC_NUMBER) // Magic number: "NanoVDB" in hex
, version()// major, minor and patch version numbers
, gridCount(0)
, codec(c)
{
}
}; // Header ( 16 bytes = 2 words )
/// @brief Data encoded for each of the grids associated with a segment.
// Grid size in memory (uint64_t) |
// Grid size on disk (uint64_t) |
// Grid name hash key (uint64_t) |
// Numer of active voxels (uint64_t) |
// Grid type (uint32_t) |
// Grid class (uint32_t) |
// Characters in grid name (uint32_t) |
// AABB in world space (2*3*double) | one per grid in file
// AABB in index space (2*3*int) |
// Size of a voxel in world units (3*double) |
// Byte size of the grid name (uint32_t) |
// Number of nodes per level (4*uint32_t) |
// Numer of active tiles per level (3*uint32_t) |
// Codec for file compression (uint16_t) |
// Padding due to 8B alignment (uint16_t) |
// Version number (uint32_t) |
struct MetaData
{// 176 bytes
uint64_t gridSize, fileSize, nameKey, voxelCount; // 4 * 8 = 32B.
GridType gridType; // 4B.
GridClass gridClass; // 4B.
BBox<Vec3d> worldBBox; // 2 * 3 * 8 = 48B.
CoordBBox indexBBox; // 2 * 3 * 4 = 24B.
Vec3R voxelSize; // 24B.
uint32_t nameSize; // 4B.
uint32_t nodeCount[4]; //4 x 4 = 16B
uint32_t tileCount[3];// 3 x 4 = 12B
Codec codec; // 2B
uint16_t padding;// 2B, due to 8B alignment from uint64_t
Version version;// 4B
}; // MetaData
struct GridMetaData : public MetaData
{
static_assert(sizeof(MetaData) == 176, "Unexpected sizeof(MetaData)");
std::string gridName;
void read(std::istream& is);
void write(std::ostream& os) const;
GridMetaData() {}
template<typename ValueT>
GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid);
uint64_t memUsage() const { return sizeof(MetaData) + nameSize; }
}; // GridMetaData
struct Segment
{
// Check assumptions made during read and write of Header and MetaData
static_assert(sizeof(Header) == 16u, "Unexpected sizeof(Header)");
Header header;
std::vector<GridMetaData> meta;
Segment(Codec c = Codec::NONE)
: header(c)
, meta()
{
}
template<typename BufferT>
void add(const GridHandle<BufferT>& h);
bool read(std::istream& is);
void write(std::ostream& os) const;
uint64_t memUsage() const;
}; // Segment
/// @brief Write a single grid to file (over-writing existing content of the file)
template<typename BufferT>
void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE, int verbose = 0);
/// @brief Write a single grid to stream (starting at the current position)
///
/// @note This method can be used to append grid to an existing stream
template<typename BufferT>
void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE);
/// @brief Write multiple grids to file (over-writing existing content of the file)
template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE, int verbose = 0);
/// @brief Writes multiple grids to stream (starting at its current position)
///
/// @note This method can be used to append multiple grids to an existing stream
template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE);
/// @brief Read the n'th grid from file (defaults to first grid)
///
/// @throw If n exceeds the number of grids in the file
template<typename BufferT = HostBuffer>
GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n = 0, int verbose = 0, const BufferT& buffer = BufferT());
/// @brief Read the n'th grid from stream (defaults to first grid)
///
/// @throw If n exceeds the number of grids in the stream
template<typename BufferT = HostBuffer>
GridHandle<BufferT> readGrid(std::istream& is, uint64_t n = 0, const BufferT& buffer = BufferT());
/// @brief Read the first grid with a specific name
///
/// @warning If not grid exists with the specified name the resulting GridHandle is empty
template<typename BufferT = HostBuffer>
GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose = 0, const BufferT& buffer = BufferT());
/// @brief Read the first grid with a specific name
template<typename BufferT = HostBuffer>
GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer = BufferT());
/// @brief Read all the grids in the file
template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose = 0, const BufferT& buffer = BufferT());
/// @brief Real all grids at the current position of the input stream
template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer = BufferT());
/// @brief Return true if the file contains a grid with the specified name
bool hasGrid(const std::string& fileName, const std::string& gridName);
/// @brief Return true if the stream contains a grid with the specified name
bool hasGrid(std::istream& is, const std::string& gridName);
/// @brief Reads and returns a vector of meta data for all the grids found in the specified file
std::vector<GridMetaData> readGridMetaData(const std::string& fileName);
/// @brief Reads and returns a vector of meta data for all the grids found in the specified stream
std::vector<GridMetaData> readGridMetaData(std::istream& is);
// --------------------------> Implementations for Internal <------------------------------------
template<typename BufferT>
fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
{
const char* data = reinterpret_cast<const char*>(handle.data());
fileSize_t total = 0, residual = handle.size();
switch (codec) {
case Codec::ZIP: {
#ifdef NANOVDB_USE_ZIP
uLongf size = compressBound(residual); // Get an upper bound on the size of the compressed data.
std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
const int status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), residual);
if (status != Z_OK)
std::runtime_error("Internal write error in ZIP");
if (size > residual)
std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
const fileSize_t outBytes = size;
os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
total += sizeof(fileSize_t) + outBytes;
#else
throw std::runtime_error("ZIP compression codec was disabled during build");
#endif
break;
}
case Codec::BLOSC: {
#ifdef NANOVDB_USE_BLOSC
do {
fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
std::unique_ptr<char[]> tmp(new char[size]);
const int count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
if (count <= 0)
std::runtime_error("Internal write error in BLOSC");
const fileSize_t outBytes = count;
os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
total += sizeof(fileSize_t) + outBytes;
data += chunk;
residual -= chunk;
} while (residual > 0);
#else
throw std::runtime_error("BLOSC compression codec was disabled during build");
#endif
break;
}
default:
os.write(data, residual);
total += residual;
}
if (!os) {
throw std::runtime_error("Failed to write Tree to file");
}
return total;
} // Internal::write
template<typename BufferT>
void Internal::read(std::istream& is, GridHandle<BufferT>& handle, Codec codec)
{
char* data = reinterpret_cast<char*>(handle.buffer().data());
fileSize_t residual = handle.buffer().size();
// read tree using optional compression
switch (codec) {
case Codec::ZIP: {
#ifdef NANOVDB_USE_ZIP
fileSize_t size;
is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
is.read(reinterpret_cast<char*>(tmp.get()), size);
uLongf numBytes = residual;
int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
if (status != Z_OK)
std::runtime_error("Internal read error in ZIP");
if (fileSize_t(numBytes) != residual)
throw std::runtime_error("UNZIP failed on byte size");
#else
throw std::runtime_error("ZIP compression codec was disabled during build");
#endif
break;
}
case Codec::BLOSC: {
#ifdef NANOVDB_USE_BLOSC
do {
fileSize_t size;
is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
std::unique_ptr<char[]> tmp(new char[size]);
is.read(reinterpret_cast<char*>(tmp.get()), size);
const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
const int count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
if (count < 1)
std::runtime_error("Internal read error in BLOSC");
if (count != int(chunk))
throw std::runtime_error("BLOSC failed on byte size");
data += size_t(chunk);
residual -= chunk;
} while (residual > 0);
#else
throw std::runtime_error("BLOSC compression codec was disabled during build");
#endif
break;
}
default:
is.read(data, residual);
}
if (!is) {
throw std::runtime_error("Failed to read Tree from file");
}
} // Internal::read
// --------------------------> Implementations for GridMetaData <------------------------------------
template<typename ValueT>
inline GridMetaData::GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid)
: MetaData{size, // gridSize
0, // fileSize
0, // nameKey
grid.activeVoxelCount(), // voxelCount
grid.gridType(), // gridType
grid.gridClass(), // gridClass
grid.worldBBox(), // worldBBox
grid.tree().bbox(), // indexBBox
grid.voxelSize(), // voxelSize
0, // nameSize
{0, 0, 0, 1}, // nodeCount[4]
{0, 0, 0}, // tileCount[3]
c, // codec
0, // padding
Version()}// version
, gridName(grid.gridName())
{
nameKey = stringHash(gridName);
nameSize = static_cast<uint32_t>(gridName.size() + 1); // include '\0'
const uint32_t* ptr = reinterpret_cast<const TreeData<3>*>(&grid.tree())->mNodeCount;
for (int i = 0; i < 3; ++i) {
MetaData::nodeCount[i] = *ptr++;
}
//MetaData::nodeCount[3] = 1;// one root node
for (int i = 0; i < 3; ++i) {
MetaData::tileCount[i] = *ptr++;
}
}
inline void GridMetaData::write(std::ostream& os) const
{
os.write(reinterpret_cast<const char*>(this), sizeof(MetaData));
os.write(gridName.c_str(), nameSize);
if (!os) {
throw std::runtime_error("Failed writing GridMetaData");
}
}
inline void GridMetaData::read(std::istream& is)
{
is.read(reinterpret_cast<char*>(this), sizeof(MetaData));
std::unique_ptr<char[]> tmp(new char[nameSize]);
is.read(reinterpret_cast<char*>(tmp.get()), nameSize);
gridName.assign(tmp.get());
if (!is) {
throw std::runtime_error("Failed reading GridMetaData");
}
}
// --------------------------> Implementations for Segment <------------------------------------
inline uint64_t Segment::memUsage() const
{
uint64_t sum = sizeof(Header);
for (auto& m : meta) {
sum += m.memUsage();
}
return sum;
}
template<typename BufferT>
inline void Segment::add(const GridHandle<BufferT>& h)
{
if (auto* grid = h.template grid<float>()) { // most common
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Vec3f>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<double>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<int32_t>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<uint32_t>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<int64_t>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<int16_t>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Vec3d>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<ValueMask>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<ValueIndex>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<bool>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Rgba8>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Fp4>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Fp8>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Fp16>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<FpN>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Vec4f>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else if (auto* grid = h.template grid<Vec4d>()) {
meta.emplace_back(h.size(), header.codec, *grid);
} else {
throw std::runtime_error("nanovdb::io::Segment::add Cannot write grid of unknown type to file");
}
header.gridCount += 1;
}
inline void Segment::write(std::ostream& os) const
{
if (header.gridCount == 0) {
throw std::runtime_error("Segment contains no grids");
} else if (!os.write(reinterpret_cast<const char*>(&header), sizeof(Header))) {
throw std::runtime_error("Failed to write Header of Segment");
}
for (auto& m : meta) {
m.write(os);
}
}
inline bool Segment::read(std::istream& is)
{
is.read(reinterpret_cast<char*>(&header), sizeof(Header));
if (is.eof()) {
return false;
}
if (!is || header.magic != NANOVDB_MAGIC_NUMBER) {
// first check for byte-swapped header magic.
if (header.magic == reverseEndianness(NANOVDB_MAGIC_NUMBER))
throw std::runtime_error("This nvdb file has reversed endianness");
throw std::runtime_error("Magic number error: This is not a valid nvdb file");
} else if ( header.version.getMajor() != NANOVDB_MAJOR_VERSION_NUMBER) {
std::stringstream ss;
if (header.version.getMajor() < NANOVDB_MAJOR_VERSION_NUMBER) {
ss << "The file contains an older version of NanoVDB: " << std::string(header.version.c_str()) << "!\n\t"
<< "Recommendation: Re-generate this NanoVDB file with this version: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X of NanoVDB";
} else {
ss << "This tool was compiled against an older version of NanoVDB: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X!\n\t"
<< "Recommendation: Re-compile this tool against the newer version: " << header.version.getMajor() << ".X of NanoVDB";
}
throw std::runtime_error("An unrecoverable error in nanovdb::Segment::read:\n\tIncompatible file format: " + ss.str());
}
meta.resize(header.gridCount);
for (auto& m : meta) {
m.read(is);
m.version = header.version;
}
return true;
}
// --------------------------> Implementations for read/write <------------------------------------
template<typename BufferT>
void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec, int verbose)
{
std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
if (!os.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
}
writeGrid<BufferT>(os, handle, codec);
if (verbose) {
std::cout << "Wrote nanovdb::Grid to file named \"" << fileName << "\"" << std::endl;
}
}
template<typename BufferT>
void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
{
Segment s(codec);
s.add(handle);
const uint64_t headerSize = s.memUsage();
std::streamoff seek = headerSize;
os.seekp(seek, std::ios_base::cur); // skip forward from the current position
s.meta[0].fileSize = Internal::write(os, handle, codec);
seek += s.meta[0].fileSize;
os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
s.write(os); // write header
os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
}
template<typename BufferT, template<typename...> class VecT>
void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec, int verbose)
{
std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
if (!os.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
}
writeGrids<BufferT, VecT>(os, handles, codec);
if (verbose) {
std::cout << "Wrote " << handles.size() << " nanovdb::Grid(s) to file named \"" << fileName << "\"" << std::endl;
}
}
template<typename BufferT, template<typename...> class VecT>
void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec)
{
Segment s(codec);
for (auto& h : handles) {
s.add(h);
}
const uint64_t headerSize = s.memUsage();
std::streamoff seek = headerSize;
os.seekp(seek, std::ios_base::cur); // skip forward from the current position
for (size_t i = 0; i < handles.size(); ++i) {
s.meta[i].fileSize = Internal::write(os, handles[i], codec);
seek += s.meta[i].fileSize;
}
os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
s.write(os); // write header
os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
}
/// @brief Read the n'th grid
template<typename BufferT>
GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n, int verbose, const BufferT& buffer)
{
std::ifstream is(fileName, std::ios::in | std::ios::binary);
if (!is.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
}
auto handle = readGrid<BufferT>(is, n, buffer);
if (verbose) {
std::cout << "Read NanoGrid # " << n << " from the file named \"" << fileName << "\"" << std::endl;
}
return handle; // is converted to r-value and return value is move constructed.
}
template<typename BufferT>
GridHandle<BufferT> readGrid(std::istream& is, uint64_t n, const BufferT& buffer)
{
Segment s;
uint64_t counter = 0;
while (s.read(is)) {
std::streamoff seek = 0;
for (auto& m : s.meta) {
if (counter == n) {
GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
is.seekg(seek, std::ios_base::cur); // skip forward from the current position
Internal::read(is, handle, s.header.codec);
return handle; // is converted to r-value and return value is move constructed.
} else {
seek += m.fileSize;
}
++counter;
}
is.seekg(seek, std::ios_base::cur); // skip forward from the current position
}
throw std::runtime_error("Grid index exceeds grid count in file");
}
/// @brief Read the first grid with a specific name
template<typename BufferT>
GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose, const BufferT& buffer)
{
std::ifstream is(fileName, std::ios::in | std::ios::binary);
if (!is.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
}
auto handle = readGrid<BufferT>(is, gridName, buffer);
if (verbose) {
if (handle) {
std::cout << "Read NanoGrid named \"" << gridName << "\" from the file named \"" << fileName << "\"" << std::endl;
} else {
std::cout << "File named \"" << fileName << "\" does not contain a grid named \"" + gridName + "\"" << std::endl;
}
}
return handle; // is converted to r-value and return value is move constructed.
}
template<typename BufferT>
GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer)
{
const auto key = stringHash(gridName);
Segment s;
while (s.read(is)) {
std::streamoff seek = 0;
for (auto& m : s.meta) {
if (m.nameKey == key && m.gridName == gridName) { // check for hash key collision
GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
is.seekg(seek, std::ios_base::cur); // rewind
Internal::read(is, handle, s.header.codec);
return handle; // is converted to r-value and return value is move constructed.
} else {
seek += m.fileSize;
}
}
is.seekg(seek, std::ios_base::cur); // skip forward from the current position
}
return GridHandle<BufferT>(); // empty handle
}
/// @brief Read all the grids
template<typename BufferT, template<typename...> class VecT>
VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose, const BufferT& buffer)
{
std::ifstream is(fileName, std::ios::in | std::ios::binary);
if (!is.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
}
auto handles = readGrids<BufferT, VecT>(is, buffer);
if (verbose) {
std::cout << "Read " << handles.size() << " NanoGrid(s) from the file named \"" << fileName << "\"" << std::endl;
}
return handles; // is converted to r-value and return value is move constructed.
}
template<typename BufferT, template<typename...> class VecT>
VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer)
{
VecT<GridHandle<BufferT>> handles;
Segment seg;
while (seg.read(is)) {
for (auto& m : seg.meta) {
GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
Internal::read(is, handle, seg.header.codec);
handles.push_back(std::move(handle)); // force move copy assignment
}
}
return handles; // is converted to r-value and return value is move constructed.
}
inline std::vector<GridMetaData> readGridMetaData(const std::string& fileName)
{
std::ifstream is(fileName, std::ios::in | std::ios::binary);
if (!is.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
}
return readGridMetaData(is); // is converted to r-value and return value is move constructed.
}
inline std::vector<GridMetaData> readGridMetaData(std::istream& is)
{
std::vector<GridMetaData> meta;
Segment seg;
while (seg.read(is)) {
std::streamoff seek = 0;
for (auto& m : seg.meta) {
meta.push_back(m);
seek += m.fileSize;
}
is.seekg(seek, std::ios_base::cur);
}
return meta; // is converted to r-value and return value is move constructed.
}
inline bool hasGrid(const std::string& fileName, const std::string& gridName)
{
std::ifstream is(fileName, std::ios::in | std::ios::binary);
if (!is.is_open()) {
throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
}
return hasGrid(is, gridName);
}
inline bool hasGrid(std::istream& is, const std::string& gridName)
{
const auto key = stringHash(gridName);
Segment s;
while (s.read(is)) {
std::streamoff seek = 0;
for (auto& m : s.meta) {
if (m.nameKey == key && m.gridName == gridName) {
return true; // check for hash key collision
}
seek += m.fileSize;
}
is.seekg(seek, std::ios_base::cur);
}
return false;
}
inline uint64_t stringHash(const char* cstr)
{
uint64_t hash = 0;
if (!cstr) {
return hash;
}
for (auto* str = reinterpret_cast<const unsigned char*>(cstr); *str; ++str) {
uint64_t overflow = hash >> (64 - 8);
hash *= 67; // Next-ish prime after 26 + 26 + 10
hash += *str + overflow;
}
return hash;
}
}
} // namespace nanovdb::io
#endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
GridHandle.h
Defines two classes, a GridRegister the defines the value type (e.g. Double, Float etc) of a NanoVDB ...
NanoVDB.h
Implements a light-weight self-contained VDB data-structure in a single file! In other words,...
__hostdev__
#define __hostdev__
Definition NanoVDB.h:192
NANOVDB_MAJOR_VERSION_NUMBER
#define NANOVDB_MAJOR_VERSION_NUMBER
Definition NanoVDB.h:123
NANOVDB_MAGIC_NUMBER
#define NANOVDB_MAGIC_NUMBER
Definition NanoVDB.h:121
nanovdb::GridHandle
This class serves to manage a raw memory buffer of a NanoVDB Grid.
Definition GridHandle.h:71
nanovdb::GridHandle::size
uint64_t size() const override
Returns the size in bytes of the raw memory buffer managed by this GridHandle's allocator.
Definition GridHandle.h:125
nanovdb::GridHandle::data
uint8_t * data() override
Returns a non-const pointer to the data.
Definition GridHandle.h:117
nanovdb::HostBuffer
This is a buffer that contains a shared or private pool to either externally or internally managed ho...
Definition HostBuffer.h:115
nanovdb::Version
Bit-compacted representation of all three version numbers.
Definition NanoVDB.h:648
hash
__hostdev__ uint32_t hash(uint32_t x)
Definition common.h:13
nanovdb::io::Internal
Internal functions for compressed read/write of a NanoVDB GridHandle into a stream.
Definition IO.h:75
nanovdb::io::Internal::write
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::read
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::MAX_SIZE
static constexpr fileSize_t MAX_SIZE
Definition IO.h:76
nanovdb::io
Definition NanoVDB.h:5629
nanovdb::io::writeGrid
void writeGrid(const std::string &fileName, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE, int verbose=0)
Write a single grid to file (over-writing existing content of the file).
Definition IO.h:517
nanovdb::io::readGrids
VecT< GridHandle< BufferT > > readGrids(const std::string &fileName, int verbose=0, const BufferT &buffer=BufferT())
Read all the grids in the file.
Definition IO.h:657
nanovdb::io::toStr
const char * toStr(Codec codec)
Definition IO.h:66
nanovdb::io::writeGrids
void writeGrids(const std::string &fileName, const VecT< GridHandle< BufferT > > &handles, Codec codec=Codec::NONE, int verbose=0)
Write multiple grids to file (over-writing existing content of the file).
Definition IO.h:545
nanovdb::io::fileSize_t
uint64_t fileSize_t
Definition IO.h:53
nanovdb::io::stringHash
uint64_t stringHash(const char *cstr)
Standard hash function to use on strings; std::hash may vary by platform/implementation and is know t...
Definition IO.h:735
nanovdb::io::readGridMetaData
std::vector< GridMetaData > readGridMetaData(const std::string &fileName)
Reads and returns a vector of meta data for all the grids found in the specified file.
Definition IO.h:685
nanovdb::io::reverseEndianness
uint64_t reverseEndianness(uint64_t val)
Return a uint64_t with its bytes reversed so we can check for endianness.
Definition IO.h:96
nanovdb::io::Codec
Codec
Optional compression codecs.
Definition IO.h:61
nanovdb::io::Codec::ZIP
@ ZIP
Definition IO.h:62
nanovdb::io::Codec::BLOSC
@ BLOSC
Definition IO.h:63
nanovdb::io::Codec::END
@ END
Definition IO.h:64
nanovdb::io::Codec::NONE
@ NONE
Definition IO.h:61
nanovdb::io::hasGrid
bool hasGrid(const std::string &fileName, const std::string &gridName)
Return true if the file contains a grid with the specified name.
Definition IO.h:709
nanovdb::io::readGrid
GridHandle< BufferT > readGrid(const std::string &fileName, uint64_t n=0, int verbose=0, const BufferT &buffer=BufferT())
Read the n'th grid from file (defaults to first grid).
Definition IO.h:578
nanovdb
Definition NanoVDB.h:208
nanovdb::NanoGrid
Grid< NanoTree< BuildT > > NanoGrid
Definition NanoVDB.h:4563
nanovdb::GridClass
GridClass
Classes (defined in OpenVDB) that are currently supported by NanoVDB.
Definition NanoVDB.h:281
nanovdb::GridType
GridType
List of types that are currently supported by NanoVDB.
Definition NanoVDB.h:243
nanovdb::Vec3R
Vec3< double > Vec3R
Definition NanoVDB.h:1287
nanovdb::CoordBBox
BBox< Coord > CoordBBox
Definition NanoVDB.h:1809
nanovdb::BBox
Definition NanoVDB.h:1655
nanovdb::io::GridMetaData::GridMetaData
GridMetaData()
Definition IO.h:166
nanovdb::io::GridMetaData::write
void write(std::ostream &os) const
Definition IO.h:396
nanovdb::io::GridMetaData::memUsage
uint64_t memUsage() const
Definition IO.h:169
nanovdb::io::GridMetaData::gridName
std::string gridName
Definition IO.h:163
nanovdb::io::GridMetaData::read
void read(std::istream &is)
Definition IO.h:405
nanovdb::io::Header
Data encoded at the head of each segment of a file or stream.
Definition IO.h:113
nanovdb::io::Header::gridCount
uint16_t gridCount
Definition IO.h:116
nanovdb::io::Header::codec
Codec codec
Definition IO.h:117
nanovdb::io::Header::magic
uint64_t magic
Definition IO.h:114
nanovdb::io::Header::version
Version version
Definition IO.h:115
nanovdb::io::Header::Header
Header(Codec c=Codec::NONE)
Definition IO.h:118
nanovdb::io::MetaData
Data encoded for each of the grids associated with a segment.
Definition IO.h:145
nanovdb::io::MetaData::worldBBox
BBox< Vec3d > worldBBox
Definition IO.h:149
nanovdb::io::MetaData::nameKey
uint64_t nameKey
Definition IO.h:146
nanovdb::io::MetaData::padding
uint16_t padding
Definition IO.h:156
nanovdb::io::MetaData::fileSize
uint64_t fileSize
Definition IO.h:146
nanovdb::io::MetaData::tileCount
uint32_t tileCount[3]
Definition IO.h:154
nanovdb::io::MetaData::gridClass
GridClass gridClass
Definition IO.h:148
nanovdb::io::MetaData::indexBBox
CoordBBox indexBBox
Definition IO.h:150
nanovdb::io::MetaData::nodeCount
uint32_t nodeCount[4]
Definition IO.h:153
nanovdb::io::MetaData::voxelSize
Vec3R voxelSize
Definition IO.h:151
nanovdb::io::MetaData::codec
Codec codec
Definition IO.h:155
nanovdb::io::MetaData::voxelCount
uint64_t voxelCount
Definition IO.h:146
nanovdb::io::MetaData::version
Version version
Definition IO.h:157
nanovdb::io::MetaData::gridType
GridType gridType
Definition IO.h:147
nanovdb::io::MetaData::nameSize
uint32_t nameSize
Definition IO.h:152
nanovdb::io::MetaData::gridSize
uint64_t gridSize
Definition IO.h:146
nanovdb::io::Segment
Definition IO.h:173
nanovdb::io::Segment::read
bool read(std::istream &is)
Definition IO.h:484
nanovdb::io::Segment::add
void add(const GridHandle< BufferT > &h)
Definition IO.h:428
nanovdb::io::Segment::write
void write(std::ostream &os) const
Definition IO.h:472
nanovdb::io::Segment::meta
std::vector< GridMetaData > meta
Definition IO.h:177
nanovdb::io::Segment::header
Header header
Definition IO.h:176
nanovdb::io::Segment::memUsage
uint64_t memUsage() const
Definition IO.h:418
nanovdb::io::Segment::Segment
Segment(Codec c=Codec::NONE)
Definition IO.h:178