buffer.hpp
Go to the documentation of this file.
#ifndef OSMIUM_MEMORY_BUFFER_HPP
#define OSMIUM_MEMORY_BUFFER_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstring>
#include <functional>
#include <iterator>
#include <memory>
#include <stdexcept>
#include <utility>
#include <osmium/memory/item.hpp>
#include <osmium/memory/item_iterator.hpp>
#include <osmium/osm/entity.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
struct buffer_is_full : public std::runtime_error {
buffer_is_full() :
std::runtime_error("Osmium buffer is full") {
}
}; // struct buffer_is_full
namespace memory {
class Buffer {
public:
// This is needed so we can call std::back_inserter() on a Buffer.
using value_type = Item;
enum class auto_grow : bool {
yes = true,
no = false
}; // enum class auto_grow
private:
std::unique_ptr<unsigned char[]> m_memory;
unsigned char* m_data;
size_t m_capacity;
size_t m_written;
size_t m_committed;
auto_grow m_auto_grow {auto_grow::no};
std::function<void(Buffer&)> m_full;
public:
Buffer() noexcept :
m_memory(),
m_data(nullptr),
m_capacity(0),
m_written(0),
m_committed(0) {
}
explicit Buffer(unsigned char* data, size_t size) :
m_memory(),
m_data(data),
m_capacity(size),
m_written(size),
m_committed(size) {
if (size % align_bytes != 0) {
throw std::invalid_argument("buffer size needs to be multiple of alignment");
}
}
explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
m_memory(),
m_data(data),
m_capacity(capacity),
m_written(committed),
m_committed(committed) {
if (capacity % align_bytes != 0) {
throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
}
if (committed % align_bytes != 0) {
throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
}
}
explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) :
m_memory(new unsigned char[capacity]),
m_data(m_memory.get()),
m_capacity(capacity),
m_written(0),
m_committed(0),
m_auto_grow(auto_grow) {
if (capacity % align_bytes != 0) {
throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
}
}
// buffers can not be copied
Buffer(const Buffer&) = delete;
Buffer& operator=(const Buffer&) = delete;
// buffers can be moved
Buffer(Buffer&&) = default;
Buffer& operator=(Buffer&&) = default;
~Buffer() = default;
unsigned char* data() const noexcept {
assert(m_data);
return m_data;
}
size_t capacity() const noexcept {
return m_capacity;
}
size_t committed() const noexcept {
return m_committed;
}
size_t written() const noexcept {
return m_written;
}
bool is_aligned() const noexcept {
assert(m_data);
return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
}
OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
assert(m_data);
m_full = full;
}
void grow(size_t size) {
assert(m_data);
if (!m_memory) {
throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
}
if (m_capacity < size) {
if (size % align_bytes != 0) {
throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
}
std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
std::copy_n(m_memory.get(), m_capacity, memory.get());
using std::swap;
swap(m_memory, memory);
m_data = m_memory.get();
m_capacity = size;
}
}
size_t commit() {
assert(m_data);
assert(is_aligned());
const size_t offset = m_committed;
m_committed = m_written;
return offset;
}
void rollback() {
assert(m_data);
m_written = m_committed;
}
size_t clear() {
const size_t committed = m_committed;
m_written = 0;
m_committed = 0;
return committed;
}
template <typename T>
T& get(const size_t offset) const {
assert(m_data);
return *reinterpret_cast<T*>(&m_data[offset]);
}
unsigned char* reserve_space(const size_t size) {
assert(m_data);
// try to flush the buffer empty first.
if (m_written + size > m_capacity && m_full) {
m_full(*this);
}
// if there's still not enough space, then try growing the buffer.
if (m_written + size > m_capacity) {
if (m_memory && (m_auto_grow == auto_grow::yes)) {
// double buffer size until there is enough space
size_t new_capacity = m_capacity * 2;
while (m_written + size > new_capacity) {
new_capacity *= 2;
}
grow(new_capacity);
} else {
throw osmium::buffer_is_full();
}
}
unsigned char* data = &m_data[m_written];
m_written += size;
return data;
}
template <typename T>
T& add_item(const T& item) {
assert(m_data);
unsigned char* target = reserve_space(item.padded_size());
std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
return *reinterpret_cast<T*>(target);
}
void add_buffer(const Buffer& buffer) {
assert(m_data && buffer);
unsigned char* target = reserve_space(buffer.committed());
std::copy_n(buffer.data(), buffer.committed(), target);
}
void push_back(const osmium::memory::Item& item) {
assert(m_data);
add_item(item);
commit();
}
template <typename T>
using t_iterator = osmium::memory::ItemIterator<T>;
template <typename T>
using t_const_iterator = osmium::memory::ItemIterator<const T>;
using iterator = t_iterator<osmium::OSMEntity>;
using const_iterator = t_const_iterator<osmium::OSMEntity>;
template <typename T>
t_iterator<T> begin() {
assert(m_data);
return t_iterator<T>(m_data, m_data + m_committed);
}
iterator begin() {
assert(m_data);
return iterator(m_data, m_data + m_committed);
}
template <typename T>
t_iterator<T> get_iterator(size_t offset) {
assert(m_data);
return t_iterator<T>(m_data + offset, m_data + m_committed);
}
iterator get_iterator(size_t offset) {
assert(m_data);
return iterator(m_data + offset, m_data + m_committed);
}
template <typename T>
t_iterator<T> end() {
assert(m_data);
return t_iterator<T>(m_data + m_committed, m_data + m_committed);
}
iterator end() {
assert(m_data);
return iterator(m_data + m_committed, m_data + m_committed);
}
template <typename T>
t_const_iterator<T> cbegin() const {
assert(m_data);
return t_const_iterator<T>(m_data, m_data + m_committed);
}
const_iterator cbegin() const {
assert(m_data);
return const_iterator(m_data, m_data + m_committed);
}
template <typename T>
t_const_iterator<T> get_iterator(size_t offset) const {
assert(m_data);
return t_const_iterator<T>(m_data + offset, m_data + m_committed);
}
const_iterator get_iterator(size_t offset) const {
assert(m_data);
return const_iterator(m_data + offset, m_data + m_committed);
}
template <typename T>
t_const_iterator<T> cend() const {
assert(m_data);
return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
}
const_iterator cend() const {
assert(m_data);
return const_iterator(m_data + m_committed, m_data + m_committed);
}
template <typename T>
t_const_iterator<T> begin() const {
return cbegin<T>();
}
const_iterator begin() const {
return cbegin();
}
template <typename T>
t_const_iterator<T> end() const {
return cend<T>();
}
const_iterator end() const {
return cend();
}
explicit operator bool() const noexcept {
return m_data != nullptr;
}
void swap(Buffer& other) {
using std::swap;
swap(m_memory, other.m_memory);
swap(m_data, other.m_data);
swap(m_capacity, other.m_capacity);
swap(m_written, other.m_written);
swap(m_committed, other.m_committed);
swap(m_auto_grow, other.m_auto_grow);
swap(m_full, other.m_full);
}
template <typename TCallbackClass>
void purge_removed(TCallbackClass* callback) {
assert(m_data);
if (begin() == end()) {
return;
}
iterator it_write = begin();
iterator next;
for (iterator it_read = begin(); it_read != end(); it_read = next) {
next = std::next(it_read);
if (!it_read->removed()) {
if (it_read != it_write) {
assert(it_read.data() >= data());
assert(it_write.data() >= data());
size_t old_offset = static_cast<size_t>(it_read.data() - data());
size_t new_offset = static_cast<size_t>(it_write.data() - data());
callback->moving_in_buffer(old_offset, new_offset);
std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
}
it_write.advance_once();
}
}
assert(it_write.data() >= data());
m_written = static_cast<size_t>(it_write.data() - data());
m_committed = m_written;
}
}; // class Buffer
inline void swap(Buffer& lhs, Buffer& rhs) {
lhs.swap(rhs);
}
inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
if (!lhs || !rhs) {
return !lhs && !rhs;
}
return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
}
inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
return ! (lhs == rhs);
}
} // namespace memory
} // namespace osmium
#endif // OSMIUM_MEMORY_BUFFER_HPP
osmium::memory::Buffer::m_written
size_t m_written
Definition: buffer.hpp:114
osmium::memory::Buffer::swap
void swap(Buffer &other)
Definition: buffer.hpp:661
osmium::memory::Buffer::cend
t_const_iterator< T > cend() const
Definition: buffer.hpp:626
osmium::memory::Buffer::begin
const_iterator begin() const
Definition: buffer.hpp:641
osmium::memory::Buffer::clear
size_t clear()
Definition: buffer.hpp:361
OSMIUM_DEPRECATED
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
osmium::memory::Buffer::is_aligned
bool is_aligned() const noexcept
Definition: buffer.hpp:260
osmium::memory::Buffer::written
size_t written() const noexcept
Definition: buffer.hpp:250
osmium::memory::Buffer::set_full_callback
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:285
osmium::memory::Buffer::get_iterator
iterator get_iterator(size_t offset)
Definition: buffer.hpp:571
osmium::memory::Buffer::grow
void grow(size_t size)
Definition: buffer.hpp:307
item_iterator.hpp
osmium::memory::Buffer::end
const_iterator end() const
Definition: buffer.hpp:650
osmium::memory::ItemIterator
Definition: item_iterator.hpp:119
osmium::operator==
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:222
osmium::memory::Buffer::m_data
unsigned char * m_data
Definition: buffer.hpp:112
osmium::memory::align_bytes
constexpr item_size_type align_bytes
Definition: item.hpp:53
osmium::memory::Buffer::Buffer
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:169
std
Definition: reader_iterator.hpp:39
osmium::memory::swap
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:721
osmium::memory::ItemIterator::advance_once
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:168
osmium::memory::Buffer::Buffer
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:197
osmium::memory::ItemIterator::data
unsigned char * data() const
Definition: item_iterator.hpp:189
osmium::memory::Buffer::end
t_iterator< T > end()
Definition: buffer.hpp:585
osmium::memory::Item
Definition: item.hpp:97
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
osmium::memory::Buffer::add_item
T & add_item(const T &item)
Definition: buffer.hpp:457
osmium::memory::Buffer::purge_removed
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:690
osmium::memory::Buffer::begin
t_iterator< T > begin()
Definition: buffer.hpp:529
osmium::memory::Buffer::add_buffer
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:474
osmium::memory::Buffer::cbegin
const_iterator cbegin() const
Definition: buffer.hpp:609
osmium::memory::Buffer::m_committed
size_t m_committed
Definition: buffer.hpp:115
osmium::memory::Buffer::data
unsigned char * data() const noexcept
Definition: buffer.hpp:224
osmium::memory::Buffer::Buffer
Buffer() noexcept
Definition: buffer.hpp:129
osmium::memory::Buffer::capacity
size_t capacity() const noexcept
Definition: buffer.hpp:233
std::end
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
osmium::memory::Buffer::reserve_space
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:417
osmium::memory::Buffer::m_auto_grow
auto_grow m_auto_grow
Definition: buffer.hpp:116
osmium::memory::Buffer::push_back
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:488
osmium::memory::Buffer::end
iterator end()
Definition: buffer.hpp:598
osmium::memory::Buffer::begin
iterator begin()
Definition: buffer.hpp:542
osmium::memory::Buffer::cend
const_iterator cend() const
Definition: buffer.hpp:631
osmium::memory::Buffer::m_capacity
size_t m_capacity
Definition: buffer.hpp:113
osmium::memory::Buffer::get_iterator
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:615
entity.hpp
osmium::memory::Buffer::committed
size_t committed() const noexcept
Definition: buffer.hpp:241
osmium::memory::Buffer::Buffer
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:147
osmium::memory::Buffer::end
t_const_iterator< T > end() const
Definition: buffer.hpp:646
item.hpp
osmium::memory::Buffer
Definition: buffer.hpp:97
osmium::buffer_is_full
Definition: buffer.hpp:58
osmium::memory::Buffer::get_iterator
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:620
osmium::memory::Buffer::cbegin
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:604
osmium::memory::Buffer::begin
t_const_iterator< T > begin() const
Definition: buffer.hpp:637
osmium::memory::Buffer::auto_grow
auto_grow
Definition: buffer.hpp:104
osmium::buffer_is_full::buffer_is_full
buffer_is_full()
Definition: buffer.hpp:60
std::begin
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
osmium::memory::Buffer::rollback
void rollback()
Definition: buffer.hpp:349
osmium::memory::Buffer::m_memory
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
osmium::operator!=
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:431
osmium::memory::Buffer::get_iterator
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:557
osmium::memory::Buffer::m_full
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:117
osmium::memory::Buffer::commit
size_t commit()
Definition: buffer.hpp:335
compatibility.hpp