Loading...
Searching...
No Matches
writer.hpp
Go to the documentation of this file.
#ifndef OSMIUM_IO_WRITER_HPP
#define OSMIUM_IO_WRITER_HPP
/*
This file is part of Osmium (https://osmcode.org/libosmium).
Copyright 2013-2025 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/detail/read_write.hpp>
#include <osmium/io/detail/write_thread.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/header.hpp>
#include <osmium/io/writer_options.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/thread/pool.hpp>
#include <osmium/thread/util.hpp>
#include <osmium/util/config.hpp>
#include <osmium/version.hpp>
#include <cassert>
#include <cstddef>
#include <exception>
#include <functional>
#include <future>
#include <initializer_list>
#include <memory>
#include <string>
#include <utility>
namespace osmium {
namespace memory {
class Item;
} //namespace memory
namespace io {
namespace detail {
inline std::size_t get_output_queue_size() noexcept {
return osmium::config::get_max_queue_size("OUTPUT", 20);
}
} // namespace detail
class Writer {
enum {
default_buffer_size = 10UL * 1024UL * 1024UL
};
osmium::io::File m_file;
detail::future_string_queue_type m_output_queue{detail::get_output_queue_size(), "raw_output"};
std::unique_ptr<osmium::io::detail::OutputFormat> m_output{nullptr};
osmium::memory::Buffer m_buffer;
osmium::io::Header m_header;
size_t m_buffer_size = default_buffer_size;
std::future<std::size_t> m_write_future;
osmium::thread::thread_handler m_thread;
// Checking the m_write_future is much more expensive then checking
// one atomic bool, so we set this bool in the write_thread when
// the writer should check the future...
std::atomic_bool m_notification{false};
enum class status {
okay = 0, // normal writing
error = 1, // some error occurred while writing
closed = 2 // close() called successfully
} m_status = status::okay;
// Has the header already bin written to the file?
bool m_header_written = false;
// This function will run in a separate thread.
static void write_thread(detail::future_string_queue_type& output_queue,
std::unique_ptr<osmium::io::Compressor>&& compressor,
std::promise<std::size_t>&& write_promise,
std::atomic_bool* notification) {
detail::WriteThread write_thread{output_queue,
std::move(compressor),
std::move(write_promise),
notification};
write_thread();
}
void write_header() {
if (m_header.get("generator").empty()) {
m_header.set("generator", "libosmium/" LIBOSMIUM_VERSION_STRING);
}
m_output->write_header(m_header);
m_header_written = true;
}
void do_write(osmium::memory::Buffer&& buffer) {
if (!m_header_written) {
write_header();
}
if (buffer && buffer.committed() > 0) {
m_output->write_buffer(std::move(buffer));
}
}
void do_flush() {
if (!m_header_written) {
write_header();
}
if (m_notification) {
osmium::thread::check_for_exception(m_write_future);
}
if (m_buffer && m_buffer.committed() > 0) {
osmium::memory::Buffer buffer{m_buffer_size,
osmium::memory::Buffer::auto_grow::no};
using std::swap;
swap(m_buffer, buffer);
m_output->write_buffer(std::move(buffer));
}
}
template <typename TFunction, typename... TArgs>
void ensure_cleanup(TFunction func, TArgs&&... args) {
if (m_status != status::okay) {
throw io_error("Can not write to writer when in status 'closed' or 'error'");
}
try {
func(std::forward<TArgs>(args)...);
} catch (...) {
m_status = status::error;
detail::add_to_queue(m_output_queue, std::current_exception());
detail::add_end_of_data_to_queue(m_output_queue);
throw;
}
}
struct options_type {
osmium::io::Header header;
overwrite allow_overwrite = overwrite::no;
fsync sync = fsync::no;
osmium::thread::Pool* pool = nullptr;
};
static void set_option(options_type& options, osmium::thread::Pool& pool) {
options.pool = &pool;
}
static void set_option(options_type& options, const osmium::io::Header& header) {
options.header = header;
}
static void set_option(options_type& options, overwrite value) {
options.allow_overwrite = value;
}
static void set_option(options_type& options, fsync value) {
options.sync = value;
}
void do_close() {
if (m_status == status::okay) {
ensure_cleanup([&]() {
do_write(std::move(m_buffer));
m_output->write_end();
m_status = status::closed;
detail::add_end_of_data_to_queue(m_output_queue);
});
}
}
public:
template <typename... TArgs>
explicit Writer(const osmium::io::File& file, TArgs&&... args) :
m_file(file.check()) {
assert(!m_file.buffer()); // XXX can't handle pseudo-files
options_type options;
(void)std::initializer_list<int>{(set_option(options, std::forward<TArgs>(args)), 0)...};
if (!options.pool) {
options.pool = &thread::Pool::default_instance();
}
m_header = options.header;
m_output = osmium::io::detail::OutputFormatFactory::instance().create_output(*options.pool, m_file, m_output_queue);
std::unique_ptr<osmium::io::Compressor> compressor =
CompressionFactory::instance().create_compressor(file.compression(),
osmium::io::detail::open_for_writing(m_file.filename(), options.allow_overwrite),
options.sync);
std::promise<std::size_t> write_promise;
m_write_future = write_promise.get_future();
m_thread = osmium::thread::thread_handler{write_thread, std::ref(m_output_queue), std::move(compressor), std::move(write_promise), &m_notification};
}
template <typename... TArgs>
explicit Writer(const std::string& filename, TArgs&&... args) :
Writer(osmium::io::File{filename}, std::forward<TArgs>(args)...) {
}
template <typename... TArgs>
explicit Writer(const char* filename, TArgs&&... args) :
Writer(osmium::io::File{filename}, std::forward<TArgs>(args)...) {
}
Writer(const Writer&) = delete;
Writer& operator=(const Writer&) = delete;
Writer(Writer&&) = delete;
Writer& operator=(Writer&&) = delete;
~Writer() noexcept {
try {
do_close();
} catch (...) { // NOLINT(bugprone-empty-catch)
// Ignore any exceptions because destructor must not throw.
}
}
size_t buffer_size() const noexcept {
return m_buffer_size;
}
void set_buffer_size(size_t size) noexcept {
m_buffer_size = size;
}
void set_header(const osmium::io::Header& header) {
m_header = header;
}
void flush() {
ensure_cleanup([&]() {
do_flush();
});
}
void operator()(osmium::memory::Buffer&& buffer) {
ensure_cleanup([&]() {
do_flush();
do_write(std::move(buffer));
});
}
void operator()(const osmium::memory::Item& item) {
ensure_cleanup([&]() {
if (!m_buffer) {
m_buffer = osmium::memory::Buffer{m_buffer_size,
osmium::memory::Buffer::auto_grow::no};
}
try {
m_buffer.push_back(item);
} catch (const osmium::buffer_is_full&) {
do_flush();
m_buffer.push_back(item);
}
});
}
std::size_t close() {
do_close();
if (m_write_future.valid()) {
return m_write_future.get();
}
return 0;
}
}; // class Writer
} // namespace io
} // namespace osmium
#endif // OSMIUM_IO_WRITER_HPP
buffer.hpp
osmium::io::CompressionFactory::create_compressor
std::unique_ptr< osmium::io::Compressor > create_compressor(const osmium::io::file_compression compression, TArgs &&... args) const
Definition compression.hpp:211
osmium::io::CompressionFactory::instance
static CompressionFactory & instance()
Definition compression.hpp:191
osmium::io::File
Definition file.hpp:72
osmium::io::File::filename
File & filename(const std::string &filename)
Definition file.hpp:312
osmium::io::File::compression
file_compression compression() const noexcept
Definition file.hpp:294
osmium::io::File::buffer
const char * buffer() const noexcept
Definition file.hpp:143
osmium::io::Header
Definition header.hpp:68
osmium::io::Writer
Definition writer.hpp:100
osmium::io::Writer::close
std::size_t close()
Definition writer.hpp:402
osmium::io::Writer::default_buffer_size
@ default_buffer_size
Definition writer.hpp:103
osmium::io::Writer::write_header
void write_header()
Definition writer.hpp:148
osmium::io::Writer::set_option
static void set_option(options_type &options, fsync value)
Definition writer.hpp:219
osmium::io::Writer::set_option
static void set_option(options_type &options, const osmium::io::Header &header)
Definition writer.hpp:211
osmium::io::Writer::set_option
static void set_option(options_type &options, osmium::thread::Pool &pool)
Definition writer.hpp:207
osmium::io::Writer::m_buffer_size
size_t m_buffer_size
Definition writer.hpp:116
osmium::io::Writer::Writer
Writer(Writer &&)=delete
osmium::io::Writer::do_flush
void do_flush()
Definition writer.hpp:167
osmium::io::Writer::Writer
Writer(const osmium::io::File &file, TArgs &&... args)
Definition writer.hpp:267
osmium::io::Writer::Writer
Writer(const std::string &filename, TArgs &&... args)
Definition writer.hpp:293
osmium::io::Writer::buffer_size
size_t buffer_size() const noexcept
Definition writer.hpp:319
osmium::io::Writer::status
status
Definition writer.hpp:127
osmium::io::Writer::status::closed
@ closed
osmium::io::Writer::status::error
@ error
osmium::io::Writer::status::okay
@ okay
osmium::io::Writer::m_thread
osmium::thread::thread_handler m_thread
Definition writer.hpp:120
osmium::io::Writer::flush
void flush()
Definition writer.hpp:348
osmium::io::Writer::operator()
void operator()(osmium::memory::Buffer &&buffer)
Definition writer.hpp:362
osmium::io::Writer::m_header_written
bool m_header_written
Definition writer.hpp:134
osmium::io::Writer::m_header
osmium::io::Header m_header
Definition writer.hpp:114
osmium::io::Writer::m_buffer
osmium::memory::Buffer m_buffer
Definition writer.hpp:112
osmium::io::Writer::m_notification
std::atomic_bool m_notification
Definition writer.hpp:125
osmium::io::Writer::~Writer
~Writer() noexcept
Definition writer.hpp:308
osmium::io::Writer::m_output
std::unique_ptr< osmium::io::detail::OutputFormat > m_output
Definition writer.hpp:110
osmium::io::Writer::set_header
void set_header(const osmium::io::Header &header)
Definition writer.hpp:337
osmium::io::Writer::set_option
static void set_option(options_type &options, overwrite value)
Definition writer.hpp:215
osmium::io::Writer::m_write_future
std::future< std::size_t > m_write_future
Definition writer.hpp:118
osmium::io::Writer::do_close
void do_close()
Definition writer.hpp:223
osmium::io::Writer::do_write
void do_write(osmium::memory::Buffer &&buffer)
Definition writer.hpp:158
osmium::io::Writer::operator=
Writer & operator=(Writer &&)=delete
osmium::io::Writer::operator=
Writer & operator=(const Writer &)=delete
osmium::io::Writer::m_output_queue
detail::future_string_queue_type m_output_queue
Definition writer.hpp:108
osmium::io::Writer::set_buffer_size
void set_buffer_size(size_t size) noexcept
Definition writer.hpp:327
osmium::io::Writer::Writer
Writer(const Writer &)=delete
osmium::io::Writer::m_file
osmium::io::File m_file
Definition writer.hpp:106
osmium::io::Writer::operator()
void operator()(const osmium::memory::Item &item)
Definition writer.hpp:376
osmium::io::Writer::ensure_cleanup
void ensure_cleanup(TFunction func, TArgs &&... args)
Definition writer.hpp:185
osmium::io::Writer::write_thread
static void write_thread(detail::future_string_queue_type &output_queue, std::unique_ptr< osmium::io::Compressor > &&compressor, std::promise< std::size_t > &&write_promise, std::atomic_bool *notification)
Definition writer.hpp:137
osmium::io::Writer::Writer
Writer(const char *filename, TArgs &&... args)
Definition writer.hpp:298
osmium::io::Writer::m_status
enum osmium::io::Writer::status m_status
osmium::memory::Item
Definition item.hpp:105
osmium::thread::Pool
Definition pool.hpp:90
osmium::thread::Pool::default_instance
static Pool & default_instance()
Definition pool.hpp:186
osmium::thread::thread_handler
Definition util.hpp:91
compression.hpp
config.hpp
error.hpp
header.hpp
file.hpp
detail
Definition attr.hpp:342
osmium::config::get_max_queue_size
std::size_t get_max_queue_size(const char *queue_name, const std::size_t default_value) noexcept
Definition config.hpp:83
osmium::io::fsync
fsync
Definition writer_options.hpp:51
osmium::io::fsync::no
@ no
osmium::io::overwrite
overwrite
Definition writer_options.hpp:43
osmium::io::overwrite::no
@ no
osmium::thread::check_for_exception
void check_for_exception(std::future< T > &future)
Definition util.hpp:57
osmium
Namespace for everything in the Osmium library.
Definition assembler.hpp:53
std
Definition location.hpp:555
pool.hpp
osmium::io::Writer::options_type
Definition writer.hpp:200
osmium::io::Writer::options_type::allow_overwrite
overwrite allow_overwrite
Definition writer.hpp:202
osmium::io::Writer::options_type::pool
osmium::thread::Pool * pool
Definition writer.hpp:204
osmium::io::Writer::options_type::sync
fsync sync
Definition writer.hpp:203
osmium::io::Writer::options_type::header
osmium::io::Header header
Definition writer.hpp:201
osmium::io_error
Definition error.hpp:46
util.hpp
version.hpp
LIBOSMIUM_VERSION_STRING
#define LIBOSMIUM_VERSION_STRING
Definition version.hpp:45
writer_options.hpp