collector.hpp
Go to the documentation of this file.
#ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
#define OSMIUM_RELATIONS_COLLECTOR_HPP
/*
This file is part of Osmium (http://osmcode.org/libosmium).
Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <iomanip>
//#include <iostream>
#include <vector>
#include <osmium/fwd.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp> // IWYU pragma: keep
#include <osmium/osm/types.hpp>
#include <osmium/handler.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/visitor.hpp>
#include <osmium/relations/detail/relation_meta.hpp>
#include <osmium/relations/detail/member_meta.hpp>
namespace osmium {
namespace relations {
template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
class Collector {
class HandlerPass1 : public osmium::handler::Handler {
TCollector& m_collector;
public:
HandlerPass1(TCollector& collector) noexcept :
m_collector(collector) {
}
void relation(const osmium::Relation& relation) {
if (m_collector.keep_relation(relation)) {
m_collector.add_relation(relation);
}
}
}; // class HandlerPass1
public:
class HandlerPass2 : public osmium::handler::Handler {
TCollector& m_collector;
public:
HandlerPass2(TCollector& collector) noexcept :
m_collector(collector) {
}
void node(const osmium::Node& node) {
if (TNodes) {
if (! m_collector.find_and_add_object(node)) {
m_collector.node_not_in_any_relation(node);
}
}
}
void way(const osmium::Way& way) {
if (TWays) {
if (! m_collector.find_and_add_object(way)) {
m_collector.way_not_in_any_relation(way);
}
}
}
void relation(const osmium::Relation& relation) {
if (TRelations) {
if (! m_collector.find_and_add_object(relation)) {
m_collector.relation_not_in_any_relation(relation);
}
}
}
void flush() {
m_collector.flush();
}
}; // class HandlerPass2
private:
HandlerPass2 m_handler_pass2;
// All relations we are interested in will be kept in this buffer
osmium::memory::Buffer m_relations_buffer;
// All members we are interested in will be kept in this buffer
osmium::memory::Buffer m_members_buffer;
std::vector<RelationMeta> m_relations;
std::vector<MemberMeta> m_member_meta[3];
int m_count_complete = 0;
typedef std::function<void(osmium::memory::Buffer&&)> callback_func_type;
callback_func_type m_callback;
static constexpr size_t initial_buffer_size = 1024 * 1024;
public:
Collector() :
m_handler_pass2(*static_cast<TCollector*>(this)),
m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
m_relations(),
m_member_meta() {
}
protected:
std::vector<MemberMeta>& member_meta(const item_type type) {
return m_member_meta[static_cast<uint16_t>(type) - 1];
}
callback_func_type callback() {
return m_callback;
}
const std::vector<RelationMeta>& relations() const {
return m_relations;
}
bool keep_relation(const osmium::Relation& /*relation*/) const {
return true;
}
bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
return true;
}
void node_not_in_any_relation(const osmium::Node& /*node*/) {
}
void way_not_in_any_relation(const osmium::Way& /*way*/) {
}
void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
}
void flush() {
}
void clean_assembled_relations() {
m_relations.erase(
std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
m_relations.end()
);
}
const osmium::Relation& get_relation(size_t offset) const {
return m_relations_buffer.get<osmium::Relation>(offset);
}
const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
return get_relation(relation_meta.relation_offset());
}
osmium::OSMObject& get_member(size_t offset) const {
return m_members_buffer.get<osmium::OSMObject>(offset);
}
private:
void add_relation(const osmium::Relation& relation) {
const size_t offset = m_relations_buffer.committed();
m_relations_buffer.add_item(relation);
RelationMeta relation_meta(offset);
int n = 0;
for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
relation_meta.increment_need_members();
} else {
member.ref(0); // set member id to zero to indicate we are not interested
}
++n;
}
assert(offset == m_relations_buffer.committed());
if (relation_meta.has_all_members()) {
m_relations_buffer.rollback();
} else {
m_relations_buffer.commit();
m_relations.push_back(std::move(relation_meta));
// std::cerr << "added relation id=" << relation.id() << "\n";
}
}
void sort_member_meta() {
/* std::cerr << "relations: " << m_relations.size() << "\n";
std::cerr << "node members: " << m_member_meta[0].size() << "\n";
std::cerr << "way members: " << m_member_meta[1].size() << "\n";
std::cerr << "relation members: " << m_member_meta[2].size() << "\n";*/
std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
}
bool find_and_add_object(const osmium::OSMObject& object) {
auto& mmv = member_meta(object.type());
auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id()));
if (osmium::relations::count_not_removed(range.first, range.second) == 0) {
// nothing found
return false;
}
{
members_buffer().add_item(object);
const size_t member_offset = members_buffer().commit();
for (auto it = range.first; it != range.second; ++it) {
it->set_buffer_offset(member_offset);
}
}
for (auto it = range.first; it != range.second; ++it) {
MemberMeta& member_meta = *it;
if (member_meta.removed()) {
break;
}
assert(member_meta.member_id() == object.id());
assert(member_meta.relation_pos() < m_relations.size());
RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
// std::cerr << " => " << member_meta.member_pos() << " < " << get_relation(relation_meta).members().size() << " (id=" << get_relation(relation_meta).id() << ")\n";
assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
// std::cerr << " add way " << member_meta.member_id() << " to rel " << get_relation(relation_meta).id() << " at pos " << member_meta.member_pos() << "\n";
relation_meta.got_one_member();
if (relation_meta.has_all_members()) {
const size_t relation_offset = member_meta.relation_pos();
static_cast<TCollector*>(this)->complete_relation(relation_meta);
clear_member_metas(relation_meta);
m_relations[relation_offset] = RelationMeta();
possibly_purge_removed_members();
}
}
// Remove MemberMetas that were marked as removed.
mmv.erase(std::remove_if(mmv.begin(), mmv.end(), [](MemberMeta& mm) {
return mm.removed();
}), mmv.end());
return true;
}
void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
const osmium::Relation& relation = get_relation(relation_meta);
for (const auto& member : relation.members()) {
if (member.ref() != 0) {
auto& mmv = member_meta(member.type());
auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(member.ref()));
assert(range.first != range.second);
// if this is the last time this object was needed
// then mark it as removed
if (osmium::relations::count_not_removed(range.first, range.second) == 1) {
get_member(range.first->buffer_offset()).set_removed(true);
}
for (auto it = range.first; it != range.second; ++it) {
if (!it->removed() && relation.id() == get_relation(it->relation_pos()).id()) {
it->remove();
break;
}
}
}
}
}
public:
uint64_t used_memory() const {
const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
const uint64_t members = nmembers * sizeof(MemberMeta);
const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
const uint64_t members_buffer_capacity = m_members_buffer.capacity();
std::cout << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
std::cout << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
std::cout << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
std::cout << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
std::cout << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
std::cout << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
std::cout << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
std::cout << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
std::cout << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
std::cout << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
std::cout << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
std::cout << " total .................................. = " << std::setw(12) << total << "\n";
std::cout << " =======================================================\n";
return relations_buffer_capacity + members_buffer_capacity + relations + members;
}
HandlerPass2& handler(const callback_func_type& callback = nullptr) {
m_callback = callback;
return m_handler_pass2;
}
osmium::memory::Buffer& members_buffer() {
return m_members_buffer;
}
size_t get_offset(osmium::item_type type, osmium::object_id_type id) {
const auto& mmv = member_meta(type);
const auto range = std::equal_range(mmv.cbegin(), mmv.cend(), MemberMeta(id));
assert(range.first != range.second);
return range.first->buffer_offset();
}
template <typename TIter>
void read_relations(TIter begin, TIter end) {
HandlerPass1 handler(*static_cast<TCollector*>(this));
osmium::apply(begin, end, handler);
sort_member_meta();
}
template <typename TSource>
void read_relations(TSource& source) {
read_relations(std::begin(source), std::end(source));
source.close();
}
void moving_in_buffer(size_t old_offset, size_t new_offset) {
const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
auto& mmv = member_meta(object.type());
auto range = std::equal_range(mmv.begin(), mmv.end(), MemberMeta(object.id()));
for (auto it = range.first; it != range.second; ++it) {
assert(it->buffer_offset() == old_offset);
it->set_buffer_offset(new_offset);
}
}
void possibly_purge_removed_members() {
++m_count_complete;
if (m_count_complete > 10000) { // XXX
// const size_t size_before = m_members_buffer.committed();
m_members_buffer.purge_removed(this);
/*
const size_t size_after = m_members_buffer.committed();
double percent = static_cast<double>(size_before - size_after);
percent /= size_before;
percent *= 100;
std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
*/
m_count_complete = 0;
}
}
std::vector<const osmium::Relation*> get_incomplete_relations() const {
std::vector<const osmium::Relation*> relations;
for (const auto& relation_meta : m_relations) {
if (!relation_meta.has_all_members()) {
relations.push_back(&get_relation(relation_meta));
}
}
return relations;
}
}; // class Collector
} // namespace relations
} // namespace osmium
#endif // OSMIUM_RELATIONS_COLLECTOR_HPP
osmium::relations::Collector::get_incomplete_relations
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:543
osmium::relations::Collector::HandlerPass2::relation
void relation(const osmium::Relation &relation)
Definition: collector.hpp:147
osmium::relations::Collector::clear_member_metas
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:416
osmium::relations::Collector::m_callback
callback_func_type m_callback
Definition: collector.hpp:183
osmium::relations::Collector::members_buffer
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:479
osmium::osm_entity_bits::type
type
Definition: entity_bits.hpp:60
osmium::Relation::members
RelationMemberList & members()
Definition: relation.hpp:177
osmium::relations::Collector::HandlerPass2::way
void way(const osmium::Way &way)
Definition: collector.hpp:139
osmium::item_type
item_type
Definition: item_type.hpp:43
osmium::relations::Collector::clean_assembled_relations
void clean_assembled_relations()
Definition: collector.hpp:289
osmium::relations::Collector::moving_in_buffer
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:503
osmium::Relation
Definition: relation.hpp:165
osmium::relations::Collector::initial_buffer_size
static constexpr size_t initial_buffer_size
Definition: collector.hpp:185
fwd.hpp
osmium::handler::Handler
Definition: handler.hpp:45
osmium::relations::Collector::keep_member
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:237
object.hpp
osmium::relations::Collector::m_members_buffer
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:169
osmium::relations::Collector::relations
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:210
osmium::handler::Handler::way
void way(const osmium::Way &) const
Definition: handler.hpp:55
osmium::relations::Collector::read_relations
void read_relations(TSource &source)
Definition: collector.hpp:498
osmium::relations::Collector::get_offset
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:483
osmium::relations::Collector::relation_not_in_any_relation
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:268
types.hpp
osmium::Way
Definition: way.hpp:65
osmium::relations::Collector::member_meta
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:202
osmium::relations::Collector::find_and_add_object
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:369
osmium::relations::Collector::HandlerPass2
Definition: collector.hpp:121
osmium::relations::Collector::HandlerPass2::HandlerPass2
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:127
item_type.hpp
osmium::relations::Collector::m_relations
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:172
osmium::relations::Collector::sort_member_meta
void sort_member_meta()
Definition: collector.hpp:352
osmium::apply
void apply(TIterator it, TIterator end, THandlers &...handlers)
Definition: visitor.hpp:234
osmium::OSMObject::end
iterator end()
Definition: object.hpp:338
osmium::relations::Collector::node_not_in_any_relation
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:248
osmium::RelationMember
Definition: relation.hpp:54
osmium::relations::Collector::HandlerPass1::m_collector
TCollector & m_collector
Definition: collector.hpp:100
osmium::object_id_type
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
osmium
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
osmium::memory::Buffer::add_item
T & add_item(const T &item)
Definition: buffer.hpp:457
osmium::relations::Collector
Definition: collector.hpp:93
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:296
osmium::handler::Handler::node
void node(const osmium::Node &) const
Definition: handler.hpp:52
osmium::memory::Buffer::purge_removed
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:690
osmium::relations::Collector::m_handler_pass2
HandlerPass2 m_handler_pass2
Definition: collector.hpp:163
osmium::relations::Collector::Collector
Collector()
Definition: collector.hpp:192
osmium::relations::Collector::callback_func_type
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:182
osmium::relations::Collector::get_member
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:307
relation.hpp
osmium::memory::Buffer::capacity
size_t capacity() const noexcept
Definition: buffer.hpp:233
osmium::relations::Collector::HandlerPass1::relation
void relation(const osmium::Relation &relation)
Definition: collector.hpp:108
std::end
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
osmium::relations::Collector::HandlerPass1
Definition: collector.hpp:98
osmium::relations::Collector::HandlerPass2::flush
void flush()
Definition: collector.hpp:155
handler.hpp
osmium::relations::Collector::m_relations_buffer
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:166
osmium::OSMObject::id
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:110
osmium::relations::Collector::used_memory
uint64_t used_memory() const
Definition: collector.hpp:442
osmium::memory::Buffer::committed
size_t committed() const noexcept
Definition: buffer.hpp:241
osmium::relations::Collector::callback
callback_func_type callback()
Definition: collector.hpp:206
osmium::memory::Buffer
Definition: buffer.hpp:97
osmium::memory::Buffer::get
T & get(const size_t offset) const
Definition: buffer.hpp:379
osmium::relations::Collector::add_relation
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:321
osmium::relations::Collector::handler
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:474
buffer.hpp
osmium::relations::Collector::flush
void flush()
Definition: collector.hpp:282
osmium::Node
Definition: node.hpp:47
osmium::relations::Collector::way_not_in_any_relation
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:258
osmium::relations::Collector::m_count_complete
int m_count_complete
Definition: collector.hpp:180
osmium::relations::Collector::HandlerPass2::node
void node(const osmium::Node &node)
Definition: collector.hpp:131
osmium::relations::Collector::possibly_purge_removed_members
void possibly_purge_removed_members()
Definition: collector.hpp:519
osmium::relations::Collector::keep_relation
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:223
osmium::relations::Collector::HandlerPass2::m_collector
TCollector & m_collector
Definition: collector.hpp:123
osmium::relations::Collector::read_relations
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:491
std::begin
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
osmium::relations::Collector::HandlerPass1::HandlerPass1
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:104
osmium::memory::Buffer::rollback
void rollback()
Definition: buffer.hpp:349
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:303
visitor.hpp
osmium::OSMObject
Definition: object.hpp:58
osmium::memory::Buffer::commit
size_t commit()
Definition: buffer.hpp:335
osmium::relations::Collector::m_member_meta
std::vector< MemberMeta > m_member_meta[3]
Definition: collector.hpp:178
osmium::RelationMemberList::size
size_type size() const noexcept
Definition: relation.hpp:158
osmium::OSMObject::begin
iterator begin()
Definition: object.hpp:334