Loading...
Searching...
No Matches
collector.hpp
Go to the documentation of this file.
#ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
#define OSMIUM_RELATIONS_COLLECTOR_HPP
/*
This file is part of Osmium (https://osmcode.org/libosmium).
Copyright 2013-2025 Jochen Topf <jochen@topf.org> and others (see README).
Boost Software License - Version 1.0 - August 17th, 2003
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#include <osmium/handler.hpp>
#include <osmium/handler/check_order.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/relations/detail/member_meta.hpp>
#include <osmium/relations/detail/relation_meta.hpp>
#include <osmium/util/iterator.hpp>
#include <osmium/visitor.hpp>
#include <algorithm>
#include <array>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <iomanip>
#include <iostream>
#include <utility>
#include <vector>
namespace osmium {
class Node;
class Way;
namespace relations {
template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
class Collector {
class HandlerPass1 : public osmium::handler::Handler {
TCollector& m_collector;
public:
explicit HandlerPass1(TCollector& collector) noexcept :
m_collector(collector) {
}
void relation(const osmium::Relation& relation) {
if (m_collector.keep_relation(relation)) {
m_collector.add_relation(relation);
}
}
}; // class HandlerPass1
public:
class HandlerPass2 : public osmium::handler::Handler {
osmium::handler::CheckOrder m_check_order;
TCollector& m_collector;
public:
explicit HandlerPass2(TCollector& collector) noexcept :
m_collector(collector) {
}
void node(const osmium::Node& node) {
if (TNodes) {
m_check_order.node(node);
if (!m_collector.find_and_add_object(node)) {
m_collector.node_not_in_any_relation(node);
}
}
}
void way(const osmium::Way& way) {
if (TWays) {
m_check_order.way(way);
if (!m_collector.find_and_add_object(way)) {
m_collector.way_not_in_any_relation(way);
}
}
}
void relation(const osmium::Relation& relation) {
if (TRelations) {
m_check_order.relation(relation);
if (!m_collector.find_and_add_object(relation)) {
m_collector.relation_not_in_any_relation(relation);
}
}
}
void flush() {
m_collector.flush();
}
}; // class HandlerPass2
private:
HandlerPass2 m_handler_pass2;
// All relations we are interested in will be kept in this buffer
osmium::memory::Buffer m_relations_buffer;
// All members we are interested in will be kept in this buffer
osmium::memory::Buffer m_members_buffer;
std::vector<RelationMeta> m_relations;
using mm_vector_type = std::vector<MemberMeta>;
using mm_iterator = mm_vector_type::iterator;
std::array<mm_vector_type, 3> m_member_meta;
int m_count_complete = 0;
using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
callback_func_type m_callback;
enum {
initial_buffer_size = 1024UL * 1024UL
};
iterator_range<mm_iterator> find_member_meta(osmium::item_type type, osmium::object_id_type id) {
auto& mmv = member_meta(type);
return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
}
public:
Collector() :
m_handler_pass2(*static_cast<TCollector*>(this)),
m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
}
protected:
std::vector<MemberMeta>& member_meta(const item_type type) {
return m_member_meta[static_cast<uint16_t>(type) - 1];
}
callback_func_type callback() {
return m_callback;
}
const std::vector<RelationMeta>& relations() const {
return m_relations;
}
bool keep_relation(const osmium::Relation& /*relation*/) const {
return true;
}
bool keep_member(const RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
return true;
}
void node_not_in_any_relation(const osmium::Node& /*node*/) {
}
void way_not_in_any_relation(const osmium::Way& /*way*/) {
}
void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
}
void flush() {
}
const osmium::Relation& get_relation(size_t offset) const {
assert(m_relations_buffer.committed() > offset);
return m_relations_buffer.get<osmium::Relation>(offset);
}
const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
return get_relation(relation_meta.relation_offset());
}
const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
return get_relation(m_relations[member_meta.relation_pos()]);
}
osmium::OSMObject& get_member(size_t offset) const {
assert(m_members_buffer.committed() > offset);
return m_members_buffer.get<osmium::OSMObject>(offset);
}
private:
void add_relation(const osmium::Relation& relation) {
const size_t offset = m_relations_buffer.committed();
m_relations_buffer.add_item(relation);
RelationMeta relation_meta{offset};
int n = 0;
for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
relation_meta.increment_need_members();
} else {
member.set_ref(0); // set member id to zero to indicate we are not interested
}
++n;
}
assert(offset == m_relations_buffer.committed());
if (relation_meta.has_all_members()) {
m_relations_buffer.rollback();
} else {
m_relations_buffer.commit();
m_relations.push_back(relation_meta);
}
}
void sort_member_meta() {
std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
}
static typename iterator_range<mm_iterator>::iterator::difference_type count_not_removed(const iterator_range<mm_iterator>& range) {
return std::count_if(range.begin(), range.end(), [](const MemberMeta& mm) {
return !mm.removed();
});
}
bool find_and_add_object(const osmium::OSMObject& object) {
auto range = find_member_meta(object.type(), object.id());
if (count_not_removed(range) == 0) {
// nothing found
return false;
}
{
members_buffer().add_item(object);
const size_t member_offset = members_buffer().commit();
for (auto& member : range) {
member.set_buffer_offset(member_offset);
}
}
for (auto& member : range) {
if (member.removed()) {
break;
}
assert(member.member_id() == object.id());
assert(member.relation_pos() < m_relations.size());
RelationMeta& relation_meta = m_relations[member.relation_pos()];
assert(member.member_pos() < get_relation(relation_meta).members().size());
relation_meta.got_one_member();
if (relation_meta.has_all_members()) {
const size_t relation_offset = member.relation_pos();
static_cast<TCollector*>(this)->complete_relation(relation_meta);
clear_member_metas(relation_meta);
m_relations[relation_offset] = RelationMeta{};
possibly_purge_removed_members();
}
}
return true;
}
void clear_member_metas(const RelationMeta& relation_meta) {
const osmium::Relation& relation = get_relation(relation_meta);
for (const auto& member : relation.members()) {
if (member.ref() != 0) {
const auto range = find_member_meta(member.type(), member.ref());
assert(!range.empty());
// if this is the last time this object was needed
// then mark it as removed
if (count_not_removed(range) == 1) {
get_member(range.begin()->buffer_offset()).set_removed(true);
}
for (auto& member_meta : range) {
if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
member_meta.remove();
break;
}
}
}
}
}
public:
uint64_t used_memory() const {
const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
const uint64_t members = nmembers * sizeof(MemberMeta);
const uint64_t relations_size = m_relations.capacity() * sizeof(RelationMeta);
const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
const uint64_t members_buffer_capacity = m_members_buffer.capacity();
std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations_size << "\n";
std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
const uint64_t total = relations_size + members + relations_buffer_capacity + members_buffer_capacity;
std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
std::cerr << " =======================================================\n";
return relations_buffer_capacity + members_buffer_capacity + relations_size + members;
}
HandlerPass2& handler(const callback_func_type& callback = nullptr) {
m_callback = callback;
return m_handler_pass2;
}
osmium::memory::Buffer& members_buffer() {
return m_members_buffer;
}
bool is_available(osmium::item_type type, osmium::object_id_type id) {
const auto range = find_member_meta(type, id);
assert(!range.empty());
return range.begin()->is_available();
}
size_t get_offset(osmium::item_type type, osmium::object_id_type id) {
const auto range = find_member_meta(type, id);
assert(!range.empty());
assert(range.begin()->is_available());
return range.begin()->buffer_offset();
}
std::pair<bool, size_t> get_availability_and_offset(osmium::item_type type, osmium::object_id_type id) {
const auto range = find_member_meta(type, id);
assert(!range.empty());
if (range.begin()->is_available()) {
return std::make_pair(true, range.begin()->buffer_offset());
}
return std::make_pair(false, 0);
}
template <typename TIter>
void read_relations(TIter begin, TIter end) {
HandlerPass1 handler_pass1{*static_cast<TCollector*>(this)};
osmium::apply(begin, end, handler_pass1);
sort_member_meta();
}
template <typename TSource>
void read_relations(TSource& source) {
using std::begin;
using std::end;
read_relations(begin(source), end(source));
source.close();
}
void moving_in_buffer(size_t old_offset, size_t new_offset) {
const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
auto range = find_member_meta(object.type(), object.id());
for (auto& member : range) {
assert(member.buffer_offset() == old_offset);
member.set_buffer_offset(new_offset);
}
}
void possibly_purge_removed_members() {
++m_count_complete;
if (m_count_complete > 10000) { // XXX
// const size_t size_before = m_members_buffer.committed();
m_members_buffer.purge_removed(this);
/*
const size_t size_after = m_members_buffer.committed();
double percent = static_cast<double>(size_before - size_after);
percent /= size_before;
percent *= 100;
std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
*/
m_count_complete = 0;
}
}
std::vector<const osmium::Relation*> get_incomplete_relations() const {
std::vector<const osmium::Relation*> incomplete_relations;
for (const auto& relation_meta : m_relations) {
if (!relation_meta.has_all_members()) {
incomplete_relations.push_back(&get_relation(relation_meta));
}
}
return incomplete_relations;
}
}; // class Collector
} // namespace relations
} // namespace osmium
#endif // OSMIUM_RELATIONS_COLLECTOR_HPP
buffer.hpp
check_order.hpp
osmium::Node
Definition node.hpp:48
osmium::OSMObject
Definition object.hpp:64
osmium::OSMObject::id
object_id_type id() const noexcept
Get ID of this object.
Definition object.hpp:118
osmium::RelationMember
Definition relation.hpp:56
osmium::Relation
Definition relation.hpp:161
osmium::Way
Definition way.hpp:72
osmium::handler::CheckOrder
Definition check_order.hpp:87
osmium::handler::CheckOrder::node
void node(const osmium::Node &node)
Definition check_order.hpp:98
osmium::handler::CheckOrder::relation
void relation(const osmium::Relation &relation)
Definition check_order.hpp:139
osmium::handler::CheckOrder::way
void way(const osmium::Way &way)
Definition check_order.hpp:120
osmium::handler::Handler
Definition handler.hpp:71
osmium::memory::Item::set_removed
void set_removed(const bool removed) noexcept
Definition item.hpp:179
osmium::relations::Collector::HandlerPass1
Definition collector.hpp:103
osmium::relations::Collector::HandlerPass1::m_collector
TCollector & m_collector
Definition collector.hpp:105
osmium::relations::Collector::HandlerPass1::HandlerPass1
HandlerPass1(TCollector &collector) noexcept
Definition collector.hpp:109
osmium::relations::Collector::HandlerPass1::relation
void relation(const osmium::Relation &relation)
Definition collector.hpp:113
osmium::relations::Collector::HandlerPass2
Definition collector.hpp:126
osmium::relations::Collector::HandlerPass2::flush
void flush()
Definition collector.hpp:164
osmium::relations::Collector::HandlerPass2::m_check_order
osmium::handler::CheckOrder m_check_order
Definition collector.hpp:128
osmium::relations::Collector::HandlerPass2::HandlerPass2
HandlerPass2(TCollector &collector) noexcept
Definition collector.hpp:133
osmium::relations::Collector::HandlerPass2::way
void way(const osmium::Way &way)
Definition collector.hpp:146
osmium::relations::Collector::HandlerPass2::m_collector
TCollector & m_collector
Definition collector.hpp:129
osmium::relations::Collector::HandlerPass2::relation
void relation(const osmium::Relation &relation)
Definition collector.hpp:155
osmium::relations::Collector::HandlerPass2::node
void node(const osmium::Node &node)
Definition collector.hpp:137
osmium::relations::Collector
Definition collector.hpp:98
osmium::relations::Collector::members_buffer
osmium::memory::Buffer & members_buffer()
Definition collector.hpp:484
osmium::relations::Collector::m_callback
callback_func_type m_callback
Definition collector.hpp:194
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(size_t offset) const
Definition collector.hpp:301
osmium::relations::Collector::clear_member_metas
void clear_member_metas(const RelationMeta &relation_meta)
Definition collector.hpp:422
osmium::relations::Collector::Collector
Collector()
Definition collector.hpp:210
osmium::relations::Collector::handler
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition collector.hpp:479
osmium::relations::Collector::sort_member_meta
void sort_member_meta()
Definition collector.hpp:365
osmium::relations::Collector::add_relation
void add_relation(const osmium::Relation &relation)
Definition collector.hpp:335
osmium::relations::Collector::relations
const std::vector< RelationMeta > & relations() const
Definition collector.hpp:226
osmium::relations::Collector::get_member
osmium::OSMObject & get_member(size_t offset) const
Definition collector.hpp:320
osmium::relations::Collector::member_meta
std::vector< MemberMeta > & member_meta(const item_type type)
Definition collector.hpp:218
osmium::relations::Collector::get_offset
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:514
osmium::relations::Collector::used_memory
uint64_t used_memory() const
Definition collector.hpp:447
osmium::relations::Collector::m_relations
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition collector.hpp:181
osmium::relations::Collector::way_not_in_any_relation
void way_not_in_any_relation(const osmium::Way &)
Definition collector.hpp:274
osmium::relations::Collector::flush
void flush()
Definition collector.hpp:298
osmium::relations::Collector::callback
callback_func_type callback()
Definition collector.hpp:222
osmium::relations::Collector::relation_not_in_any_relation
void relation_not_in_any_relation(const osmium::Relation &)
Definition collector.hpp:284
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition collector.hpp:316
osmium::relations::Collector::initial_buffer_size
@ initial_buffer_size
Definition collector.hpp:197
osmium::relations::Collector::mm_iterator
mm_vector_type::iterator mm_iterator
Definition collector.hpp:188
osmium::relations::Collector::is_available
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:499
osmium::relations::Collector::m_members_buffer
osmium::memory::Buffer m_members_buffer
Definition collector.hpp:178
osmium::relations::Collector::find_and_add_object
bool find_and_add_object(const osmium::OSMObject &object)
Definition collector.hpp:384
osmium::relations::Collector::get_availability_and_offset
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:537
osmium::relations::Collector::keep_member
bool keep_member(const RelationMeta &, const osmium::RelationMember &) const
Definition collector.hpp:253
osmium::relations::Collector::get_incomplete_relations
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition collector.hpp:600
osmium::relations::Collector::get_relation
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition collector.hpp:309
osmium::relations::Collector::keep_relation
bool keep_relation(const osmium::Relation &) const
Definition collector.hpp:239
osmium::relations::Collector::m_count_complete
int m_count_complete
Definition collector.hpp:191
osmium::relations::Collector::callback_func_type
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition collector.hpp:193
osmium::relations::Collector::read_relations
void read_relations(TIter begin, TIter end)
Definition collector.hpp:547
osmium::relations::Collector::count_not_removed
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition collector.hpp:371
osmium::relations::Collector::moving_in_buffer
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition collector.hpp:561
osmium::relations::Collector::node_not_in_any_relation
void node_not_in_any_relation(const osmium::Node &)
Definition collector.hpp:264
osmium::relations::Collector::m_handler_pass2
HandlerPass2 m_handler_pass2
Definition collector.hpp:172
osmium::relations::Collector::possibly_purge_removed_members
void possibly_purge_removed_members()
Definition collector.hpp:576
osmium::relations::Collector::m_relations_buffer
osmium::memory::Buffer m_relations_buffer
Definition collector.hpp:175
osmium::relations::Collector::mm_vector_type
std::vector< MemberMeta > mm_vector_type
Definition collector.hpp:187
osmium::relations::Collector::m_member_meta
std::array< mm_vector_type, 3 > m_member_meta
Definition collector.hpp:189
osmium::relations::Collector::find_member_meta
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition collector.hpp:200
osmium::relations::Collector::read_relations
void read_relations(TSource &source)
Definition collector.hpp:554
handler.hpp
item_type.hpp
iterator.hpp
osmium
Namespace for everything in the Osmium library.
Definition assembler.hpp:53
osmium::apply
void apply(TIterator it, TIterator end, THandlers &&... handlers)
Definition visitor.hpp:326
osmium::object_id_type
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition types.hpp:45
osmium::item_type
item_type
Definition item_type.hpp:45
osmium::item_type::relation
@ relation
osmium::item_type::node
@ node
osmium::item_type::way
@ way
osmium::make_range
iterator_range< It > make_range(P &&p) noexcept
Definition iterator.hpp:68
object.hpp
relation.hpp
osmium::iterator_range
Definition iterator.hpp:42
osmium::iterator_range::begin
It begin() const noexcept
Definition iterator.hpp:50
osmium::iterator_range::end
It end() const noexcept
Definition iterator.hpp:54
types.hpp
visitor.hpp