diff --git a/src/Makefile.am b/src/Makefile.am index 545c5f06c..088645201 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,7 +210,7 @@ bin_bbox_brim_query_LDADD = libexpatwrapper.la bin_translate_xapi_SOURCES = expat/translate_xapi.cc expat/escape_xml.cc bin_translate_xapi_LDADD = -distdir = osm-3s_v0.7.58 +distdir = osm-3s_v0.7.58.5 nobase_dist_HEADERS = \ expat/escape_json.h\ diff --git a/src/configure.ac b/src/configure.ac index 9c0ad00f4..850b222b6 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([OverpassAPI], [0.7.58], [roland@olbricht.nrw]) +AC_INIT([OverpassAPI], [0.7.58.5], [roland@olbricht.nrw]) AC_CONFIG_SRCDIR([template_db/file_blocks.h]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/src/overpass_api/core/settings.cc b/src/overpass_api/core/settings.cc index 3c8399964..1013c4a5a 100644 --- a/src/overpass_api/core/settings.cc +++ b/src/overpass_api/core/settings.cc @@ -99,7 +99,7 @@ Basic_Settings::Basic_Settings() base_directory("./"), logfile_name("transactions.log"), shared_name_base("/osm3s_v0.7.58"), - version("0.7.58"), + version("0.7.59"), source_hash("4f7e2b3b87ff98dc98f61d4f01b2127578d79aef"), #ifdef HAVE_LZ4 compression_method(File_Blocks_Index_Base::LZ4_COMPRESSION), diff --git a/src/overpass_api/data/filter_by_tags.h b/src/overpass_api/data/filter_by_tags.h index 7be532a60..be20aec3b 100644 --- a/src/overpass_api/data/filter_by_tags.h +++ b/src/overpass_api/data/filter_by_tags.h @@ -79,7 +79,8 @@ template< class Id_Type > std::map< Id_Type, std::pair< uint64, Uint31_Index > > collect_attic_kv( std::vector< std::pair< std::string, std::string > >::const_iterator kvit, uint64 timestamp, Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >& tags_db, - Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db) + Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db, + std::vector< std::pair< Id_Type, Uint31_Index > >* relevant_ids = 0) { std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id; std::set< Tag_Index_Global > tag_req = get_kv_req(kvit->first, kvit->second); @@ -87,13 +88,19 @@ std::map< Id_Type, std::pair< uint64, Uint31_Index > > collect_attic_kv( for (typename Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >::Discrete_Iterator it2(tags_db.discrete_begin(tag_req.begin(), tag_req.end())); !(it2 == tags_db.discrete_end()); ++it2) - timestamp_per_id[it2.object().id] = std::make_pair(NOW, it2.object().idx); + { + if (!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u)))) + timestamp_per_id[it2.object().id] = std::make_pair(NOW, it2.object().idx); + } for (typename Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >::Discrete_Iterator it2(attic_tags_db.discrete_begin(tag_req.begin(), tag_req.end())); !(it2 == attic_tags_db.discrete_end()); ++it2) { - if (it2.object().timestamp > timestamp) + if (it2.object().timestamp > timestamp && + (!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u))))) { std::pair< uint64, Uint31_Index >& ref = timestamp_per_id[it2.object().id]; if (ref.first == 0 || it2.object().timestamp < ref.first) @@ -165,20 +172,25 @@ template< class Id_Type > std::map< Id_Type, std::pair< uint64, Uint31_Index > > collect_attic_kregv( std::vector< std::pair< std::string, Regular_Expression* > >::const_iterator krit, uint64 timestamp, Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >& tags_db, - Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db) + Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db, + std::vector< std::pair< Id_Type, Uint31_Index > >* relevant_ids = 0) { std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id; Ranges< Tag_Index_Global > ranges = get_k_req(krit->first); for (auto it2 = tags_db.range_begin(ranges); !(it2 == tags_db.range_end()); ++it2) { - if (krit->second->matches(it2.index().value)) + if ((!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u)))) + && krit->second->matches(it2.index().value)) timestamp_per_id[it2.object().id] = std::make_pair(NOW, it2.object().idx); } for (auto it2 = attic_tags_db.range_begin(ranges); !(it2 == attic_tags_db.range_end()); ++it2) { if (it2.object().timestamp > timestamp && it2.index().value != void_tag_value() + && (!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u)))) && krit->second->matches(it2.index().value)) { std::pair< uint64, Uint31_Index >& ref = timestamp_per_id[it2.object().id]; diff --git a/src/overpass_api/osm-backend/area_updater.cc b/src/overpass_api/osm-backend/area_updater.cc index 3060c3af8..33d91506c 100644 --- a/src/overpass_api/osm-backend/area_updater.cc +++ b/src/overpass_api/osm-backend/area_updater.cc @@ -159,13 +159,15 @@ void Area_Updater::prepare_delete_tags Ranges< Tag_Index_Local > ranges; for (auto it = to_delete_coarse.begin(); it != to_delete_coarse.end(); ++it) ranges.push_back({ it->first, "", "" }, { it->first + 1, "", "" }); - + ranges.sort(); + // iterate over the result Block_Backend< Tag_Index_Local, Uint32_Index > areas_db (transaction->data_index(area_settings().AREA_TAGS_LOCAL)); Tag_Index_Local current_index; Tag_Entry< uint32 > tag_entry; current_index.index = 0xffffffff; + std::set< Area::Id_Type >* handle = 0; for (auto it = areas_db.range_begin(ranges); !(it == areas_db.range_end()); ++it) { if (!(current_index == it.index())) @@ -177,16 +179,18 @@ void Area_Updater::prepare_delete_tags tag_entry.key = it.index().key; tag_entry.value = it.index().value; tag_entry.ids.clear(); + + handle = &to_delete_coarse[it.index().index]; } - std::set< Area::Id_Type >& handle(to_delete_coarse[it.index().index]); - if (handle.find(it.object().val()) != handle.end()) + if (handle->find(it.object().val()) != handle->end()) tag_entry.ids.push_back(it.object().val()); } if ((current_index.index != 0xffffffff) && (!tag_entry.ids.empty())) tags_to_delete.push_back(tag_entry); } + Area_Location* binary_search_for_id (std::vector< std::pair< Area_Location, Uint31_Index > >& vect, uint32 id) { @@ -206,58 +210,6 @@ Area_Location* binary_search_for_id return 0; } -void Area_Updater::prepare_tags - (std::vector< Tag_Entry< uint32 > >& tags_to_delete, - const std::map< uint32, std::vector< uint32 > >& to_delete) -{ - // make indices appropriately coarse - std::map< uint32, std::set< uint32 > > to_delete_coarse; - for (std::map< uint32, std::vector< uint32 > >::const_iterator - it(to_delete.begin()); it != to_delete.end(); ++it) - { - std::set< uint32 >& handle(to_delete_coarse[it->first & 0xffffff00]); - for (std::vector< uint32 >::const_iterator it2(it->second.begin()); - it2 != it->second.end(); ++it2) - handle.insert(*it2); - } - - // formulate range query - Ranges< Tag_Index_Local > ranges; - for (auto it = to_delete_coarse.begin(); it != to_delete_coarse.end(); ++it) - ranges.push_back({ it->first, "", "" }, { it->first + 1, "", "" }); - - // iterate over the result - Block_Backend< Tag_Index_Local, Uint32_Index > areas_db - (transaction->data_index(area_settings().AREA_TAGS_LOCAL)); - Tag_Index_Local current_index; - Tag_Entry< uint32 > tag_entry; - current_index.index = 0xffffffff; - for (auto it = areas_db.range_begin(ranges); !(it == areas_db.range_end()); ++it) - { - if (!(current_index == it.index())) - { - if ((current_index.index != 0xffffffff) && (!tag_entry.ids.empty())) - tags_to_delete.push_back(tag_entry); - current_index = it.index(); - tag_entry.index = it.index().index; - tag_entry.key = it.index().key; - tag_entry.value = it.index().value; - tag_entry.ids.clear(); - } - - std::set< uint32 >& handle(to_delete_coarse[it.index().index]); - if (handle.find(it.object().val()) != handle.end()) - { - Area_Location* area(binary_search_for_id(areas_to_insert, it.object().val())); - if (area != 0) - area->tags.push_back(std::make_pair(it.index().key, it.index().value)); - tag_entry.ids.push_back(it.object().val()); - } - } - if ((current_index.index != 0xffffffff) && (!tag_entry.ids.empty())) - tags_to_delete.push_back(tag_entry); -} - void Area_Updater::update_area_tags_local (const std::vector< Tag_Entry< uint32 > >& tags_to_delete) { diff --git a/src/overpass_api/osm-backend/area_updater.h b/src/overpass_api/osm-backend/area_updater.h index ec8961007..e878081e0 100644 --- a/src/overpass_api/osm-backend/area_updater.h +++ b/src/overpass_api/osm-backend/area_updater.h @@ -86,9 +86,6 @@ struct Area_Updater : public Area_Usage_Listener void prepare_delete_tags (std::vector< Tag_Entry< uint32 > >& tags_to_delete, const std::map< Uint31_Index, std::set< Area_Skeleton > >& to_delete); - void prepare_tags - (std::vector< Tag_Entry< uint32 > >& tags_to_delete, - const std::map< uint32, std::vector< uint32 > >& to_delete); void update_area_tags_local (const std::vector< Tag_Entry< uint32 > >& tags_to_delete); void update_area_tags_global diff --git a/src/overpass_api/statements/coord_query.cc b/src/overpass_api/statements/coord_query.cc index c80fbe47e..f7f93f85a 100644 --- a/src/overpass_api/statements/coord_query.cc +++ b/src/overpass_api/statements/coord_query.cc @@ -387,7 +387,7 @@ void Coord_Query_Statement::execute(Resource_Manager& rman) while (!tai.is_end() && tai.get_idx().val() < idx.val()) tai.next(); if (tai.is_end()) - continue; + break; if (cur_it->first == tai.get_idx()) { diff --git a/src/overpass_api/statements/query.cc b/src/overpass_api/statements/query.cc index 8d9db3b0f..b6d9ee5e5 100644 --- a/src/overpass_api/statements/query.cc +++ b/src/overpass_api/statements/query.cc @@ -178,7 +178,7 @@ void filter_id_list( { if (key_regex.matches(it.index().key) && it.index().value != void_tag_value() && val_regex.matches(it.index().value) && (!filtered || - binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it.object().id, Uint31_Index(0u))))) + std::binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it.object().id, Uint31_Index(0u))))) new_ids.push_back(std::make_pair(it.object().id, it.object().idx)); if (!filtered && new_ids.size() == 1024*1024) @@ -216,7 +216,7 @@ void filter_id_list( { if (key_regex.matches(it.index().key) && it.index().value != void_tag_value() && val_regex.matches(it.index().value) && - (!filtered || binary_search(old_ids.begin(), old_ids.end(), it.object()))) + (!filtered || std::binary_search(old_ids.begin(), old_ids.end(), it.object()))) new_ids.push_back(it.object()); } @@ -238,7 +238,7 @@ void filter_id_list( for (typename Container::const_iterator it = container.begin(); it != container.end(); ++it) { if (!filtered || - binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it->first, Uint31_Index(0u)))) + std::binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it->first, Uint31_Index(0u)))) new_ids.push_back(std::make_pair(it->first, it->second.second)); } @@ -431,6 +431,106 @@ std::vector< Id_Type > Query_Statement::collect_ids } +template< typename Id_Type > +class Id_Remover +{ +public: + Id_Remover(std::vector< std::pair< Id_Type, Uint31_Index > >& ids_) : ids(ids_), found(ids.size(), false) {} + ~Id_Remover() + { + auto t_it = ids.begin(); + decltype(found.size()) i = 0; + for (auto s_it = ids.begin(); s_it != ids.end(); ++s_it) + { + if (!found[i++]) + { + *t_it = *s_it; + ++t_it; + } + } + ids.erase(t_it, ids.end()); + } + void remove(Id_Type id) + { + auto it = std::lower_bound(ids.begin(), ids.end(), std::pair< Id_Type, Uint31_Index >{ id, Uint31_Index(0u) }); + if (it != ids.end() && it->first == id) + found[std::distance(ids.begin(), it)] = true; + } + +private: + std::vector< std::pair< Id_Type, Uint31_Index > >& ids; + std::vector< bool > found; +}; + + +template< class Id_Type > +void Query_Statement::filter_non_ids + (std::vector< std::pair< Id_Type, Uint31_Index > >& ids, + const File_Properties& file_prop, const File_Properties& attic_file_prop, + Resource_Manager& rman, uint64 timestamp) +{ + if (key_nvalues.empty() && key_nregexes.empty()) + return; + + Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > > tags_db + (rman.get_transaction()->data_index(&file_prop)); + Optional< Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > > > attic_tags_db + (timestamp == NOW ? 0 : + new Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > > + (rman.get_transaction()->data_index(&attic_file_prop))); + Id_Remover< Id_Type > idr(ids); + + // Handle Key-Non-Value pairs + for (std::vector< std::pair< std::string, std::string > >::const_iterator knvit = key_nvalues.begin(); + knvit != key_nvalues.end(); ++knvit) + { + if (timestamp == NOW) + { + std::set< Tag_Index_Global > tag_req = get_kv_req(knvit->first, knvit->second); + for (typename Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >::Discrete_Iterator + it2(tags_db.discrete_begin(tag_req.begin(), tag_req.end())); + !(it2 == tags_db.discrete_end()); ++it2) + idr.remove(it2.object().id); + } + else + { + std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id + = collect_attic_kv(knvit, timestamp, tags_db, *attic_tags_db.obj, &ids); + + for (typename std::map< Id_Type, std::pair< uint64, Uint31_Index > >::const_iterator + it = timestamp_per_id.begin(); it != timestamp_per_id.end(); ++it) + idr.remove(it->first); + } + rman.health_check(*this); + } + + // Handle Key-Regular-Expression-Non-Value pairs + for (std::vector< std::pair< std::string, Regular_Expression* > >::const_iterator knrit = key_nregexes.begin(); + knrit != key_nregexes.end(); ++knrit) + { + if (timestamp == NOW) + { + Ranges< Tag_Index_Global > ranges = get_k_req(knrit->first); + for (auto it2 = tags_db.range_begin(ranges); !it2.is_end(); ++it2) + { + if (knrit->second->matches(it2.index().value)) + idr.remove(it2.object().id); + } + } + else + { + std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id + = collect_attic_kregv(knrit, timestamp, tags_db, *attic_tags_db.obj, &ids); + + for (typename std::map< Id_Type, std::pair< uint64, Uint31_Index > >::const_iterator + it = timestamp_per_id.begin(); it != timestamp_per_id.end(); ++it) + idr.remove(it->first); + } + rman.health_check(*this); + } +} + + template< class Id_Type > std::vector< std::pair< Id_Type, Uint31_Index > > Query_Statement::collect_non_ids (const File_Properties& file_prop, const File_Properties& attic_file_prop, @@ -804,7 +904,7 @@ void filter_by_ids( for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(it->second.begin(), it->second.end(), eit->id)) + if (std::binary_search(it->second.begin(), it->second.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -825,7 +925,7 @@ void filter_by_ids( for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(it->second.begin(), it->second.end(), eit->id)) + if (std::binary_search(it->second.begin(), it->second.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -837,7 +937,7 @@ void filter_by_ids( for (typename std::vector< Attic< TObject > >::const_iterator eit = attic_item_it->second.begin(); eit != attic_item_it->second.end(); ++eit) { - if (binary_search(it->second.begin(), it->second.end(), eit->id)) + if (std::binary_search(it->second.begin(), it->second.end(), eit->id)) attic_result[attic_item_it->first.val()].push_back(*eit); } ++attic_item_it; @@ -1041,7 +1141,7 @@ void Query_Statement::filter_by_tags for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) + if (std::binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -1091,7 +1191,7 @@ void Query_Statement::filter_by_tags for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) + if (std::binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -1228,34 +1328,17 @@ void Query_Statement::progress_1( && (!keys.empty() || !key_regexes.empty() || !regkey_regexes.empty()))) { bool result_valid = true; - ids.invert = false; std::vector< std::pair< Id_Type, Uint31_Index > > id_idxs = collect_ids< Skeleton, Id_Type >(file_prop, attic_file_prop, rman, timestamp, check_keys_late, result_valid); + ids.invert = !result_valid; if (!key_nvalues.empty() || (check_keys_late != prefer_ranges && !key_nregexes.empty())) + filter_non_ids< Id_Type >(id_idxs, file_prop, attic_file_prop, rman, timestamp); + + for (auto it = id_idxs.begin(); it != id_idxs.end(); ++it) { - std::vector< std::pair< Id_Type, Uint31_Index > > non_ids - = collect_non_ids< Id_Type >(file_prop, attic_file_prop, rman, timestamp); - std::vector< std::pair< Id_Type, Uint31_Index > > diff_ids(id_idxs.size()); - diff_ids.erase(set_difference(id_idxs.begin(), id_idxs.end(), non_ids.begin(), non_ids.end(), - diff_ids.begin()), diff_ids.end()); - ids.ids.clear(); - range_vec.clear(); - for (typename std::vector< std::pair< Id_Type, Uint31_Index > >::const_iterator it = diff_ids.begin(); - it != diff_ids.end(); ++it) - { - ids.ids.push_back(it->first); - range_vec.push_back(it->second); - } - } - else - { - for (typename std::vector< std::pair< Id_Type, Uint31_Index > >::const_iterator it = id_idxs.begin(); - it != id_idxs.end(); ++it) - { - ids.ids.push_back(it->first); - range_vec.push_back(it->second); - } + ids.ids.push_back(it->first); + range_vec.push_back(it->second); } if (ids.ids.empty() && result_valid) diff --git a/src/overpass_api/statements/query.h b/src/overpass_api/statements/query.h index 8f3e74ee0..52c477f67 100644 --- a/src/overpass_api/statements/query.h +++ b/src/overpass_api/statements/query.h @@ -190,6 +190,12 @@ class Query_Statement : public Output_Statement (const File_Properties& file_prop, Resource_Manager& rman, Query_Filter_Strategy check_keys_late); + template< class Id_Type > + void filter_non_ids + (std::vector< std::pair< Id_Type, Uint31_Index > >& ids, + const File_Properties& file_prop, const File_Properties& attic_file_prop, + Resource_Manager& rman, uint64 timestamp); + template< class Id_Type > std::vector< std::pair< Id_Type, Uint31_Index > > collect_non_ids (const File_Properties& file_prop, const File_Properties& attic_file_prop,