From 65ed8299c9df0301485da46eed6815e0bf279fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Sun, 17 Apr 2022 13:42:42 -0300 Subject: [PATCH 01/11] feat: Compute dijkstra and insert into UBODT if solicited OD pair has not been calculated --- src/mm/fmm/fmm_algorithm.cpp | 2 +- src/mm/fmm/fmm_app.hpp | 2 +- src/mm/fmm/ubodt.cpp | 75 ++++++++++++++++++++++++++++++----- src/mm/fmm/ubodt.hpp | 22 ++++++++-- src/network/network_graph.cpp | 70 +++++++++++++++++++++----------- src/network/network_graph.hpp | 13 ++++++ 6 files changed, 145 insertions(+), 39 deletions(-) diff --git a/src/mm/fmm/fmm_algorithm.cpp b/src/mm/fmm/fmm_algorithm.cpp index 1bf1f84..d22c934 100644 --- a/src/mm/fmm/fmm_algorithm.cpp +++ b/src/mm/fmm/fmm_algorithm.cpp @@ -264,7 +264,7 @@ double FastMapMatch::get_sp_dist( // Transition on the same OD nodes sp_dist = ca->edge->length - ca->offset + cb->offset; } else { - Record *r = ubodt_->look_up(ca->edge->target, cb->edge->source); + Record *r = ubodt_->look_up_or_make(ca->edge->target, cb->edge->source); // No sp path exist from O to D. if (r == nullptr) return std::numeric_limits::infinity(); // calculate original SP distance diff --git a/src/mm/fmm/fmm_app.hpp b/src/mm/fmm/fmm_app.hpp index 3d8d6fc..9efbcd8 100644 --- a/src/mm/fmm/fmm_app.hpp +++ b/src/mm/fmm/fmm_app.hpp @@ -30,7 +30,7 @@ class FMMApp { config_(config), network_(config_.network_config), ng_(network_), - ubodt_(UBODT::read_ubodt_file(config_.ubodt_file)){}; + ubodt_(UBODT::read_ubodt_file(config_.ubodt_file, ng_)){}; /** * Run the fmm program */ diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index a53e983..383568f 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -18,8 +18,8 @@ using namespace FMM; using namespace FMM::CORE; using namespace FMM::NETWORK; using namespace FMM::MM; -UBODT::UBODT(int buckets_arg, int multiplier_arg) : - buckets(buckets_arg), multiplier(multiplier_arg) { +UBODT::UBODT(int buckets_arg, int multiplier_arg, NetworkGraph graph_arg) : + buckets(buckets_arg), multiplier(multiplier_arg), graph(graph_arg) { SPDLOG_TRACE("Intialization UBODT with buckets {} multiplier {}", buckets, multiplier); hashtable = (Record **) malloc(sizeof(Record *) * buckets); @@ -59,11 +59,64 @@ Record *UBODT::look_up(NodeIndex source, NodeIndex target) const { return r; } +Record* UBODT::look_up_or_make(NETWORK::NodeIndex source, + NETWORK::NodeIndex target) { + Record *r = look_up(source, target); + // No transition exist from source to target + if (r == nullptr) { + Heap Q; + PredecessorMap pmap; + DistanceMap dmap; + + graph.shortest_path_dijkstra(&Q, &pmap, &dmap, this, source, target); + + std::stack route; + + for (auto iter = pmap.begin(); iter != pmap.end(); ++iter) { + NodeIndex cur_node = iter->first; + if (!Q.contain_node(cur_node) && !look_up(source, cur_node)) { + // If node isn't in the heap, it means that pmap and dmap hold truthful information about distance + // If there is no look_up value, it must be stored + + NodeIndex prev_n = iter->second; + // Create a stack with the route from source to cur_node + for(NodeIndex u=cur_node; u != source; u=pmap[u]) + route.push(u); + + // Insert ods from the former to the latter + // The former is always the source + NodeIndex former = source; + while(former != cur_node && !look_up(former, cur_node)) { + // Repeat until the route reached cur_node + // or the rest of the route is already in the table + + r = (Record*) malloc(sizeof(Record)); + r->source = former; + r->target = cur_node; + + // First node of this route will be the former of the next + NodeIndex s = former; + former = route.top(); + route.pop(); + r->first_n = former; + r->prev_n = prev_n; + r->next_e = graph.get_edge_index(s, former, dmap[former]-dmap[s]); + r->cost = dmap[cur_node]-dmap[s]; + r->next = nullptr; + insert(r); + } + } + } + r = look_up(source, target); + } + return r; +} + std::vector UBODT::look_sp_path(NodeIndex source, - NodeIndex target) const { + NodeIndex target) { std::vector edges; if (source == target) { return edges; } - Record *r = look_up(source, target); + Record *r = look_up_or_make(source, target); // No transition exist from source to target if (r == nullptr) { return edges; } while (r->first_n != target) { @@ -77,7 +130,7 @@ std::vector UBODT::look_sp_path(NodeIndex source, C_Path UBODT::construct_complete_path(int traj_id, const TGOpath &path, const std::vector &edges, std::vector *indices, - double reverse_tolerance) const { + double reverse_tolerance) { C_Path cpath; if (!indices->empty()) indices->clear(); if (path.empty()) return cpath; @@ -185,13 +238,13 @@ int UBODT::find_prime_number(double value) { } std::shared_ptr UBODT::read_ubodt_file(const std::string &filename, - int multiplier) { + NETWORK::NetworkGraph graph, int multiplier) { std::shared_ptr ubodt = nullptr; auto start_time = UTIL::get_current_time(); if (UTIL::check_file_extension(filename,"bin")){ - ubodt = read_ubodt_binary(filename,multiplier); + ubodt = read_ubodt_binary(filename, graph, multiplier); } else if (UTIL::check_file_extension(filename,"csv,txt")) { - ubodt = read_ubodt_csv(filename,multiplier); + ubodt = read_ubodt_csv(filename, graph, multiplier); } else { std::string message = (boost::format("File format not supported: %1%") % filename).str(); SPDLOG_CRITICAL(message); @@ -204,13 +257,14 @@ std::shared_ptr UBODT::read_ubodt_file(const std::string &filename, } std::shared_ptr UBODT::read_ubodt_csv(const std::string &filename, + const NETWORK::NetworkGraph graph, int multiplier) { SPDLOG_INFO("Reading UBODT file (CSV format) from {}", filename); long rows = estimate_ubodt_rows(filename); int buckets = find_prime_number(rows / LOAD_FACTOR); SPDLOG_TRACE("Estimated buckets {}", buckets); int progress_step = 1000000; - std::shared_ptr table = std::make_shared(buckets, multiplier); + std::shared_ptr table = std::make_shared(buckets, multiplier, graph); FILE *stream = fopen(filename.c_str(), "r"); long NUM_ROWS = 0; char line[BUFFER_LINE]; @@ -245,13 +299,14 @@ std::shared_ptr UBODT::read_ubodt_csv(const std::string &filename, } std::shared_ptr UBODT::read_ubodt_binary(const std::string &filename, + const NETWORK::NetworkGraph graph, int multiplier) { SPDLOG_INFO("Reading UBODT file (binary format) from {}", filename); long rows = estimate_ubodt_rows(filename); int progress_step = 1000000; SPDLOG_TRACE("Estimated rows is {}", rows); int buckets = find_prime_number(rows / LOAD_FACTOR); - std::shared_ptr table = std::make_shared(buckets, multiplier); + std::shared_ptr table = std::make_shared(buckets, multiplier, graph); long NUM_ROWS = 0; std::ifstream ifs(filename.c_str()); // Check byte offset diff --git a/src/mm/fmm/ubodt.hpp b/src/mm/fmm/ubodt.hpp index 06c0099..ba40ff0 100644 --- a/src/mm/fmm/ubodt.hpp +++ b/src/mm/fmm/ubodt.hpp @@ -11,6 +11,7 @@ #define FMM_UBODT_H_ #include "network/type.hpp" +#include "network/network_graph.hpp" #include "mm/transition_graph.hpp" #include "util/debug.hpp" @@ -42,8 +43,9 @@ class UBODT { * @param buckets_arg Bucket number * @param multiplier_arg A multiplier used for querying, recommended to be * the number of nodes in the graph. + * @param network The NetworkGraph whose OD pairs the UBODT represents */ - UBODT(int buckets_arg, int multiplier_arg); + UBODT(int buckets_arg, int multiplier_arg, NETWORK::NetworkGraph network); ~UBODT(); /** * Look up the row according to a source node and a target node @@ -54,6 +56,16 @@ class UBODT { */ Record *look_up(NETWORK::NodeIndex source, NETWORK::NodeIndex target) const; + /** + * Look up the row according to a source node and a target node. + * If it hasn't been calculated, calculates it. + * @param source source node + * @param target target node + * @return A row in the ubodt if the od pair is found, otherwise nullptr + * is returned. + */ + Record *look_up_or_make(NETWORK::NodeIndex source, NETWORK::NodeIndex target); + /** * Look up a shortest path (SP) containing edges from source to target. * In case that SP is not found, empty is returned. @@ -62,7 +74,7 @@ class UBODT { * @return a shortest path connecting source to target */ std::vector look_sp_path(NETWORK::NodeIndex source, - NETWORK::NodeIndex target) const; + NETWORK::NodeIndex target); /** * Construct the complete path (a vector of edge ID) from an optimal path @@ -79,7 +91,7 @@ class UBODT { C_Path construct_complete_path(int traj_id, const TGOpath &path, const std::vector &edges, std::vector *indices, - double reverse_tolerance) const; + double reverse_tolerance); /** * Get the upperbound of the UBODT * @return upperbound value @@ -112,6 +124,7 @@ class UBODT { * @return A shared pointer to the UBODT data. */ static std::shared_ptr read_ubodt_file(const std::string &filename, + const NETWORK::NetworkGraph graph, int multiplier = 50000); /** * Read UBODT from a CSV file @@ -120,6 +133,7 @@ class UBODT { * @return A shared pointer to the UBODT data. */ static std::shared_ptr read_ubodt_csv(const std::string &filename, + const NETWORK::NetworkGraph graph, int multiplier = 50000); /** @@ -129,6 +143,7 @@ class UBODT { * @return A shared pointer to the UBODT data. */ static std::shared_ptr read_ubodt_binary(const std::string &filename, + const NETWORK::NetworkGraph graph, int multiplier = 50000); /** * Estimate the number of rows in a file @@ -153,6 +168,7 @@ class UBODT { long long num_rows=0; // multiplier to get a unique ID double delta = 0.0; Record **hashtable; + NETWORK::NetworkGraph graph; }; } } diff --git a/src/network/network_graph.cpp b/src/network/network_graph.cpp index 14f9e00..51c8191 100644 --- a/src/network/network_graph.cpp +++ b/src/network/network_graph.cpp @@ -3,6 +3,8 @@ #include "network/network.hpp" #include "util/debug.hpp" +#include "mm/fmm/ubodt.hpp" + #include #include #include @@ -58,42 +60,62 @@ std::vector NetworkGraph::shortest_path_dijkstra( Heap Q; PredecessorMap pmap; DistanceMap dmap; + + shortest_path_dijkstra(&Q, &pmap, &dmap, nullptr, source, target); + // Backtrack from target to source + return back_track(source, target, pmap, dmap); +} + +void NetworkGraph::shortest_path_dijkstra(Heap *Q, PredecessorMap *pmap, + DistanceMap *dmap, MM::UBODT *odTable, NodeIndex source, NodeIndex target) const { // Initialization - Q.push(source, 0); - pmap.insert({source, source}); - dmap.insert({source, 0}); + Q->push(source, 0); + pmap->insert({source, source}); + dmap->insert({source, 0}); OutEdgeIterator out_i, out_end; double temp_dist = 0; // Dijkstra search - while (!Q.empty()) { - HeapNode node = Q.top(); - Q.pop(); + while (!Q->empty()) { + HeapNode node = Q->top(); + Q->pop(); NodeIndex u = node.index; if (u == target) break; for (boost::tie(out_i, out_end) = boost::out_edges(u, g); out_i != out_end; ++out_i) { EdgeDescriptor e = *out_i; NodeIndex v = boost::target(e, g); - temp_dist = node.value + g[e].length; - auto iter = dmap.find(v); - if (iter != dmap.end()) { - // dmap contains node v - if (iter->second > temp_dist) { - // a smaller distance is found for v - pmap[v] = u; - dmap[v] = temp_dist; - Q.decrease_key(v, temp_dist); - } - } else { - // dmap does not contain v - Q.push(v, temp_dist); - pmap.insert({v, u}); - dmap.insert({v, temp_dist}); - } + + auto insert = [](NodeIndex u, NodeIndex v, double temp_dist, + Heap *Q, PredecessorMap *pmap, DistanceMap *dmap) { + auto iter = dmap->find(v); + if (iter != dmap->end()) { + // dmap contains node v + if (iter->second > temp_dist) { + // a smaller distance is found for v + (*pmap)[v] = u; + (*dmap)[v] = temp_dist; + Q->decrease_key(v, temp_dist); + } + } else { + // dmap does not contain v + Q->push(v, temp_dist); + pmap->insert( { v, u }); + dmap->insert( { v, temp_dist }); + } + }; + + // Check if a better path was already calculated + if (odTable != nullptr) { + MM::Record *r = odTable->look_up(u, v); + if (r != nullptr && r->cost < g[e].length) { + insert(r->prev_n, v, node.value + r->cost, Q, pmap, dmap); + continue; + } + } + + insert(u, v, node.value + g[e].length, Q, pmap, dmap); } } - // Backtrack from target to source - return back_track(source, target, pmap, dmap); } double NetworkGraph::calc_heuristic_dist( diff --git a/src/network/network_graph.hpp b/src/network/network_graph.hpp index 078c3de..e753c79 100644 --- a/src/network/network_graph.hpp +++ b/src/network/network_graph.hpp @@ -27,6 +27,9 @@ #include "network/network.hpp" namespace FMM { + +namespace MM {class UBODT;} + namespace NETWORK { /** * Graph class of the network @@ -46,6 +49,16 @@ class NetworkGraph { */ std::vector shortest_path_dijkstra( NodeIndex source, NodeIndex target) const; + + /** + * Dijkstra Shortest path query from source to target + * @param source source node queried + * @param target target node queried + * @param pmap predecessor map updated to store the routing result + * @param dmap distance map updated to store the routing result + */ + void shortest_path_dijkstra(Heap *Q, PredecessorMap *pmap, + DistanceMap *dmap, MM::UBODT *odTable, NodeIndex source, NodeIndex target) const; /** * Calculate heuristic distance from p1 to p2,which is used in Astar routing. * @param p1 From 1ae0180ffc0913f51fa3eb39dac5eae124c97dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Sat, 16 Apr 2022 15:31:50 -0300 Subject: [PATCH 02/11] feat: Allow fmm to be run without UBODT, as it may be built on-the-run --- src/mm/fmm/fmm_app_config.cpp | 4 ---- src/mm/fmm/ubodt.cpp | 7 +++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/mm/fmm/fmm_app_config.cpp b/src/mm/fmm/fmm_app_config.cpp index e3cab64..624c60f 100644 --- a/src/mm/fmm/fmm_app_config.cpp +++ b/src/mm/fmm/fmm_app_config.cpp @@ -126,10 +126,6 @@ bool FMMAppConfig::validate() const if (!fmm_config.validate()) { return false; } - if (!UTIL::file_exists(ubodt_file)) { - SPDLOG_CRITICAL("UBODT file not exists {}", ubodt_file); - return false; - } SPDLOG_DEBUG("Validating done"); return true; }; diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index 383568f..bd93b1d 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -241,8 +241,11 @@ std::shared_ptr UBODT::read_ubodt_file(const std::string &filename, NETWORK::NetworkGraph graph, int multiplier) { std::shared_ptr ubodt = nullptr; auto start_time = UTIL::get_current_time(); - if (UTIL::check_file_extension(filename,"bin")){ - ubodt = read_ubodt_binary(filename, graph, multiplier); + if(!UTIL::file_exists(filename)) { + int buckets = find_prime_number(multiplier / LOAD_FACTOR); + ubodt = std::make_shared(buckets, multiplier, graph); + } else if (UTIL::check_file_extension(filename,"bin")){ + ubodt = read_ubodt_binary(filename,graph,multiplier); } else if (UTIL::check_file_extension(filename,"csv,txt")) { ubodt = read_ubodt_csv(filename, graph, multiplier); } else { From 878e149a263885c5ca9cdddee4882761ad1b4aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Sun, 17 Apr 2022 10:15:00 -0300 Subject: [PATCH 03/11] fix(docker): Remove unnecessary files from docker build and test image after build --- .dockerignore | 101 ++++++++++++++++++++++++++++++++++++++++++++++ docker/Dockerfile | 5 ++- 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..309acd1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,101 @@ +# folders +bin/ +input/ +output/ +dist/ +test/bin/ +test/log/ + +# Doc files + +docs/html + +# Mac + +**/.DS_Store + +# Mac + +**/.DS_Store + +# Cmake + +build/ +cmake-build-debug/ +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +# Projects +*.cbp +*.depend +*.layout +.kdev4/ +*.kdev4 +.idea/ + +# log file +*.log + + +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su + +# Binary files + +*.bin +*.binary + +# bash file to sync file +sync.sh + + +# Jupyternotebook +.ipynb_checkpoints +*/.ipynb_checkpoints/* +*.pyc + +# Docker specifics +Dockerfile +.git diff --git a/docker/Dockerfile b/docker/Dockerfile index fa3b964..996bf6b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,13 +7,14 @@ RUN apt-get update && apt-get install -y \ make libbz2-dev libexpat1-dev swig python-dev RUN add-apt-repository -y ppa:ubuntugis/ppa && apt-get -q update RUN apt-get -y install gdal-bin libgdal-dev -RUN mkdir -p /fmm COPY . /fmm WORKDIR /fmm -RUN rm -rf build RUN mkdir -p build && \ cd build && \ cmake .. && \ make install + +RUN cd example/python && python fmm_test.py + EXPOSE 8080 CMD From ec705e6ce7c6d37b6b33c4401ad606c421606551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Mon, 18 Apr 2022 00:12:10 -0300 Subject: [PATCH 04/11] dep: Update cpp version to c++14 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e2aeb3..72ff3cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ project(fmm) set(CMAKE_BUILD_TYPE "Release") set(CMAKE_CXX_FLAGS "-O3 -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE") -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 14) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build") list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") From 8b00d8e60a8be12f9a62b7987ebb9f5cea3534da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Tue, 19 Apr 2022 23:34:59 -0300 Subject: [PATCH 05/11] fix: Change presumed row number when ubodt table is not loaded --- src/mm/fmm/ubodt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index bd93b1d..73b517d 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -242,7 +242,7 @@ std::shared_ptr UBODT::read_ubodt_file(const std::string &filename, std::shared_ptr ubodt = nullptr; auto start_time = UTIL::get_current_time(); if(!UTIL::file_exists(filename)) { - int buckets = find_prime_number(multiplier / LOAD_FACTOR); + int buckets = find_prime_number(graph.get_num_vertices()*graph.get_num_vertices()); ubodt = std::make_shared(buckets, multiplier, graph); } else if (UTIL::check_file_extension(filename,"bin")){ ubodt = read_ubodt_binary(filename,graph,multiplier); From 7d2adfc5e6a15d471385ce9093f53b4bd683bb60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Tue, 19 Apr 2022 23:38:14 -0300 Subject: [PATCH 06/11] fix: Optimize fmm_algorithm with ubodt generaion for multiple threads --- src/mm/fmm/fmm_algorithm.cpp | 2 +- src/mm/fmm/ubodt.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mm/fmm/fmm_algorithm.cpp b/src/mm/fmm/fmm_algorithm.cpp index d22c934..f040135 100644 --- a/src/mm/fmm/fmm_algorithm.cpp +++ b/src/mm/fmm/fmm_algorithm.cpp @@ -198,7 +198,7 @@ std::string FastMapMatch::match_gps_file( std::vector trajectories = reader.read_next_N_trajectories(buffer_trajectories_size); int trajectories_fetched = trajectories.size(); - #pragma omp parallel for + #pragma omp parallel for schedule (guided) for (int i = 0; i < trajectories_fetched; ++i) { Trajectory &trajectory = trajectories[i]; int points_in_tr = trajectory.geom.get_num_points(); diff --git a/src/mm/fmm/ubodt.hpp b/src/mm/fmm/ubodt.hpp index ba40ff0..fbdc28e 100644 --- a/src/mm/fmm/ubodt.hpp +++ b/src/mm/fmm/ubodt.hpp @@ -167,7 +167,7 @@ class UBODT { const int buckets; // number of buckets long long num_rows=0; // multiplier to get a unique ID double delta = 0.0; - Record **hashtable; + std::atomic hashtable; NETWORK::NetworkGraph graph; }; } From 506cb772227100f5780d775de1bcaa2b0189e74b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Sun, 24 Apr 2022 10:52:07 -0300 Subject: [PATCH 07/11] feat: Implement UBODT::look_k_sp_path --- src/mm/fmm/ubodt.cpp | 268 +++++++++++++++++++++++++++++++++++++++++++ src/mm/fmm/ubodt.hpp | 12 ++ 2 files changed, 280 insertions(+) diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index 73b517d..636cd13 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -127,6 +127,274 @@ std::vector UBODT::look_sp_path(NodeIndex source, return edges; } +std::vector> UBODT::look_k_sp_path( + const NETWORK::NodeIndex source, const NETWORK::NodeIndex target, int K) { + // Return empty list if target and source are the same + std::vector> returnV; + if (source == target) {return returnV;} + + std::vector> routes; + routes.push_back({look_up_or_make(source, target)}); + + struct RoutesNode { + std::vector index; /**< Index of a node in the heap */ + double value; /**< Value of a node in the heap */ + bool operator<(const RoutesNode &rhs) const { + return value < rhs.value; + } + }; + FibHeap route_candidates; + + std::vector> temporaryRecords; + + const Graph_T g = graph.get_boost_graph(); + + /** Lambda functions to help code reusability inside the function **/ + auto nextNode = [this](Record* &r, std::vector::iterator &recordIterator, const NodeIndex currentNode){ + // If at the end of an OD pair, get the next pair + if(currentNode == r->target) + r = *++recordIterator; + + // get route from current node to the target, and get next node + r = look_up(currentNode, r->target); + return r-> first_n; + }; + + auto getOutEdges = [g](NodeIndex source){ + static OutEdgeIterator out_i, out_end; + std::map out_edges; + for(boost::tie(out_i, out_end) = boost::out_edges(source, g); + out_i != out_end; out_i++){ + EdgeDescriptor e = *out_i; + out_edges[g[e].index] = e; + } + return out_edges; + }; + + auto getMinimalPathAfterEdge = [this, g, target, &temporaryRecords](const EdgeDescriptor e, const NodeIndex previousNode, const std::vector &basePath){ + NodeIndex nodeAfterEdge = boost::target(e, g); + + if(nodeAfterEdge == previousNode) + throw std::invalid_argument("Loop edge"); + + if(!basePath.empty() && nodeAfterEdge == basePath.back()->prev_n) + throw std::invalid_argument("Edge goes backward"); + + Record *nextOD = look_up_or_make(nodeAfterEdge, target); + if (nextOD == nullptr) + throw std::invalid_argument("No od pair"); + + EdgeProperty eProperty = g[e]; + RoutesNode returnV; + returnV.value = eProperty.length + nextOD -> cost; + for(Record* record : basePath) { + returnV.value += record->cost; + returnV.index.push_back(record); + } + + // push the edge as a route + Record *edgeRecord = look_up_or_make(previousNode, nodeAfterEdge); + // If edge is not the sp, the edge must be pushed, mocking a sp + if(edgeRecord->next_e != eProperty.index) { + std::unique_ptr tempRecord(new Record); + tempRecord->source = previousNode; + tempRecord->prev_n = previousNode; + tempRecord->target = nodeAfterEdge; + tempRecord->first_n = nodeAfterEdge; + tempRecord->next_e = eProperty.index; + tempRecord->cost = eProperty.length; + tempRecord->next = nullptr; + temporaryRecords.push_back(std::move(tempRecord)); + edgeRecord = temporaryRecords.back().get(); + } + returnV.index.push_back(edgeRecord); + // Push the last part of the route + returnV.index.push_back(nextOD); + return returnV; + }; + + auto isSubRoute = [this, &nextNode](std::vector &route1, + std::vector &smallerRoute, Record* &r, const NodeIndex lastNodeBase) { + auto iterR = route1.begin(); + r = *iterR; + auto iterBasePath = smallerRoute.begin(); + Record *basePathRecord = *iterBasePath; + + // basePath contains all record, from + // its source to its target. Then, advance + while (iterBasePath + 1 != smallerRoute.end() && r == basePathRecord) { + r = *++iterR; + basePathRecord = *++iterBasePath; + } + + // Walk through route1 to check if the route matches it + NodeIndex nextNodeRoute = r->source; + NodeIndex nextNodeBase = basePathRecord->source; + while (nextNodeBase != lastNodeBase + && nextNodeBase == nextNodeRoute) { // Path still didn't get to spurNode + nextNodeRoute = nextNode(r, iterR, nextNodeRoute); + nextNodeBase = nextNode(basePathRecord, iterBasePath, + nextNodeBase); + } + + // Update r, if applicable + if(nextNodeRoute == r->target && iterR+1 != route1.end()) + r = *++iterR; + if(r != nullptr && nextNodeRoute != r->target) + r = look_up(nextNodeRoute, r->target); + + return nextNodeBase == nextNodeRoute; + }; + /** End of lambda functions **/ + + // Implementation of Yen's algorithm + // https://doi.org/10.1287/mnsc.17.11.712 + for(int k=1; k latestRoute = routes.at(k-1); + + // The spur node is the ith node of latestRoute + auto spurNodeIter = latestRoute.begin(); + Record* spurNodeRecord = *spurNodeIter; + NodeIndex spurNode = spurNodeRecord->source; + std::vector basePath; + // Repeat until the last node + while(spurNode != target){ + + // Load all edges that come out of the spur node + std::map out_edges = getOutEdges(spurNode); + + // Eliminate edges of the spurNode used by routes + // that use basePath integrally + for(auto route : routes) { + // Special case of spurNode=source + Record *r = route.front(); + if(basePath.empty()){ + out_edges.erase(r->next_e); + continue; + } + + // Erase edge of next + if(isSubRoute(route, basePath, r, spurNode)) + out_edges.erase(r->next_e); + } + + // Calculate shortest path from source to destination + //without some edges out of spurNode + for(auto edge: out_edges) { + try { + auto candidate = getMinimalPathAfterEdge(edge.second, spurNode, basePath); + route_candidates.push(candidate); + } catch (std::invalid_argument const& ex) { + continue; + } + } + + /** Update spurnode and basepath **/ + auto formerSpurNodeIter = spurNodeIter; + spurNode = nextNode(spurNodeRecord, spurNodeIter, spurNode); + // Check if the sp inside latestRoute changed + if(formerSpurNodeIter != spurNodeIter || basePath.empty()) + basePath.push_back(look_up((*spurNodeIter)->source, spurNode)); + else + basePath.back() = look_up((*spurNodeIter)->source, spurNode); + } + + /** Test if the candidate is valid **/ + // It will be invalid if the last od passes its spur node + while (!route_candidates.empty()) { + std::vector topCandidate = route_candidates.top().index; + route_candidates.pop(); + + auto lastIter = topCandidate.end()-2; + NodeIndex candidateSpurNode = (*lastIter++)->source; + auto candidateIter=topCandidate.begin(); + Record *r = *candidateIter; + + // Nodes before and including spur + std::set nodesUntilSpur; + while(candidateIter!=lastIter) { + nodesUntilSpur.insert(r->source); + while(r->first_n != r->target){ + r = look_up(r->first_n, r->target); + nodesUntilSpur.insert(r->source); + } + r = *++candidateIter; + } + + while(r->first_n != target && nodesUntilSpur.find(r->source) == nodesUntilSpur.end()) { + r = look_up(r->first_n, target); + } + + if(nodesUntilSpur.find(r->source) == nodesUntilSpur.end()) { + // Valid candidate + // Check if not already included + bool uniquePath = true; + for(auto route : routes) { + Record *r; + if(isSubRoute(route, topCandidate, r, target)) { + uniquePath = false; + break; + } + } + //Route already included, trying again + if(uniquePath) { + routes.push_back(topCandidate); + // End route_candidates loop + break; + } + } else { + // Candidate contains some node twice. + NodeIndex duplicatedNode = r->source; + // Get last record, since this is the record + // where the mistake is. + r = topCandidate.back(); + // The process of finding a new route consists + // of splitting the last record in from two to three records + // The source of the first one must be firstSource + NodeIndex firstSource = r->source; + + while(r->source != duplicatedNode) { //Next lastSource must not be the duplicated node + // If there are three records, the last one's source will be lastSource + NodeIndex lastSource = r->source; + + // make route as basepath-> (firstSource,newSource) -> (newSource,target) + // Append all route, except the last element + std::vector candidateBasePath(topCandidate.begin(), candidateIter); + if(firstSource != lastSource) + candidateBasePath.push_back(look_up(firstSource, lastSource)); + + for(auto edge: getOutEdges(lastSource)) { + if(edge.first == r->next_e) + continue; + + try { + auto newCandidate = getMinimalPathAfterEdge(edge.second, lastSource, candidateBasePath); + route_candidates.push(newCandidate); + } catch (std::invalid_argument const& ex) { + continue; + } + } + r = look_up(r->first_n, r->target); + } + } + } + } + for (auto route: routes){ + std::vector edges; + auto iterR = route.begin(); + Record *r = *iterR; + NodeIndex currentNode = r->source; + while(currentNode != target){ + currentNode = nextNode(r, iterR, currentNode); + edges.push_back(r->next_e); + } + returnV.push_back(edges); + } + + return returnV;//routes; + +} + C_Path UBODT::construct_complete_path(int traj_id, const TGOpath &path, const std::vector &edges, std::vector *indices, diff --git a/src/mm/fmm/ubodt.hpp b/src/mm/fmm/ubodt.hpp index fbdc28e..77b4ba0 100644 --- a/src/mm/fmm/ubodt.hpp +++ b/src/mm/fmm/ubodt.hpp @@ -76,6 +76,18 @@ class UBODT { std::vector look_sp_path(NETWORK::NodeIndex source, NETWORK::NodeIndex target); + /** + * Returns a vector with, at most, the k shortest paths from a source ot target + * In case that SP is not found, an empty vector is returned. + * @param source source node + * @param target target node + * @return vector of vectors with the shortest paths connecting source to target + */ + std::vector> look_k_sp_path( + NETWORK::NodeIndex source, + NETWORK::NodeIndex target, + int k); + /** * Construct the complete path (a vector of edge ID) from an optimal path * (a vector of optimal nodes in the transition graph) From e19d43608bbfdb46764eb0088d50496679dc505e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Mon, 2 May 2022 06:13:10 -0300 Subject: [PATCH 08/11] feat: Preallocate routes vector in UBODT::look_k_sp_path --- src/mm/fmm/ubodt.cpp | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index 636cd13..8054bcb 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -133,8 +133,8 @@ std::vector> UBODT::look_k_sp_path( std::vector> returnV; if (source == target) {return returnV;} - std::vector> routes; - routes.push_back({look_up_or_make(source, target)}); + std::vector> routes(K); + routes.at(0) = {look_up_or_make(source, target)}; struct RoutesNode { std::vector index; /**< Index of a node in the heap */ @@ -247,9 +247,10 @@ std::vector> UBODT::look_k_sp_path( }; /** End of lambda functions **/ + int k=1; // Implementation of Yen's algorithm // https://doi.org/10.1287/mnsc.17.11.712 - for(int k=1; k latestRoute = routes.at(k-1); // The spur node is the ith node of latestRoute @@ -265,16 +266,16 @@ std::vector> UBODT::look_k_sp_path( // Eliminate edges of the spurNode used by routes // that use basePath integrally - for(auto route : routes) { + for(int i=0; inext_e); continue; } // Erase edge of next - if(isSubRoute(route, basePath, r, spurNode)) + if(isSubRoute(routes.at(i), basePath, r, spurNode)) out_edges.erase(r->next_e); } @@ -328,17 +329,15 @@ std::vector> UBODT::look_k_sp_path( if(nodesUntilSpur.find(r->source) == nodesUntilSpur.end()) { // Valid candidate // Check if not already included - bool uniquePath = true; - for(auto route : routes) { - Record *r; - if(isSubRoute(route, topCandidate, r, target)) { - uniquePath = false; + int i = 0; + Record *r; + for(; i> UBODT::look_k_sp_path( } } } + if(route_candidates.empty()) + break; } - for (auto route: routes){ + for (int i=0; i edges; - auto iterR = route.begin(); + auto iterR = routes.at(i).begin(); Record *r = *iterR; NodeIndex currentNode = r->source; while(currentNode != target){ From e5b5ae78c337e70c5162510b33a2465f80af96ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Wed, 11 May 2022 23:20:07 -0300 Subject: [PATCH 09/11] feat: Add swig interface to look_k_sp_path() results --- python/fmm.i | 1 + 1 file changed, 1 insertion(+) diff --git a/python/fmm.i b/python/fmm.i index 344df2f..e625827 100644 --- a/python/fmm.i +++ b/python/fmm.i @@ -57,6 +57,7 @@ using namespace FMM::CONFIG; %template(UnsignedIntVector) std::vector; %template(DoubleVector) std::vector; %template(PyCandidateVector) std::vector; +%template(UnsignedIntVectorVector) std::vector>; // %template(DoubleVVector) vector >; // %template(DoubleVVVector) vector > >; // %template(IntSet) set; From a874f17d316f91ded4022e6158af3d60a15d12c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Mon, 4 Jul 2022 19:26:08 -0300 Subject: [PATCH 10/11] feat: Adds UBODT::look_bfs_le_path() --- src/mm/fmm/ubodt.cpp | 273 +++++++++++++++++++++++++++++++++++++++++++ src/mm/fmm/ubodt.hpp | 13 ++- 2 files changed, 285 insertions(+), 1 deletion(-) diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index 8054bcb..0ddd963 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -127,6 +127,279 @@ std::vector UBODT::look_sp_path(NodeIndex source, return edges; } +std::vector> UBODT::look_bfs_le_path( + const NETWORK::NodeIndex source, + const NETWORK::NodeIndex target, int size) { + // Return empty list if target and source are the same + std::vector> returnV; + if (source == target) {return returnV;} + + // depth will always be less than the size + std::vector>> routes(size); + routes.at(0) = {{look_up_or_make(source, target)}}; + + struct RoutesNode { + std::vector index; /**< Index of a node in the heap */ + double value; /**< Value of a node in the heap */ + bool operator<(const RoutesNode &rhs) const { + return value < rhs.value; + } + }; + + std::vector> temporaryRecords; + int routesSize = 0; + + const Graph_T g = graph.get_boost_graph(); + + /** Lambda functions to help code reusability inside the function **/ + auto nextNode = [this](Record* &r, std::vector::iterator &recordIterator, const NodeIndex currentNode){ + // If at the end of an OD pair, get the next pair + if(currentNode == r->target) + r = *++recordIterator; + + // get route from current node to the target, and get next node + r = look_up(currentNode, r->target); + return r-> first_n; + }; + + auto getOutEdges = [g](NodeIndex source){ + static OutEdgeIterator out_i, out_end; + std::map out_edges; + for(boost::tie(out_i, out_end) = boost::out_edges(source, g); + out_i != out_end; out_i++){ + EdgeDescriptor e = *out_i; + out_edges[g[e].index] = e; + } + return out_edges; + }; + + auto getMinimalPathAfterEdge = [this, g, target, &temporaryRecords](const EdgeDescriptor e, const NodeIndex previousNode, const std::vector &basePath){ + NodeIndex nodeAfterEdge = boost::target(e, g); + + if(nodeAfterEdge == previousNode) + throw std::invalid_argument("Loop edge"); + + if(!basePath.empty() && nodeAfterEdge == basePath.back()->prev_n) + throw std::invalid_argument("Edge goes backward"); + + Record *nextOD = look_up_or_make(nodeAfterEdge, target); + if (nextOD == nullptr) + throw std::invalid_argument("No od pair"); + + EdgeProperty eProperty = g[e]; + RoutesNode returnV; + returnV.value = eProperty.length + nextOD -> cost; + for(Record* record : basePath) { + returnV.value += record->cost; + returnV.index.push_back(record); + } + + // push the edge as a route + Record *edgeRecord = look_up_or_make(previousNode, nodeAfterEdge); + // If edge is not the sp, the edge must be pushed, mocking a sp + if(edgeRecord->next_e != eProperty.index) { + std::unique_ptr tempRecord(new Record); + tempRecord->source = previousNode; + tempRecord->prev_n = previousNode; + tempRecord->target = nodeAfterEdge; + tempRecord->first_n = nodeAfterEdge; + tempRecord->next_e = eProperty.index; + tempRecord->cost = eProperty.length; + tempRecord->next = nullptr; + temporaryRecords.push_back(std::move(tempRecord)); + edgeRecord = temporaryRecords.back().get(); + } + returnV.index.push_back(edgeRecord); + // Push the last part of the route + returnV.index.push_back(nextOD); + return returnV; + }; + + auto isSubRoute = [this, &nextNode](std::vector &route1, + std::vector &smallerRoute, Record* &r, const NodeIndex lastNodeBase) { + auto iterR = route1.begin(); + r = *iterR; + auto iterBasePath = smallerRoute.begin(); + Record *basePathRecord = *iterBasePath; + + // basePath contains all record, from + // its source to its target. Then, advance + while (iterBasePath + 1 != smallerRoute.end() && r == basePathRecord) { + r = *++iterR; + basePathRecord = *++iterBasePath; + } + + // Walk through route1 to check if the route matches it + NodeIndex nextNodeRoute = r->source; + NodeIndex nextNodeBase = basePathRecord->source; + while (nextNodeBase != lastNodeBase + && nextNodeBase == nextNodeRoute) { // Path still didn't get to spurNode + nextNodeRoute = nextNode(r, iterR, nextNodeRoute); + nextNodeBase = nextNode(basePathRecord, iterBasePath, + nextNodeBase); + } + + // Update r, if applicable + if(nextNodeRoute == r->target && iterR+1 != route1.end()) + r = *++iterR; + if(r != nullptr && nextNodeRoute != r->target) + r = look_up(nextNodeRoute, r->target); + + return nextNodeBase == nextNodeRoute; + }; + /** End of lambda functions **/ + + for(int i=0; routesSize examinedRoute = routes.at(i).at(j); + + // The spur node is the ith node of latestRoute + auto spurNodeIter = examinedRoute.begin(); + Record* spurNodeRecord = *spurNodeIter; + NodeIndex spurNode = spurNodeRecord->source; + std::vector basePath; + // Repeat until the last node + while(spurNode != target) { + + // Load all edges that come out of the spur node + std::map out_edges = getOutEdges(spurNode); + + // Remove next edge + // Special case of spurNode=source + Record *r = examinedRoute.front(); + if(basePath.empty()){ + out_edges.erase(r->next_e); + } else if(isSubRoute(examinedRoute, basePath, r, spurNode)) { + out_edges.erase(r->next_e); + } + + // Calculate shortest path from source to destination + //without some edges out of spurNode + FibHeap route_candidates; + for(auto edge: out_edges) { + try { + auto candidate = getMinimalPathAfterEdge(edge.second, spurNode, basePath); + route_candidates.push(candidate); + } catch (std::invalid_argument const& ex) { + continue; + } + } + + while (!route_candidates.empty()) { + std::vector topCandidate = route_candidates.top().index; + route_candidates.pop(); + + auto lastIter = topCandidate.end()-2; + NodeIndex candidateSpurNode = (*lastIter++)->source; + auto candidateIter=topCandidate.begin(); + Record *r = *candidateIter; + + // Nodes before and including spur + std::set nodesUntilSpur; + while(candidateIter!=lastIter) { + nodesUntilSpur.insert(r->source); + while(r->first_n != r->target){ + r = look_up(r->first_n, r->target); + nodesUntilSpur.insert(r->source); + } + r = *++candidateIter; + } + + while(r->first_n != target && nodesUntilSpur.find(r->source) == nodesUntilSpur.end()) { + r = look_up(r->first_n, target); + } + + if(nodesUntilSpur.find(r->source) == nodesUntilSpur.end()) { + // Valid candidate + // Check if not already included + bool isPathUnique = true; + Record *r; + for(int i2=0; i2<=i+1; i2++) { + for(int j2=0; j2source; + // Get last record, since this is the record + // where the mistake is. + r = topCandidate.back(); + // The process of finding a new route consists + // of splitting the last record in from two to three records + // The source of the first one must be firstSource + NodeIndex firstSource = r->source; + + while(r->source != duplicatedNode) { //Next lastSource must not be the duplicated node + // If there are three records, the last one's source will be lastSource + NodeIndex lastSource = r->source; + + // make route as basepath-> (firstSource,newSource) -> (newSource,target) + // Append all route, except the last element + std::vector candidateBasePath(topCandidate.begin(), candidateIter); + if(firstSource != lastSource) + candidateBasePath.push_back(look_up(firstSource, lastSource)); + + for(auto edge: getOutEdges(lastSource)) { + if(edge.first == r->next_e) + continue; + + try { + auto newCandidate = getMinimalPathAfterEdge(edge.second, lastSource, candidateBasePath); + route_candidates.push(newCandidate); + } catch (std::invalid_argument const& ex) { + continue; + } + } + r = look_up(r->first_n, r->target); + } + } + } + + /** Update spurnode and basepath **/ + auto formerSpurNodeIter = spurNodeIter; + spurNode = nextNode(spurNodeRecord, spurNodeIter, spurNode); + // Check if the sp inside latestRoute changed + if(formerSpurNodeIter != spurNodeIter || basePath.empty()) + basePath.push_back(look_up((*spurNodeIter)->source, spurNode)); + else + basePath.back() = look_up((*spurNodeIter)->source, spurNode); + } + } + } + + /** Test if the candidate is valid **/ + // It will be invalid if the last od passes its spur node + + for(int i=0; i edges; + auto iterR = routes.at(i).at(j).begin(); + Record *r = *iterR; + NodeIndex currentNode = r->source; + while(currentNode != target){ + currentNode = nextNode(r, iterR, currentNode); + edges.push_back(r->next_e); + } + returnV.push_back(edges); + } + } + + return returnV;//routes; +} + std::vector> UBODT::look_k_sp_path( const NETWORK::NodeIndex source, const NETWORK::NodeIndex target, int K) { // Return empty list if target and source are the same diff --git a/src/mm/fmm/ubodt.hpp b/src/mm/fmm/ubodt.hpp index 77b4ba0..5e2d460 100644 --- a/src/mm/fmm/ubodt.hpp +++ b/src/mm/fmm/ubodt.hpp @@ -77,7 +77,7 @@ class UBODT { NETWORK::NodeIndex target); /** - * Returns a vector with, at most, the k shortest paths from a source ot target + * Returns a vector with, at most, the k shortest paths from source to target * In case that SP is not found, an empty vector is returned. * @param source source node * @param target target node @@ -88,6 +88,17 @@ class UBODT { NETWORK::NodeIndex target, int k); + /** + * Returns a vector with paths from a source o target using the breadth-first search on link elimination + * @param source source node + * @param target target node + * @param size size of generated set + * @return vector of vectors with the shortest paths connecting source to target + */ + std::vector> look_bfs_le_path( + NETWORK::NodeIndex source, + NETWORK::NodeIndex target, int size); + /** * Construct the complete path (a vector of edge ID) from an optimal path * (a vector of optimal nodes in the transition graph) From e4f65f80ee83d1c7d6f4d59ff8c1fe11af691192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fl=C3=A1vio=20Tapaj=C3=B3s?= Date: Wed, 6 Jul 2022 12:48:12 -0300 Subject: [PATCH 11/11] fix: Limit the maximum number of tries when reaching for the optimal path on UBODT::look_bfs_le_path() --- src/mm/fmm/ubodt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mm/fmm/ubodt.cpp b/src/mm/fmm/ubodt.cpp index 0ddd963..348fa05 100644 --- a/src/mm/fmm/ubodt.cpp +++ b/src/mm/fmm/ubodt.cpp @@ -285,7 +285,7 @@ std::vector> UBODT::look_bfs_le_path( } } - while (!route_candidates.empty()) { + while (!route_candidates.empty() && route_candidates.n < 10e3) { std::vector topCandidate = route_candidates.top().index; route_candidates.pop();