reduce contention on parallel insertions (#5)

libgalois/include/galois/LargeVector.h

@@ -29,7 +29,7 @@ template <typename T>
 class LargeVector : public boost::noncopyable {
 private:
   size_t m_capacity;
-  size_t m_size;
+  size_t volatile m_size;
   T* m_data;
 
   int m_fd;
@@ -68,9 +68,9 @@ class LargeVector : public boost::noncopyable {
     std::cout << "new_cap = " << new_cap << "\tmmap_size = " << mmap_size
               << std::endl;
 
-    m_data =
-        static_cast<T*>(mmap(nullptr, mmap_size, PROT_READ | PROT_WRITE,
-                             MAP_SHARED | MAP_HUGETLB | MAP_HUGE_2MB, m_fd, 0));
+    m_data = static_cast<T*>(
+        mmap(nullptr, mmap_size, PROT_READ | PROT_WRITE,
+             MAP_SHARED | MAP_HUGETLB | MAP_HUGE_2MB | MAP_POPULATE, m_fd, 0));
     if (m_data == MAP_FAILED)
       throw std::runtime_error(std::string("mmap failed: ") +
                                std::strerror(errno));

libgalois/include/galois/graphs/LS_LC_CSR_Graph.h

@@ -23,6 +23,7 @@
 #include <unordered_set>
 #include <iterator>
 #include <cstddef>
+#include <atomic>
 
 #include <boost/range/iterator_range_core.hpp>
 #include <boost/range/counting_range.hpp>
@@ -78,6 +79,8 @@ class LS_LC_CSR_Graph : private boost::noncopyable {
   // must be held to acquire m_edges_lock.
   SpinLock m_edges_lock;
 
+  std::atomic_uint64_t m_edges_tail = ATOMIC_VAR_INIT(0);
+
   // returns a reference to the metadata for the pointed-to edge
   inline EdgeMetadata& getEdgeMetadata(EdgeHandle const& handle) {
     return getEdgeMetadata(handle.buffer, handle.index);
@@ -118,18 +121,22 @@ class LS_LC_CSR_Graph : private boost::noncopyable {
     // Copies the edge list to the end of m_edges[1], prepending
     // the new edges.
 
-    vertex_meta.lock();
+    vertex_meta.lock(); // prevents compaction
     {
       uint64_t const new_degree = vertex_meta.degree + dsts.size();
-      uint64_t new_begin;
-      m_edges_lock.lock();
-      {
-        new_begin = m_edges[1].size();
-        m_edges[1].resize(new_begin + new_degree);
-      }
-      m_edges_lock.unlock();
+      uint64_t const new_begin =
+          m_edges_tail.fetch_add(new_degree, std::memory_order_relaxed);
       uint64_t const new_end = new_begin + new_degree;
 
+      if (m_edges[1].size() < new_end) {
+        m_edges_lock.lock();
+        {
+          if (m_edges[1].size() < new_end)
+            m_edges[1].resize(std::max(m_edges[1].size() * 2, new_end));
+        }
+        m_edges_lock.unlock();
+      }
+
       // insert new edges
       std::transform(dsts.begin(), dsts.end(), &getEdgeMetadata(1, new_begin),
                      [](VertexTopologyID dst) {
@@ -242,6 +249,7 @@ class LS_LC_CSR_Graph : private boost::noncopyable {
     {
       m_edges[0].resize(0);
       swap(m_edges[0], m_edges[1]);
+      m_edges_tail.store(0, std::memory_order_relaxed); // fine because lock
     }
     m_edges_lock.unlock();