Merge pull request #106 from DARMA-tasking/8-adapt-input-and-output-t…

…o-use-compressed-json-format

#8: Adapt input and output to use compressed JSON format

CMakeLists.txt

@@ -33,6 +33,7 @@ option(VT_TV_PYTHON_BINDINGS_ENABLED "Build vt-tv with Python bindings" OFF)
 option(VT_TV_OPENMP_ENABLED "Build vt-tv with openMP support" ON)
 option(VT_TV_TESTS_ENABLED "Build vt-tv with unit tests" ON)
 option(VT_TV_COVERAGE_ENABLED "Build vt-tv with coverage" OFF)
+set(VT_TV_N_THREADS "2" CACHE STRING "Number of OpenMP threads to use")
 
 # add -fPIC to all targets (if building with nanobind)
 if(VT_TV_PYTHON_BINDINGS_ENABLED)
@@ -41,18 +42,20 @@ endif()
 
 option(VT_TV_WERROR_ENABLED "Build vt-tv with warnings as errors" OFF)
 
-set(VT_TV_N_THREADS "2" CACHE STRING "Number of OpenMP threads to use")
-
 include(cmake/load_packages.cmake)
 
 if(APPLE AND NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
   add_compile_options(-ffat-lto-objects)
 endif()
 
 add_definitions(-DVT_TV_N_THREADS=${VT_TV_N_THREADS})
-add_definitions(-DVT_TV_OPENMP_ENABLED=${VT_TV_OPENMP_ENABLED})
 add_compile_definitions(SRC_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
 add_compile_definitions(BUILD_DIR="${CMAKE_BINARY_DIR}")
+if(VT_TV_OPENMP_ENABLED)
+  add_definitions(-DVT_TV_OPENMP_ENABLED=1)
+else()
+  add_definitions(-DVT_TV_OPENMP_ENABLED=0)
+endif()
 
 add_custom_target(vt_tv_examples)
 if (VT_TV_TESTS_ENABLED)

bindings/python/tv.h

@@ -65,10 +65,8 @@
 #include <filesystem>
 #include <map>
 
-#ifdef VT_TV_OPENMP_ENABLED
 #if VT_TV_OPENMP_ENABLED
-    #include <omp.h>
-#endif
+#include <omp.h>
 #endif
 
 namespace vt::tv::bindings::python {

src/vt-tv/utility/parse_render.cc

@@ -71,39 +71,50 @@ void ParseRender::parseAndRender(
         input_dir += '/';
       }
 
-      int64_t n_ranks =
-        config["input"]["n_ranks"].as<int64_t>(); // signed for omp parallel for
-
       // Read JSON file and input data
       std::filesystem::path p = input_dir;
       std::string path = std::filesystem::absolute(p).string();
 
+      // Collect all file paths into a vector
+      std::vector<std::filesystem::path> data_files;
+      for (const auto& entry : std::filesystem::directory_iterator(input_dir)) {
+        data_files.push_back(entry.path());
+      }
+
       info = std::make_unique<Info>();
 
-#ifdef VT_TV_OPENMP_ENABLED
 #if VT_TV_OPENMP_ENABLED
-#ifdef VT_TV_N_THREADS
       const int threads = VT_TV_N_THREADS;
-#else
-      const int threads = 2;
-#endif // VT_TV_N_THREADS
       omp_set_num_threads(threads);
       fmt::print("vt-tv: Using {} threads\n", threads);
 #pragma omp parallel for
-#endif
 #endif // VT_TV_OPENMP_ENABLED
-      for (int64_t rank = 0; rank < n_ranks; rank++) {
+
+      for (uint64_t i = 0; i < data_files.size(); i++) {
+        auto filepath = data_files[i].string();
+        auto filename = data_files[i].filename().string();
+
+        int64_t rank;
+        auto first_dot = filename.find(".");
+        auto next_dot = filename.find(".", first_dot + 1);
+
+        rank =
+          std::stoll(filename.substr(first_dot + 1, next_dot - first_dot - 1));
+
         fmt::print("Reading file for rank {}\n", rank);
         utility::JSONReader reader{static_cast<NodeType>(rank)};
-        reader.readFile(input_dir + "data." + std::to_string(rank) + ".json");
+        reader.readFile(filepath);
         auto tmpInfo = reader.parse();
-#ifdef VT_TV_OPENMP_ENABLED
 #if VT_TV_OPENMP_ENABLED
 #pragma omp critical
-#endif
 #endif
         { info->addInfo(tmpInfo->getObjectInfo(), tmpInfo->getRank(rank)); }
       }
+      std::size_t n_ranks = config["input"]["n_ranks"].as<std::size_t>();
+      if (info->getNumRanks() != n_ranks) {
+        throw std::runtime_error("Number of ranks does not match expected value.");
+      }
+      fmt::print("Num ranks={}\n", info->getNumRanks());
     }
 
     std::array<std::string, 3> qoi_request = {
@@ -148,8 +159,6 @@ void ParseRender::parseAndRender(
 
       output_file_stem = config["output"]["file_stem"].as<std::string>("vttv");
 
-      fmt::print("Num ranks={}\n", info->getNumRanks());
-
       if (config["output"]["window_size"]) {
         win_size = config["output"]["window_size"].as<uint64_t>(2000);
         // Update font_size with new win_size

src/vt-tv/utility/parse_render.h

@@ -51,11 +51,9 @@
 #include <limits>
 #include <memory>
 
-#ifdef VT_TV_OPENMP_ENABLED
 #if VT_TV_OPENMP_ENABLED
 #include <omp.h>
 #endif
-#endif
 namespace vt::tv::utility {
 
 /**

tests/unit/util.h

@@ -55,6 +55,10 @@
 #include <tuple>
 #include <variant>
 
+#if VT_TV_OPENMP_ENABLED
+#include <omp.h>
+#endif
+
 #include <fmt-vt/format.h>
 
 #include <gmock/gmock.h>

tests/unit/utility/test_json_reader.cc

@@ -56,14 +56,12 @@ using JSONReader = vt::tv::utility::JSONReader;
  */
 struct JSONReaderTest : public ::testing::Test { };
 
-TEST_F(JSONReaderTest, test_json_reader_1) {
-  std::filesystem::path p =
-    std::filesystem::path(SRC_DIR) / "data/lb_test_data";
+void test_json_reader(std::filesystem::path p, std::string suffix) {
   std::string path = std::filesystem::absolute(p).string();
 
   NodeType rank = 0;
   JSONReader reader{rank};
-  reader.readFile(path + "/data.0.json");
+  reader.readFile(path + "/data.0" + suffix);
   auto info = reader.parse();
 
   auto const& obj_info = info->getObjectInfo();
@@ -115,9 +113,19 @@ TEST_F(JSONReaderTest, test_json_reader_1) {
   }
 }
 
+TEST_F(JSONReaderTest, test_json_reader_1) {
+  std::filesystem::path p = std::filesystem::path(SRC_DIR) / "data/lb_test_data" ;
+  test_json_reader(p, ".json");
+}
+
+TEST_F(JSONReaderTest, test_json_reader_compressed) {
+  std::filesystem::path p = std::filesystem::path(SRC_DIR) / "data/lb_test_data_compressed" ;
+  test_json_reader(p, ".json.br");
+}
+
 TEST_F(JSONReaderTest, test_json_reader_metadata_attributes) {
   std::filesystem::path p =
-    std::filesystem::path(SRC_DIR) / "data/lb_test_data";
+    std::filesystem::path(SRC_DIR) / "data/reader_test_data";
   std::string path = std::filesystem::absolute(p).string();
 
   NodeType rank = 0;
@@ -141,7 +149,7 @@ TEST_F(JSONReaderTest, test_json_reader_metadata_attributes) {
 
 TEST_F(JSONReaderTest, test_json_reader_object_info_attributes) {
   std::filesystem::path p =
-    std::filesystem::path(SRC_DIR) / "data/lb_test_data";
+    std::filesystem::path(SRC_DIR) / "data/reader_test_data";
   std::string path = std::filesystem::absolute(p).string();
 
   NodeType rank = 0;
@@ -192,7 +200,7 @@ TEST_F(JSONReaderTest, test_json_reader_qoi_serializer) {
 
 TEST_F(JSONReaderTest, test_json_reader_object_work_user_defined) {
   std::filesystem::path p =
-    std::filesystem::path(SRC_DIR) / "data/lb_test_data";
+    std::filesystem::path(SRC_DIR) / "data/reader_test_data";
   std::string path = std::filesystem::absolute(p).string();
 
   NodeType rank = 0;