diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a94e3a0f4..24877e06f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ option(VT_TV_PYTHON_BINDINGS_ENABLED "Build vt-tv with Python bindings" OFF) option(VT_TV_OPENMP_ENABLED "Build vt-tv with openMP support" ON) option(VT_TV_TESTS_ENABLED "Build vt-tv with unit tests" ON) option(VT_TV_COVERAGE_ENABLED "Build vt-tv with coverage" OFF) +set(VT_TV_N_THREADS "2" CACHE STRING "Number of OpenMP threads to use") # add -fPIC to all targets (if building with nanobind) if(VT_TV_PYTHON_BINDINGS_ENABLED) @@ -41,8 +42,6 @@ endif() option(VT_TV_WERROR_ENABLED "Build vt-tv with warnings as errors" OFF) -set(VT_TV_N_THREADS "2" CACHE STRING "Number of OpenMP threads to use") - include(cmake/load_packages.cmake) if(APPLE) @@ -50,9 +49,13 @@ if(APPLE) endif() add_definitions(-DVT_TV_N_THREADS=${VT_TV_N_THREADS}) -add_definitions(-DVT_TV_OPENMP_ENABLED=${VT_TV_OPENMP_ENABLED}) add_compile_definitions(SRC_DIR="${CMAKE_CURRENT_SOURCE_DIR}") add_compile_definitions(BUILD_DIR="${CMAKE_BINARY_DIR}") +if(VT_TV_OPENMP_ENABLED) + add_definitions(-DVT_TV_OPENMP_ENABLED=1) +else() + add_definitions(-DVT_TV_OPENMP_ENABLED=0) +endif() add_custom_target(vt_tv_examples) add_custom_target(vt_tv_tests) diff --git a/src/vt-tv/utility/parse_render.cc b/src/vt-tv/utility/parse_render.cc index a639a7c7f7..4c1a2fd190 100644 --- a/src/vt-tv/utility/parse_render.cc +++ b/src/vt-tv/utility/parse_render.cc @@ -60,9 +60,6 @@ void ParseRender::parseAndRender(PhaseType phase_id, std::unique_ptr info) std::string input_dir = config["input"]["directory"].as(); std::filesystem::path input_path(input_dir); - // Allow user to point to specific files (e.g. json.br) - std::string file_suffix = config["input"]["suffix"].as("json"); - // If it's a relative path, prepend the SRC_DIR if (input_path.is_relative()) { input_path = std::filesystem::path(SRC_DIR) / input_path; @@ -80,34 +77,44 @@ void ParseRender::parseAndRender(PhaseType phase_id, std::unique_ptr info) std::filesystem::path p = input_dir; std::string path = std::filesystem::absolute(p).string(); + // Collect all file paths into a vector + std::vector data_files; + for (const auto& entry : std::filesystem::directory_iterator(input_dir)) { + data_files.push_back(entry.path()); + } + info = std::make_unique(); - #ifdef VT_TV_OPENMP_ENABLED #if VT_TV_OPENMP_ENABLED - #ifdef VT_TV_N_THREADS - const int threads = VT_TV_N_THREADS; - #else - const int threads = 2; - #endif // VT_TV_N_THREADS - omp_set_num_threads(threads); - fmt::print("vt-tv: Using {} threads\n", threads); - # pragma omp parallel for - #endif + const int threads = VT_TV_N_THREADS; + omp_set_num_threads(threads); + fmt::print("vt-tv: Using {} threads\n", threads); + # pragma omp parallel for #endif // VT_TV_OPENMP_ENABLED - for (int64_t rank = 0; rank < n_ranks; rank++) { - fmt::print("Reading file for rank {}\n", rank); - utility::JSONReader reader{static_cast(rank)}; - reader.readFile(input_dir + "data." + std::to_string(rank) + "." + file_suffix); - auto tmpInfo = reader.parse(); - #ifdef VT_TV_OPENMP_ENABLED - #if VT_TV_OPENMP_ENABLED - #pragma omp critical - #endif - #endif - { + + for (uint64_t i = 0; i < data_files.size(); i++) { + auto filepath = data_files[i].string(); + auto filename = data_files[i].filename().string(); + + int64_t rank; + auto first_dot = filename.find("."); + auto next_dot = filename.find(".", first_dot + 1); + + rank = std::stoll(filename.substr(first_dot + 1, next_dot - first_dot - 1)); + + fmt::print("Reading file for rank {}\n", rank); + utility::JSONReader reader{static_cast(rank)}; + reader.readFile(filepath); + auto tmpInfo = reader.parse(); + #if VT_TV_OPENMP_ENABLED + # pragma omp critical + #endif + { info->addInfo(tmpInfo->getObjectInfo(), tmpInfo->getRank(rank)); - } } + } + assert(info->getNumRanks() == static_cast(n_ranks)); + fmt::print("Num ranks={}\n", info->getNumRanks()); } std::array qoi_request = { @@ -153,8 +160,6 @@ void ParseRender::parseAndRender(PhaseType phase_id, std::unique_ptr info) output_file_stem = config["output"]["file_stem"].as("vttv"); - fmt::print("Num ranks={}\n", info->getNumRanks()); - if (config["output"]["window_size"]) { win_size = config["output"]["window_size"].as(2000); // Update font_size with new win_size diff --git a/src/vt-tv/utility/parse_render.h b/src/vt-tv/utility/parse_render.h index 9011f1e1c4..6f2b78691c 100644 --- a/src/vt-tv/utility/parse_render.h +++ b/src/vt-tv/utility/parse_render.h @@ -51,10 +51,8 @@ #include #include -#ifdef VT_TV_OPENMP_ENABLED #if VT_TV_OPENMP_ENABLED - #include -#endif +#include #endif namespace vt::tv::utility { diff --git a/tests/data/lb_test_data/data.0.json.br b/tests/data/lb_test_data_compressed/data.0.json.br similarity index 100% rename from tests/data/lb_test_data/data.0.json.br rename to tests/data/lb_test_data_compressed/data.0.json.br diff --git a/tests/data/lb_test_data/data.1.json.br b/tests/data/lb_test_data_compressed/data.1.json.br similarity index 100% rename from tests/data/lb_test_data/data.1.json.br rename to tests/data/lb_test_data_compressed/data.1.json.br diff --git a/tests/data/lb_test_data/data.2.json.br b/tests/data/lb_test_data_compressed/data.2.json.br similarity index 100% rename from tests/data/lb_test_data/data.2.json.br rename to tests/data/lb_test_data_compressed/data.2.json.br diff --git a/tests/data/lb_test_data/data.3.json.br b/tests/data/lb_test_data_compressed/data.3.json.br similarity index 100% rename from tests/data/lb_test_data/data.3.json.br rename to tests/data/lb_test_data_compressed/data.3.json.br diff --git a/tests/data/lb_test_data/reader_test_data.json b/tests/data/reader_test_data/reader_test_data.json similarity index 100% rename from tests/data/lb_test_data/reader_test_data.json rename to tests/data/reader_test_data/reader_test_data.json diff --git a/tests/data/synthetic_attributes/data.0.json.br b/tests/data/synthetic_attributes/data.0.json.br deleted file mode 100644 index 2715bdca53..0000000000 Binary files a/tests/data/synthetic_attributes/data.0.json.br and /dev/null differ diff --git a/tests/data/synthetic_attributes/data.1.json.br b/tests/data/synthetic_attributes/data.1.json.br deleted file mode 100644 index f7f9363b33..0000000000 Binary files a/tests/data/synthetic_attributes/data.1.json.br and /dev/null differ diff --git a/tests/data/synthetic_attributes/data.2.json.br b/tests/data/synthetic_attributes/data.2.json.br deleted file mode 100644 index dd92cb9e46..0000000000 Binary files a/tests/data/synthetic_attributes/data.2.json.br and /dev/null differ diff --git a/tests/data/synthetic_attributes/data.3.json.br b/tests/data/synthetic_attributes/data.3.json.br deleted file mode 100644 index 7bab15b998..0000000000 Binary files a/tests/data/synthetic_attributes/data.3.json.br and /dev/null differ diff --git a/tests/data/synthetic_attributes_compressed/data.0.json.br b/tests/data/synthetic_attributes_compressed/data.0.json.br new file mode 100644 index 0000000000..727acbd611 Binary files /dev/null and b/tests/data/synthetic_attributes_compressed/data.0.json.br differ diff --git a/tests/data/synthetic_attributes_compressed/data.1.json.br b/tests/data/synthetic_attributes_compressed/data.1.json.br new file mode 100644 index 0000000000..636557650f Binary files /dev/null and b/tests/data/synthetic_attributes_compressed/data.1.json.br differ diff --git a/tests/data/synthetic_attributes_compressed/data.2.json.br b/tests/data/synthetic_attributes_compressed/data.2.json.br new file mode 100644 index 0000000000..a0e42b4d16 Binary files /dev/null and b/tests/data/synthetic_attributes_compressed/data.2.json.br differ diff --git a/tests/data/synthetic_attributes_compressed/data.3.json.br b/tests/data/synthetic_attributes_compressed/data.3.json.br new file mode 100644 index 0000000000..8f745dfe08 Binary files /dev/null and b/tests/data/synthetic_attributes_compressed/data.3.json.br differ diff --git a/tests/unit/utility/test_json_reader.cc b/tests/unit/utility/test_json_reader.cc index 01d26dd5c3..ecbc3d53cd 100644 --- a/tests/unit/utility/test_json_reader.cc +++ b/tests/unit/utility/test_json_reader.cc @@ -112,8 +112,8 @@ TEST_F(JSONReaderTest, test_json_reader_1) { } } -TEST_F(JSONReaderTest, test_json_reader_metadata_attributes) { - std::filesystem::path p = std::filesystem::path(SRC_DIR) / "tests/data/lb_test_data" ; +TEST_F(TestJSONReader, test_json_reader_metadata_attributes) { + std::filesystem::path p = std::filesystem::path(SRC_DIR) / "tests/unit/reader_test_data" ; std::string path = std::filesystem::absolute(p).string(); NodeType rank = 0;