From 11a28eca4c2c17222ec120a4793e2cd69741e835 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Tue, 2 Jul 2024 13:12:43 -0600 Subject: [PATCH 1/9] added voxel file read initialization --- single-node-refactor/src/common/mesh_inputs.h | 2 +- single-node-refactor/src/common/region.h | 20 +- single-node-refactor/src/driver.h | 550 ++++++++++++++++-- single-node-refactor/src/input/parse_yaml.cpp | 77 ++- 4 files changed, 601 insertions(+), 48 deletions(-) diff --git a/single-node-refactor/src/common/mesh_inputs.h b/single-node-refactor/src/common/mesh_inputs.h index 32cb03e81..8638a2562 100644 --- a/single-node-refactor/src/common/mesh_inputs.h +++ b/single-node-refactor/src/common/mesh_inputs.h @@ -80,7 +80,7 @@ struct mesh_input_t int num_dims = 3; ///< Number of dimensions for the mesh mesh_input::source source = mesh_input::none; ///< Source of mesh, file or generate std::string file_path = ""; ///< Absolute path of mesh file - mesh_input::type type; ///< Type of mesh to generate if + mesh_input::type type; ///< Type of mesh to generate if double origin[3] = {0.0, 0.0, 0.0}; ///< Mesh origin for generating a mesh double length[3] = {0.0, 0.0, 0.0}; ///< x,y,z length of generated mesh diff --git a/single-node-refactor/src/common/region.h b/single-node-refactor/src/common/region.h index a9bdcee39..93e62d4e7 100644 --- a/single-node-refactor/src/common/region.h +++ b/single-node-refactor/src/common/region.h @@ -44,7 +44,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // ============================================================================== namespace region { - // for tagging boundary faces + // for tagging volumes to paint material onto the mesh enum vol_tag { no_volume = 0, @@ -52,7 +52,10 @@ namespace region box = 2, // tag all elements inside a box cylinder = 3, // tag all elements inside a cylinder sphere = 4, // tag all elements inside a sphere - readVoxelFile = 5, // tag all elements in a voxel mesh input WARING: Currently unimplemented + readVoxelFile = 5, // tag all elements in a voxel mesh (structured VTK) + readPolycrystalFile = 6, // tag all elements in a polycrystallince voxel mesh (structured VTK) + readSTLFile = 7, // read a STL file and voxelize it + readVTKFile = 8, // tag all elements in a VTK mesh (unstructured mesh) }; } // end of namespace @@ -62,7 +65,7 @@ static std::map region_type_map { "box", region::box }, { "sphere", region::sphere }, { "cylinder", region::cylinder }, - { "readVoxelFile", region::readVoxelFile } + { "voxel_file", region::readVoxelFile } }; ///////////////////////////////////////////////////////////////////////////// @@ -77,6 +80,8 @@ struct reg_fill_t // type region::vol_tag volume; ///< Type of volume for this region eg. global, box, sphere, planes, etc. + std::string file_path = ""; ///< Absolute path of mesh file + // material id size_t material_id; ///< Material ID for this region @@ -92,6 +97,11 @@ struct reg_fill_t double radius1 = 0.0; ///< Inner radius to fill for sphere double radius2 = 0.0; ///< Outer radius to fill for sphere + // scale parameters for mesh files + double scale_x = 1.0; + double scale_y = 1.0; + double scale_z = 1.0; + // initial conditions init_conds::init_velocity_conds velocity; ///< Initial conditions for this region WARNING: Currently unimplemented @@ -115,6 +125,7 @@ struct reg_fill_t static std::vector str_region_inps { "type", + "file_path", "material_id", "x1", "x2", @@ -124,6 +135,9 @@ static std::vector str_region_inps "z2", "radius1", "radius2", + "scale_x", + "scale_y", + "scale_z", "velocity", "u", "v", diff --git a/single-node-refactor/src/driver.h b/single-node-refactor/src/driver.h index 57ea633ec..b41431fdf 100644 --- a/single-node-refactor/src/driver.h +++ b/single-node-refactor/src/driver.h @@ -45,12 +45,68 @@ void fill_regions(DCArrayKokkos&, - Material_t, - mesh_t, - node_t, - MaterialPoint_t, - GaussPoint_t, - corner_t); + Material_t&, + mesh_t&, + node_t&, + MaterialPoint_t&, + GaussPoint_t&, + corner_t&); + +// ============================================================================== +// Functions to read voxel mesh +// ============================================================================== +void user_voxel_init(DCArrayKokkos& elem_values, + double& dx, + double& dy, + double& dz, + double& orig_x, + double& orig_y, + double& orig_z, + size_t& voxel_num_i, + size_t& voxel_num_j, + size_t& voxel_num_k, + double scale_x, + double scale_y, + double scale_z, + std::string mesh_file); + +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_id_device +/// +/// \brief This gives the index value of the point or the elem +/// +/// Assumes that the grid has an i,j,k structure +/// the elem = i + (j)*(num_points_i-1) + (k)*(num_points_i-1)*(num_points_j-1) +/// the point = i + (j)*num_points_i + (k)*num_points_i*num_points_j +/// +/// \param i index +/// \param j index +/// \param k index +/// \param Number of i indices +/// \param Number of j indices +/// +///////////////////////////////////////////////////////////////////////////// +KOKKOS_FUNCTION +int get_id_device(int i, int j, int k, int num_i, int num_j) +{ + return i + j * num_i + k * num_i * num_j; +} + +// for string delimiter parsing +std::vector split(std::string s, std::string delimiter); + +// retrieves multiple values between [ ] +std::vector extract_list(std::string str); + +const std::string WHITESPACE = " "; + +std::string ltrim(const std::string& s); + +std::string rtrim(const std::string& s); + +std::string trim(const std::string& s); + class Driver @@ -273,29 +329,64 @@ class Driver /// ///////////////////////////////////////////////////////////////////////////// void fill_regions(DCArrayKokkos& region_fills, - Material_t Materials, - mesh_t mesh, - node_t node, - MaterialPoint_t MaterialPoints, - GaussPoint_t GaussPoints, - corner_t corner) + Material_t& Materials, + mesh_t& mesh, + node_t& node, + MaterialPoint_t& MaterialPoints, + GaussPoint_t& GaussPoints, + corner_t& corner) { int num_fills = region_fills.size(); printf("Num Fills's = %d\n", num_fills); + // --------------------------------------------- + // variables from a voxel file + // --------------------------------------------- + DCArrayKokkos voxel_elem_mat_id; // 1 or 0 if material exist, or it is the material_id + double voxel_dx, voxel_dy, voxel_dz; // voxel mesh resolution, set by input file + double orig_x, orig_y, orig_z; // origin of voxel elem center mesh, set by input file + size_t voxel_num_i, voxel_num_j, voxel_num_k; // num voxel elements in each direction, set by input file + + DCArrayKokkos read_voxel_file(num_fills); // check to see if readVoxelFile + FOR_ALL(f_id, 0, num_fills, { + if (region_fills(f_id).volume == region::readVoxelFile) + { + read_voxel_file(f_id) = region::readVoxelFile; // read the voxel file + } + // add other mesh voxel files + else + { + read_voxel_file(f_id) = 0; + } + }); // end parallel for + read_voxel_file.update_host(); // copy to CPU if code is to read a file + Kokkos::fence(); + // --------------------------------------------- + + + for (int f_id = 0; f_id < num_fills; f_id++) { - // // voxel mesh setup - // if (read_voxel_file.host(f_id) == 1) - // { - // // read voxel mesh - // user_voxel_init(voxel_elem_values, - // voxel_dx, voxel_dy, voxel_dz, - // orig_x, orig_y, orig_z, - // voxel_num_i, voxel_num_j, voxel_num_k); - - // // copy values read from file to device - // voxel_elem_values.update_device(); - // } // endif + + // ---- + // voxel mesh setup + if (read_voxel_file.host(f_id) == region::readVoxelFile) + { + // read voxel mesh to get the values in the fcn interface + user_voxel_init(voxel_elem_mat_id, + voxel_dx, voxel_dy, voxel_dz, + orig_x, orig_y, orig_z, + voxel_num_i, voxel_num_j, voxel_num_k, + region_fills(f_id).scale_x, + region_fills(f_id).scale_y, + region_fills(f_id).scale_z, + region_fills(f_id).file_path); + + // copy values read from file to device + voxel_elem_mat_id.update_device(); + } // endif + // add else if for other mesh reads including STL-2-voxel + + int num_elems = mesh.num_elems; // parallel loop over elements in mesh @@ -389,25 +480,32 @@ class Driver case region::readVoxelFile: { + fill_this = 0; // default is no, don't fill it - // // find the closest element in the voxel mesh to this element - // double i0_real = (elem_coords[0] - orig_x) / (voxel_dx); - // double j0_real = (elem_coords[1] - orig_y) / (voxel_dy); - // double k0_real = (elem_coords[2] - orig_z) / (voxel_dz); + // find the closest element in the voxel mesh to this element + double i0_real = (elem_coords[0] - orig_x - region_fills(f_id).origin[0]) / (voxel_dx); + double j0_real = (elem_coords[1] - orig_y - region_fills(f_id).origin[1]) / (voxel_dy); + double k0_real = (elem_coords[2] - orig_z - region_fills(f_id).origin[2]) / (voxel_dz); - // int i0 = (int)i0_real; - // int j0 = (int)j0_real; - // int k0 = (int)k0_real; + int i0 = (int)i0_real; + int j0 = (int)j0_real; + int k0 = (int)k0_real; - // // look for the closest element in the voxel mesh - // int elem_id0 = get_id(i0, j0, k0, voxel_num_i, voxel_num_j); + // look for the closest element in the voxel mesh + int elem_id0 = get_id_device(i0, j0, k0, voxel_num_i, voxel_num_j); + + // if voxel mesh overlaps this mesh, then fill it if =1 + if (elem_id0 < voxel_elem_mat_id.size() && elem_id0 >= 0 && + i0 >= 0 && j0 >= 0 && k0 >= 0 && + i0 < voxel_num_i && j0 < voxel_num_j && k0 < voxel_num_k) { + // voxel mesh elem values = 0 or 1 + fill_this = voxel_elem_mat_id(elem_id0); // values from file + + } // end if + + break; - // // if voxel mesh overlaps this mesh, then fill it if =1 - // if (elem_id0 < voxel_elem_values.size() && elem_id0 >= 0) { - // // voxel mesh elem values = 0 or 1 - // fill_this = voxel_elem_values(elem_id0); // values from file - // } // end if } // end case case region::no_volume: { @@ -678,3 +776,379 @@ class Driver Kokkos::fence(); } // end fill regions + + +// ----------------------------------------------------------------------------- +// The function to read a voxel vtk file from Dream3d and intialize the mesh +// ------------------------------------------------------------------------------ +void user_voxel_init(DCArrayKokkos& elem_values, + double& dx, + double& dy, + double& dz, + double& orig_x, + double& orig_y, + double& orig_z, + size_t& num_elems_i, + size_t& num_elems_j, + size_t& num_elems_k, + double scale_x, + double scale_y, + double scale_z, + std::string mesh_file) +{ + std::string MESH = mesh_file; // user specified + + std::ifstream in; // FILE *in; + in.open(MESH); + + // check to see of a mesh was supplied when running the code + if (in) + { + printf("\nReading the 3D voxel mesh: "); + std::cout << MESH << std::endl; + } + else + { + std::cout << "\n\n**********************************\n\n"; + std::cout << " ERROR:\n"; + std::cout << " Voxel vtk input does not exist \n"; + std::cout << "**********************************\n\n" << std::endl; + std::exit(EXIT_FAILURE); + } // end if + + size_t i; // used for writing information to file + size_t point_id; // the global id for the point + size_t elem_id; // the global id for the elem + size_t this_point; // a local id for a point in a elem (0:7 for a Hexahedral elem) + + size_t num_points_i; + size_t num_points_j; + size_t num_points_k; + + size_t num_dims = 3; + + std::string token; + + bool found = false; + + // look for POINTS + i = 0; + while (found == false) { + std::string str; + std::string delimiter = " "; + std::getline(in, str); + std::vector v = split(str, delimiter); + + // looking for the following text: + // POINTS %d float + if (v[0] == "DIMENSIONS") + { + num_points_i = std::stoi(v[1]); + num_points_j = std::stoi(v[2]); + num_points_k = std::stoi(v[3]); + printf("Num voxel nodes read in = %zu, %zu, %zu\n", num_points_i, num_points_j, num_points_k); + + found = true; + } // end if + + if (i > 1000) + { + printf("ERROR: Failed to find POINTS \n"); + break; + } // end if + + i++; + } // end while + + found = false; + + int num_points = num_points_i * num_points_j * num_points_k; + CArray pt_coords_x(num_points_i); + CArray pt_coords_y(num_points_j); + CArray pt_coords_z(num_points_k); + + while (found == false) { + std::string str; + std::string str0; + std::string delimiter = " "; + std::getline(in, str); + std::vector v = split(str, delimiter); + + // looking for the following text: + if (v[0] == "X_COORDINATES") + { + size_t num_saved = 0; + + while (num_saved < num_points_i - 1) { + // get next line + std::getline(in, str0); + + // remove starting and trailing spaces + str = trim(str0); + std::vector v_coords = split(str, delimiter); + + // loop over the contents of the vector v_coords + for (size_t this_point = 0; this_point < v_coords.size(); this_point++) + { + pt_coords_x(num_saved) = scale_x*std::stod(v_coords[this_point]); + num_saved++; + } // end for + } // end while + + found = true; + } // end if + + if (i > 1000) + { + printf("ERROR: Failed to find X_COORDINATES \n"); + break; + } // end if + + i++; + } // end while + found = false; + + while (found == false) { + std::string str; + std::string str0; + std::string delimiter = " "; + std::getline(in, str); + std::vector v = split(str, delimiter); + + // looking for the following text: + if (v[0] == "Y_COORDINATES") + { + size_t num_saved = 0; + + while (num_saved < num_points_j - 1) { + // get next line + std::getline(in, str0); + + // remove starting and trailing spaces + str = trim(str0); + std::vector v_coords = split(str, delimiter); + + // loop over the contents of the vector v_coords + for (size_t this_point = 0; this_point < v_coords.size(); this_point++) + { + pt_coords_y(num_saved) = scale_y*std::stod(v_coords[this_point]); + num_saved++; + } // end for + } // end while + + found = true; + } // end if + + if (i > 1000) + { + printf("ERROR: Failed to find Y_COORDINATES \n"); + break; + } // end if + + i++; + } // end while + found = false; + + while (found == false) { + std::string str; + std::string str0; + std::string delimiter = " "; + std::getline(in, str); + std::vector v = split(str, delimiter); + + // looking for the following text: + if (v[0] == "Z_COORDINATES") + { + size_t num_saved = 0; + + while (num_saved < num_points_k - 1) { + // get next line + std::getline(in, str0); + + // remove starting and trailing spaces + str = trim(str0); + std::vector v_coords = split(str, delimiter); + + // loop over the contents of the vector v_coords + for (size_t this_point = 0; this_point < v_coords.size(); this_point++) + { + pt_coords_z(num_saved) = scale_z*std::stod(v_coords[this_point]); + num_saved++; + } // end for + } // end while + + found = true; + } // end if + + if (i > 1000) + { + printf("ERROR: Failed to find Z_COORDINATES \n"); + break; + } // end if + + i++; + } // end while + found = false; + + size_t num_elems; + num_elems_i = num_points_i - 1; + num_elems_j = num_points_j - 1; + num_elems_k = num_points_k - 1; + + // center to center distance between first and last elem along each edge + double Lx = (pt_coords_x(num_points_i - 2) - pt_coords_x(0)); + double Ly = (pt_coords_y(num_points_j - 2) - pt_coords_y(0)); + double Lz = (pt_coords_z(num_points_k - 2) - pt_coords_z(0)); + + // spacing between elems + dx = Lx / ((double) num_elems_i); + dy = Ly / ((double) num_elems_j); + dz = Lz / ((double) num_elems_k); + + // element mesh origin + orig_x = 0.5 * (pt_coords_x(0) + pt_coords_x(1)), + orig_y = 0.5 * (pt_coords_y(0) + pt_coords_y(1)), + orig_z = 0.5 * (pt_coords_z(0) + pt_coords_z(1)), + + // look for CELLS + i = 0; + while (found == false) { + std::string str; + std::getline(in, str); + + std::string delimiter = " "; + std::vector v = split(str, delimiter); + + // looking for the following text: + // CELLS num_elems size + if (v[0] == "CELL_DATA") + { + num_elems = std::stoi(v[1]); + printf("Num voxel elements read in %zu\n", num_elems); + + found = true; + } // end if + + if (i > 1000) + { + printf("ERROR: Failed to find CELL_DATA \n"); + break; + } // end if + + i++; + } // end while + found = false; + + // allocate memory for element voxel values + elem_values = DCArrayKokkos(num_elems); + + // reading the cell data + while (found == false) { + std::string str; + std::string str0; + + std::string delimiter = " "; + std::getline(in, str); + std::vector v = split(str, delimiter); + + // looking for the following text: + if (v[0] == "LOOKUP_TABLE") + { + size_t num_saved = 0; + + while (num_saved < num_elems - 1) { + // get next line + std::getline(in, str0); + + // remove starting and trailing spaces + str = trim(str0); + std::vector v_values = split(str, delimiter); + + // loop over the contents of the vector v_coords + for (size_t this_elem = 0; this_elem < v_values.size(); this_elem++) + { + // save integers (0 or 1) to host side + elem_values.host(num_saved) = std::stoi(v_values[this_elem]); + num_saved++; + } // end for + + // printf(" done with one row of data \n"); + } // end while + + found = true; + } // end if + + if (i > 1000) + { + printf("ERROR: Failed to find LOOKUP_TABLE data \n"); + break; + } // end if + + i++; + } // end while + found = false; + + printf("\n"); + + in.close(); +} // end routine + + +// Code from stackover flow for string delimiter parsing +std::vector split(std::string s, std::string delimiter) +{ + size_t pos_start = 0, pos_end, delim_len = delimiter.length(); + std::string token; + std::vector res; + + while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) { + token = s.substr(pos_start, pos_end - pos_start); + pos_start = pos_end + delim_len; + res.push_back(token); + } + + res.push_back(s.substr(pos_start)); + return res; +} // end of split + + +// retrieves multiple values between [ ] +std::vector extract_list(std::string str) +{ + // replace '[' with a space and ']' with a space + std::replace(str.begin(), str.end(), '[', ' '); + std::replace(str.begin(), str.end(), ']', ' '); + + std::vector str_values; + std::vector values; + + // exact the str values into a vector + str_values = split(str, ","); + + // convert the text values into double values + for (auto& word : str_values) + { + values.push_back(atof(word.c_str()) ); + } // end for + + return values; +} // end of extract_list + + +std::string ltrim(const std::string& s) +{ + size_t start = s.find_first_not_of(WHITESPACE); + return (start == std::string::npos) ? "" : s.substr(start); +} + + +std::string rtrim(const std::string& s) +{ + size_t end = s.find_last_not_of(WHITESPACE); + return (end == std::string::npos) ? "" : s.substr(0, end + 1); +} + +std::string trim(const std::string& s) +{ + return rtrim(ltrim(s)); +} \ No newline at end of file diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index a12e545ad..454785722 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1109,6 +1109,42 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) region_fills(reg_id).z2 = z2; }); } // z2 + else if (a_word.compare("scale_x") == 0) { + // outer plane + + double scale_x = root["regions"][reg_id]["fill_volume"]["scale_x"].As(); + if (VERBOSE) { + std::cout << "\tscale_x = " << scale_x << std::endl; + } + + RUN({ + region_fills(reg_id).scale_x = scale_x; + }); + } // scale_x + else if (a_word.compare("scale_y") == 0) { + // outer plane + + double scale_y = root["regions"][reg_id]["fill_volume"]["scale_y"].As(); + if (VERBOSE) { + std::cout << "\tscale_y = " << scale_y << std::endl; + } + + RUN({ + region_fills(reg_id).scale_y = scale_y; + }); + } // scale_y + else if (a_word.compare("scale_z") == 0) { + // outer plane + + double scale_z = root["regions"][reg_id]["fill_volume"]["scale_z"].As(); + if (VERBOSE) { + std::cout << "\tscale_z = " << scale_z << std::endl; + } + + RUN({ + region_fills(reg_id).scale_z = scale_z; + }); + } // scale_z else if (a_word.compare("velocity") == 0) { // velocity fill region type @@ -1256,7 +1292,6 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) region_fills(reg_id).volume = region::no_volume; }); break; - default: RUN({ @@ -1273,13 +1308,26 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) throw std::runtime_error("**** Region Volume Fill Type Not Understood ****"); break; } // end switch - } + } + else{ std::cout << "ERROR: invalid input: " << type << std::endl; throw std::runtime_error("**** Volume Fill Not Understood ****"); } // end if } // end volume fill type + // Get mesh file path + else if (a_word.compare("file_path") == 0) { + // region volume fill type + std::string path = root["regions"][reg_id]["fill_volume"]["file_path"].As(); + + if (VERBOSE) { + std::cout << "\tfile_path = " << path << std::endl; + } + + region_fills(reg_id).file_path = path; // saving the absolute file path + + } // end file path // else if (a_word.compare("origin") == 0) { std::string origin = root["regions"][reg_id]["fill_volume"]["origin"].As(); @@ -1326,6 +1374,27 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) } } // end for words in material + // ----------------------------------------------- + // check for consistency in input settings + + // if the following is true, stop simulation; must add all mesh read options + if (region_fills(reg_id).volume == region::readVoxelFile && region_fills(reg_id).file_path.empty()) { + std::cout << "ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file" << std::endl; + } + + // add all mesh read options here + if (region_fills(reg_id).volume != region::readVoxelFile ) { + // this means it is a geometric definition of the region + + // check to see if a file path was set + if(region_fills(reg_id).file_path.size()>0){ + std::cout << "ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region" << std::endl; + exit(0); + } + } + // ----------------------------------------------- + + } // end loop over regions } // end of function to parse region @@ -1706,8 +1775,6 @@ void parse_materials(Yaml::Node& root, Material_t& Materials) Yaml::Node & mat_global_vars_yaml = root["materials"][mat_id]["material"][a_word]; size_t num_global_vars = mat_global_vars_yaml.Size(); - - std::cout << "*** parsing num global eos vars = " << num_global_vars << std::endl; RUN({ Materials.num_eos_global_vars(mat_id) = num_global_vars; @@ -1741,8 +1808,6 @@ void parse_materials(Yaml::Node& root, Material_t& Materials) Yaml::Node & mat_global_vars_yaml = root["materials"][mat_id]["material"][a_word]; size_t num_global_vars = mat_global_vars_yaml.Size(); - - std::cout << "*** parsing num global eos vars = " << num_global_vars << std::endl; RUN({ Materials.num_strength_global_vars(mat_id) = num_global_vars; From 76c56359ad9471fd78ca314475b9b9a9c34577b5 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Wed, 10 Jul 2024 15:18:00 -0600 Subject: [PATCH 2/9] fixed bug with gpu memory --- single-node-refactor/src/common/region.h | 25 +++++--- .../src/common/simulation_parameters.h | 4 +- single-node-refactor/src/driver.h | 15 +++-- single-node-refactor/src/input/parse_yaml.cpp | 59 +++++++++++-------- single-node-refactor/src/input/parse_yaml.h | 3 +- 5 files changed, 65 insertions(+), 41 deletions(-) diff --git a/single-node-refactor/src/common/region.h b/single-node-refactor/src/common/region.h index 93e62d4e7..7c8a83da7 100644 --- a/single-node-refactor/src/common/region.h +++ b/single-node-refactor/src/common/region.h @@ -80,8 +80,6 @@ struct reg_fill_t // type region::vol_tag volume; ///< Type of volume for this region eg. global, box, sphere, planes, etc. - std::string file_path = ""; ///< Absolute path of mesh file - // material id size_t material_id; ///< Material ID for this region @@ -97,11 +95,6 @@ struct reg_fill_t double radius1 = 0.0; ///< Inner radius to fill for sphere double radius2 = 0.0; ///< Outer radius to fill for sphere - // scale parameters for mesh files - double scale_x = 1.0; - double scale_y = 1.0; - double scale_z = 1.0; - // initial conditions init_conds::init_velocity_conds velocity; ///< Initial conditions for this region WARNING: Currently unimplemented @@ -119,6 +112,24 @@ struct reg_fill_t double origin[3] = {0.0, 0.0, 0.0}; ///< Origin for region }; + +///////////////////////////////////////////////////////////////////////////// +/// +/// \struct reg_fill_host_t +/// +/// \brief Geometry data, on the cpu only, for regions of materials/states +/// +///////////////////////////////////////////////////////////////////////////// +struct reg_fill_host_t +{ + std::string file_path; ///< path of mesh file + + // scale parameters for input mesh files + double scale_x = 1.0; + double scale_y = 1.0; + double scale_z = 1.0; +}; + // ---------------------------------- // valid inputs for a material fill // ---------------------------------- diff --git a/single-node-refactor/src/common/simulation_parameters.h b/single-node-refactor/src/common/simulation_parameters.h index a70dfc180..0287ba764 100644 --- a/single-node-refactor/src/common/simulation_parameters.h +++ b/single-node-refactor/src/common/simulation_parameters.h @@ -62,7 +62,9 @@ struct SimulationParameters_t std::vector solver_inputs; ///< Solvers to use during the simulation - DCArrayKokkos region_fills; ///< Region data for simulation mesh, set the initial conditions + CArrayKokkos region_fills; ///< Region data for simulation mesh, set the initial conditions + + CArray region_fills_host; ///< Region data on CPU, set the initial conditions }; // simulation_parameters_t diff --git a/single-node-refactor/src/driver.h b/single-node-refactor/src/driver.h index b41431fdf..6afe4115f 100644 --- a/single-node-refactor/src/driver.h +++ b/single-node-refactor/src/driver.h @@ -44,7 +44,8 @@ #include "state.h" -void fill_regions(DCArrayKokkos&, +void fill_regions(CArrayKokkos&, + CArray &, Material_t&, mesh_t&, node_t&, @@ -228,6 +229,7 @@ class Driver //fill_regions(); fill_regions(SimulationParamaters.region_fills, + SimulationParamaters.region_fills_host, Materials, mesh, node, @@ -328,7 +330,8 @@ class Driver /// \brief Fills mesh regions based on YAML input /// ///////////////////////////////////////////////////////////////////////////// - void fill_regions(DCArrayKokkos& region_fills, + void fill_regions(CArrayKokkos& region_fills, + CArray ®ion_fills_host, Material_t& Materials, mesh_t& mesh, node_t& node, @@ -376,10 +379,10 @@ class Driver voxel_dx, voxel_dy, voxel_dz, orig_x, orig_y, orig_z, voxel_num_i, voxel_num_j, voxel_num_k, - region_fills(f_id).scale_x, - region_fills(f_id).scale_y, - region_fills(f_id).scale_z, - region_fills(f_id).file_path); + region_fills_host(f_id).scale_x, + region_fills_host(f_id).scale_y, + region_fills_host(f_id).scale_z, + region_fills_host(f_id).file_path); // copy values read from file to device voxel_elem_mat_id.update_device(); diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 454785722..2ccec85cf 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -313,7 +313,7 @@ void parse_yaml(Yaml::Node& root, SimulationParameters_t& SimulationParamaters, std::cout << "Parsing YAML regions:" << std::endl; } // parse the region yaml text into a vector of region_fills - parse_regions(root, SimulationParamaters.region_fills); + parse_regions(root, SimulationParamaters.region_fills, SimulationParamaters.region_fills_host); if (VERBOSE) { printf("\n"); @@ -898,13 +898,16 @@ void parse_output_options(Yaml::Node& root, output_options_t& output_options) // ================================================================================= // Parse Fill regions // ================================================================================= -void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) +void parse_regions(Yaml::Node& root, + CArrayKokkos& region_fills, + CArray ®ion_fills_host) { Yaml::Node& region_yaml = root["regions"]; size_t num_regions = region_yaml.Size(); - region_fills = DCArrayKokkos(num_regions , "sim_param.region_fills"); + region_fills = CArrayKokkos(num_regions , "sim_param.region_fills"); + region_fills_host = CArray(num_regions); // loop over the fill regions specified @@ -1117,9 +1120,8 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tscale_x = " << scale_x << std::endl; } - RUN({ - region_fills(reg_id).scale_x = scale_x; - }); + region_fills_host(reg_id).scale_x = scale_x; + } // scale_x else if (a_word.compare("scale_y") == 0) { // outer plane @@ -1129,9 +1131,8 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tscale_y = " << scale_y << std::endl; } - RUN({ - region_fills(reg_id).scale_y = scale_y; - }); + region_fills_host(reg_id).scale_y = scale_y; + } // scale_y else if (a_word.compare("scale_z") == 0) { // outer plane @@ -1141,9 +1142,8 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tscale_z = " << scale_z << std::endl; } - RUN({ - region_fills(reg_id).scale_z = scale_z; - }); + region_fills_host(reg_id).scale_z = scale_z; + } // scale_z else if (a_word.compare("velocity") == 0) { @@ -1325,7 +1325,7 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tfile_path = " << path << std::endl; } - region_fills(reg_id).file_path = path; // saving the absolute file path + region_fills_host(reg_id).file_path = path; // saving the absolute file path } // end file path // @@ -1376,21 +1376,28 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) // ----------------------------------------------- // check for consistency in input settings - - // if the following is true, stop simulation; must add all mesh read options - if (region_fills(reg_id).volume == region::readVoxelFile && region_fills(reg_id).file_path.empty()) { - std::cout << "ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file" << std::endl; - } - // add all mesh read options here - if (region_fills(reg_id).volume != region::readVoxelFile ) { - // this means it is a geometric definition of the region + // check to see if a file path is empty + if(region_fills_host(reg_id).file_path.empty()){ - // check to see if a file path was set - if(region_fills(reg_id).file_path.size()>0){ - std::cout << "ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region" << std::endl; - exit(0); - } + RUN({ + // if the following is true, stop simulation; must add all mesh read options + if (region_fills(reg_id).volume == region::readVoxelFile) { + printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); + exit(0); + } + }); + } // end if check + + // check to see if a file path was set + if(region_fills_host(reg_id).file_path.size()>0){ + RUN({ + if (region_fills(reg_id).volume != region::readVoxelFile){ + // this means it is a geometric definition of the region + printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); + exit(0); + } + }); } // ----------------------------------------------- diff --git a/single-node-refactor/src/input/parse_yaml.h b/single-node-refactor/src/input/parse_yaml.h index 23dbacef5..af11ef464 100644 --- a/single-node-refactor/src/input/parse_yaml.h +++ b/single-node-refactor/src/input/parse_yaml.h @@ -50,6 +50,7 @@ struct SimulationParameters_t; struct solver_input_t; struct mesh_input_t; struct reg_fill_t; +struct reg_fill_host_t; struct output_options_t; struct Material_t; @@ -105,7 +106,7 @@ void parse_mesh_input(Yaml::Node& root, mesh_input_t& mesh_input); void parse_output_options(Yaml::Node& root, output_options_t& output_options); // parse the region text -void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills); +void parse_regions(Yaml::Node& root, CArrayKokkos& region_fills, CArray ®ion_fills_host); // parse the region text void parse_materials(Yaml::Node& root, Material_t& Materials); From 4c33dfc1b462aa44a01b8090242e394d6e881f3b Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Wed, 10 Jul 2024 16:50:05 -0600 Subject: [PATCH 3/9] updated exit error to work with gpus --- single-node-refactor/src/input/parse_yaml.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 2ccec85cf..d3ba743b4 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1376,28 +1376,41 @@ void parse_regions(Yaml::Node& root, // ----------------------------------------------- // check for consistency in input settings + DCArrayKokkos error_value(1); // check to see if a file path is empty if(region_fills_host(reg_id).file_path.empty()){ RUN({ + error_value(0) = 0; + // if the following is true, stop simulation; must add all mesh read options if (region_fills(reg_id).volume == region::readVoxelFile) { printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); - exit(0); } }); + error_value.update_host(); + Kokkos::fence(); + + if(error_value(0)==1) exit(0); + } // end if check // check to see if a file path was set if(region_fills_host(reg_id).file_path.size()>0){ RUN({ + error_value(0) = 0; + if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); exit(0); } }); + error_value.update_host(); + Kokkos::fence(); + + if(error_value(0)==1) exit(0); } // ----------------------------------------------- From 3f7b96f6876dc5158e43f017cf275a0c0eca6f80 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Wed, 10 Jul 2024 17:00:14 -0600 Subject: [PATCH 4/9] updated exit error to work with gpus --- single-node-refactor/src/input/parse_yaml.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index d3ba743b4..04ad9fd63 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1404,7 +1404,6 @@ void parse_regions(Yaml::Node& root, if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); - exit(0); } }); error_value.update_host(); From c6a7f524de9e66dfde8aa51d6942376f34519a29 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:22:12 -0500 Subject: [PATCH 5/9] ENH: Testing Kokkos::abort --- single-node-refactor/src/input/parse_yaml.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 04ad9fd63..15cefd374 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1387,12 +1387,17 @@ void parse_regions(Yaml::Node& root, // if the following is true, stop simulation; must add all mesh read options if (region_fills(reg_id).volume == region::readVoxelFile) { printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); + + Kokkos::abort("********************************************************************************************\n + ERROR: \n + When using a file to initialize a region, a file_path must be set to point to the mesh file\n + ********************************************************************************************") } }); error_value.update_host(); Kokkos::fence(); - if(error_value(0)==1) exit(0); + //if(error_value.host(0)==1) exit(0); } // end if check @@ -1404,12 +1409,17 @@ void parse_regions(Yaml::Node& root, if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); + + Kokkos::abort("********************************************************************************************\n + ERROR: \n + hen a geometric entity defines the region, a mesh file cannot be passed to set the region\n + ********************************************************************************************") } }); error_value.update_host(); Kokkos::fence(); - if(error_value(0)==1) exit(0); + // if(error_value(0)==1) exit(0); } // ----------------------------------------------- From d0b27be2a51870fbc3bb41eedea4dbde3282a53b Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:25:36 -0500 Subject: [PATCH 6/9] BUG: Tweak string in Kokkos::abort --- single-node-refactor/src/input/parse_yaml.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 15cefd374..a43f2f745 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1388,10 +1388,10 @@ void parse_regions(Yaml::Node& root, if (region_fills(reg_id).volume == region::readVoxelFile) { printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); - Kokkos::abort("********************************************************************************************\n - ERROR: \n - When using a file to initialize a region, a file_path must be set to point to the mesh file\n - ********************************************************************************************") + Kokkos::abort("********************************************************************************************\n" + "ERROR: \n" + "When using a file to initialize a region, a file_path must be set to point to the mesh file\n" + "********************************************************************************************") } }); error_value.update_host(); @@ -1410,10 +1410,10 @@ void parse_regions(Yaml::Node& root, // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); - Kokkos::abort("********************************************************************************************\n - ERROR: \n - hen a geometric entity defines the region, a mesh file cannot be passed to set the region\n - ********************************************************************************************") + Kokkos::abort("********************************************************************************************\n" + "ERROR: \n" + "When a geometric entity defines the region, a mesh file cannot be passed to set the region\n" + "********************************************************************************************") } }); error_value.update_host(); From dc5fb21011b66f650d65be203ea740f88cb42ca9 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:28:06 -0500 Subject: [PATCH 7/9] BUG: Add missing semicolon --- single-node-refactor/src/input/parse_yaml.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index a43f2f745..b21f3d1bf 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1391,7 +1391,7 @@ void parse_regions(Yaml::Node& root, Kokkos::abort("********************************************************************************************\n" "ERROR: \n" "When using a file to initialize a region, a file_path must be set to point to the mesh file\n" - "********************************************************************************************") + "********************************************************************************************"); } }); error_value.update_host(); @@ -1413,7 +1413,7 @@ void parse_regions(Yaml::Node& root, Kokkos::abort("********************************************************************************************\n" "ERROR: \n" "When a geometric entity defines the region, a mesh file cannot be passed to set the region\n" - "********************************************************************************************") + "********************************************************************************************"); } }); error_value.update_host(); From 75b7664790330c9931dcd27c272fbf7ad8a08430 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:36:03 -0500 Subject: [PATCH 8/9] BUG: Fix style of error messages --- single-node-refactor/src/input/parse_yaml.cpp | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index b21f3d1bf..447886306 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1376,54 +1376,33 @@ void parse_regions(Yaml::Node& root, // ----------------------------------------------- // check for consistency in input settings - DCArrayKokkos error_value(1); // check to see if a file path is empty if(region_fills_host(reg_id).file_path.empty()){ RUN({ - error_value(0) = 0; - // if the following is true, stop simulation; must add all mesh read options if (region_fills(reg_id).volume == region::readVoxelFile) { - printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); - - Kokkos::abort("********************************************************************************************\n" + Kokkos::abort("\n********************************************************************************************\n" "ERROR: \n" "When using a file to initialize a region, a file_path must be set to point to the mesh file\n" - "********************************************************************************************"); + "********************************************************************************************\n"); } }); - error_value.update_host(); - Kokkos::fence(); - - //if(error_value.host(0)==1) exit(0); - } // end if check // check to see if a file path was set if(region_fills_host(reg_id).file_path.size()>0){ RUN({ - error_value(0) = 0; - if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region - printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); - - Kokkos::abort("********************************************************************************************\n" + Kokkos::abort("\n********************************************************************************************\n" "ERROR: \n" "When a geometric entity defines the region, a mesh file cannot be passed to set the region\n" - "********************************************************************************************"); + "********************************************************************************************\n"); } }); - error_value.update_host(); - Kokkos::fence(); - - // if(error_value(0)==1) exit(0); } - // ----------------------------------------------- - - } // end loop over regions } // end of function to parse region From 270a5bb3fc66746d4af83dc01c62e3945ed68e17 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Thu, 11 Jul 2024 07:54:49 -0600 Subject: [PATCH 9/9] fixed mesh_t names to be elem --- single-node-refactor/src/common/mesh.h | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/single-node-refactor/src/common/mesh.h b/single-node-refactor/src/common/mesh.h index 8e1660ea5..a491619ed 100644 --- a/single-node-refactor/src/common/mesh.h +++ b/single-node-refactor/src/common/mesh.h @@ -111,15 +111,15 @@ void bubble_sort(size_t arr[], const size_t num) } // end for i } // end function -struct zones_in_MaterialPoint_t +struct zones_in_elem_t { private: size_t num_zones_in_elem_; public: - zones_in_MaterialPoint_t() { + zones_in_elem_t() { }; - zones_in_MaterialPoint_t(const size_t num_zones_in_elem_inp) { + zones_in_elem_t(const size_t num_zones_in_elem_inp) { this->num_zones_in_elem_ = num_zones_in_elem_inp; }; @@ -138,15 +138,15 @@ struct zones_in_MaterialPoint_t }; // if material points are defined strictly internal to the element. -struct legendre_in_MaterialPoint_t +struct legendre_in_elem_t { private: size_t num_leg_gauss_in_elem_; public: - legendre_in_MaterialPoint_t() { + legendre_in_elem_t() { }; - legendre_in_MaterialPoint_t(const size_t num_leg_gauss_in_elem_inp) { + legendre_in_elem_t(const size_t num_leg_gauss_in_elem_inp) { this->num_leg_gauss_in_elem_ = num_leg_gauss_in_elem_inp; }; @@ -165,15 +165,15 @@ struct legendre_in_MaterialPoint_t }; /// if material points are defined at element interfaces -struct lobatto_in_MaterialPoint_t +struct lobatto_in_elem_t { private: size_t num_lob_gauss_in_elem_; public: - lobatto_in_MaterialPoint_t() { + lobatto_in_elem_t() { }; - lobatto_in_MaterialPoint_t(const size_t num_lob_gauss_in_elem_inp) { + lobatto_in_elem_t(const size_t num_lob_gauss_in_elem_inp) { this->num_lob_gauss_in_elem_ = num_lob_gauss_in_elem_inp; }; @@ -249,9 +249,9 @@ struct mesh_t CArrayKokkos surfs_in_elem; ///< Surfaces on an element // CArrayKokkos zones_in_elem; ///< Zones in an element - zones_in_MaterialPoint_t zones_in_elem; ///< Zones in an element - lobatto_in_MaterialPoint_t lobatto_in_elem; ///< Gauss Lobatto points in an element - legendre_in_MaterialPoint_t legendre_in_elem; ///< Gauss Legendre points in an element + zones_in_elem_t zones_in_elem; ///< Zones in an element + lobatto_in_elem_t lobatto_in_elem; ///< Gauss Lobatto points in an element + legendre_in_elem_t legendre_in_elem; ///< Gauss Legendre points in an element // ---- Node Data Definitions ---- // size_t num_nodes; ///< Number of nodes in the mesh @@ -348,10 +348,10 @@ struct mesh_t nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "mesh.nodes_in_elem"); corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "mesh.corners_in_elem"); - zones_in_elem = zones_in_MaterialPoint_t(num_zones_in_elem); + zones_in_elem = zones_in_elem_t(num_zones_in_elem); surfs_in_elem = CArrayKokkos(num_elems, num_surfs_in_elem, "mesh.surfs_in_zone"); nodes_in_zone = CArrayKokkos(num_zones, num_nodes_in_zone, "mesh.nodes_in_zone"); - legendre_in_elem = legendre_in_MaterialPoint_t(num_leg_gauss_in_elem); + legendre_in_elem = legendre_in_elem_t(num_leg_gauss_in_elem); return; }; // end method